"""Pre-specified tests motivated by Fable summaries 1-4; fresh source records."""
from common import *
import re,json
g,t=load();valid=g[g.valid].copy();paid=valid[~valid.free].copy();main=paid[~paid.explicit].copy()
rows=[]
for label,d in [('all',valid),('paid',paid),('paid_nonexplicit',main)]:
    for year,x in d[d.year.between(2019,2026)].groupby('year'):
        rows.append(dict(population=label,year=year,window='year' if year<2026 else 'through_September5',**stats(x)))
    for year in [2025,2026]:rows.append(dict(population=label,year=year,window='January-August',**stats(d[d.year.eq(year)&d.month.le(8)])))
save(rows,'supply.csv')
rows=[]
for label,d in [('all',valid),('paid',paid),('paid_nonexplicit_2023_25',main[main.year.between(2023,2025)])]:
    for metric in ['reviews','revenue_formula']:
        v=d.reviews if metric=='reviews' else (d.reviews*d.price*35.9).dropna()
        for frac in [.01,.05]:rows.append(dict(population=label,metric=metric,n=len(v),top_fraction=frac,share=v.nlargest(int(np.ceil(len(v)*frac))).sum()/v.sum()))
save(rows,'concentration.csv')
tags=['Arcade','Minimalist','Abstract','Score Attack','Precision Platformer','Roguelike','Roguelite','Action Roguelike','Roguelike Deckbuilder','Bullet Heaven','Auto Battler','Loot','Idler','Incremental','Desktop Companion','Shop Keeper','Boomer Shooter','FMV','Horror','Colony Sim','Anime','Turn-Based Tactics']
pairs=[['Online Co-Op','Roguelite'],['Online Co-Op','Action Roguelike'],['Roguelite','Loot'],['Local Co-Op','Action Roguelike'],['Retro','Idler'],['Card Game','Base Building'],['Roguelike','Inventory Management'],['Roguelite','Mystery'],['Loot','Idler'],['Anime','Roguelike Deckbuilder']]
periods={'2019-21':(2019,2021),'2023-24':(2023,2024),'2025':(2025,2025),'2023-25':(2023,2025),'2026JanAug':(2026,2026)}
membership={(tuple(ns),rank):member(t,ns,rank) for ns in [[n] for n in tags]+pairs for rank in [7,10,20]}
rows=[];pairrows=[];members=[]
for rank in [7,10,20]:
    for period,(y1,y2) in periods.items():
        pool=main[main.year.between(y1,y2)&(main.month.le(8) if y1==2026 else True)]
        rows.append(dict(rank=rank,period=period,group='BASELINE',**stats(pool)))
        for ns in [[n] for n in tags]+pairs:
            ids=membership[(tuple(ns),rank)];a=pool[pool.appid.isin(ids)];b=pool[~pool.appid.isin(ids)]
            row=dict(rank=rank,period=period,group=' + '.join(ns),**stats(a),**compare(a,b))
            for field in ['r100','r556','h50']:
                cheap=a[a.price.le(10)];row['cheap_'+field]=float(cheap[field].mean()) if len(cheap) else None
            row['cheap_n']=len(a[a.price.le(10)]);rows.append(row)
            if len(ns)==2:
                for n in ns:
                    component=pool[pool.appid.isin(membership[((n,),rank)] if ((n,),rank) in membership else member(t,[n],rank))&~pool.appid.isin(ids)]
                    pairrows.append(dict(rank=rank,period=period,group=' + '.join(ns),comparison=n+' without pair',comparison_n=len(component),comparison_r100_rate=component.r100.mean(),comparison_h50_rate=component.h50.mean(),**compare(a,component)))
                if period in ['2023-25','2026JanAug'] and rank in [7,20]:
                    members.extend(dict(group=' + '.join(ns),rank=rank,period=period,**z) for z in a[['appid','name','date','price','reviews','pct','dev','h50']].to_dict('records'))
save(rows,'tags_and_pairs.csv');save(pairrows,'pair_component_checks.csv');save(members,'pair_members.csv')
pool=main[main.year.between(2023,2025)]
rows=[]
for band,a in pool.groupby('price_band',observed=True):
    rows.append(dict(group='all',price_band=band,**stats(a)))
    for tag in ['Idler','Roguelike Deckbuilder','Action Roguelike','Horror','Auto Battler','Bullet Heaven']:
        rows.append(dict(group=tag,price_band=band,**stats(a[a.appid.isin(member(t,[tag],20))])))
save(rows,'prices.csv')
rows=[]
for feat,mask in [('controller',pool.controller),('10_languages',pool.lang_count.ge(10)),('demo',pool.demo),('achievements_support',pool.achievements),('selfpub',pool.selfpub),('currently_ea',pool.ea)]:
    a=pool[mask];b=pool[~mask]
    for val,x in [(True,a),(False,b)]:rows.append(dict(feature=feat,value=val,**stats(x),**(compare(x,b if val else a,cols=('year','price_band','primary')))))
save(rows,'features.csv')
NOUNS=['items','weapons','upgrades','abilities','passives','characters','classes','relics','cards','spells','perks','skills','units','heroes','artifacts','talents','augments','builds','traits','runes']
pattern=re.compile(r'(?:over\s+|more than\s+)?(\d{2,4})\s*\+?\s*(?:unique\s+|different\s+|distinct\s+|powerful\s+|collectible\s+|playable\s+|craftable\s+)*('+ '|'.join(NOUNS)+r')\b',re.I)
def claims(s):
    d={}
    for n,noun in pattern.findall(s):
        n=int(n);noun=noun.lower()
        if 10<=n<=5000:d[noun]=max(d.get(noun,0),n)
    return d
q=pool.copy();q['claims']=q.desc.map(claims);q['claim_total']=q.claims.map(lambda d:sum(d.values()))
build_names=['Roguelike','Roguelite','Action Roguelike','Roguelike Deckbuilder','Deckbuilding','Auto Battler','Bullet Heaven','Loot','Action RPG','Dungeon Crawler','Idler','Incremental']
buildids=set(t[t.tag.isin(build_names)].appid);q['build']=q.appid.isin(buildids)
rows=[];arms=[]
for label,base in [('all',q),('build_tags',q[q.build])]:
    for feat,mask in [('any',base.claim_total.gt(0)),('300plus',base.claim_total.ge(300))]+[(n,base.claims.map(lambda d:n in d)) for n in ['characters','items','weapons']]:
        a=base[mask];b=base[~mask]
        for control in [('quarter','price_band'),('year','price_band','primary','length_band')]:
            rows.append(dict(population=label,claim=feat,control='+'.join(control),**stats(a),**compare(a,b,cols=control)))
    for arm,x in [('h50',base[base.h50]),('low10',base[base.reviews.lt(10)]),('middle',base[~base.h50&base.reviews.ge(10)])]:
        arms.append(dict(population=label,arm=arm,n=len(x),any_claim=float(x.claim_total.gt(0).mean()),big300=float(x.claim_total.ge(300).mean())))
save(rows,'quantity_claims.csv');save(arms,'quantity_arm_prevalence.csv')
q.drop(columns=['desc','claims']).to_parquet(R/'claim_members.parquet',index=False)
rows=[]
for term in ['incremental','short incremental','synergies','playstyle','hundreds','co-op','craft','build','expand','levels','simple','puzzle','score','obstacles','controls','reflexes','precision','arcade','experiment']:
    for label,base in [('2023-25',pool),('2026JanAug',main[main.year.eq(2026)&main.month.le(8)])]:
        mask=base.desc.str.contains(r'\b'+re.escape(term)+r'\b',case=False,regex=True);a=base[mask];b=base[~mask]
        rows.append(dict(term=term,period=label,**stats(a),**compare(a,b,cols=('year','price_band','primary','length_band'))))
save(rows,'description_words.csv')
print('Market tests complete',flush=True)