from pathlib import Path
from collections import Counter
from itertools import combinations
import pandas as pd,numpy as np,json
from scipy.stats import hypergeom
R=Path(__file__).resolve().parent;g=pd.read_parquet(R/'games.parquet');t=pd.read_parquet(R.parent/'fable-verification-2026-09-06/tags.parquet')
tags=t.sort_values('rank').groupby('appid').tag.agg(list).to_dict();g['tag_n']=g.appid.map(lambda a:len(tags.get(a,[])));g['activity_n']=g.activity_tags.map(len)
rows=[];coverage=[]
def div(c):
    v=np.array(list(c.values()),float);p=v/v.sum();return dict(effective_labels=float(np.exp(-sum(p*np.log(p)))),top10_share=float(np.sort(p)[-10:].sum()))
coarse={'Action','Adventure','RPG','Strategy','Simulation','Puzzle','Sports','Racing','Horror','Visual Novel'}
for y in range(2019,2026):
    base=g[g.year.eq(y)]
    coverage.append(dict(year=y,n=len(base),median_tags=base.tag_n.median(),mean_tags=base.tag_n.mean(),share_le5=base.tag_n.le(5).mean(),share_ge15=base.tag_n.ge(15).mean(),share_exact20=base.tag_n.eq(20).mean(),mean_activity_tags=base.activity_n.mean()))
    configs=[('exact20_tags',base[base.tag_n.eq(20)],5),('first3_activity',base[base.activity_n.ge(3)],3),('100plus_reviews',base[base.reviews.ge(100)],5),('all',base,5)]
    for name,d,k in configs:
        c=Counter();pair=Counter();cc=Counter();primary=Counter()
        for a in d.itertuples():
            ts=a.activity_tags[:k]
            for tag in ts:c[tag]+=1/len(ts)
            pair.update(combinations(sorted(ts),2))
            ct=[x for x in tags.get(a.appid,[]) if x in coarse]
            for tag in ct:cc[tag]+=1/len(ct)
            primary[a.primary_activity]+=1
        N=len(d);cv=np.array(list(pair.values()));n=min(1000,N)
        rows.append(dict(year=y,configuration=name,n=N,activity_budget=k,rarefaction_n=n,expected_pairs_in_1000=hypergeom.sf(0,N,cv,n).sum(),effective_activity=div(c)['effective_labels'],primary_effective=div(primary)['effective_labels'],coarse_effective=div(cc)['effective_labels']))
pd.DataFrame(rows).to_csv(R/'coverage_sensitivities.csv',index=False);pd.DataFrame(coverage).to_csv(R/'tag_coverage.csv',index=False)
# How growth is distributed across primary labels, rather than total tag votes.
primary=pd.read_csv(R/'primary_activity.csv');a=primary[primary.period.eq('2021')].set_index('activity');b=primary[primary.period.eq('2025')].set_index('activity')
out=a[['n','share']].join(b[['n','share']],how='outer',lsuffix='_2021',rsuffix='_2025').fillna(0)
out['change_n']=out.n_2025-out.n_2021;out['change_share_pp']=100*(out.share_2025-out.share_2021);out['share_of_net_growth']=out.change_n/(out.n_2025.sum()-out.n_2021.sum());out.sort_values('change_n',ascending=False).to_csv(R/'growth_by_primary_activity.csv')
# Previously occupied versus previously unoccupied *current-tag* combinations.
pairs=pd.read_csv(R/'pair_prevalence.csv');old=pairs[pairs.period.eq('2021')].set_index(['tag_a','tag_b']);new=pairs[pairs.period.eq('2025')].set_index(['tag_a','tag_b']);z=old[['n']].join(new[['n']],how='outer',lsuffix='_2021',rsuffix='_2025').fillna(0)
z['newly_observed']=z.n_2021.eq(0)&z.n_2025.gt(0);z.to_csv(R/'pair_occupancy_changes.csv')
print('Coverage sensitivities complete')