from pathlib import Path
from collections import Counter
from itertools import combinations
import pandas as pd,numpy as np,json
from scipy.stats import hypergeom
R=Path(__file__).resolve().parent
g=pd.read_parquet(R.parent/'fable-verification-2026-09-06/games.parquet');t=pd.read_parquet(R.parent/'fable-verification-2026-09-06/tags.parquet')
# Fixed before examining outcomes. A transparent activity/genre label set, not
# a classifier of true mechanics or creativity. Art/setting/mood labels omitted.
activity=set('Action|Action RPG|Action RTS|Action Roguelike|Action-Adventure|Adventure|Arcade|Arena Shooter|Auto Battler|Automation|Automobile Sim|Base Building|Baseball|Basketball|Battle Royale|Beat \'em up|Billiards|Board Game|Boomer Shooter|Boss Rush|Bowling|Boxing|Building|Bullet Heaven|Bullet Hell|CRPG|Card Battler|Card Game|Character Action Game|Chess|Choose Your Own Adventure|City Builder|Cleaning|Collectathon|Colony Sim|Combat Racing|Cooking|Crafting|Creature Collector|Cricket|Dating Sim|Deckbuilding|Decorating|Desktop Companion|Detective|Dice|Diplomacy|Driving|Dungeon Crawler|Economy|Education|Escape Room|Exploration|Extraction Shooter|FMV|FPS|Farming|Farming Sim|Fighting|Fishing|Flight|Football (American)|Football (Soccer)|Gambling|Game Development|God Game|Golf|Grand Strategy|Hack and Slash|Hacking|Heist|Hero Shooter|Hidden Object|Hobby Sim|Hockey|Horror|Hunting|Idler|Immersive Sim|Incremental|Interactive Fiction|Inventory Management|Investigation|JRPG|Job Simulator|Life Sim|Logic|Loot|Looter Shooter|MMORPG|MOBA|Mahjong|Management|Match 3|Medical Sim|Metroidvania|Mini Golf|Mining|Musou|Mystery|Mystery Dungeon|Naval Combat|On-Rails Shooter|Open World Survival Craft|Organizing|Otome|Outbreak Sim|Parkour|Party Game|Party-Based RPG|Pinball|Platformer|Point & Click|Poker|Political Sim|Precision Platformer|Programming|Puzzle|Puzzle Platformer|RPG|RTS|Racing|Real Time Tactics|Resource Management|Rhythm|Roguelike|Roguelike Deckbuilder|Roguelite|Rugby|Runner|Sailing|Sandbox|Shoot \'Em Up|Shooter|Shop Keeper|Skateboarding|Skiing|Sniper|Snooker|Snowboarding|Social Deduction|Sokoban|Solitaire|Souls-like|Space Sim|Sports|Stealth|Strategy|Strategy RPG|Survival|Survival Horror|Tabletop|Tactical|Tactical RPG|Tennis|Third-Person Shooter|Thriller|Time Management|Top-Down Shooter|Tower Defense|Trading|Trading Card Game|Traditional Roguelike|Transportation|Trivia|Turn-Based Strategy|Turn-Based Tactics|Twin Stick Shooter|Typing|Vehicular Combat|Visual Novel|Volleyball|Walking Simulator|Wargame|Word Game|Wrestling|4X|2D Fighter|3D Fighter|2D Platformer|3D Platformer'.split('|'))
unknown=activity-set(t.tag);assert not unknown,unknown
admin={'Indie','Singleplayer','Multiplayer','Early Access','Free to Play','Controller','Great Soundtrack','Masterpiece','Cult Classic','Classic','Kickstarter','Crowdfunded','Steam Achievements','Software','Utilities'}
taglists=t.sort_values('rank').groupby('appid').tag.agg(list).to_dict()
g=g[g.valid&~g.free&~g.explicit&g.year.between(2019,2026)&((g.year<2026)|g.month.le(8))].copy()
g['activity_tags']=g.appid.map(lambda a:[x for x in taglists.get(a,[]) if x in activity])
g['primary_activity']=g.activity_tags.map(lambda x:x[0] if x else 'unclassified')
g['content_tags']=g.appid.map(lambda a:[x for x in taglists.get(a,[]) if x not in admin])
g.to_parquet(R/'games.parquet',index=False)
spec={'scope':'Paid released games outside explicit descriptors 3/4; snapshot dates corrected as prior study; 2019-2025 full years, 2026 Jan-Aug separate','activity_tags':sorted(activity),'content_admin_exclusions':sorted(admin),'rarefaction_n':3000,'activity_pair_definition':'any two of the first five ranked activity tags among the returned top20','tag_weighting':'one total vote per game spread over its retained tags','interpretation':'Diversity of current advertised labels attached to release cohorts. Not historical tag vocabularies, actual mechanic diversity, originality, lineage, or gameplay verification.'}
(R/'study_spec.json').write_text(json.dumps(spec,indent=2))
def diversity(c):
    a=np.array(list(c.values()),float);p=a/a.sum();return {'effective_shannon':float(np.exp(-(p*np.log(p)).sum())),'effective_simpson':float(1/(p*p).sum()),'top10_share':float(np.sort(p)[-10:].sum()),'observed_labels':len(a)}
rows=[];primary=[];tagrows=[];pairsrows=[];pair_members=[]
periods=[(str(y),g[g.year.eq(y)]) for y in range(2019,2026)]+[(f'{y}JanAug',g[g.year.eq(y)&g.month.le(8)]) for y in [2025,2026]]
for period,d in periods:
    for representation in ['all_top20','activity_tags','primary_activity']:
        c=Counter();presence=Counter()
        for r in d.itertuples():
            tags=taglists.get(r.appid,[]) if representation=='all_top20' else r.activity_tags if representation=='activity_tags' else [r.primary_activity]
            if not tags:continue
            for tag in tags:c[tag]+=1/len(tags);presence[tag]+=1
        rows.append(dict(period=period,representation=representation,n=len(d),**diversity(c)))
        if representation=='activity_tags':
            tagrows.extend(dict(period=period,tag=tag,n=n,share=n/len(d)) for tag,n in presence.items())
    pc=d.primary_activity.value_counts();primary.extend(dict(period=period,activity=tag,n=int(n),share=n/len(d)) for tag,n in pc.items())
    counts=Counter();weighted=Counter()
    for r in d.itertuples():
        ps=list(combinations(sorted(r.activity_tags[:5]),2))
        for pair in ps:counts[pair]+=1;weighted[pair]+=1/len(ps)
    N=len(d);n=3000;cv=np.array(list(counts.values()))
    expected1=float(hypergeom.sf(0,N,cv,n).sum());expected5=float(hypergeom.sf(4,N,cv,n).sum())
    pairsrows.append(dict(period=period,n=N,raw_pairs=len(counts),raw_pairs_ge5=int((cv>=5).sum()),raw_pairs_ge20=int((cv>=20).sum()),expected_pairs_in_3000=expected1,expected_pairs_ge5_in_3000=expected5,**diversity(weighted)))
    pair_members.extend(dict(period=period,tag_a=a,tag_b=b,n=k,share=k/N) for (a,b),k in counts.items())
pd.DataFrame(rows).to_csv(R/'diversity.csv',index=False);pd.DataFrame(primary).to_csv(R/'primary_activity.csv',index=False);pd.DataFrame(tagrows).to_csv(R/'activity_prevalence.csv',index=False);pd.DataFrame(pairsrows).to_csv(R/'pair_diversity.csv',index=False);pd.DataFrame(pair_members).to_csv(R/'pair_prevalence.csv',index=False)
print('Variety normalization complete',len(g),'games',len(activity),'activity labels',flush=True)