from pathlib import Path
import pandas as pd,numpy as np,json,re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import MultiLabelBinarizer
R=Path(__file__).resolve().parent;g=pd.read_parquet(R/'games.parquet')
g['creator']=g.dev_full.where(g.dev_full.ne(''),g.appid.map(lambda a:'unknown:'+str(a)))
rows=[];examples=[];members=[]
def summarize(v):return dict(median=float(np.median(v)),p25=float(np.quantile(v,.25)),p75=float(np.quantile(v,.75)),p90=float(np.quantile(v,.9)),ge08=float(np.mean(v>=.8)),ge09=float(np.mean(v>=.9)),exact=float(np.mean(v>=.99999)))
def evaluate(a,mat,kind,config,seed,year):
    if kind=='tags':
        inter=(mat@mat.T).toarray().astype(float);length=np.asarray(mat.sum(axis=1)).ravel();scores=inter/(length[:,None]+length[None,:]-inter)
    else:scores=(mat@mat.T).toarray()
    np.fill_diagonal(scores,-1);best=scores.max(axis=1)
    creators=a.creator.to_numpy();same=creators[:,None]==creators[None,:];scores[same]=-1
    idx=scores.argmax(axis=1);different=scores[np.arange(len(a)),idx]
    rows.append(dict(year=year,kind=kind,configuration=config,seed=seed,n=len(a),comparison='all_other_games',**summarize(best)))
    rows.append(dict(year=year,kind=kind,configuration=config,seed=seed,n=len(a),comparison='different_credit',**summarize(different)))
    publishers=a.pub.fillna('').to_numpy();same_pub=(publishers[:,None]==publishers[None,:])&(publishers[:,None]!='')
    strict=scores.copy();strict[same_pub]=-1
    rows.append(dict(year=year,kind=kind,configuration=config,seed=seed,n=len(a),comparison='different_credit_and_publisher',**summarize(strict.max(axis=1))))
    if config=='main' and seed==77007:
        members.extend(dict(year=year,kind=kind,appid=r.appid) for r in a.itertuples())
        chosen=set(np.argsort(different)[-10:].tolist())|set(np.random.default_rng(seed+year).choice(len(a),10,replace=False).tolist())
        for i in sorted(chosen):
            j=int(idx[i]);x=a.iloc[i];y=a.iloc[j]
            examples.append(dict(year=year,kind=kind,appid=x.appid,name=x['name'],neighbor_appid=y.appid,neighbor_name=y['name'],similarity=float(different[i]),creator=x.creator,neighbor_creator=y.creator,description=x.short,neighbor_description=y.short,tags='; '.join(x.content_tags),neighbor_tags='; '.join(y.content_tags)))
for year in [2019,2021,2023,2025]:
    base=g[g.year.eq(year)].copy()
    for config,n,seed in [('main',3000,77007),('replicate',2000,77008),('replicate',2000,77009),('one_per_credit',3000,77010)]:
        eligible=base[base.content_tags.map(len).ge(5)].copy()
        if config=='one_per_credit':eligible=eligible.sample(frac=1,random_state=seed).drop_duplicates('creator')
        a=eligible.sort_values('appid').sample(min(n,len(eligible)),random_state=seed+year).reset_index(drop=True)
        mlb=MultiLabelBinarizer(sparse_output=True);mat=mlb.fit_transform(a.content_tags).astype(np.int16)
        evaluate(a,mat,'tags',config,seed,year)
    print('tag neighbors',year,flush=True)
# The primary text comparison uses short pitches to avoid full-description-length
# effects. It is lexical overlap on English-oriented text, not semantic novelty.
stops={'the','and','you','your','a','to','of','in','is','with','for','game','will','an','as','on'}
def english_pitch(s):
    words=re.findall(r"[a-z]+",str(s).lower());return len(words)>=15 and sum(w in stops for w in words)/len(words)>=.08 and sum(ord(c)<128 for c in str(s))/max(1,len(str(s)))>=.8
text=g[g.year.le(2025)&g.short.map(english_pitch)].copy()
vec=TfidfVectorizer(lowercase=True,ngram_range=(1,2),min_df=3,max_df=.7,max_features=60000,stop_words='english',dtype=np.float32)
matrix=vec.fit_transform(text.short);text['matrix_row']=np.arange(len(text));coverage=[]
for year in [2019,2021,2023,2025]:
    base=text[text.year.eq(year)];coverage.append(dict(year=year,all_n=int(g.year.eq(year).sum()),eligible_text_n=len(base)))
    for config,n,seed in [('main',3000,77007),('replicate',2000,77008),('replicate',2000,77009),('one_per_credit',3000,77010)]:
        eligible=base
        if config=='one_per_credit':eligible=eligible.sample(frac=1,random_state=seed).drop_duplicates('creator')
        a=eligible.sort_values('appid').sample(min(n,len(eligible)),random_state=seed+year).reset_index(drop=True)
        evaluate(a,matrix[a.matrix_row.to_numpy()],'short_pitch',config,seed,year)
    print('pitch neighbors',year,flush=True)
pd.DataFrame(rows).to_csv(R/'neighbor_summary.csv',index=False);pd.DataFrame(examples).to_csv(R/'neighbor_examples.csv',index=False);pd.DataFrame(members).to_csv(R/'neighbor_sample_members.csv',index=False);pd.DataFrame(coverage).to_csv(R/'text_coverage.csv',index=False)
(R/'text_spec.json').write_text(json.dumps({'representation':'TF-IDF short-pitch words and bigrams, English-oriented screen, common fixed vocabulary/IDF fitted on 2019-2025 current pitches','vocabulary_size':len(vec.vocabulary_),'screen':'at least15 Latin words, >=8% common English function/game words, >=80% ASCII characters','scope':'Textual positioning similarity, not verified design similarity, copied games, AI text, novelty or historical influence'},indent=2))