from pathlib import Path
import pandas as pd,requests,json,time,re,html
R=Path(__file__).resolve().parent
assert not json.loads((R/'collection_policy.json').read_text())['active'],'Run only after the history collector closes'
links=pd.read_csv(R/'developer_links.csv');rows=[]
cached={r['requested_url']:r for r in pd.read_csv(R/'developer_link_checks.csv').to_dict('records')} if (R/'developer_link_checks.csv').exists() else {}
for row in links.itertuples():
    if row.developer_page_url in cached:rows.append(cached[row.developer_page_url]);continue
    time.sleep(2);url=row.developer_page_url;rec={'credit':row.credit,'requested_url':url}
    try:
        resp=requests.get(url,timeout=(10,35));rec['status']=resp.status_code;rec['final_url']=resp.url
        match=re.search(r'<title>(.*?)</title>',resp.text,re.S|re.I);rec['page_title']=html.unescape(match[1].strip()) if match else None
        rec['verified_response']=resp.status_code==200
        if resp.status_code!=200:rows.append(rec);break
    except requests.RequestException as e:rec['error']=str(e);rows.append(rec);break
    rows.append(rec)
pd.DataFrame(rows).to_csv(R/'developer_link_checks.csv',index=False);print('Developer-page checks',len(rows),'of',len(links))