import json, subprocess, time
from datetime import date
from pathlib import Path

WRAPPER='/root/clawd/skills/x-twitter/scripts/x_twitter.py'
PROFILE='datathrop'
START=date(2011,7,1)
CUT=date(2026,3,13)

seen=set()
if Path('/workspace/datathrop_progress.json').exists():
    p=json.load(open('/workspace/datathrop_progress.json'))
    seen=set(p.get('seen_ids',[]))
if Path('/workspace/extra_missing.jsonl').exists():
    for l in Path('/workspace/extra_missing.jsonl').read_text(encoding='utf-8').splitlines():
        try:
            d=json.loads(l)
            if d.get('id'):
                seen.add(d['id'])
        except:
            pass

cur=START
cnt=0
while cur < CUT:
    nxt = date(cur.year + (cur.month // 12), cur.month % 12 + 1, 1)
    q = f"from:datathrop exclude:replies since:{cur.isoformat()} until:{nxt.isoformat()}"
    done=False
    for attempt in range(1,5):
        p = subprocess.run(
            ['python3',WRAPPER,'--profile-key',PROFILE,'search',q,'--count','100','--all','--max-pages','12'],
            capture_output=True,text=True,timeout=120)
        if p.returncode == 0:
            r = (p.stdout or '').strip()
            if r:
                data = json.loads(r)
                tweets = data.get('tweets',[]) if isinstance(data, dict) else data
                add=0
                for t in tweets:
                    if isinstance(t,dict):
                        tid=t.get('id')
                        if tid and tid not in seen:
                            seen.add(tid)
                            add += 1
                            cnt += 1
                if add:
                    print(f"{cur:%Y-%m} +{add}")
            done=True
            break
        err=(p.stderr or '') + (p.stdout or '')
        if '429' in err:
            print(f"429 {cur:%Y-%m} attempt {attempt}")
            time.sleep(90)
            continue
        print(f"error {cur:%Y-%m}")
        break
    if (not done) and attempt==4:
        print(f"skip {cur:%Y-%m}")

    cur = nxt
    time.sleep(2)

print('total_new', cnt)