#!/usr/bin/env python3
"""Find remaining tweets via different search operators."""
import json, subprocess, time, sys
from datetime import date, timedelta

WRAPPER = "/root/clawd/skills/x-twitter/scripts/x_twitter.py"
PROFILE = "datathrop"
SEEN_FILE = "/workspace/datathrop_progress.json"
OUTPUT_FILE = "/workspace/found_extra.jsonl"
DELAY = 3.0

with open(SEEN_FILE) as f:
    progress = json.load(f)
seen = set(progress.get('seen_ids', []))
print(f"Already seen: {len(seen)} IDs", flush=True)

new_all = []

def search(query, label=""):
    global new_all
    r = subprocess.run(
        ['python3', WRAPPER, '--profile-key', PROFILE, 'search', query, '--count', '100'],
        capture_output=True, text=True, timeout=60
    )
    if r.returncode != 0:
        if '429' in r.stderr:
            print(f"  429 ({label}), wait 90s", flush=True)
            time.sleep(90)
            return 0
        return 0
    try:
        data = json.loads(r.stdout)
        tweets = data if isinstance(data, list) else data.get('tweets', [])
    except:
        return 0
    
    count = 0
    for t in tweets:
        if not isinstance(t, dict): continue
        tid = t.get('id')
        if tid and tid not in seen:
            seen.add(tid)
            new_all.append(t)
            count += 1
    if count:
        print(f"  {label}: +{count} new (total new: {len(new_all)})", flush=True)
    return count

start = date(2011, 7, 1)
end = date(2026, 3, 13)

# Monthly with filter:replies
print("=== filter:replies monthly ===", flush=True)
month = start
while month < end:
    nm = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    search(f"from:datathrop filter:replies since:{month} until:{nm}", f"rep {month:%Y-%m}")
    month = nm
    time.sleep(DELAY)

# Monthly exclude:retweets
print("=== exclude:retweets monthly ===", flush=True)
month = start
while month < end:
    nm = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    search(f"from:datathrop exclude:retweets since:{month} until:{nm}", f"nort {month:%Y-%m}")
    month = nm
    time.sleep(DELAY)

# Monthly filter:links
print("=== filter:links monthly ===", flush=True)
month = start
while month < end:
    nm = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    search(f"from:datathrop filter:links since:{month} until:{nm}", f"link {month:%Y-%m}")
    month = nm
    time.sleep(DELAY)

# Monthly filter:media
print("=== filter:media monthly ===", flush=True)
month = start
while month < end:
    nm = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    search(f"from:datathrop filter:media since:{month} until:{nm}", f"med {month:%Y-%m}")
    month = nm
    time.sleep(DELAY)

# Monthly min_faves:1
print("=== min_faves:1 monthly ===", flush=True)
month = start
while month < end:
    nm = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    search(f"from:datathrop min_faves:1 since:{month} until:{nm}", f"pop {month:%Y-%m}")
    month = nm
    time.sleep(DELAY)

# Quarterly with broader operators
print("=== quarterly from:datathrop OR to:datathrop ===", flush=True)
quarter_start = start
while quarter_start < end:
    qend = quarter_start + timedelta(days=90)
    search(f"from:datathrop since:{quarter_start} until:{qend}", f"q {quarter_start}")
    quarter_start = qend
    time.sleep(DELAY)

# Delete what we found
print(f"\nFound {len(new_all)} new tweets. Deleting...", flush=True)
deleted = 0
failed = 0
for t in new_all:
    tid = t.get('id')
    text = t.get('text', '')
    is_rt = text.startswith('RT @')
    url = f"https://x.com/datathrop/status/{tid}"
    cmd = 'unretweet' if is_rt else 'delete'
    r = subprocess.run(
        ['python3', WRAPPER, '--profile-key', PROFILE, cmd, url],
        capture_output=True, text=True, timeout=60
    )
    ok = False
    if r.returncode == 0:
        try:
            d = json.loads(r.stdout)
            if isinstance(d, dict) and d.get('ok'): ok = True
        except: pass
    if 'not found' in r.stderr.lower() or 'not found' in r.stdout.lower(): ok = True
    if ok:
        deleted += 1
    else:
        failed += 1
        print(f"  FAIL {tid}", flush=True)
    time.sleep(2)

print(f"\n=== EXTRA DONE: found={len(new_all)} deleted={deleted} failed={failed} ===", flush=True)