#!/usr/bin/env python3
from __future__ import annotations

import json
import subprocess
import time
from datetime import date
from pathlib import Path

WRAPPER = "/root/clawd/skills/x-twitter/scripts/x_twitter.py"
PROFILE = "datathrop"
START = date(2011, 7, 1)
CUTOFF = date(2026, 3, 13)
MAX_RETRIES = 4
RETRY_DELAY = 90
DELAY_BETWEEN_QUERIES = 2
DELAY_BETWEEN_DELETES = 1

PROGRESS = Path("/workspace/datathrop_progress.json")
OUT_JSONL = Path("/workspace/extra_missing.jsonl")
FAILED_TXT = Path("/workspace/extra_missing_failed.txt")

if PROGRESS.exists():
    progress = json.loads(PROGRESS.read_text(encoding="utf-8"))
    seen = set(progress.get("seen_ids", []))
else:
    progress = {"next_day": "2011-07-01", "found": 0, "deleted": 0, "failed": 0, "seen_ids": []}
    seen = set()

# Load already found ids
if OUT_JSONL.exists():
    for line in OUT_JSONL.read_text(encoding="utf-8").splitlines():
        try:
            t = json.loads(line)
            tid = t.get("id")
            if tid:
                seen.add(tid)
        except Exception:
            pass


def run_query(query: str):
    for attempt in range(1, MAX_RETRIES + 1):
        proc = subprocess.run(
            [
                "python3",
                WRAPPER,
                "--profile-key",
                PROFILE,
                "search",
                query,
                "--count",
                "100",
                "--all",
                "--max-pages",
                "12",
            ],
            capture_output=True,
            text=True,
            timeout=120,
        )
        if proc.returncode == 0:
            raw = (proc.stdout or "").strip()
            if not raw:
                return []
            try:
                data = json.loads(raw)
            except Exception as e:
                print(f"JSON parse fail for '{query}': {e}")
                return []
            tweets = data.get("tweets", []) if isinstance(data, dict) else data
            return [t for t in tweets if isinstance(t, dict)]

        err = (proc.stderr or "") + (proc.stdout or "")
        if "429" in err:
            print(f"429 on {query}, attempt {attempt}/{MAX_RETRIES}, wait {RETRY_DELAY}s", flush=True)
            if attempt < MAX_RETRIES:
                time.sleep(RETRY_DELAY)
                continue
            print(f"SKIP after retries: {query}", flush=True)
            return []
        print(f"ERROR {proc.returncode} on {query}: {err.strip()[:240]}", flush=True)
        return []

    return []


def save_jsonl(rows):
    if not rows:
        return
    mode = "a" if OUT_JSONL.exists() else "w"
    with OUT_JSONL.open(mode, encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r) + "\n")


def delete_tweet(tid: str, text: str) -> bool:
    op = "unretweet" if str(text).startswith("RT @") else "delete"
    url = f"https://x.com/datathrop/status/{tid}"
    p = subprocess.run(["python3", WRAPPER, "--profile-key", PROFILE, op, url], capture_output=True, text=True, timeout=120)
    if p.returncode != 0:
        err = (p.stderr or "") + (p.stdout or "")
        if "not found" in err.lower() or "does not exist" in err.lower():
            return True
        return False
    try:
        d = json.loads(p.stdout)
        return isinstance(d, dict) and d.get("ok")
    except Exception:
        return '"status_code": 200' in p.stdout or "200" in p.stdout


new_found = []

print("=== scanning by month: filter:replies ===", flush=True)
cur = START
while cur < CUTOFF:
    nxt = date(cur.year + (cur.month // 12), cur.month % 12 + 1, 1)
    q = f"from:datathrop filter:replies since:{cur.isoformat()} until:{nxt.isoformat()}"
    tweets = run_query(q)
    added = []
    for t in tweets:
        tid = t.get("id")
        if tid and tid not in seen:
            seen.add(tid)
            new_found.append(t)
            added.append(t)
    if added:
        save_jsonl(added)
        print(f"{cur:%Y-%m}: +{len(added)} new replies", flush=True)
    cur = nxt
    time.sleep(DELAY_BETWEEN_QUERIES)

print(f"FOUND replies total={len(new_found)}", flush=True)

if not new_found:
    print("No new replies found.")
    progress["seen_ids"] = list(seen)[-8000:]
    PROGRESS.write_text(json.dumps(progress), encoding="utf-8")
    raise SystemExit(0)

ok = fail = 0
fails = []

for i, t in enumerate(new_found, 1):
    tid = t.get("id")
    text = t.get("text", "")
    if not tid:
        fail += 1
        fails.append("<missing-id>")
        continue
    if delete_tweet(tid, text):
        ok += 1
        print(f"DEL {i}/{len(new_found)} OK {tid}", flush=True)
    else:
        fail += 1
        fails.append(tid)
        print(f"DEL {i}/{len(new_found)} FAIL {tid}", flush=True)
    time.sleep(DELAY_BETWEEN_DELETES)

print(f"=== DONE replies scan/delete: found={len(new_found)} ok={ok} fail={len(fails)} ===", flush=True)
if fails:
    FAILED_TXT.write_text("\n".join(fails) + "\n", encoding="utf-8")

# update main progress counters roughly
progress["found"] = progress.get("found", 0) + len(new_found)
progress["deleted"] = progress.get("deleted", 0) + ok
progress["failed"] = progress.get("failed", 0) + fail
progress["seen_ids"] = list(seen)[-8000:]
PROGRESS.write_text(json.dumps(progress), encoding="utf-8")