#!/usr/bin/env python3
"""Find additional @datathrop posts not caught by first pass.
Focus on replies/search operators that can be omitted by default queries."""

from __future__ import annotations

import json
import subprocess
import time
from datetime import date
from pathlib import Path

WRAPPER = "/root/clawd/skills/x-twitter/scripts/x_twitter.py"
PROFILE = "datathrop"
PROGRESS = Path("/workspace/datathrop_progress.json")
OUT_NEW = Path("/workspace/extra_missing.jsonl")

# Read current seen ids from original run to avoid re-processing known tweets.
if PROGRESS.exists():
    with PROGRESS.open("r", encoding="utf-8") as f:
        progress = json.load(f)
    seen = set(progress.get("seen_ids", []))
else:
    progress = {"next_day": "2011-07-01", "found": 0, "deleted": 0, "failed": 0, "seen_ids": []}
    seen = set()


def run_search(query: str, label: str):
    """Run one search query and return list of tweet dicts."""
    while True:
        r = subprocess.run(
            [
                "python3",
                WRAPPER,
                "--profile-key",
                PROFILE,
                "search",
                query,
                "--count",
                "100",
                "--all",
                "--max-pages",
                "12",
            ],
            capture_output=True,
            text=True,
            timeout=120,
        )
        if r.returncode != 0:
            if "429" in (r.stderr + r.stdout):
                print(f"{label}: 429, wait 90s")
                time.sleep(90)
                continue
            print(f"{label}: ERROR {r.returncode}")
            return []

        raw = (r.stdout or "").strip()
        if not raw:
            return []
        try:
            data = json.loads(raw)
        except Exception:
            print(f"{label}: JSON parse error")
            return []

        tweets = data.get("tweets", []) if isinstance(data, dict) else data
        return [t for t in tweets if isinstance(t, dict)]


def save_new(lines):
    if not lines:
        return
    mode = "a" if OUT_NEW.exists() else "w"
    with OUT_NEW.open(mode, encoding="utf-8") as f:
        for t in lines:
            f.write(json.dumps(t) + "\n")


start = date(2011, 7, 1)
cutoff = date(2026, 3, 13)

all_new = []

# 1) Replies first: these were not reliably surfaced in the first pass.
print("=== pass: filter:replies by month ===")
cur = start
while cur < cutoff:
    nxt = date(cur.year + (cur.month // 12), cur.month % 12 + 1, 1)
    q = f"from:datathrop filter:replies since:{cur.isoformat()} until:{nxt.isoformat()}"
    tweets = run_search(q, f"replies {cur}")
    added = 0
    for t in tweets:
        tid = t.get("id")
        if not tid or tid in seen:
            continue
        seen.add(tid)
        all_new.append(t)
        added += 1
    if added:
        print(f"  +{added} replies for {cur:%Y-%m}")
        save_new(tweets)
    cur = nxt
    time.sleep(2.2)

# 2) Non-replies explicitly, to cross-check for any operator misses.
print("=== pass: exclude:replies by month ===")
cur = start
while cur < cutoff:
    nxt = date(cur.year + (cur.month // 12), cur.month % 12 + 1, 1)
    q = f"from:datathrop exclude:replies since:{cur.isoformat()} until:{nxt.isoformat()}"
    tweets = run_search(q, f"exclude:replies {cur}")
    added = 0
    for t in tweets:
        tid = t.get("id")
        if not tid or tid in seen:
            continue
        seen.add(tid)
        all_new.append(t)
        added += 1
    if added:
        print(f"  +{added} non-replies for {cur:%Y-%m}")
        save_new(tweets)
    cur = nxt
    time.sleep(2.2)

# 3) Raw query fallback: include both replies and retweets and sort by operator.
print("=== pass: from:datathrop by month ===")
cur = start
while cur < cutoff:
    nxt = date(cur.year + (cur.month // 12), cur.month % 12 + 1, 1)
    q = f"from:datathrop since:{cur.isoformat()} until:{nxt.isoformat()}"
    tweets = run_search(q, f"from {cur}")
    added = 0
    for t in tweets:
        tid = t.get("id")
        if not tid or tid in seen:
            continue
        seen.add(tid)
        all_new.append(t)
        added += 1
    if added:
        print(f"  +{added} raw for {cur:%Y-%m}")
        save_new(tweets)
    cur = nxt
    time.sleep(2.2)

print(f"FOUND NEW={len(all_new)}")

# Persist seen ids (trim for file size)
progress["seen_ids"] = list(seen)[-5000:]
with PROGRESS.open("w", encoding="utf-8") as f:
    json.dump(progress, f)

if not all_new:
    print("NO NEW CANDIDATES")
    raise SystemExit(0)

# Delete everything found.
ok = 0
failed = []

for i, tweet in enumerate(all_new, 1):
    tid = tweet.get("id")
    if not tid:
        continue
    text = tweet.get("text", "")
    is_rt = str(text).startswith("RT @")
    url = f"https://x.com/datathrop/status/{tid}"
    cmd = "unretweet" if is_rt else "delete"

    r = subprocess.run(
        ["python3", WRAPPER, "--profile-key", PROFILE, cmd, url],
        capture_output=True,
        text=True,
        timeout=120,
    )
    stdout = r.stdout or ""
    stderr = r.stderr or ""

    success = False
    if r.returncode == 0:
        try:
            o = json.loads(stdout)
            if isinstance(o, dict) and o.get("ok"):
                success = True
        except Exception:
            if "200" in stdout:
                success = True

    if ("not found" in stderr.lower()) or ("not found" in stdout.lower()):
        success = True

    if success:
        ok += 1
        print(f"{i}/{len(all_new)} OK {tid}")
    else:
        failed.append(tid)
        print(f"{i}/{len(all_new)} FAIL {tid}")
    time.sleep(2)

print(f"=== EXTRA RUN DONE ok={ok} fail={len(failed)} total={len(all_new)} ===")
if failed:
    Path("/workspace/extra_missing_failed.txt").write_text("\n".join(failed) + "\n", encoding="utf-8")