#!/usr/bin/env python3
"""
Find the remaining ~1319 tweets that the day-windowed search missed.
Try multiple search strategies:
1. filter:replies (only replies)
2. filter:links (tweets with links)  
3. filter:media (tweets with media)
4. min_faves:0 (everything with at least 0 likes)
5. exclude:retweets (non-RTs only)
6. Different keyword/hashtag combinations
7. Longer time windows (monthly with different offsets)
"""
import json, subprocess, time, sys
from datetime import date, timedelta

WRAPPER = "/root/clawd/skills/x-twitter/scripts/x_twitter.py"
PROFILE = "datathrop"
SEEN_FILE = "/workspace/datathrop_progress.json"
OUTPUT_FILE = "/workspace/missing_tweets.jsonl"
LOG_FILE = "/workspace/find_missing.log"
DELAY = 3.0

# Load already-seen IDs
with open(SEEN_FILE) as f:
    progress = json.load(f)
seen = set(progress.get('seen_ids', []))
print(f"Already seen: {len(seen)} IDs")

new_tweets = []

def search(query, label=""):
    """Run a search and return new (unseen) tweets."""
    global new_tweets
    for attempt in range(5):
        r = subprocess.run(
            ['python3', WRAPPER, '--profile-key', PROFILE, 'search', query, '--count', '100', '--all'],
            capture_output=True, text=True, timeout=120
        )
        if r.returncode != 0:
            if '429' in r.stderr:
                print(f"  429 ({label}), waiting 90s...")
                time.sleep(90)
                continue
            print(f"  ERROR ({label}): {r.stderr[:200]}")
            return 0
        
        try:
            data = json.loads(r.stdout)
            tweets = data if isinstance(data, list) else data.get('tweets', [])
        except:
            return 0
        
        count = 0
        for t in tweets:
            if not isinstance(t, dict):
                continue
            tid = t.get('id')
            if tid and tid not in seen:
                seen.add(tid)
                new_tweets.append(t)
                count += 1
        
        if count:
            print(f"  {label}: +{count} new (total new: {len(new_tweets)})")
            with open(OUTPUT_FILE, 'a') as f:
                for t in tweets:
                    if isinstance(t, dict) and t.get('id'):
                        f.write(json.dumps(t) + '\n')
        return count
    
    print(f"  GAVE UP ({label})")
    return 0

# Strategy 1: Monthly windows with filter:replies (replies often missed)
print("\n=== Strategy 1: filter:replies monthly ===")
start = date(2011, 7, 1)
end = date(2026, 3, 13)
month = start
while month < end:
    next_month = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    q = f"from:datathrop filter:replies since:{month.isoformat()} until:{next_month.isoformat()}"
    search(q, f"replies {month:%Y-%m}")
    month = next_month
    time.sleep(DELAY)

# Strategy 2: Monthly windows with exclude:retweets 
print("\n=== Strategy 2: exclude:retweets monthly ===")
month = start
while month < end:
    next_month = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    q = f"from:datathrop exclude:retweets since:{month.isoformat()} until:{next_month.isoformat()}"
    search(q, f"no-rt {month:%Y-%m}")
    month = next_month
    time.sleep(DELAY)

# Strategy 3: Monthly with filter:links
print("\n=== Strategy 3: filter:links monthly ===")
month = start
while month < end:
    next_month = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    q = f"from:datathrop filter:links since:{month.isoformat()} until:{next_month.isoformat()}"
    search(q, f"links {month:%Y-%m}")
    month = next_month
    time.sleep(DELAY)

# Strategy 4: Monthly with filter:media
print("\n=== Strategy 4: filter:media monthly ===")
month = start
while month < end:
    next_month = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    q = f"from:datathrop filter:media since:{month.isoformat()} until:{next_month.isoformat()}"
    search(q, f"media {month:%Y-%m}")
    month = next_month
    time.sleep(DELAY)

# Strategy 5: Monthly with min_faves:1 (popular tweets that might be indexed differently)
print("\n=== Strategy 5: min_faves:1 monthly ===")
month = start
while month < end:
    next_month = date(month.year + month.month // 12, month.month % 12 + 1, 1)
    q = f"from:datathrop min_faves:1 since:{month.isoformat()} until:{next_month.isoformat()}"
    search(q, f"popular {month:%Y-%m}")
    month = next_month
    time.sleep(DELAY)

# Strategy 6: Try half-day windows for years that had many tweets
# High-activity years: 2013, 2014, 2015, 2017, 2018, 2019
print("\n=== Strategy 6: Half-day windows for high-activity periods ===")
for year in [2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021]:
    # Try with different search operators that might surface different tweets
    q = f"from:datathrop OR to:datathrop since:{year}-01-01 until:{year+1}-01-01"
    search(q, f"from/to {year}")
    time.sleep(DELAY)

print(f"\n=== TOTAL NEW FOUND: {len(new_tweets)} ===")