#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
阶段2: 逐页整页截图爬取。
从 SQLite 读取所有 level2 叶子(url + screenshot_path), 依次:
  随机 5-15s 延时 -> 导航到 url -> 等待加载 -> 整页滚动拼接截图 -> 保存到树形目录
支持断点续跑(已存在截图文件则跳过)。进度写 db/crawl_progress.log
用法: python3 crawl_shots.py [start_index]
"""
import os, sys, json, time, random, sqlite3, urllib.request
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from capture_full import capture, ev

DAEMON = "http://127.0.0.1:10086/command"
SESSION = "2haohr"
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DB = os.path.join(BASE, "db", "research.db")
LOG = os.path.join(BASE, "db", "crawl_progress.log")

def navigate(url):
    body = json.dumps({"action": "navigate", "args": {"url": url, "newTab": False}, "session": SESSION}).encode()
    req = urllib.request.Request(DAEMON, data=body, headers={"Content-Type": "application/json"})
    with urllib.request.urlopen(req, timeout=45) as resp:
        return json.loads(resp.read().decode())

def wait_ready(max_wait=8):
    """等待页面加载完成(readyState + 主内容出现)"""
    t0 = time.time()
    while time.time() - t0 < max_wait:
        try:
            rs = ev("(()=>document.readyState)()", SESSION)
            if rs == "complete":
                time.sleep(1.2)
                return True
        except Exception:
            pass
        time.sleep(0.6)
    return False

def log(msg):
    line = time.strftime("%H:%M:%S ") + msg
    print(line, flush=True)
    with open(LOG, "a") as f:
        f.write(line + "\n")

def main():
    start = int(sys.argv[1]) if len(sys.argv) > 1 else 0
    conn = sqlite3.connect(DB); conn.row_factory = sqlite3.Row
    rows = conn.execute(
        "SELECT node_key,name,full_path,url,screenshot_path FROM features WHERE level=2 ORDER BY sort_order, id"
    ).fetchall()
    # 稳定顺序: 按 full_path
    rows = sorted(rows, key=lambda r: r["full_path"] or "")
    conn.close()
    total = len(rows)
    log(f"=== crawl start, total={total}, start_index={start} ===")
    done = 0; skipped = 0; failed = 0
    for i, r in enumerate(rows):
        if i < start:
            continue
        rel = r["screenshot_path"]
        out = os.path.join(BASE, rel)
        if os.path.exists(out) and os.path.getsize(out) > 3000:
            skipped += 1
            continue
        delay = random.uniform(5, 15)
        log(f"[{i+1}/{total}] wait {delay:.1f}s -> {r['full_path']}")
        time.sleep(delay)
        try:
            navigate(r["url"])
            wait_ready()
            time.sleep(random.uniform(0.8, 2.0))  # 额外抖动
            res = capture(SESSION, out, 16)
            done += 1
            log(f"    OK {r['name']} -> {rel}")
        except Exception as e:
            failed += 1
            log(f"    FAIL {r['name']} url={r['url']} err={str(e)[:150]}")
    log(f"=== DONE done={done} skipped={skipped} failed={failed} total={total} ===")

if __name__ == "__main__":
    main()