first commit

2026-09-03 15:35:23 +08:00
commit 9b26161bcf
510 changed files with 29402 additions and 0 deletions
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+阶段2: 逐页整页截图爬取。
+从 SQLite 读取所有 level2 叶子(url + screenshot_path), 依次:
+  随机 5-15s 延时 -> 导航到 url -> 等待加载 -> 整页滚动拼接截图 -> 保存到树形目录
+支持断点续跑(已存在截图文件则跳过)。进度写 db/crawl_progress.log
+用法: python3 crawl_shots.py [start_index]
+"""
+import os, sys, json, time, random, sqlite3, urllib.request
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from capture_full import capture, ev
+
+DAEMON = "http://127.0.0.1:10086/command"
+SESSION = "2haohr"
+BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DB = os.path.join(BASE, "db", "research.db")
+LOG = os.path.join(BASE, "db", "crawl_progress.log")
+
+def navigate(url):
+    body = json.dumps({"action": "navigate", "args": {"url": url, "newTab": False}, "session": SESSION}).encode()
+    req = urllib.request.Request(DAEMON, data=body, headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=45) as resp:
+        return json.loads(resp.read().decode())
+
+def wait_ready(max_wait=8):
+    """等待页面加载完成(readyState + 主内容出现)"""
+    t0 = time.time()
+    while time.time() - t0 < max_wait:
+        try:
+            rs = ev("(()=>document.readyState)()", SESSION)
+            if rs == "complete":
+                time.sleep(1.2)
+                return True
+        except Exception:
+            pass
+        time.sleep(0.6)
+    return False
+
+def log(msg):
+    line = time.strftime("%H:%M:%S ") + msg
+    print(line, flush=True)
+    with open(LOG, "a") as f:
+        f.write(line + "\n")
+
+def main():
+    start = int(sys.argv[1]) if len(sys.argv) > 1 else 0
+    conn = sqlite3.connect(DB); conn.row_factory = sqlite3.Row
+    rows = conn.execute(
+        "SELECT node_key,name,full_path,url,screenshot_path FROM features WHERE level=2 ORDER BY sort_order, id"
+    ).fetchall()
+    # 稳定顺序: 按 full_path
+    rows = sorted(rows, key=lambda r: r["full_path"] or "")
+    conn.close()
+    total = len(rows)
+    log(f"=== crawl start, total={total}, start_index={start} ===")
+    done = 0; skipped = 0; failed = 0
+    for i, r in enumerate(rows):
+        if i < start:
+            continue
+        rel = r["screenshot_path"]
+        out = os.path.join(BASE, rel)
+        if os.path.exists(out) and os.path.getsize(out) > 3000:
+            skipped += 1
+            continue
+        delay = random.uniform(5, 15)
+        log(f"[{i+1}/{total}] wait {delay:.1f}s -> {r['full_path']}")
+        time.sleep(delay)
+        try:
+            navigate(r["url"])
+            wait_ready()
+            time.sleep(random.uniform(0.8, 2.0))  # 额外抖动
+            res = capture(SESSION, out, 16)
+            done += 1
+            log(f"    OK {r['name']} -> {rel}")
+        except Exception as e:
+            failed += 1
+            log(f"    FAIL {r['name']} url={r['url']} err={str(e)[:150]}")
+    log(f"=== DONE done={done} skipped={skipped} failed={failed} total={total} ===")
+
+if __name__ == "__main__":
+    main()