83 lines
3.0 KiB
Python
83 lines
3.0 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
阶段2: 逐页整页截图爬取。
|
||
|
|
从 SQLite 读取所有 level2 叶子(url + screenshot_path), 依次:
|
||
|
|
随机 5-15s 延时 -> 导航到 url -> 等待加载 -> 整页滚动拼接截图 -> 保存到树形目录
|
||
|
|
支持断点续跑(已存在截图文件则跳过)。进度写 db/crawl_progress.log
|
||
|
|
用法: python3 crawl_shots.py [start_index]
|
||
|
|
"""
|
||
|
|
import os, sys, json, time, random, sqlite3, urllib.request
|
||
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
|
|
from capture_full import capture, ev
|
||
|
|
|
||
|
|
DAEMON = "http://127.0.0.1:10086/command"
|
||
|
|
SESSION = "2haohr"
|
||
|
|
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
|
|
DB = os.path.join(BASE, "db", "research.db")
|
||
|
|
LOG = os.path.join(BASE, "db", "crawl_progress.log")
|
||
|
|
|
||
|
|
def navigate(url):
|
||
|
|
body = json.dumps({"action": "navigate", "args": {"url": url, "newTab": False}, "session": SESSION}).encode()
|
||
|
|
req = urllib.request.Request(DAEMON, data=body, headers={"Content-Type": "application/json"})
|
||
|
|
with urllib.request.urlopen(req, timeout=45) as resp:
|
||
|
|
return json.loads(resp.read().decode())
|
||
|
|
|
||
|
|
def wait_ready(max_wait=8):
|
||
|
|
"""等待页面加载完成(readyState + 主内容出现)"""
|
||
|
|
t0 = time.time()
|
||
|
|
while time.time() - t0 < max_wait:
|
||
|
|
try:
|
||
|
|
rs = ev("(()=>document.readyState)()", SESSION)
|
||
|
|
if rs == "complete":
|
||
|
|
time.sleep(1.2)
|
||
|
|
return True
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
time.sleep(0.6)
|
||
|
|
return False
|
||
|
|
|
||
|
|
def log(msg):
|
||
|
|
line = time.strftime("%H:%M:%S ") + msg
|
||
|
|
print(line, flush=True)
|
||
|
|
with open(LOG, "a") as f:
|
||
|
|
f.write(line + "\n")
|
||
|
|
|
||
|
|
def main():
|
||
|
|
start = int(sys.argv[1]) if len(sys.argv) > 1 else 0
|
||
|
|
conn = sqlite3.connect(DB); conn.row_factory = sqlite3.Row
|
||
|
|
rows = conn.execute(
|
||
|
|
"SELECT node_key,name,full_path,url,screenshot_path FROM features WHERE level=2 ORDER BY sort_order, id"
|
||
|
|
).fetchall()
|
||
|
|
# 稳定顺序: 按 full_path
|
||
|
|
rows = sorted(rows, key=lambda r: r["full_path"] or "")
|
||
|
|
conn.close()
|
||
|
|
total = len(rows)
|
||
|
|
log(f"=== crawl start, total={total}, start_index={start} ===")
|
||
|
|
done = 0; skipped = 0; failed = 0
|
||
|
|
for i, r in enumerate(rows):
|
||
|
|
if i < start:
|
||
|
|
continue
|
||
|
|
rel = r["screenshot_path"]
|
||
|
|
out = os.path.join(BASE, rel)
|
||
|
|
if os.path.exists(out) and os.path.getsize(out) > 3000:
|
||
|
|
skipped += 1
|
||
|
|
continue
|
||
|
|
delay = random.uniform(5, 15)
|
||
|
|
log(f"[{i+1}/{total}] wait {delay:.1f}s -> {r['full_path']}")
|
||
|
|
time.sleep(delay)
|
||
|
|
try:
|
||
|
|
navigate(r["url"])
|
||
|
|
wait_ready()
|
||
|
|
time.sleep(random.uniform(0.8, 2.0)) # 额外抖动
|
||
|
|
res = capture(SESSION, out, 16)
|
||
|
|
done += 1
|
||
|
|
log(f" OK {r['name']} -> {rel}")
|
||
|
|
except Exception as e:
|
||
|
|
failed += 1
|
||
|
|
log(f" FAIL {r['name']} url={r['url']} err={str(e)[:150]}")
|
||
|
|
log(f"=== DONE done={done} skipped={skipped} failed={failed} total={total} ===")
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|