432 lines
18 KiB
Python
432 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
阶段B Part1: 重走全部叶子页 -> 去遮挡 -> 重拍主截图 -> 安全抓取 4 类更深界面。
|
|
|
|
安全约束(硬性):
|
|
浏览器是用户本人登录的真实 Chrome。深层抓取一律"打开 -> 截图 -> 换页重载",
|
|
绝不点击 保存/删除/提交/确定/发送 等会产生真实数据变更的按钮(见 BAD 黑名单)。
|
|
为杜绝状态污染: 每抓一个深层界面前都重新导航到干净页面, 只打开一个目标后立即截图,
|
|
不做任何"确认"操作; 下一个深层界面靠再次重载复位。
|
|
|
|
4 类深层界面:
|
|
add 主操作弹窗(顶部工具栏 新增/添加/新建/创建/录入/发起 表单)
|
|
detail 行详情/编辑(首个数据行的 详情/查看/编辑)
|
|
filter 筛选面板(#icon-filter / 文本"筛选")
|
|
columns 自定义列/表头设置(#icon-custom-columns / 文本 自定义列/列设置/表头设置)
|
|
menu_* 更多菜单里已发现且安全的配置类入口(page_extras 白名单)
|
|
|
|
存储:
|
|
主截图: 覆盖 features.screenshot_path
|
|
深层截图: 同目录 <base>__<kind>.jpg, 记录到 page_shots(node_key,full_path,url,kind,label,path,ts)
|
|
断点续跑: page_shots 已有该 node_key 的 kind=main 记录则整页跳过(除非 --force)。
|
|
用法: python3 rewalk.py [start_index] [--force]
|
|
"""
|
|
import os, sys, json, time, sqlite3, urllib.request
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from capture_full import capture, ev, shot, call, HIDE_JS
|
|
|
|
SESSION = "2haohr"
|
|
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
DB = os.path.join(BASE, "db", "research.db")
|
|
LOG = os.path.join(BASE, "db", "rewalk_progress.log")
|
|
|
|
# ---- 操作黑名单: 深层抓取时任何匹配的按钮都绝不点击 ----
|
|
BAD = (r"删除|移除|停用|注销|作废|清空|批量|放弃|提交|保存|确定|确认|发送|禁用|冻结|解除|"
|
|
r"终止|结束|撤销|重置|清除|导入|上传|同步|生成|发放|归档|封存|开除|辞退|下架|驳回|"
|
|
r"通过|拒绝|办理|恢复|启用|锁定|解锁|导出|下载|离职|开票|付款|转账|扣款|发起流程")
|
|
|
|
# ---- 更多菜单安全白名单: 仅打开明确的配置/查看类界面 ----
|
|
MENU_SAFE = r"设置|字段|配置|规则|对比|说明|模板|方案|切换视图|调整排序|自定义列|列设置|表头|视图|统计口径"
|
|
|
|
|
|
def dbc():
|
|
c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
|
|
|
|
|
|
def log(msg):
|
|
line = time.strftime("%H:%M:%S ") + msg
|
|
print(line, flush=True)
|
|
with open(LOG, "a") as f:
|
|
f.write(line + "\n")
|
|
|
|
|
|
def navigate(url):
|
|
body = json.dumps({"action": "navigate", "args": {"url": url, "newTab": False}, "session": SESSION}).encode()
|
|
last = None
|
|
for attempt in range(4):
|
|
try:
|
|
req = urllib.request.Request(DAEMON_URL, data=body, headers={"Content-Type": "application/json"})
|
|
with _OPENER.open(req, timeout=45) as resp:
|
|
return json.loads(resp.read().decode())
|
|
except Exception as e:
|
|
last = e
|
|
time.sleep(1.5 + attempt * 1.5)
|
|
raise last
|
|
|
|
|
|
DAEMON_URL = "http://127.0.0.1:10086/command"
|
|
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
|
|
|
|
def wait_ready(max_wait=9):
|
|
t0 = time.time()
|
|
while time.time() - t0 < max_wait:
|
|
try:
|
|
if ev("(()=>document.readyState)()", SESSION) == "complete":
|
|
time.sleep(1.2); return True
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.6)
|
|
return False
|
|
|
|
|
|
# 内容稳定探测: docH(整页高度) + '共N条'数据计数
|
|
SETTLE_JS = r"""(()=>{const de=document.documentElement;const m=(document.body.innerText||'').match(/共\s*([\d,]+)\s*条/);return JSON.stringify({docH:de.scrollHeight,cnt:m?m[1].replace(/,/g,''):null});})()"""
|
|
|
|
|
|
def settle_content(max_wait=12.0):
|
|
"""截图前等内容稳定: 轮询 docH 直到连续两次不变, 且(若页面有'共N条')数据>0。
|
|
列表/报表页的表格数据是异步加载的, 加载完成后 docH 才会增长到真实高度;
|
|
若在'共0条/加载中'时截图, 会只截到首屏空表(没截全)。"""
|
|
t0 = time.time(); last = -1; stable = 0
|
|
while time.time() - t0 < max_wait:
|
|
try:
|
|
m = json.loads(ev(SETTLE_JS, SESSION) or "{}")
|
|
except Exception:
|
|
m = {}
|
|
h = int(m.get("docH") or 0)
|
|
cnt = m.get("cnt")
|
|
data_ok = (cnt is None) or (str(cnt).isdigit() and int(cnt) > 0)
|
|
stable = stable + 1 if (h == last and h > 0) else 0
|
|
last = h
|
|
if stable >= 2 and data_ok and time.time() - t0 > 1.5:
|
|
return h
|
|
time.sleep(0.5)
|
|
return last
|
|
|
|
|
|
# 主截图去遮挡: 点"立即刷新"(安全重载数据) + 跳过新手引导 + DOM 隐藏遮罩/协议层 + 隐藏浮窗
|
|
DEOCCLUDE_JS = r"""
|
|
(()=>{
|
|
const vis=el=>{const r=el.getBoundingClientRect();return r.width>0&&r.height>0&&getComputedStyle(el).visibility!=='hidden';};
|
|
let acts=[];
|
|
const refresh=[...document.querySelectorAll('a,button,span')].find(x=>x.childElementCount===0&&/立即刷新/.test((x.innerText||'').trim())&&vis(x));
|
|
if(refresh){refresh.click();acts.push('refresh');}
|
|
const skip=[...document.querySelectorAll('a,button,span,div')].find(x=>x.childElementCount===0&&/^(跳过|我知道了|知道了|跳过引导|不再提示|下次再说|以后再说)$/.test((x.innerText||'').trim())&&vis(x));
|
|
if(skip){skip.click();acts.push('skip');}
|
|
// DOM 隐藏引导遮罩/协议弹层(纯客户端, 不产生服务端动作)
|
|
const masksel=['[class*=guide]','[class*=Guide]','[class*=driver]','[class*=intro]','[class*=newbie]','[class*=Newbie]','[class*=agreement]','[class*=Agreement]','[class*=guideMask]','[class*=coach]'];
|
|
masksel.forEach(s=>document.querySelectorAll(s).forEach(el=>{const r=el.getBoundingClientRect();const p=getComputedStyle(el).position;if((p==='fixed'||p==='absolute')&&r.width>window.innerWidth*0.4&&r.height>window.innerHeight*0.3){el.style.setProperty('display','none','important');acts.push('mask');}}));
|
|
return JSON.stringify(acts);
|
|
})()
|
|
"""
|
|
|
|
|
|
def deocclude_main():
|
|
"""返回是否触发了 refresh(需重新等待)。"""
|
|
try:
|
|
acts = json.loads(ev(DEOCCLUDE_JS, SESSION) or "[]")
|
|
except Exception:
|
|
acts = []
|
|
ev(HIDE_JS, SESSION)
|
|
return "refresh" in acts
|
|
|
|
|
|
def save_shot(out_abs):
|
|
img = shot(SESSION)
|
|
os.makedirs(os.path.dirname(out_abs), exist_ok=True)
|
|
img.save(out_abs, "JPEG", quality=80)
|
|
return img.size
|
|
|
|
|
|
# ---------- 4 类深层触发 (均带 BAD 黑名单过滤, 只打开不确认) ----------
|
|
OPEN_ADD = r"""
|
|
(()=>{
|
|
const bad=/%BAD%/; const add=/(新增|添加|新建|创建|录入|发起)/;
|
|
const cands=[...document.querySelectorAll('button,a')].filter(x=>{
|
|
const t=(x.innerText||'').trim(); const r=x.getBoundingClientRect();
|
|
return t&&t.length<=8&&add.test(t)&&!bad.test(t)&&r.width>0&&r.height>0&&r.top<340;});
|
|
cands.sort((a,b)=>a.getBoundingClientRect().top-b.getBoundingClientRect().top);
|
|
if(!cands.length) return '';
|
|
const t=(cands[0].innerText||'').trim(); cands[0].click(); return t;
|
|
})()
|
|
""".replace("%BAD%", BAD)
|
|
|
|
OPEN_DETAIL = r"""
|
|
(()=>{
|
|
const bad=/%BAD%/; const want=/^(详情|查看|编辑|查看详情|详情信息)$/;
|
|
const els=[...document.querySelectorAll('span,a,button')].filter(x=>{
|
|
const t=(x.innerText||'').trim(); const r=x.getBoundingClientRect();
|
|
return want.test(t)&&!bad.test(t)&&x.childElementCount<=1&&r.width>0&&r.height>0&&r.top>150&&r.top<window.innerHeight;});
|
|
els.sort((a,b)=>{const ra=a.getBoundingClientRect(),rb=b.getBoundingClientRect();return ra.top-rb.top||ra.left-rb.left;});
|
|
if(!els.length) return '';
|
|
const t=(els[0].innerText||'').trim(); els[0].click(); return t;
|
|
})()
|
|
""".replace("%BAD%", BAD)
|
|
|
|
OPEN_FILTER = r"""
|
|
(()=>{
|
|
let target=null;
|
|
document.querySelectorAll('use').forEach(u=>{const h=u.getAttribute('xlink:href')||u.getAttribute('href')||'';if(/#icon-filter|#icon-shaixuan|filter/i.test(h)&&!target){target=u;}});
|
|
if(!target){target=[...document.querySelectorAll('button,span,div')].find(x=>x.childElementCount<=2&&/^筛选$/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);}
|
|
if(!target) return '';
|
|
let el=target; for(let i=0;i<4&⪙i++){if(el.tagName==='BUTTON'||el.getAttribute('role')==='button'||/btn|button/i.test(el.className||'')){break;}el=el.parentElement;}
|
|
(el||target).click(); return 'filter';
|
|
})()
|
|
"""
|
|
|
|
OPEN_COLUMNS = r"""
|
|
(()=>{
|
|
let target=null;
|
|
document.querySelectorAll('use').forEach(u=>{const h=u.getAttribute('xlink:href')||u.getAttribute('href')||'';if(/#icon-custom-columns|#icon-column|#icon-setting-col|columns/i.test(h)&&!target){target=u;}});
|
|
if(!target){target=[...document.querySelectorAll('button,span,div')].find(x=>x.childElementCount<=2&&/^(自定义列|列设置|表头设置|表头字段|字段设置)$/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);}
|
|
if(!target) return '';
|
|
let el=target; for(let i=0;i<4&⪙i++){if(el.tagName==='BUTTON'||el.getAttribute('role')==='button'||/btn|button/i.test(el.className||'')){break;}el=el.parentElement;}
|
|
(el||target).click(); return 'columns';
|
|
})()
|
|
"""
|
|
|
|
CLICK_MORE = r"""
|
|
(()=>{
|
|
const el=[...document.querySelectorAll('button,a,span,div')].find(x=>x.childElementCount<=2&&/^(更多操作|更多功能|···|\.\.\.|⋯)$/.test((x.innerText||'').replace(/\s/g,''))&&x.getBoundingClientRect().width>0&&x.getBoundingClientRect().top>60&&x.getBoundingClientRect().top<420);
|
|
if(el){el.click();return 'clicked';}return 'none';
|
|
})()
|
|
"""
|
|
|
|
CLICK_MENU_ITEM = r"""
|
|
(()=>{
|
|
const bad=/%BAD%/; const label=%LABEL%;
|
|
const el=[...document.querySelectorAll('[class*=dropdown] li,[class*=Dropdown] li,[class*=popper] li,[class*=Popover] li,[role=menu] li,[class*=dropdown] a,[class*=menuItem],li,[role=menuitem]')].find(x=>{
|
|
const t=(x.innerText||'').trim(); return t===label&&!bad.test(t)&&x.getBoundingClientRect().width>0;});
|
|
if(el){el.click();return 'ok';}return 'miss';
|
|
})()
|
|
"""
|
|
|
|
|
|
def modal_visible():
|
|
js = ("(()=>{const s=['.el-overlay','.el-dialog__wrapper','.el-drawer','.el-drawer__container',"
|
|
"'[class*=dialog]','[class*=Dialog]','[class*=drawer]','[class*=Drawer]','[class*=popper]',"
|
|
"'[class*=Popover]','[role=dialog]'];for(const q of s){for(const el of document.querySelectorAll(q)){"
|
|
"const r=el.getBoundingClientRect();if(r.width>120&&r.height>120&&getComputedStyle(el).display!=='none')return true;}}return false;})()")
|
|
try:
|
|
return bool(ev(js, SESSION))
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
# 统计当前可见的"浮层面板"(弹窗/抽屉/筛选抑扇/popover)数量, 用于点击前后对比判定是否弹出新面板
|
|
PANEL_COUNT_JS = r"""
|
|
(()=>{let n=0;const W=innerWidth,H=innerHeight;
|
|
document.querySelectorAll('div,aside,section,ul').forEach(el=>{
|
|
const s=getComputedStyle(el);const r=el.getBoundingClientRect();const z=parseInt(s.zIndex)||0;
|
|
const floaty=(s.position==='fixed'||s.position==='absolute');
|
|
if(floaty&&s.display!=='none'&&r.width>=200&&r.width<=W*0.99&&r.height>=160&&r.top<H-30&&r.bottom>60&&
|
|
(z>=8||/dialog|drawer|popper|popover|modal|mask|panel|Dialog|Drawer/.test(el.className||''))){n++;}
|
|
});return n;})()
|
|
"""
|
|
|
|
|
|
def panels():
|
|
try:
|
|
return int(ev(PANEL_COUNT_JS, SESSION) or 0)
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def fresh_page(url):
|
|
"""导航到干净页面: 重载 + 去遮挡 + (若刷新)再等待。"""
|
|
navigate(url); wait_ready()
|
|
if deocclude_main():
|
|
wait_ready(); deocclude_main()
|
|
settle_content()
|
|
time.sleep(0.3)
|
|
|
|
|
|
def cur_url():
|
|
try:
|
|
return ev("(()=>location.href)()", SESSION) or ""
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def url_path(u):
|
|
return (u or "").split("?")[0].split("#")[0]
|
|
|
|
|
|
def deep_capture(url, node_key, full_path, out_base, kind, open_js, label, conn, accept_route=True):
|
|
"""重载到干净页 -> 执行 open_js -> 判定出现(整页跳转表单 或 弹层) -> 截图 -> 记录。绝不做确认。"""
|
|
rel_dir = os.path.dirname(out_base)
|
|
base = os.path.splitext(os.path.basename(out_base))[0]
|
|
fname = "%s__%s.jpg" % (base, kind)
|
|
rel_path = os.path.join(rel_dir, fname)
|
|
out_abs = os.path.join(BASE, rel_path)
|
|
if os.path.exists(out_abs) and os.path.getsize(out_abs) > 3000:
|
|
return None
|
|
fresh_page(url)
|
|
before = url_path(cur_url())
|
|
b_panels = panels()
|
|
clicked = ev(open_js, SESSION)
|
|
if not clicked:
|
|
return None
|
|
time.sleep(1.3)
|
|
after = url_path(cur_url())
|
|
routed = after != before
|
|
if routed and not accept_route:
|
|
return None # filter/columns 不应发生跳转: 判为误点, 不记录
|
|
if not routed:
|
|
opened = modal_visible() or panels() > b_panels
|
|
if not opened:
|
|
return None # 点了没反馈, 放弃, 不记录
|
|
if routed:
|
|
wait_ready(); time.sleep(0.6); deocclude_main()
|
|
ev(HIDE_JS, SESSION) # 去掉可能盖在上面的浮窗
|
|
try:
|
|
if routed:
|
|
capture(SESSION, out_abs, 16) # 整页表单: 滚动拼接
|
|
else:
|
|
save_shot(out_abs) # 弹层: 单屏视口
|
|
except Exception as e:
|
|
log(" shot-fail %s %s: %s" % (kind, clicked, str(e)[:80]))
|
|
return None
|
|
conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
|
|
(node_key, full_path, url, kind, clicked if isinstance(clicked, str) else label, rel_path,
|
|
time.strftime("%Y-%m-%d %H:%M:%S")))
|
|
conn.commit()
|
|
return clicked
|
|
|
|
|
|
def process_page(r, conn, force):
|
|
url = r["url"]; node_key = r["node_key"]; full_path = r["full_path"]
|
|
rel_main = r["screenshot_path"]
|
|
out_main = os.path.join(BASE, rel_main)
|
|
|
|
# 断点续跑: 已有 main 记录且未 force 则跳过
|
|
if not force:
|
|
done = conn.execute("SELECT 1 FROM page_shots WHERE node_key=? AND kind='main' LIMIT 1",
|
|
(node_key,)).fetchone()
|
|
if done:
|
|
return "skip"
|
|
|
|
# ---- 主截图: 去遮挡后整页拼接, 覆盖原图 ----
|
|
fresh_page(url)
|
|
try:
|
|
capture(SESSION, out_main, 16)
|
|
except Exception as e:
|
|
log(" main-capture FAIL %s: %s" % (full_path, str(e)[:100]))
|
|
return "fail"
|
|
conn.execute("DELETE FROM page_shots WHERE node_key=? AND kind='main'", (node_key,))
|
|
conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
|
|
(node_key, full_path, url, "main", "主界面", rel_main, time.strftime("%Y-%m-%d %H:%M:%S")))
|
|
conn.commit()
|
|
|
|
tally = ["main"]
|
|
# ---- 4 类深层 ----
|
|
for kind, js, acc in (("filter", OPEN_FILTER, False), ("columns", OPEN_COLUMNS, False),
|
|
("add", OPEN_ADD, True), ("detail", OPEN_DETAIL, True)):
|
|
try:
|
|
c = deep_capture(url, node_key, full_path, rel_main, kind, js, None, conn, accept_route=acc)
|
|
if c:
|
|
tally.append("%s(%s)" % (kind, c if isinstance(c, str) else ""))
|
|
except Exception as e:
|
|
log(" deep %s err: %s" % (kind, str(e)[:80]))
|
|
|
|
# ---- 更多菜单安全入口(最多 3 个) ----
|
|
import re as _re
|
|
extras = conn.execute("SELECT label FROM page_extras WHERE node_key=?", (node_key,)).fetchall()
|
|
safe = []
|
|
for e in extras:
|
|
lab = (e["label"] or "").strip()
|
|
if lab and _re.search(MENU_SAFE, lab) and not _re.search(BAD, lab) and lab not in safe:
|
|
safe.append(lab)
|
|
for lab in safe[:3]:
|
|
item_js = CLICK_MENU_ITEM.replace("%BAD%", BAD).replace("%LABEL%", json.dumps(lab, ensure_ascii=False))
|
|
try:
|
|
c = deep_capture_menu(url, node_key, full_path, rel_main, lab, item_js, conn)
|
|
if c:
|
|
tally.append("menu(%s)" % lab)
|
|
except Exception as e:
|
|
log(" menu %s err: %s" % (lab, str(e)[:80]))
|
|
|
|
log(" OK %s [%s]" % (full_path, ", ".join(tally)))
|
|
return "done"
|
|
|
|
|
|
def deep_capture_menu(url, node_key, full_path, out_base, label, item_js, conn):
|
|
"""更多菜单类: 重载 -> 打开更多 -> 点安全项 -> 截图。"""
|
|
rel_dir = os.path.dirname(out_base)
|
|
base = os.path.splitext(os.path.basename(out_base))[0]
|
|
safe_label = label.replace("/", "_").replace(" ", "")[:16]
|
|
rel_path = os.path.join(rel_dir, "%s__menu_%s.jpg" % (base, safe_label))
|
|
out_abs = os.path.join(BASE, rel_path)
|
|
if os.path.exists(out_abs) and os.path.getsize(out_abs) > 3000:
|
|
return None
|
|
fresh_page(url)
|
|
b_panels = panels()
|
|
ok = False
|
|
for attempt in range(3):
|
|
if ev(CLICK_MORE, SESSION) != "clicked":
|
|
break
|
|
time.sleep(1.0 + attempt * 0.6)
|
|
if ev(item_js, SESSION) == "ok":
|
|
ok = True; break
|
|
ev("(()=>{document.body.click();return 1;})()", SESSION)
|
|
time.sleep(0.4)
|
|
if not ok:
|
|
return None
|
|
time.sleep(1.3)
|
|
after = url_path(cur_url())
|
|
routed = after != url_path(url)
|
|
if not routed and not (modal_visible() or panels() > b_panels):
|
|
return None
|
|
if routed:
|
|
wait_ready(); time.sleep(0.6); deocclude_main()
|
|
ev(HIDE_JS, SESSION)
|
|
try:
|
|
if routed:
|
|
capture(SESSION, out_abs, 16)
|
|
else:
|
|
save_shot(out_abs)
|
|
except Exception:
|
|
return None
|
|
conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
|
|
(node_key, full_path, url, "menu", label, rel_path, time.strftime("%Y-%m-%d %H:%M:%S")))
|
|
conn.commit()
|
|
return label
|
|
|
|
|
|
def main():
|
|
args = [a for a in sys.argv[1:] if a != "--force"]
|
|
force = "--force" in sys.argv
|
|
start = int(args[0]) if args else 0
|
|
|
|
conn = dbc()
|
|
conn.execute("PRAGMA journal_mode=DELETE")
|
|
rows = [dict(r) for r in conn.execute(
|
|
"SELECT node_key,name,full_path,url,screenshot_path FROM features "
|
|
"WHERE level=2 AND url IS NOT NULL AND url<>'' ORDER BY sort_order,id")]
|
|
rows = sorted(rows, key=lambda r: r["full_path"] or "")
|
|
total = len(rows)
|
|
log("=== rewalk start total=%d start=%d force=%s ===" % (total, start, force))
|
|
done = skip = fail = 0
|
|
for i, r in enumerate(rows):
|
|
if i < start:
|
|
continue
|
|
log("[%d/%d] %s" % (i + 1, total, r["full_path"]))
|
|
try:
|
|
st = process_page(r, conn, force)
|
|
except Exception as e:
|
|
st = "fail"; log(" PAGE-FAIL %s: %s" % (r["full_path"], str(e)[:120]))
|
|
if st == "done":
|
|
done += 1
|
|
elif st == "skip":
|
|
skip += 1
|
|
else:
|
|
fail += 1
|
|
conn.execute("PRAGMA journal_mode=DELETE"); conn.commit(); conn.close()
|
|
log("=== DONE done=%d skip=%d fail=%d total=%d ===" % (done, skip, fail, total))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|