#!/usr/bin/env python3 # -*- coding: utf-8 -*- """全站列表页深度采集器 (任务1 + 任务3)。 对每个有 URL 的叶子功能页: 1) 点「更多/更多操作」等入口, 抓取下拉菜单里的隐藏功能项(发现之前未扫描到的界面); 2) 点「自定义列/设置字段/选择表头」(#icon-custom-columns 或文字)弹窗, 抓取全部可选字段 (含"系统字段/自定义字段"分组), 远超截图与导出Excel的可见字段; 3) 兜底: 抓取默认渲染出的可见列头。 结果写 SQLite: page_extras(隐藏入口) + field_specs(完整字段字典)。 复用非均匀拟人间隔与登录态检测。""" import json, urllib.request, time, os, random, sqlite3, collections D = "http://127.0.0.1:10086/command"; S = "2haohr" DB = "db/research.db"; LOG = "db/collect_fields.log" def call(a, args, t=90): b = json.dumps({"action": a, "args": args, "session": S}).encode() r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"}) return json.loads(urllib.request.urlopen(r, timeout=t).read().decode()) def ev(code, t=90): try: return call("evaluate", {"code": code}, t).get("data", {}).get("value") except Exception as e: return "EXC:" + str(e)[:80] def log(m): line = time.strftime("%H:%M:%S ") + m print(line, flush=True) with open(LOG, "a") as f: f.write(line + "\n") def db(): c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c def ensure(): c = db() c.execute("""CREATE TABLE IF NOT EXISTS page_extras( id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT, kind TEXT, label TEXT, ts TEXT)""") c.execute("""CREATE TABLE IF NOT EXISTS field_specs( id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT, field TEXT, field_group TEXT, source TEXT, sort_order INTEGER, ts TEXT)""") c.execute("PRAGMA journal_mode=DELETE") c.commit(); c.close() def clear_page(nk): c = db() c.execute("DELETE FROM page_extras WHERE node_key=?", (nk,)) c.execute("DELETE FROM field_specs WHERE node_key=?", (nk,)) c.commit(); c.close() def save_extras(nk, fp, url, items): c = db(); ts = time.strftime("%Y-%m-%d %H:%M:%S") for kind, label in items: c.execute("INSERT INTO page_extras(node_key,full_path,url,kind,label,ts) VALUES(?,?,?,?,?,?)", (nk, fp, url, kind, label, ts)) c.commit(); c.close() def save_fields(nk, fp, url, fields, source): c = db(); ts = time.strftime("%Y-%m-%d %H:%M:%S") for i, (grp, name) in enumerate(fields): c.execute("INSERT INTO field_specs(node_key,full_path,url,field,field_group,source,sort_order,ts)" " VALUES(?,?,?,?,?,?,?,?)", (nk, fp, url, name, grp, source, i, ts)) c.commit(); c.close() def human_delay(kind="page"): r = random.random() if kind == "page": d = random.uniform(3.5, 8.0) if r < 0.7 else random.uniform(8.0, 18.0) else: d = random.uniform(1.0, 3.0) if r < 0.75 else random.uniform(3.0, 7.0) d = max(0.7, d + random.gauss(0, 0.4)) time.sleep(d); return d def logged_out(): h = ev("location.href") return isinstance(h, str) and "/login" in h # ---- JS 片段 ---- CLICK_MORE = r"""(()=>{const kws=['更多','更多操作','更多设置']; const b=[...document.querySelectorAll('button,a,span,div')].find(x=>x.childElementCount===0&&kws.includes((x.innerText||'').trim())&&x.getBoundingClientRect().width>0); if(!b)return 'no-more';b.click();return 'clicked';})()""" # 抓取下拉/浮层菜单里的操作项: 限 dropdown/popper/menu/popover 容器(可见), # 且单容器项数<=15(超过则为筛选/树选择面板, 非操作菜单, 丢弃) MENU_ITEMS = r"""(()=>{const BAD=/搜索|条\/页|进入员工端|进入官网|退出登录|个性化设置|按.*排序|升序|降序|排在前|排在后|帮助/; const boxes=[...document.querySelectorAll('[class*=dropdown],[class*=Dropdown],[class*=popper],[class*=popover],[class*=Popover],[role=menu]')] .filter(m=>{const r=m.getBoundingClientRect();return r.width>0&&r.height>0;}); let out=[]; for(const box of boxes){ const its=[...box.querySelectorAll('li,[role=menuitem],[class*=item],[class*=option]')] .map(e=>(e.innerText||'').replace(/\s+/g,'').trim()).filter(t=>t&&t.length>1&&t.length<16&&!BAD.test(t)); const uniq=[...new Set(its)]; if(uniq.length>=1&&uniq.length<=15)out=out.concat(uniq); // 超过15视为筛选面板污染, 丢弃 } return JSON.stringify([...new Set(out)]);})()""" # 点自定义列/设置字段/选择表头 入口(图标或文字) CLICK_COLSET = r"""(()=>{ const u=[...document.querySelectorAll('use')].find(u=>{const h=(u.getAttribute('xlink:href')||u.getAttribute('href')||'');return h==='#icon-custom-columns'||h==='#icon-setting'&&false;}); if(u){const b=u.closest('button,a,[role=button],span,div');if(b){b.click();return 'icon';}} const kw=/^(自定义列|设置字段|字段设置|选择表头|表头设置|设置花名册字段|列设置|自定义字段)$/; const t=[...document.querySelectorAll('button,a,span,div,li')].find(x=>x.childElementCount===0&&kw.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0); if(t){t.click();return 'txt';} return 'none';})()""" # 从"选择表头/自定义列"弹窗按顺序抓字段(复选框标签法, 已验证=花名册81字段干净) # 限定弹窗容器(含"选择表头|已选"且checkbox>3且宽180-700高>200), walk遍历childNodes: # text节点匹配GRPS作分组标题, LABEL标签作字段, NOISE集过滤噪音 GRAB_FIELDS = r"""(()=>{ const pop=[...document.querySelectorAll('div,aside,section')].find(m=>{const t=m.innerText||'';const r=m.getBoundingClientRect(); return r.width>180&&r.width<700&&r.height>200&&/选择表头|已选/.test(t)&&m.querySelectorAll('input[type=checkbox]').length>3;}); if(!pop)return 'no-modal'; const GRPS=['系统字段','自定义字段','基础字段','常用字段','其他字段','个人信息','工作信息','合同信息']; const NOISE=new Set(['选择表头','没有搜索到相关内容','全部','清除','取消','确定','恢复默认','重置','搜索','已选']); let seq=[],cur=''; const walk=el=>{for(const c of el.childNodes){ if(c.nodeType===3){const t=(c.textContent||'').trim();if(t&&t.length<15){if(GRPS.includes(t))cur=t;else if(!NOISE.has(t)&&!/^已选/.test(t))seq.push([cur,t]);}} else if(c.nodeType===1){if(c.tagName==='LABEL'){const t=(c.innerText||'').trim();if(t&&t.length<15&&!NOISE.has(t))seq.push([cur,t]);}else walk(c);}}}; walk(pop); const seen=new Set(),out=[];for(const [g,n] of seq){if(seen.has(n))continue;seen.add(n);out.push([g,n]);} return JSON.stringify(out);})()""" # 处理"页面内容已过期, 请点击 立即刷新"横幅(标签闲置后出现) REFRESH = r"""(()=>{const b=[...document.querySelectorAll('a,button,span')].find(x=>x.childElementCount===0&&/立即刷新/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);if(b){b.click();return 'refreshed';}return 'none';})()""" CLOSE_MODAL = r"""(()=>{const b=[...document.querySelectorAll('button,a')].find(x=>{const t=(x.innerText||'').trim();return (t==='取消'||t==='关闭')&&x.getBoundingClientRect().width>0;}); if(b){b.click();return 'closed';}document.body.click();return 'bodyclick';})()""" # 兜底: 默认渲染的可见列头(过滤 操作/序号/全选/日期列等非字段噪音) VISIBLE_COLS = r"""(()=>{const BAD=/^(操作|序号|全选|选择|处理)$|^\d+日|^\d+$/; const heads=[...document.querySelectorAll('[role=columnheader],th,[class*=headerCell],[class*=header_] [class*=cell_],[class*=thead] [class*=cell]')] .map(e=>(e.innerText||'').replace(/\s+/g,'').trim()).filter(t=>t&&t.length<20&&!BAD.test(t)); return JSON.stringify([...new Set(heads)]);})()""" def process(p, idx, total): fp, url, nk = p["full_path"], p["url"], p["node_key"] log("[%d/%d] %s" % (idx, total, fp)) call("navigate", {"url": url, "newTab": False}) time.sleep(4) for _ in range(4): if not logged_out(): break time.sleep(2) if logged_out(): log(" !! 登录态失效, 中止"); return -1 # 处理"页面内容已过期"横幅(标签闲置后出现) if ev(REFRESH) == "refreshed": time.sleep(3) clear_page(nk) # ---- 任务1: 更多菜单隐藏入口(菜单渲染有时滞后, 重试抓取) ---- extras = [] for attempt in range(3): if ev(CLICK_MORE) != "clicked": break time.sleep(1.2 + attempt * 0.8) try: items = json.loads(ev(MENU_ITEMS)) except Exception: items = [] if items: for it in items: extras.append(("more_menu", it)) ev(CLOSE_MODAL); time.sleep(0.5) break ev(CLOSE_MODAL); time.sleep(0.4) if extras: save_extras(nk, fp, url, extras) log(" 更多菜单项: %d 个 -> %s" % (len(extras), "、".join(x[1] for x in extras[:8]))) # ---- 任务3: 完整字段 ---- fields = []; source = "" if ev(CLICK_COLSET) != "none": time.sleep(1.5) raw = ev(GRAB_FIELDS) if isinstance(raw, str) and raw.startswith("["): try: fields = json.loads(raw) except Exception: fields = [] source = "custom_columns" ev(CLOSE_MODAL); time.sleep(0.5) if not fields: # 兜底可见列 try: cols = json.loads(ev(VISIBLE_COLS)) except Exception: cols = [] fields = [("", c) for c in cols] source = "visible_cols" if fields: save_fields(nk, fp, url, fields, source) log(" 字段: %d 个 (来源=%s) -> %s" % (len(fields), source, "、".join(f[1] for f in fields[:10]))) else: log(" 无字段入口(可能非表格页)") return len(fields) def main(): ensure() call("find_tab", {"url": "2haohr.com"}) c = db() rows = [dict(r) for r in c.execute("SELECT node_key,parent_key,name,full_path,url FROM features")] c.close() kids = collections.defaultdict(int) for r in rows: if r["parent_key"]: kids[r["parent_key"]] += 1 leaves = [r for r in rows if kids.get(r["node_key"], 0) == 0 and r["url"]] log("=== 全站字段采集, 共 %d 个叶子页 ===" % len(leaves)) total_f = 0 for i, p in enumerate(leaves, 1): try: n = process(p, i, len(leaves)) if n == -1: log("=== 中止 ==="); break total_f += max(0, n) except Exception as e: log(" 页面异常: %s" % str(e)[:100]) human_delay("page") log("=== DONE 累计字段=%d ===" % total_f) if __name__ == "__main__": main()