240 lines
11 KiB
Python
240 lines
11 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
# -*- coding: utf-8 -*-
|
||
|
|
"""全站列表页深度采集器 (任务1 + 任务3)。
|
||
|
|
对每个有 URL 的叶子功能页:
|
||
|
|
1) 点「更多/更多操作」等入口, 抓取下拉菜单里的隐藏功能项(发现之前未扫描到的界面);
|
||
|
|
2) 点「自定义列/设置字段/选择表头」(#icon-custom-columns 或文字)弹窗, 抓取全部可选字段
|
||
|
|
(含"系统字段/自定义字段"分组), 远超截图与导出Excel的可见字段;
|
||
|
|
3) 兜底: 抓取默认渲染出的可见列头。
|
||
|
|
结果写 SQLite: page_extras(隐藏入口) + field_specs(完整字段字典)。
|
||
|
|
复用非均匀拟人间隔与登录态检测。"""
|
||
|
|
import json, urllib.request, time, os, random, sqlite3, collections
|
||
|
|
|
||
|
|
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
||
|
|
DB = "db/research.db"; LOG = "db/collect_fields.log"
|
||
|
|
|
||
|
|
|
||
|
|
def call(a, args, t=90):
|
||
|
|
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
||
|
|
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
||
|
|
return json.loads(urllib.request.urlopen(r, timeout=t).read().decode())
|
||
|
|
|
||
|
|
|
||
|
|
def ev(code, t=90):
|
||
|
|
try:
|
||
|
|
return call("evaluate", {"code": code}, t).get("data", {}).get("value")
|
||
|
|
except Exception as e:
|
||
|
|
return "EXC:" + str(e)[:80]
|
||
|
|
|
||
|
|
|
||
|
|
def log(m):
|
||
|
|
line = time.strftime("%H:%M:%S ") + m
|
||
|
|
print(line, flush=True)
|
||
|
|
with open(LOG, "a") as f:
|
||
|
|
f.write(line + "\n")
|
||
|
|
|
||
|
|
|
||
|
|
def db():
|
||
|
|
c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
|
||
|
|
|
||
|
|
|
||
|
|
def ensure():
|
||
|
|
c = db()
|
||
|
|
c.execute("""CREATE TABLE IF NOT EXISTS page_extras(
|
||
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
|
||
|
|
kind TEXT, label TEXT, ts TEXT)""")
|
||
|
|
c.execute("""CREATE TABLE IF NOT EXISTS field_specs(
|
||
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
|
||
|
|
field TEXT, field_group TEXT, source TEXT, sort_order INTEGER, ts TEXT)""")
|
||
|
|
c.execute("PRAGMA journal_mode=DELETE")
|
||
|
|
c.commit(); c.close()
|
||
|
|
|
||
|
|
|
||
|
|
def clear_page(nk):
|
||
|
|
c = db()
|
||
|
|
c.execute("DELETE FROM page_extras WHERE node_key=?", (nk,))
|
||
|
|
c.execute("DELETE FROM field_specs WHERE node_key=?", (nk,))
|
||
|
|
c.commit(); c.close()
|
||
|
|
|
||
|
|
|
||
|
|
def save_extras(nk, fp, url, items):
|
||
|
|
c = db(); ts = time.strftime("%Y-%m-%d %H:%M:%S")
|
||
|
|
for kind, label in items:
|
||
|
|
c.execute("INSERT INTO page_extras(node_key,full_path,url,kind,label,ts) VALUES(?,?,?,?,?,?)",
|
||
|
|
(nk, fp, url, kind, label, ts))
|
||
|
|
c.commit(); c.close()
|
||
|
|
|
||
|
|
|
||
|
|
def save_fields(nk, fp, url, fields, source):
|
||
|
|
c = db(); ts = time.strftime("%Y-%m-%d %H:%M:%S")
|
||
|
|
for i, (grp, name) in enumerate(fields):
|
||
|
|
c.execute("INSERT INTO field_specs(node_key,full_path,url,field,field_group,source,sort_order,ts)"
|
||
|
|
" VALUES(?,?,?,?,?,?,?,?)", (nk, fp, url, name, grp, source, i, ts))
|
||
|
|
c.commit(); c.close()
|
||
|
|
|
||
|
|
|
||
|
|
def human_delay(kind="page"):
|
||
|
|
r = random.random()
|
||
|
|
if kind == "page":
|
||
|
|
d = random.uniform(3.5, 8.0) if r < 0.7 else random.uniform(8.0, 18.0)
|
||
|
|
else:
|
||
|
|
d = random.uniform(1.0, 3.0) if r < 0.75 else random.uniform(3.0, 7.0)
|
||
|
|
d = max(0.7, d + random.gauss(0, 0.4))
|
||
|
|
time.sleep(d); return d
|
||
|
|
|
||
|
|
|
||
|
|
def logged_out():
|
||
|
|
h = ev("location.href")
|
||
|
|
return isinstance(h, str) and "/login" in h
|
||
|
|
|
||
|
|
|
||
|
|
# ---- JS 片段 ----
|
||
|
|
CLICK_MORE = r"""(()=>{const kws=['更多','更多操作','更多设置'];
|
||
|
|
const b=[...document.querySelectorAll('button,a,span,div')].find(x=>x.childElementCount===0&&kws.includes((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);
|
||
|
|
if(!b)return 'no-more';b.click();return 'clicked';})()"""
|
||
|
|
|
||
|
|
# 抓取下拉/浮层菜单里的操作项: 限 dropdown/popper/menu/popover 容器(可见),
|
||
|
|
# 且单容器项数<=15(超过则为筛选/树选择面板, 非操作菜单, 丢弃)
|
||
|
|
MENU_ITEMS = r"""(()=>{const BAD=/搜索|条\/页|进入员工端|进入官网|退出登录|个性化设置|按.*排序|升序|降序|排在前|排在后|帮助/;
|
||
|
|
const boxes=[...document.querySelectorAll('[class*=dropdown],[class*=Dropdown],[class*=popper],[class*=popover],[class*=Popover],[role=menu]')]
|
||
|
|
.filter(m=>{const r=m.getBoundingClientRect();return r.width>0&&r.height>0;});
|
||
|
|
let out=[];
|
||
|
|
for(const box of boxes){
|
||
|
|
const its=[...box.querySelectorAll('li,[role=menuitem],[class*=item],[class*=option]')]
|
||
|
|
.map(e=>(e.innerText||'').replace(/\s+/g,'').trim()).filter(t=>t&&t.length>1&&t.length<16&&!BAD.test(t));
|
||
|
|
const uniq=[...new Set(its)];
|
||
|
|
if(uniq.length>=1&&uniq.length<=15)out=out.concat(uniq); // 超过15视为筛选面板污染, 丢弃
|
||
|
|
}
|
||
|
|
return JSON.stringify([...new Set(out)]);})()"""
|
||
|
|
|
||
|
|
# 点自定义列/设置字段/选择表头 入口(图标或文字)
|
||
|
|
CLICK_COLSET = r"""(()=>{
|
||
|
|
const u=[...document.querySelectorAll('use')].find(u=>{const h=(u.getAttribute('xlink:href')||u.getAttribute('href')||'');return h==='#icon-custom-columns'||h==='#icon-setting'&&false;});
|
||
|
|
if(u){const b=u.closest('button,a,[role=button],span,div');if(b){b.click();return 'icon';}}
|
||
|
|
const kw=/^(自定义列|设置字段|字段设置|选择表头|表头设置|设置花名册字段|列设置|自定义字段)$/;
|
||
|
|
const t=[...document.querySelectorAll('button,a,span,div,li')].find(x=>x.childElementCount===0&&kw.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);
|
||
|
|
if(t){t.click();return 'txt';}
|
||
|
|
return 'none';})()"""
|
||
|
|
|
||
|
|
# 从"选择表头/自定义列"弹窗按顺序抓字段(复选框标签法, 已验证=花名册81字段干净)
|
||
|
|
# 限定弹窗容器(含"选择表头|已选"且checkbox>3且宽180-700高>200), walk遍历childNodes:
|
||
|
|
# text节点匹配GRPS作分组标题, LABEL标签作字段, NOISE集过滤噪音
|
||
|
|
GRAB_FIELDS = r"""(()=>{
|
||
|
|
const pop=[...document.querySelectorAll('div,aside,section')].find(m=>{const t=m.innerText||'';const r=m.getBoundingClientRect();
|
||
|
|
return r.width>180&&r.width<700&&r.height>200&&/选择表头|已选/.test(t)&&m.querySelectorAll('input[type=checkbox]').length>3;});
|
||
|
|
if(!pop)return 'no-modal';
|
||
|
|
const GRPS=['系统字段','自定义字段','基础字段','常用字段','其他字段','个人信息','工作信息','合同信息'];
|
||
|
|
const NOISE=new Set(['选择表头','没有搜索到相关内容','全部','清除','取消','确定','恢复默认','重置','搜索','已选']);
|
||
|
|
let seq=[],cur='';
|
||
|
|
const walk=el=>{for(const c of el.childNodes){
|
||
|
|
if(c.nodeType===3){const t=(c.textContent||'').trim();if(t&&t.length<15){if(GRPS.includes(t))cur=t;else if(!NOISE.has(t)&&!/^已选/.test(t))seq.push([cur,t]);}}
|
||
|
|
else if(c.nodeType===1){if(c.tagName==='LABEL'){const t=(c.innerText||'').trim();if(t&&t.length<15&&!NOISE.has(t))seq.push([cur,t]);}else walk(c);}}};
|
||
|
|
walk(pop);
|
||
|
|
const seen=new Set(),out=[];for(const [g,n] of seq){if(seen.has(n))continue;seen.add(n);out.push([g,n]);}
|
||
|
|
return JSON.stringify(out);})()"""
|
||
|
|
|
||
|
|
# 处理"页面内容已过期, 请点击 立即刷新"横幅(标签闲置后出现)
|
||
|
|
REFRESH = r"""(()=>{const b=[...document.querySelectorAll('a,button,span')].find(x=>x.childElementCount===0&&/立即刷新/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);if(b){b.click();return 'refreshed';}return 'none';})()"""
|
||
|
|
|
||
|
|
CLOSE_MODAL = r"""(()=>{const b=[...document.querySelectorAll('button,a')].find(x=>{const t=(x.innerText||'').trim();return (t==='取消'||t==='关闭')&&x.getBoundingClientRect().width>0;});
|
||
|
|
if(b){b.click();return 'closed';}document.body.click();return 'bodyclick';})()"""
|
||
|
|
|
||
|
|
# 兜底: 默认渲染的可见列头(过滤 操作/序号/全选/日期列等非字段噪音)
|
||
|
|
VISIBLE_COLS = r"""(()=>{const BAD=/^(操作|序号|全选|选择|处理)$|^\d+日|^\d+$/;
|
||
|
|
const heads=[...document.querySelectorAll('[role=columnheader],th,[class*=headerCell],[class*=header_] [class*=cell_],[class*=thead] [class*=cell]')]
|
||
|
|
.map(e=>(e.innerText||'').replace(/\s+/g,'').trim()).filter(t=>t&&t.length<20&&!BAD.test(t));
|
||
|
|
return JSON.stringify([...new Set(heads)]);})()"""
|
||
|
|
|
||
|
|
|
||
|
|
def process(p, idx, total):
|
||
|
|
fp, url, nk = p["full_path"], p["url"], p["node_key"]
|
||
|
|
log("[%d/%d] %s" % (idx, total, fp))
|
||
|
|
call("navigate", {"url": url, "newTab": False})
|
||
|
|
time.sleep(4)
|
||
|
|
for _ in range(4):
|
||
|
|
if not logged_out():
|
||
|
|
break
|
||
|
|
time.sleep(2)
|
||
|
|
if logged_out():
|
||
|
|
log(" !! 登录态失效, 中止"); return -1
|
||
|
|
# 处理"页面内容已过期"横幅(标签闲置后出现)
|
||
|
|
if ev(REFRESH) == "refreshed":
|
||
|
|
time.sleep(3)
|
||
|
|
clear_page(nk)
|
||
|
|
|
||
|
|
# ---- 任务1: 更多菜单隐藏入口(菜单渲染有时滞后, 重试抓取) ----
|
||
|
|
extras = []
|
||
|
|
for attempt in range(3):
|
||
|
|
if ev(CLICK_MORE) != "clicked":
|
||
|
|
break
|
||
|
|
time.sleep(1.2 + attempt * 0.8)
|
||
|
|
try:
|
||
|
|
items = json.loads(ev(MENU_ITEMS))
|
||
|
|
except Exception:
|
||
|
|
items = []
|
||
|
|
if items:
|
||
|
|
for it in items:
|
||
|
|
extras.append(("more_menu", it))
|
||
|
|
ev(CLOSE_MODAL); time.sleep(0.5)
|
||
|
|
break
|
||
|
|
ev(CLOSE_MODAL); time.sleep(0.4)
|
||
|
|
if extras:
|
||
|
|
save_extras(nk, fp, url, extras)
|
||
|
|
log(" 更多菜单项: %d 个 -> %s" % (len(extras), "、".join(x[1] for x in extras[:8])))
|
||
|
|
|
||
|
|
# ---- 任务3: 完整字段 ----
|
||
|
|
fields = []; source = ""
|
||
|
|
if ev(CLICK_COLSET) != "none":
|
||
|
|
time.sleep(1.5)
|
||
|
|
raw = ev(GRAB_FIELDS)
|
||
|
|
if isinstance(raw, str) and raw.startswith("["):
|
||
|
|
try:
|
||
|
|
fields = json.loads(raw)
|
||
|
|
except Exception:
|
||
|
|
fields = []
|
||
|
|
source = "custom_columns"
|
||
|
|
ev(CLOSE_MODAL); time.sleep(0.5)
|
||
|
|
if not fields:
|
||
|
|
# 兜底可见列
|
||
|
|
try:
|
||
|
|
cols = json.loads(ev(VISIBLE_COLS))
|
||
|
|
except Exception:
|
||
|
|
cols = []
|
||
|
|
fields = [("", c) for c in cols]
|
||
|
|
source = "visible_cols"
|
||
|
|
if fields:
|
||
|
|
save_fields(nk, fp, url, fields, source)
|
||
|
|
log(" 字段: %d 个 (来源=%s) -> %s" % (len(fields), source, "、".join(f[1] for f in fields[:10])))
|
||
|
|
else:
|
||
|
|
log(" 无字段入口(可能非表格页)")
|
||
|
|
return len(fields)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
ensure()
|
||
|
|
call("find_tab", {"url": "2haohr.com"})
|
||
|
|
c = db()
|
||
|
|
rows = [dict(r) for r in c.execute("SELECT node_key,parent_key,name,full_path,url FROM features")]
|
||
|
|
c.close()
|
||
|
|
kids = collections.defaultdict(int)
|
||
|
|
for r in rows:
|
||
|
|
if r["parent_key"]:
|
||
|
|
kids[r["parent_key"]] += 1
|
||
|
|
leaves = [r for r in rows if kids.get(r["node_key"], 0) == 0 and r["url"]]
|
||
|
|
log("=== 全站字段采集, 共 %d 个叶子页 ===" % len(leaves))
|
||
|
|
total_f = 0
|
||
|
|
for i, p in enumerate(leaves, 1):
|
||
|
|
try:
|
||
|
|
n = process(p, i, len(leaves))
|
||
|
|
if n == -1:
|
||
|
|
log("=== 中止 ==="); break
|
||
|
|
total_f += max(0, n)
|
||
|
|
except Exception as e:
|
||
|
|
log(" 页面异常: %s" % str(e)[:100])
|
||
|
|
human_delay("page")
|
||
|
|
log("=== DONE 累计字段=%d ===" % total_f)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|