first commit
This commit is contained in:
Michelle Zan
Vendored
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""干净验证：报表页点下载 -> 捕获CDN直链 -> 页面内同步XHR取字节 -> base64回传 -> 落盘。"""
|
||||
import json, urllib.request, base64, time, os
|
||||
|
||||
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
||||
|
||||
def call(a, args, timeout=120):
|
||||
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
||||
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
||||
return json.loads(urllib.request.urlopen(r, timeout=timeout).read().decode())
|
||||
|
||||
def ev(code):
|
||||
return call("evaluate", {"code": code}).get("data", {}).get("value")
|
||||
|
||||
# 单一、幂等的 hook（navigate 后页面重载，hook 会被清）
|
||||
HOOK = r"""(()=>{
|
||||
if(window.__H)return 'already';
|
||||
window.__H=1; window.__cap=[];
|
||||
const ac=HTMLAnchorElement.prototype.click;
|
||||
HTMLAnchorElement.prototype.click=function(){
|
||||
try{const h=this.href||'';const d=this.getAttribute('download')||'';
|
||||
if(h && (/prv-cdn|attname=|\.xlsx|\.xls|\.docx|\.doc|\.zip|\.pdf/i.test(h)||d))
|
||||
window.__cap.push(h);}catch(e){}
|
||||
return ac.apply(this,arguments);
|
||||
};
|
||||
return 'hooked';
|
||||
})()"""
|
||||
|
||||
CLICK_DL = r"""(()=>{const cand=['直接下载','下载','生成excel','导出Excel','导出'];
|
||||
for(const t of cand){const b=[...document.querySelectorAll('button,a,[role=button]')]
|
||||
.find(x=>(x.innerText||'').trim()===t && x.getBoundingClientRect().width>0);
|
||||
if(b){b.click();return t;}}return 'no-btn';})()"""
|
||||
|
||||
# 弹窗里确认导出（选择部门后 生成excel/确定/下载）
|
||||
CONFIRM = r"""(()=>{const cand=['生成excel','确定','下载','导出','确认'];
|
||||
const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer]')];
|
||||
for(const m of ms){const r=m.getBoundingClientRect();
|
||||
if(r.width>200&&r.height>100&&getComputedStyle(m).display!=='none'){
|
||||
for(const t of cand){const b=[...m.querySelectorAll('button,a')]
|
||||
.find(x=>(x.innerText||'').trim()===t);
|
||||
if(b){b.click();return 'modal:'+t;}}}}return 'no-modal';})()"""
|
||||
|
||||
FETCH_B64 = r"""(()=>{try{
|
||||
if(!window.__cap.length)return 'NO_HREF';
|
||||
const u=window.__cap[window.__cap.length-1];
|
||||
const x=new XMLHttpRequest();x.open('GET',u,false);
|
||||
x.overrideMimeType('text/plain;charset=x-user-defined');x.send();
|
||||
if(x.status!==200)return 'HTTP'+x.status;
|
||||
const s=x.responseText;let out='';const CH=8192;
|
||||
for(let i=0;i<s.length;i+=CH){let c='';const e=Math.min(i+CH,s.length);
|
||||
for(let j=i;j<e;j++)c+=String.fromCharCode(s.charCodeAt(j)&0xff);out+=c;}
|
||||
window.__b64=btoa(out);
|
||||
const m=u.match(/attname=([^&]+)/);let name=m?decodeURIComponent(m[1]):'file';
|
||||
return JSON.stringify({size:s.length,name:name,url:u.slice(0,60)});
|
||||
}catch(e){return 'EX '+e.message;}})()"""
|
||||
|
||||
def main():
|
||||
call("find_tab", {"url": "2haohr.com"})
|
||||
call("navigate", {"url": "https://2haohr.com/personnel/employee/personnel-report/home", "newTab": False})
|
||||
# 轮询等下载按钮出现
|
||||
COUNT = r"""(()=>{let n=0;document.querySelectorAll('button,a,[role=button]').forEach(x=>{const t=(x.innerText||'').trim();if(['直接下载','下载','生成excel','导出Excel','导出'].includes(t)&&x.getBoundingClientRect().width>0)n++;});return n;})()"""
|
||||
for i in range(15):
|
||||
time.sleep(2)
|
||||
c = ev(COUNT)
|
||||
print(" wait btn count=", c)
|
||||
if isinstance(c, int) and c > 0:
|
||||
break
|
||||
print("hook:", ev(HOOK))
|
||||
print("click:", ev(CLICK_DL))
|
||||
time.sleep(2.5)
|
||||
print("confirm:", ev(CONFIRM))
|
||||
time.sleep(2.5)
|
||||
meta = ev(FETCH_B64)
|
||||
print("fetch:", meta)
|
||||
if isinstance(meta, str) and meta.startswith("{"):
|
||||
info = json.loads(meta)
|
||||
b64 = ev("window.__b64")
|
||||
data = base64.b64decode(b64)
|
||||
os.makedirs("downloads/_test", exist_ok=True)
|
||||
out = "downloads/_test/probe_" + info["name"]
|
||||
open(out, "wb").write(data)
|
||||
print("落盘:", out, len(data), "bytes")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""代码级探测：在代表性报表/文档页扫描下载入口的 DOM 特征，并观察点击后的生成过程。
|
||||
不落盘，只输出发现，用于设计正式下载器。"""
|
||||
import json, urllib.request, time
|
||||
|
||||
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
||||
|
||||
def call(a, args, timeout=120):
|
||||
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
||||
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
||||
return json.loads(urllib.request.urlopen(r, timeout=timeout).read().decode())
|
||||
|
||||
def ev(code, timeout=120):
|
||||
try:
|
||||
return call("evaluate", {"code": code}, timeout).get("data", {}).get("value")
|
||||
except Exception as e:
|
||||
return "EXC:" + str(e)[:80]
|
||||
|
||||
# 增强 hook：捕获 a.click / window.open / location 赋值 / blob URL / fetch(下载类)
|
||||
HOOK = r"""(()=>{
|
||||
window.__cap=[]; window.__evt=[];
|
||||
const push=(t,v)=>{try{window.__cap.push({t:t,v:(''+v).slice(0,220)});}catch(e){}};
|
||||
const ac=HTMLAnchorElement.prototype.click;
|
||||
HTMLAnchorElement.prototype.click=function(){try{const h=this.href||'';const d=this.getAttribute('download')||'';
|
||||
if(h&&(/prv-cdn|attname=|\.xlsx|\.xls|\.docx|\.doc|\.zip|blob:/i.test(h)||d))push('a.click',h);}catch(e){}
|
||||
return ac.apply(this,arguments);};
|
||||
const wo=window.open;window.open=function(u){if(u)push('open',u);return wo.apply(this,arguments);};
|
||||
const oc=URL.createObjectURL;URL.createObjectURL=function(b){try{push('blob','size='+(b&&b.size)+' type='+(b&&b.type));}catch(e){}return oc.apply(this,arguments);};
|
||||
const of=window.fetch;window.fetch=function(u,o){const s=''+(u&&u.url||u);if(/export|download|excel|report|generate|create.*task/i.test(s))push('fetch',s);return of.apply(this,arguments);};
|
||||
const os=XMLHttpRequest.prototype.open;XMLHttpRequest.prototype.open=function(m,u){const s=''+u;if(/export|download|excel|report|generate/i.test(s))push('xhr',s);return os.apply(this,arguments);};
|
||||
return 'hooked';
|
||||
})()"""
|
||||
|
||||
# 代码级扫描：不依赖可见文字，扫 title/aria-label/class/svg use/data-*
|
||||
SCAN = r"""(()=>{
|
||||
const KW=/下载|导出|excel|xlsx|导表|生成表|report|download|export/i;
|
||||
const out=[];const seen=new Set();
|
||||
const els=document.querySelectorAll('button,a,[role=button],i,span,div,[class*=icon],[class*=download],[class*=export],[class*=btn]');
|
||||
els.forEach(el=>{
|
||||
const t=(el.innerText||'').trim();
|
||||
const g=n=>el.getAttribute?(el.getAttribute(n)||''):'';
|
||||
const title=g('title')||g('aria-label')||g('data-title')||g('alt');
|
||||
let cls='';try{cls=(el.className&&el.className.baseVal!==undefined)?el.className.baseVal:(el.className||'');}catch(e){}
|
||||
const useEl=el.querySelector&&el.querySelector('use,svg');
|
||||
let use='';if(useEl){use=useEl.getAttribute('xlink:href')||useEl.getAttribute('href')||useEl.getAttribute('data-icon')||'';}
|
||||
const hit=KW.test(t)||KW.test(title)||KW.test(''+cls)||KW.test(use);
|
||||
if(!hit)return;
|
||||
const r=el.getBoundingClientRect();if(r.width<=0||r.height<=0)return;
|
||||
const key=el.tagName+'|'+t.slice(0,8)+'|'+title.slice(0,10)+'|'+(''+cls).slice(0,24);
|
||||
if(seen.has(key))return;seen.add(key);
|
||||
out.push({tag:el.tagName,txt:t.slice(0,12),title:(''+title).slice(0,18),cls:(''+cls).slice(0,44),use:(''+use).slice(0,26),xy:[Math.round(r.x),Math.round(r.y)]});
|
||||
});
|
||||
return JSON.stringify(out.slice(0,50));
|
||||
})()"""
|
||||
|
||||
# 观察 toast/loading 文案（生成过程提示）
|
||||
TOAST = r"""(()=>{const a=[];document.querySelectorAll('[class*=message],[class*=Message],[class*=toast],[class*=Toast],[class*=notify],[class*=loading],[class*=Loading],[class*=tip]').forEach(el=>{const t=(el.innerText||'').trim();if(t&&t.length<40&&el.getBoundingClientRect().width>0)a.push(t);});return JSON.stringify([...new Set(a)].slice(0,10));})()"""
|
||||
|
||||
def probe(url, label):
|
||||
print("\n" + "=" * 70)
|
||||
print("PAGE:", label, url)
|
||||
call("navigate", {"url": url, "newTab": False})
|
||||
time.sleep(7)
|
||||
print("scan-entries:", ev(SCAN))
|
||||
print("hook:", ev(HOOK))
|
||||
|
||||
def main():
|
||||
call("find_tab", {"url": "2haohr.com"})
|
||||
targets = [
|
||||
("https://2haohr.com/attendance/stat/monthly", "月考勤表(整表导出)"),
|
||||
("https://2haohr.com/social-insurance/reports", "社保报表"),
|
||||
("https://2haohr.com/oa/regulation", "企业制度(文档)"),
|
||||
]
|
||||
for url, label in targets:
|
||||
probe(url, label)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""探测任务中心(taskCenter)：结构、任务项状态、下载链路。并测试整表页的导出图标(hover tooltip)。"""
|
||||
import json, urllib.request, base64, time, os
|
||||
|
||||
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
||||
|
||||
def call(a, args, timeout=120):
|
||||
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
||||
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
||||
return json.loads(urllib.request.urlopen(r, timeout=timeout).read().decode())
|
||||
|
||||
def ev(code, timeout=120):
|
||||
try:
|
||||
return call("evaluate", {"code": code}, timeout).get("data", {}).get("value")
|
||||
except Exception as e:
|
||||
return "EXC:" + str(e)[:80]
|
||||
|
||||
HOOK = r"""(()=>{window.__cap=[];const ac=HTMLAnchorElement.prototype.click;
|
||||
HTMLAnchorElement.prototype.click=function(){try{const h=this.href||'';if(h&&/prv-cdn|attname=|\.xlsx|\.xls|\.docx|\.doc|\.zip/i.test(h))window.__cap.push(h);}catch(e){}return ac.apply(this,arguments);};return 'ok';})()"""
|
||||
|
||||
# 读取任务中心：定位 taskCenter 容器，列出每个任务项(名称/状态/是否有下载)
|
||||
TASKS = r"""(()=>{
|
||||
const tc=document.querySelector('[class*=taskCenter]');
|
||||
if(!tc)return 'NO_TASKCENTER';
|
||||
const items=[];
|
||||
// 任务项通常是含 info_ 名称 + right_ 状态 的行
|
||||
tc.querySelectorAll('[class*=info_]').forEach(info=>{
|
||||
const row=info.closest('[class*=item],li,tr')||info.parentElement;
|
||||
const name=(info.innerText||'').replace(/\n+/g,' ').trim().slice(0,40);
|
||||
let dl='';const a=row&&row.querySelector('a');
|
||||
const stat=row?(row.innerText||'').replace(/\n+/g,'|').slice(0,40):'';
|
||||
items.push({name:name,row:stat});
|
||||
});
|
||||
return JSON.stringify({open:getComputedStyle(tc).display,cnt:items.length,items:items.slice(0,10)});
|
||||
})()"""
|
||||
|
||||
# 点击任务中心里第 idx 个“下载”链接
|
||||
CLICK_TASK_DL = r"""(function(idx){
|
||||
const tc=document.querySelector('[class*=taskCenter]');if(!tc)return 'NO_TC';
|
||||
const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
|
||||
if(idx>=dls.length)return 'IDX_OOB len='+dls.length;
|
||||
dls[idx].click();return 'clicked '+idx+'/'+dls.length;
|
||||
})(%d)"""
|
||||
|
||||
FETCH_LAST = r"""(()=>{try{if(!window.__cap.length)return 'NO_HREF';
|
||||
const u=window.__cap[window.__cap.length-1];const x=new XMLHttpRequest();x.open('GET',u,false);
|
||||
x.overrideMimeType('text/plain;charset=x-user-defined');x.send();if(x.status!==200)return 'HTTP'+x.status;
|
||||
const s=x.responseText;let out='';const CH=8192;for(let i=0;i<s.length;i+=CH){let c='';const e=Math.min(i+CH,s.length);for(let j=i;j<e;j++)c+=String.fromCharCode(s.charCodeAt(j)&0xff);out+=c;}
|
||||
window.__b64=btoa(out);const m=u.match(/attname=([^&]+)/);let name=m?decodeURIComponent(m[1]):('f_'+Date.now());
|
||||
return JSON.stringify({size:s.length,name:name});}catch(e){return 'EX '+e.message;}})()"""
|
||||
|
||||
def main():
|
||||
call("find_tab", {"url": "2haohr.com"})
|
||||
print("== 当前任务中心 ==")
|
||||
print(ev(TASKS))
|
||||
ev(HOOK)
|
||||
# 实测抓第一个任务下载
|
||||
print("click:", ev(CLICK_TASK_DL % 0))
|
||||
time.sleep(2.5)
|
||||
meta = ev(FETCH_LAST)
|
||||
print("fetch:", meta)
|
||||
if isinstance(meta, str) and meta.startswith("{"):
|
||||
info = json.loads(meta); data = base64.b64decode(ev("window.__b64||''"))
|
||||
os.makedirs("downloads/_test", exist_ok=True)
|
||||
out = "downloads/_test/task_" + info["name"]
|
||||
open(out, "wb").write(data); print("落盘:", out, len(data))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""聚焦探测“生成过程”：招聘报表页点下载图标 -> 看参数窗内容 -> 确定 -> 观察 toast/任务中心出现与完成。"""
|
||||
import json, urllib.request, time
|
||||
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
||||
def call(a, args, t=90):
|
||||
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
||||
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
||||
return json.loads(urllib.request.urlopen(r, timeout=t).read().decode())
|
||||
def ev(c):
|
||||
try: return call("evaluate", {"code": c}).get("data", {}).get("value")
|
||||
except Exception as e: return "EXC:" + str(e)[:80]
|
||||
|
||||
HOOK = r"""(()=>{window.__cap=window.__cap||[];if(!HTMLAnchorElement.prototype.__hk){const ac=HTMLAnchorElement.prototype.click;
|
||||
HTMLAnchorElement.prototype.click=function(){try{const h=this.href||'';if(h&&/-cdn\.2haohr|attname=|\.xlsx|\.xls|\.docx/i.test(h))window.__cap.push(h);}catch(e){}return ac.apply(this,arguments);};HTMLAnchorElement.prototype.__hk=1;}return 'ok';})()"""
|
||||
CLICK_ICON = r"""(()=>{const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-download');
|
||||
for(const u of us){const b=u.closest('button,a,[role=button]');if(b){b.click();return 'ok';}}return 'none';})()"""
|
||||
# 参数窗全文 + 按钮列表
|
||||
MODAL = r"""(()=>{const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')];
|
||||
const vis=ms.filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>80&&getComputedStyle(m).display!=='none';});
|
||||
if(!vis.length)return 'NO_MODAL';const m=vis[vis.length-1];
|
||||
const btns=[...m.querySelectorAll('button,a')].map(b=>(b.innerText||'').trim()).filter(t=>t&&t.length<10);
|
||||
return JSON.stringify({text:(m.innerText||'').replace(/\n{2,}/g,'\n').slice(0,400),btns:[...new Set(btns)]});})()"""
|
||||
CONFIRM = r"""(()=>{const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')];
|
||||
const vis=ms.filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>80&&getComputedStyle(m).display!=='none';});
|
||||
if(!vis.length)return 'NO_MODAL';const m=vis[vis.length-1];
|
||||
for(const t of ['确定','确认','下载','导出','生成']){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0);if(b){b.click();return 'click:'+t;}}return 'noconfirm';})()"""
|
||||
TOAST = r"""(()=>{const a=[];document.querySelectorAll('[class*=message],[class*=Message],[class*=toast],[class*=notify],[class*=Notification]').forEach(el=>{const t=(el.innerText||'').trim();if(t&&t.length<60&&el.getBoundingClientRect().width>0)a.push(t);});return JSON.stringify([...new Set(a)]);})()"""
|
||||
TC = r"""(()=>{const tc=document.querySelector('[class*=taskCenter]');if(!tc)return 'NO_TC';
|
||||
const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
|
||||
return JSON.stringify({dl:dls.length,text:(tc.innerText||'').replace(/\n{2,}/g,'\n').slice(0,300)});})()"""
|
||||
|
||||
def main():
|
||||
call("find_tab", {"url": "2haohr.com"})
|
||||
call("navigate", {"url": "https://2haohr.com/recruit/statistics/recruit-report", "newTab": False})
|
||||
time.sleep(7)
|
||||
ev(HOOK)
|
||||
print("icon click:", ev(CLICK_ICON))
|
||||
time.sleep(2)
|
||||
print("MODAL:", ev(MODAL))
|
||||
print("confirm:", ev(CONFIRM))
|
||||
# 观察生成过程
|
||||
for i in range(10):
|
||||
time.sleep(3)
|
||||
cap = ev("(window.__cap&&window.__cap.length)||0")
|
||||
print(" t+%ds cap=%s toast=%s" % (3*(i+1), cap, ev(TOAST)))
|
||||
if isinstance(cap, int) and cap > 0:
|
||||
print(" >>> 捕获直链!"); break
|
||||
print("TASK_CENTER:", ev(TC))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""快速判定“无导出入口”报表页的真实结构：等待15s后统计 #icon-download / 文字下载按钮 / 是否需先查询。"""
|
||||
import json, urllib.request, time
|
||||
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
||||
def call(a, args, t=90):
|
||||
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
||||
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
||||
return json.loads(urllib.request.urlopen(r, timeout=t).read().decode())
|
||||
def ev(c):
|
||||
try: return call("evaluate", {"code": c}).get("data", {}).get("value")
|
||||
except Exception as e: return "EXC:" + str(e)[:80]
|
||||
|
||||
STAT = r"""(()=>{
|
||||
let icon=0;document.querySelectorAll('use').forEach(u=>{const h=u.getAttribute('xlink:href')||u.getAttribute('href');if(h==='#icon-download')icon++;});
|
||||
const btns=[...document.querySelectorAll('button,a,[role=button]')].map(b=>(b.innerText||'').trim()).filter(t=>t&&t.length<12&&/下载|导出|生成|查询|搜索|excel/i.test(t));
|
||||
const bcnt={};btns.forEach(t=>bcnt[t]=(bcnt[t]||0)+1);
|
||||
// 是否存在数据表格
|
||||
const tbl=document.querySelectorAll('table,[class*=table],[class*=Table]').length;
|
||||
const empty=/暂无数据|请选择|请先|no data/i.test(document.body.innerText||'');
|
||||
return JSON.stringify({icon_download:icon,dl_btns:bcnt,tables:tbl,has_empty_hint:empty});})()"""
|
||||
|
||||
def probe(url, label):
|
||||
call("navigate", {"url": url, "newTab": False})
|
||||
time.sleep(15)
|
||||
print("[%s] %s" % (label, url))
|
||||
print(" ", ev(STAT))
|
||||
|
||||
def main():
|
||||
call("find_tab", {"url": "2haohr.com"})
|
||||
for url, label in [
|
||||
("https://2haohr.com/attendance/stat/monthly", "月考勤表"),
|
||||
("https://2haohr.com/attendance/stat/report", "考勤报表"),
|
||||
("https://2haohr.com/social-insurance/reports", "社保报表"),
|
||||
("https://2haohr.com/org/org-forms", "组织报表"),
|
||||
("https://2haohr.com/payroll/salary/report", "薪酬报表"),
|
||||
]:
|
||||
probe(url, label)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""单页验证：仅对人事报表页跑 process_page，确认 CONFIRM 修复后能产出真实报表 Excel。"""
|
||||
import dl_export as X
|
||||
|
||||
def main():
|
||||
X.ensure()
|
||||
X.call("find_tab", {"url": "2haohr.com"})
|
||||
p = {
|
||||
"full_path": "02_人事 / 人事报表",
|
||||
"url": "https://2haohr.com/personnel/employee/personnel-report/home",
|
||||
"node_key": "test_personnel_report",
|
||||
"screenshot_path": None,
|
||||
}
|
||||
n = X.process_page(p, 1, 1)
|
||||
X.log("单页测试产出: %d 个文件, SAVED=%d" % (n, len(X.SAVED)))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
读取 db/menu_map.json, 结合领域释义, 构建功能树写入 SQLite(features 表)。
|
||||
层级: level0=顶层模块, level1=分组, level2=菜单页(叶子)
|
||||
并为每个叶子分配期望截图路径 screenshots/<序号_模块>/<分组>/<名称>.jpg
|
||||
"""
|
||||
import json, os, re, sqlite3
|
||||
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
MAP = os.path.join(BASE, "db", "menu_map.json")
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
ORIGIN = "https://2haohr.com"
|
||||
|
||||
# ---- 模块级 释义/理解 ----
|
||||
MOD = {
|
||||
"工作台": ("HR 登录后的首页驾驶舱，聚合日程、待办、招聘/考勤/入离职概况、快捷入口、人事提醒与常用工具。", "系统门户与导航中枢，是使用频率最高的页面；开发时应做成可配置卡片式仪表盘。"),
|
||||
"人事": ("员工全生命周期管理中心，含员工档案、花名册、入转调离、合同、奖惩等员工关系事务。", "HR 系统的核心主数据模块，其它模块(薪酬/考勤/社保)都依赖此处的员工主数据。"),
|
||||
"组织": ("企业组织架构管理，含部门、岗位、职务、职级、工作地点、员工标签等组织主数据。", "组织与岗位主数据是权限、审批流、薪酬绩效归属的基础，须优先建设。"),
|
||||
"招聘": ("从职位发布、简历/候选人管理、面试到入职的全流程招聘管理(ATS)。", "可独立成招聘子系统；与人事入职管理衔接形成招录闭环。"),
|
||||
"考勤假期": ("考勤打卡、排班、考勤统计与假期(休假/余额/方案)管理。", "规则复杂度高(排班/加班/假期计算)，是薪酬核算的关键数据来源。"),
|
||||
"薪酬": ("工资表核算、发放、员工薪资档案、计薪规则与薪酬报表分析。", "涉及资金与个税，准确性与权限要求最高；依赖考勤与社保数据。"),
|
||||
"审批": ("企业内各类事务的审批流引擎，含表单、模板库、代理、记录与交接。", "是贯穿全系统的通用能力(入职/请假/报销等都走审批)，应做成可配置工作流平台。"),
|
||||
"培训学习": ("线下培训、在线学习(课程/直播)、考试练习一体化的企业培训管理。", "偏 LMS 学习管理系统，可作为增值模块，与绩效/晋升联动。"),
|
||||
"绩效": ("绩效考核方案、试用期/晋升考核、绩效档案与面谈记录管理。", "考核模板与指标库配置复杂；结果可回流薪酬(绩效工资)与人事(晋升)。"),
|
||||
"社保公积金": ("参保账单、账单核对、参保档案与社保报表分析。", "地区政策差异大；数据对接社保代理或电子税务局，与薪酬个税强相关。"),
|
||||
"行政管理": ("企业公告、制度、新闻、证明开具、会议室预定、问卷调查等行政 OA 事务。", "轻量 OA 能力，提升全员协作体验，非 HR 核心但常用。"),
|
||||
"电子合同": ("劳动合同在线签署、合同档案库、印章与合同模板管理。", "需对接第三方 CA/电子签名平台，具法律效力，替代纸质合同。"),
|
||||
"背景调查": ("入职前的第三方背景调查服务入口(学历/工作履历/征信等核验)。", "属增值付费服务，通常对接外部背调供应商，单页服务入口。"),
|
||||
"人才测评": ("岗位胜任力/性格等人才测评的第三方服务入口。", "增值付费服务，对接外部测评供应商，用于招聘与人才盘点。"),
|
||||
"员工福利": ("员工福利商城、积分管理、商品专区与健康(平安好医生)等福利运营。", "员工激励与福利发放渠道，偏运营，可作为增值模块。"),
|
||||
"人事仪表盘": ("自定义人事数据看板与面向管理层的经营报告。", "BI 可视化能力，聚合各模块指标，服务决策。"),
|
||||
"第三方对接": ("与企业微信、钉钉、飞书、携程商旅等外部平台的对接配置。", "集成能力，实现组织同步/消息推送/差旅等，是生态打通的关键。"),
|
||||
"培训商城": ("外部培训课程商城入口(采购付费课程)。", "疑似外部商城，映射异常；实际为付费课程采购入口，按单页外链处理。"),
|
||||
"短信发送": ("批量短信发送与发送记录查询。", "通知触达通道，需短信通道额度，常被审批/考勤等模块调用。"),
|
||||
"电子工资条": ("工资条在线发送、员工确认反馈、发放记录与统计。", "解决工资条保密发放与确认留痕，依赖薪酬工资表数据。"),
|
||||
"智能薪酬": ("新一代薪酬核算平台：数据采集、薪酬档案、个税、成本、报表全流程。", "比传统‘薪酬’模块更完整，含数据采集与成本分析，是薪酬能力的升级版。"),
|
||||
"个税对接": ("人员信息报送、个税申报、纳税单位管理，对接自然人电子税务局。", "合规刚需，与薪酬核算联动完成代扣代缴。"),
|
||||
"AI员工助手": ("面向员工的 AI 问答助手、知识库管理与使用统计。", "AI 能力，降低 HR 重复答疑成本；知识库质量决定效果。"),
|
||||
"应用管理": ("企业已开通应用/增值服务的管理与开通入口。", "应用中心/权限开关，管理各功能模块的启用状态。"),
|
||||
}
|
||||
|
||||
# 单页模块(无侧边栏)——给它们补一个叶子指向落地页
|
||||
SINGLE_PAGE = {
|
||||
"工作台": ("工作台首页", "/desk"),
|
||||
"背景调查": ("背景调查服务", "/hrService/background-investigation/entry"),
|
||||
"人才测评": ("人才测评服务", "/hrService/employee-assessment/"),
|
||||
"应用管理": ("应用管理", "/application-manage/application/list"),
|
||||
"培训商城": ("培训商城(外部课程采购)", "/training-mall"),
|
||||
}
|
||||
# 映射异常需忽略其抓到的错误分组的模块
|
||||
FORCE_SINGLE = {"培训商城"}
|
||||
|
||||
# ---- 叶子级 释义/理解, 键: "名称" 或 "模块/名称"(消歧) ----
|
||||
DESC = {
|
||||
# 人事/员工管理
|
||||
"员工花名册": ("全体员工的核心名册列表，支持按部门/状态筛选、字段自定义、批量导入导出。", "员工主数据入口，几乎所有人事操作从此发起；字段可配置是关键需求。"),
|
||||
"员工档案库": ("员工完整电子档案(证件/合同/学历/家庭等)的集中存储与调阅。", "360°档案，强调附件管理与字段完整率统计。"),
|
||||
"人事/统计分析": ("按维度对在职/结构/异动等人事数据的图表统计分析。", "偏 BI 图表，服务人事盘点。"),
|
||||
"人事报表": ("各类标准人事报表(在职、异动、合同到期等)的生成与导出。", "满足周期性上报需求，需支持自定义与导出。"),
|
||||
"万能人事表格": ("自定义字段与筛选组合生成任意人事数据表格。", "灵活取数工具，降低对开发出报表的依赖。"),
|
||||
"人事动态": ("入职/转正/离职/生日等人事事件的动态时间线。", "事件流，可驱动关怀通知与提醒。"),
|
||||
"入职管理": ("待入职/入职办理流程，含入职登记、资料收集与开通账号。", "与招聘‘已入职’衔接；支持员工自助填写(AI 辅助)。"),
|
||||
"转正管理": ("试用期到期提醒与转正申请/审批办理。", "与绩效试用期考核联动。"),
|
||||
"离职管理": ("离职申请、离职办理、交接与离职档案。", "涉及权限回收、薪酬结算、社保停缴等联动。"),
|
||||
"人事异动": ("部门/岗位/职级调动等异动记录与审批。", "组织变动留痕，影响薪酬与汇报关系。"),
|
||||
"关怀通知": ("生日/周年/证件到期等关怀提醒与自动通知。", "员工体验运营，可配置触发规则与文案。"),
|
||||
"人事/合同管理": ("劳动合同签订、续签、到期提醒与合同台账。", "与电子合同模块可打通；合同到期预警是刚需。"),
|
||||
"沟通记录": ("HR 与员工的沟通/谈话记录留存。", "员工关系管理留痕，支撑争议处理。"),
|
||||
"任职记录": ("员工历史任职(岗位/部门)变化记录。", "履历追溯，来源于人事异动。"),
|
||||
"培训经历": ("员工参加培训的经历记录。", "与培训模块数据互通。"),
|
||||
"奖惩记录": ("员工奖励与处罚记录管理。", "影响绩效与晋升评估。"),
|
||||
"考察期管理": ("特定员工考察期的设置与跟踪。", "试用/观察期管理的补充。"),
|
||||
"实习生管理": ("实习生专项档案与转正管理。", "针对实习用工的差异化流程。"),
|
||||
# 组织
|
||||
"组织管理": ("部门树的增删改与部门负责人、编制等管理。", "组织主数据根节点，权限与审批的部门维度基础。"),
|
||||
"架构图": ("可视化组织架构图展示与调整。", "直观呈现汇报关系，支持导出。"),
|
||||
"工作地点": ("企业办公/工作地点的维护。", "关联考勤打卡范围与员工工作城市。"),
|
||||
"岗位管理": ("岗位设置、岗位职责与编制管理。", "岗位是招聘/薪酬/绩效的归属维度。"),
|
||||
"职务管理": ("职务序列与职务信息管理。", "与职级配合构成职级体系。"),
|
||||
"员工标签": ("为员工打自定义标签用于分组/筛选。", "灵活分群，服务精细化运营与权限。"),
|
||||
"职级管理": ("职级/职等体系维护。", "薪酬带宽与晋升通道的基础。"),
|
||||
"组织报表": ("组织维度(编制/人数/结构)的报表。", "支持组织盘点。"),
|
||||
# 招聘
|
||||
"招聘/概览": ("招聘整体数据概览(职位/候选人/面试等看板)。", "招聘经理入口页。"),
|
||||
"招聘中": ("处于招聘流程中的候选人列表与阶段管理。", "ATS 核心看板，支持阶段流转。"),
|
||||
"已入职": ("已录用入职的候选人。", "与人事入职管理衔接。"),
|
||||
"已淘汰": ("被淘汰候选人归档。", "复用人才库，支持再激活。"),
|
||||
"招聘职位": ("招聘岗位(JD)的发布与管理。", "对接招聘门户与渠道分发。"),
|
||||
"招聘门户": ("企业招聘官网/微信招聘门户配置。", "候选人投递入口，支持自定义。"),
|
||||
"面试日程": ("面试安排、通知与面试评价。", "支持面试官协同与日程提醒。"),
|
||||
"人才库": ("企业人才储备库。", "沉淀历史候选人，降低复招成本。"),
|
||||
"招聘报表": ("招聘漏斗/周期/渠道效果报表。", "衡量招聘效率。"),
|
||||
"招聘分析": ("招聘数据多维分析。", "优化招聘策略。"),
|
||||
"招聘团队": ("招聘协作团队与角色权限设置。", "多角色协同招聘。"),
|
||||
"招聘工具": ("招聘相关工具(如筛选/AI)配置。", "提效工具集。"),
|
||||
"招聘渠道": ("招聘发布渠道与聚合配置。", "对接外部招聘网站。"),
|
||||
"招聘/模板设置": ("招聘环节的邮件/短信/评价模板。", "标准化沟通。"),
|
||||
"更多设置": ("招聘模块其它设置项。", "杂项配置。"),
|
||||
# 考勤
|
||||
"每日考勤": ("按日展示全员打卡与异常。", "考勤日常监控入口。"),
|
||||
"月考勤表": ("按月汇总的考勤统计表。", "薪酬核算的考勤依据。"),
|
||||
"明细记录": ("原始打卡明细记录查询。", "异常核查与申诉依据。"),
|
||||
"考勤/考勤报表": ("各类考勤统计报表。", "周期性考勤上报。"),
|
||||
"字段管理": ("考勤表字段自定义。", "适配企业统计口径。"),
|
||||
"考勤确认": ("月末考勤结果确认锁定。", "确认后进入薪酬核算。"),
|
||||
"考勤分组": ("按人群划分考勤规则分组。", "不同班制人群差异化管理。"),
|
||||
"排班管理": ("排班表编制与发布。", "复杂班制的核心，支持轮班。"),
|
||||
"考勤规则": ("上下班/加班/迟到等规则配置。", "考勤计算引擎的规则中心。"),
|
||||
"打卡方式": ("GPS/WiFi/人脸等打卡方式配置。", "决定打卡合规与防作弊。"),
|
||||
"考勤设置": ("考勤模块的其它设置。", "杂项配置。"),
|
||||
"休假一览": ("全员请假记录总览。", "假期使用监控。"),
|
||||
"手动调整": ("手动调整假期余额/记录。", "异常修正入口。"),
|
||||
"假期余额": ("各员工各类假期余额。", "请假审批的额度依据。"),
|
||||
"假期方案": ("年假/事病假等假期规则方案。", "假期计算的规则中心。"),
|
||||
"假期设置": ("假期模块其它设置。", "杂项配置。"),
|
||||
# 薪酬(传统)
|
||||
"月工资表": ("按月生成的工资核算表。", "薪酬核算主表，支持公式与批量。"),
|
||||
"工资发放": ("工资发放确认与银行代发。", "对接银行代发，资金安全关键。"),
|
||||
"员工薪资": ("员工薪资档案(基本工资/结构)。", "计薪基础数据。"),
|
||||
"薪酬/薪酬报表": ("薪酬相关报表。", "成本与发放统计。"),
|
||||
"薪酬/薪酬分析": ("薪酬多维分析。", "人力成本洞察。"),
|
||||
"计薪规则": ("计薪周期与规则配置。", "薪酬引擎规则。"),
|
||||
"薪酬/薪酬设置": ("薪酬模块设置。", "结构/权限配置。"),
|
||||
"年终奖计算": ("年终奖专项计税计算。", "年度一次性奖金个税处理。"),
|
||||
"发送工资条": ("发送电子工资条(跳转电子工资条模块)。", "与电子工资条模块共用。"),
|
||||
# 审批
|
||||
"审批表单": ("各类审批单据的发起与列表。", "通用审批入口。"),
|
||||
"审批模板库": ("预置审批流程模板。", "快速套用标准流程。"),
|
||||
"审批代理": ("审批人不在时的代理设置。", "保障流程不中断。"),
|
||||
"审批记录": ("历史审批记录查询。", "留痕与追溯。"),
|
||||
"审批交接": ("审批任务交接处理。", "人员变动时的流程转交。"),
|
||||
# 培训
|
||||
"培训计划": ("线下培训计划制定与管理。", "培训组织入口。"),
|
||||
"开课日程": ("培训课程排期日程。", "线下课程时间管理。"),
|
||||
"培训记录": ("培训完成记录。", "回流员工培训经历。"),
|
||||
"课程库": ("在线课程资源库。", "LMS 内容中心。"),
|
||||
"学习计划": ("为员工指派学习计划。", "驱动在线学习。"),
|
||||
"在线直播": ("在线直播课安排。", "远程培训能力。"),
|
||||
"学习进度": ("员工学习进度跟踪。", "学习效果监控。"),
|
||||
"学习记录": ("在线学习历史记录。", "留痕与统计。"),
|
||||
"题库管理": ("考试题目库维护。", "考试基础数据。"),
|
||||
"试卷管理": ("试卷组卷与管理。", "考试内容配置。"),
|
||||
"考试安排": ("考试计划安排。", "组织在线考试。"),
|
||||
"考试记录": ("考试成绩记录。", "培训效果评估。"),
|
||||
"讲师设置": ("培训讲师维护。", "线下培训资源。"),
|
||||
"团队设置": ("培训管理团队设置。", "权限协作。"),
|
||||
"培训学习/自定义设置": ("培训模块自定义设置。", "杂项配置。"),
|
||||
# 绩效
|
||||
"绩效/概览": ("绩效整体概览看板。", "绩效经理入口。"),
|
||||
"绩效考核": ("发起与管理绩效考核方案。", "绩效核心，含指标与流程。"),
|
||||
"历史绩效考核": ("历史考核归档查询。", "结果追溯。"),
|
||||
"试用期考核": ("试用期专项考核。", "与转正联动。"),
|
||||
"晋升考核": ("晋升评估考核。", "与职级晋升联动。"),
|
||||
"考核组管理": ("考核分组管理。", "按人群组织考核。"),
|
||||
"员工绩效档案": ("员工历史绩效档案。", "绩效履历。"),
|
||||
"绩效考核记录": ("考核过程与结果记录。", "留痕。"),
|
||||
"绩效面谈记录": ("绩效面谈内容记录。", "改进跟踪。"),
|
||||
"指标库": ("绩效指标(KPI/OKR)库。", "考核内容基础。"),
|
||||
"考核模板库": ("考核表模板库。", "快速套用。"),
|
||||
"考核团队": ("绩效管理团队设置。", "权限协作。"),
|
||||
"绩效/自定义设置": ("绩效模块自定义设置。", "杂项配置。"),
|
||||
# 社保
|
||||
"参保账单": ("每月社保公积金参保账单。", "缴费核对入口。"),
|
||||
"账单核对": ("账单与实缴核对。", "差异排查。"),
|
||||
"参保档案": ("员工参保档案。", "参保状态主数据。"),
|
||||
"社保报表": ("社保缴纳报表。", "上报与统计。"),
|
||||
"社保分析": ("社保成本分析。", "成本洞察。"),
|
||||
"参保规则": ("参保基数/比例规则。", "地区政策配置核心。"),
|
||||
# 行政
|
||||
"企业公告": ("企业公告发布与阅读。", "全员触达。"),
|
||||
"企业制度": ("企业规章制度发布。", "制度沉淀。"),
|
||||
"新闻动态": ("企业新闻动态。", "文化运营。"),
|
||||
"开具证明": ("在职/收入等证明在线开具。", "员工自助高频需求。"),
|
||||
"会议预定记录": ("会议室预定记录。", "资源使用查询。"),
|
||||
"会议室管理": ("会议室资源维护。", "行政资源管理。"),
|
||||
"会议操作记录": ("会议室操作日志。", "留痕。"),
|
||||
"问卷管理": ("企业问卷调查发起与统计。", "满意度/调研工具。"),
|
||||
"问卷模板库": ("问卷模板库。", "快速创建。"),
|
||||
"问卷操作记录": ("问卷操作日志。", "留痕。"),
|
||||
# 电子合同
|
||||
"电子合同/概览": ("电子合同数据概览。", "签署情况看板。"),
|
||||
"签署中": ("签署流程进行中的合同。", "待办跟踪。"),
|
||||
"已签署": ("已完成签署合同。", "生效合同管理。"),
|
||||
"已作废": ("作废合同归档。", "留痕。"),
|
||||
"合同档案库": ("电子合同档案库。", "集中存储与检索。"),
|
||||
"企业印章": ("企业电子印章管理。", "签署授权核心。"),
|
||||
"合同模板": ("合同模板管理。", "标准化签署。"),
|
||||
"合同包": ("合同套餐/额度包。", "计费额度管理。"),
|
||||
# 员工福利
|
||||
"员工福利概况": ("福利运营数据概览。", "福利运营入口。"),
|
||||
"员工积分管理": ("员工积分发放与管理。", "激励体系。"),
|
||||
"商品专区管理": ("福利商城商品管理。", "福利兑换运营。"),
|
||||
"平安好医生": ("对接平安好医生健康服务。", "健康福利增值。"),
|
||||
# 仪表盘
|
||||
"我的仪表盘": ("个人自定义数据看板。", "BI 自助看板。"),
|
||||
"报告老板": ("面向管理层的经营人事报告。", "高层决策报告。"),
|
||||
# 第三方对接
|
||||
"企业微信": ("对接企业微信(组织同步/消息)。", "生态集成。"),
|
||||
"钉钉": ("对接钉钉。", "生态集成。"),
|
||||
"飞书": ("对接飞书。", "生态集成。"),
|
||||
"携程商旅": ("对接携程商旅(差旅)。", "差旅报销联动。"),
|
||||
# 短信
|
||||
"短信发送": ("批量发送短信通知。", "触达通道。"),
|
||||
"发送记录": ("短信发送记录。", "留痕与计费。"),
|
||||
# 电子工资条
|
||||
"电子工资条/发送工资条": ("发送电子工资条给员工。", "保密发放。"),
|
||||
"发放记录": ("工资条发放记录。", "留痕。"),
|
||||
"反馈记录": ("员工对工资条的确认/异议。", "确认闭环。"),
|
||||
"电子工资条/统计分析": ("工资条发送与确认统计。", "触达率监控。"),
|
||||
# 智能薪酬
|
||||
"智能薪酬/概览": ("智能薪酬数据概览。", "薪酬看板。"),
|
||||
"薪酬核算": ("智能化薪酬核算。", "核心核算引擎。"),
|
||||
"员工考勤": ("采集考勤数据用于计薪。", "数据采集环节。"),
|
||||
"业务提报": ("采集业务提成/绩效等计薪数据。", "数据采集环节。"),
|
||||
"薪酬档案": ("员工薪酬档案。", "计薪基础。"),
|
||||
"减除费用确认": ("个税减除费用确认。", "个税计算前置。"),
|
||||
"税前扣除": ("专项附加扣除等税前扣除。", "个税合规。"),
|
||||
"个税记录": ("个税计算记录。", "代扣代缴留痕。"),
|
||||
"成本明细": ("人力成本明细。", "成本核算。"),
|
||||
"薪酬报表": ("智能薪酬报表。", "统计。"),
|
||||
"薪酬分析": ("智能薪酬分析。", "成本洞察。"),
|
||||
"薪酬结构": ("薪酬结构(薪资项)配置。", "计薪规则基础。"),
|
||||
"基础设置": ("智能薪酬基础设置。", "公司级配置。"),
|
||||
"成本设置": ("成本核算规则设置。", "成本分摊配置。"),
|
||||
# 个税
|
||||
"人员信息报送": ("向税务报送人员信息。", "个税申报前置。"),
|
||||
"个税申报": ("个人所得税申报。", "合规刚需。"),
|
||||
"纳税单位管理": ("纳税主体单位管理。", "多主体申报。"),
|
||||
# AI
|
||||
"AI员工助手/AI员工助手": ("面向员工的 AI 问答助手。", "降低 HR 答疑成本。"),
|
||||
"知识库管理": ("AI 助手知识库维护。", "决定回答质量。"),
|
||||
"员工助手设置": ("AI 助手行为设置。", "配置。"),
|
||||
"总体使用情况": ("AI 助手使用统计。", "效果度量。"),
|
||||
}
|
||||
|
||||
def desc_for(module, group, name):
|
||||
for k in (f"{module}/{name}", name):
|
||||
if k in DESC:
|
||||
return DESC[k]
|
||||
return (f"{module}模块下「{name}」相关功能页。", "映射自侧边栏菜单，待补充细化。")
|
||||
|
||||
def sanitize(s):
|
||||
s = re.sub(r'[\\/:*?"<>|]', "_", s).strip()
|
||||
return s[:40] or "page"
|
||||
|
||||
def main():
|
||||
data = json.load(open(MAP))
|
||||
conn = sqlite3.connect(DB); conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
# 先清空(可重复执行)
|
||||
cur.execute("DELETE FROM features")
|
||||
|
||||
def up(obj):
|
||||
keys = list(obj.keys()); cols = ",".join(keys); ph = ",".join(["?"]*len(keys))
|
||||
cur.execute(f"INSERT OR REPLACE INTO features ({cols}) VALUES ({ph})", [obj[k] for k in keys])
|
||||
|
||||
mod_order = list(data.keys())
|
||||
for mi, module in enumerate(mod_order, 1):
|
||||
v = data[module]
|
||||
md, mu = MOD.get(module, (f"{module}模块。", ""))
|
||||
mod_key = f"mod::{module}"
|
||||
seq_mod = f"{mi:02d}_{sanitize(module)}"
|
||||
up(dict(node_key=mod_key, parent_key="", level=0, name=module,
|
||||
full_path=module, url=v.get("loc",""), description=md, understanding=mu,
|
||||
sort_order=mi, keep=1))
|
||||
groups = v.get("groups", [])
|
||||
if module in FORCE_SINGLE:
|
||||
groups = []
|
||||
# 单页模块补叶子
|
||||
if not groups and module in SINGLE_PAGE:
|
||||
nm, path = SINGLE_PAGE[module]
|
||||
d, u = desc_for(module, "", nm)
|
||||
leaf_key = f"{module}::{path}"
|
||||
rel = f"screenshots/{seq_mod}/{sanitize(nm)}.jpg"
|
||||
up(dict(node_key=leaf_key, parent_key=mod_key, level=2, name=nm,
|
||||
full_path=f"{module} / {nm}", url=ORIGIN+path if path.startswith('/') else path,
|
||||
description=d, understanding=u, screenshot_path=rel, sort_order=1, keep=1))
|
||||
continue
|
||||
gi = 0
|
||||
for g in groups:
|
||||
gtitle = g.get("title") or "常规"
|
||||
gi += 1
|
||||
grp_key = f"{mod_key}::grp::{gi}_{gtitle}"
|
||||
up(dict(node_key=grp_key, parent_key=mod_key, level=1, name=gtitle,
|
||||
full_path=f"{module} / {gtitle}", url="", description=f"{module}模块下「{gtitle}」功能分组。",
|
||||
understanding="", sort_order=gi, keep=1))
|
||||
for ii, it in enumerate(g["items"], 1):
|
||||
nm = it["name"]; href = it["href"]
|
||||
url = href if href.startswith("http") else ORIGIN + href
|
||||
d, u = desc_for(module, gtitle, nm)
|
||||
leaf_key = f"{module}::{href}"
|
||||
rel = f"screenshots/{seq_mod}/{sanitize(gtitle)}/{ii:02d}_{sanitize(nm)}.jpg"
|
||||
up(dict(node_key=leaf_key, parent_key=grp_key, level=2, name=nm,
|
||||
full_path=f"{module} / {gtitle} / {nm}", url=url,
|
||||
description=d, understanding=u, screenshot_path=rel, sort_order=ii, keep=1))
|
||||
conn.commit()
|
||||
# 统计
|
||||
for lv in (0,1,2):
|
||||
n = cur.execute("SELECT COUNT(*) FROM features WHERE level=?", (lv,)).fetchone()[0]
|
||||
print(f"level{lv}: {n}")
|
||||
print("total:", cur.execute("SELECT COUNT(*) FROM features").fetchone()[0])
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
整页滚动截图 (scroll & stitch) —— 针对 2号人事部 window 整页滚动 + fixed 顶栏
|
||||
|
||||
用法:
|
||||
python3 capture_full.py <session> <out_path> [max_tiles]
|
||||
|
||||
原理:
|
||||
1. 隐藏帮助/反馈/AI助手/购买浮层等固定挂件, 避免遮挡
|
||||
2. 读取滚动指标, 探测顶部 fixed 顶栏高度
|
||||
3. 逐屏滚动截图(视口图), 除首屏外裁掉与固定顶栏重复的部分
|
||||
4. 纵向拼接为整页长图
|
||||
依赖: PIL(Pillow), 通过 daemon(127.0.0.1:10086) 调用 evaluate / screenshot
|
||||
"""
|
||||
import sys, os, json, time, base64, io, urllib.request
|
||||
|
||||
DAEMON = "http://127.0.0.1:10086/command"
|
||||
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def call(action, args, session):
|
||||
body = json.dumps({"action": action, "args": args, "session": session}).encode()
|
||||
req = urllib.request.Request(DAEMON, data=body, headers={"Content-Type": "application/json"})
|
||||
with _OPENER.open(req, timeout=40) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
def ev(code, session):
|
||||
r = call("evaluate", {"code": code}, session)
|
||||
return r.get("data", {}).get("value")
|
||||
|
||||
def shot(session):
|
||||
r = call("screenshot", {"format": "jpeg", "quality": 70}, session)
|
||||
b64 = r.get("data", {}).get("data")
|
||||
if not b64:
|
||||
raise RuntimeError("no screenshot data: " + json.dumps(r)[:200])
|
||||
from PIL import Image
|
||||
return Image.open(io.BytesIO(base64.b64decode(b64))).convert("RGB")
|
||||
|
||||
HIDE_JS = r"""
|
||||
(()=>{
|
||||
const sels = ['[class*="helpBox"]','[class*="aiAssistant"]','[class*="product_"]',
|
||||
'[class*="feedback"]','[class*="Feedback"]','[class*="floatBar"]',
|
||||
'[class*="sideBar"]','[class*="robot"]','[class*="advert"]'];
|
||||
let n=0;
|
||||
sels.forEach(s=>document.querySelectorAll(s).forEach(el=>{
|
||||
const r=el.getBoundingClientRect();
|
||||
if(r.width<600 && getComputedStyle(el).position==='fixed'){el.style.setProperty('display','none','important');n++;}
|
||||
}));
|
||||
// 反馈竖条
|
||||
document.querySelectorAll('div,a,button').forEach(el=>{
|
||||
if(el.children.length<=3 && /^反\s*馈$/.test((el.innerText||'').replace(/\s/g,'')) && getComputedStyle(el).position==='fixed'){el.style.setProperty('display','none','important');n++;}
|
||||
});
|
||||
return n;
|
||||
})()
|
||||
"""
|
||||
|
||||
METRICS_JS = r"""
|
||||
(()=>{
|
||||
let hdr=0;
|
||||
const walk=(el,d)=>{ if(d>5)return; for(const c of el.children){ const s=getComputedStyle(c); const r=c.getBoundingClientRect();
|
||||
if((s.position==='fixed'||s.position==='sticky') && r.top<=1 && r.height>10 && r.height<200 && r.width>window.innerWidth*0.6){ if(r.height>hdr)hdr=r.height; }
|
||||
walk(c,d+1);} };
|
||||
walk(document.body,0);
|
||||
return JSON.stringify({winH:window.innerHeight, winW:window.innerWidth,
|
||||
docH:Math.max(document.documentElement.scrollHeight,document.body.scrollHeight), hdr:Math.round(hdr)});
|
||||
})()
|
||||
"""
|
||||
|
||||
def scroll_to(y, session):
|
||||
ev(f"(()=>{{window.scrollTo(0,{y});return window.scrollY;}})()", session)
|
||||
time.sleep(0.35)
|
||||
return ev("(()=>window.scrollY)()", session)
|
||||
|
||||
# 探测主内容区的内部滚动容器(表格/列表在固定高度面板内自行滚动的情况)
|
||||
DETECT_SCROLLER_JS = r"""
|
||||
(()=>{
|
||||
const W=innerWidth,H=innerHeight; let best=null,score=0;
|
||||
document.querySelectorAll('div,main,section,ul').forEach(el=>{
|
||||
const s=getComputedStyle(el); const oy=s.overflowY;
|
||||
if(oy!=='auto'&&oy!=='scroll') return;
|
||||
const extra=el.scrollHeight-el.clientHeight; if(extra<40) return;
|
||||
const r=el.getBoundingClientRect();
|
||||
if(r.width<W*0.4||r.height<H*0.25) return;
|
||||
if(r.bottom<H*0.4||r.top>H*0.85) return;
|
||||
const sc=r.width*r.height+extra*300;
|
||||
if(sc>score){score=sc;best=el;}
|
||||
});
|
||||
if(!best) return JSON.stringify({found:false});
|
||||
document.querySelectorAll('[data-capscroll]').forEach(e=>e.removeAttribute('data-capscroll'));
|
||||
best.setAttribute('data-capscroll','1');
|
||||
const r=best.getBoundingClientRect();
|
||||
let hs=0;
|
||||
best.querySelectorAll(':scope>*,:scope>*>*,:scope>*>*>*').forEach(c=>{
|
||||
const cs=getComputedStyle(c); const cr=c.getBoundingClientRect();
|
||||
if((cs.position==='sticky'||cs.position==='fixed')&&cr.top<=r.top+3&&cr.height>10&&cr.height<160&&cr.width>r.width*0.5){ if(cr.height>hs)hs=cr.height; }
|
||||
});
|
||||
return JSON.stringify({found:true,left:Math.round(r.left),top:Math.round(r.top),
|
||||
width:Math.round(r.width),height:Math.round(r.height),
|
||||
scrollHeight:best.scrollHeight,clientHeight:best.clientHeight,hdr:Math.round(hs)});
|
||||
})()
|
||||
"""
|
||||
|
||||
def scroll_inner(y, session):
|
||||
return ev("(()=>{const e=document.querySelector('[data-capscroll]');if(!e)return 0;e.scrollTop=%d;return Math.round(e.scrollTop);})()" % int(y), session)
|
||||
|
||||
|
||||
def _inner_stitch(session, sc, scale, max_tiles):
|
||||
"""对内部滚动容器逐屏滚动 -> 裁剪容器区域 -> 纵向拼接(含粘性表头去重)。"""
|
||||
from PIL import Image
|
||||
left, top = sc["left"], sc["top"]
|
||||
width, height = sc["width"], sc["height"]
|
||||
hs = sc.get("hdr", 0) or 0
|
||||
maxScroll = max(0, sc["scrollHeight"] - sc["clientHeight"])
|
||||
box = (int(round(left * scale)), int(round(top * scale)),
|
||||
int(round((left + width) * scale)), int(round((top + height) * scale)))
|
||||
|
||||
def tile():
|
||||
return shot(session).crop(box)
|
||||
|
||||
scroll_inner(0, session); time.sleep(0.35)
|
||||
first = tile()
|
||||
parts = [first]
|
||||
prev_bottom = height
|
||||
target = height - hs
|
||||
guard = 0
|
||||
while guard < max_tiles:
|
||||
guard += 1
|
||||
actualY = int(scroll_inner(target, session) or 0); time.sleep(0.3)
|
||||
img = tile()
|
||||
crop_logical = prev_bottom - actualY
|
||||
if crop_logical < 0:
|
||||
crop_logical = 0
|
||||
crop_px = int(round(crop_logical * scale))
|
||||
if crop_px >= img.height:
|
||||
crop_px = int(round(hs * scale))
|
||||
parts.append(img.crop((0, crop_px, img.width, img.height)))
|
||||
prev_bottom = actualY + height
|
||||
if actualY >= maxScroll - 1:
|
||||
break
|
||||
target = prev_bottom - hs
|
||||
total_h = sum(p.height for p in parts)
|
||||
stitched = Image.new("RGB", (first.width, total_h), (255, 255, 255))
|
||||
y = 0
|
||||
for p in parts:
|
||||
stitched.paste(p, (0, y)); y += p.height
|
||||
scroll_inner(0, session)
|
||||
return stitched, guard + 1
|
||||
|
||||
def capture(session, out_path, max_tiles=14):
|
||||
from PIL import Image
|
||||
|
||||
ev(HIDE_JS, session)
|
||||
m = json.loads(ev(METRICS_JS, session))
|
||||
winH, winW, docH = m["winH"], m["winW"], m["docH"]
|
||||
hdr = m["hdr"] or 0
|
||||
maxY = max(0, docH - winH)
|
||||
|
||||
# 首屏
|
||||
scroll_to(0, session)
|
||||
first = shot(session)
|
||||
scale = first.width / winW # 物理/逻辑
|
||||
hdr_px = int(round(hdr * scale))
|
||||
|
||||
tiles = [(first, 0, first.height)] # (img, cropTop_px, used content bottom logical)
|
||||
prev_bottom = winH # 已覆盖内容底部(逻辑)
|
||||
ntiles = 1
|
||||
mode = "single"
|
||||
|
||||
if docH <= winH + 4:
|
||||
# window 不滚动: 尝试内部滚动容器(表格/列表页)
|
||||
sc = {}
|
||||
try:
|
||||
sc = json.loads(ev(DETECT_SCROLLER_JS, session) or "{}")
|
||||
except Exception:
|
||||
sc = {}
|
||||
if sc.get("found") and (sc["scrollHeight"] - sc["clientHeight"]) > 40:
|
||||
stitched, ntiles = _inner_stitch(session, sc, scale, max_tiles)
|
||||
mode = "inner"
|
||||
else:
|
||||
stitched = first
|
||||
else:
|
||||
parts = [first]
|
||||
target = winH - hdr
|
||||
guard = 0
|
||||
last_y = 0
|
||||
while guard < max_tiles:
|
||||
guard += 1
|
||||
actualY = scroll_to(target, session)
|
||||
# 页面实际没有继续前进(虚拟网格/内滚容器/滚动被重置): 停止, 避免重复拼接同一屏
|
||||
if actualY <= last_y + 8:
|
||||
break
|
||||
img = shot(session)
|
||||
crop_logical = prev_bottom - actualY
|
||||
if crop_logical < 0:
|
||||
crop_logical = 0
|
||||
crop_px = int(round(crop_logical * scale))
|
||||
if crop_px >= img.height:
|
||||
crop_px = hdr_px
|
||||
parts.append(img.crop((0, crop_px, img.width, img.height)))
|
||||
prev_bottom = actualY + winH
|
||||
last_y = actualY
|
||||
if actualY >= maxY - 1:
|
||||
break
|
||||
target = prev_bottom - hdr
|
||||
total_h = sum(p.height for p in parts)
|
||||
stitched = Image.new("RGB", (first.width, total_h), (255, 255, 255))
|
||||
y = 0
|
||||
for p in parts:
|
||||
stitched.paste(p, (0, y)); y += p.height
|
||||
ntiles = len(parts)
|
||||
mode = "window" if len(parts) > 1 else "single"
|
||||
|
||||
scroll_to(0, session)
|
||||
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
||||
stitched.save(out_path, "JPEG", quality=78)
|
||||
print(json.dumps({"path": out_path, "size": [stitched.width, stitched.height],
|
||||
"tiles": ntiles, "mode": mode,
|
||||
"docH": docH, "winH": winH, "hdr": hdr}, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
_session = sys.argv[1]
|
||||
_out = sys.argv[2]
|
||||
_mt = int(sys.argv[3]) if len(sys.argv) > 3 else 14
|
||||
capture(_session, _out, _mt)
|
||||
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""全站列表页深度采集器 (任务1 + 任务3)。
|
||||
对每个有 URL 的叶子功能页:
|
||||
1) 点「更多/更多操作」等入口, 抓取下拉菜单里的隐藏功能项(发现之前未扫描到的界面);
|
||||
2) 点「自定义列/设置字段/选择表头」(#icon-custom-columns 或文字)弹窗, 抓取全部可选字段
|
||||
(含"系统字段/自定义字段"分组), 远超截图与导出Excel的可见字段;
|
||||
3) 兜底: 抓取默认渲染出的可见列头。
|
||||
结果写 SQLite: page_extras(隐藏入口) + field_specs(完整字段字典)。
|
||||
复用非均匀拟人间隔与登录态检测。"""
|
||||
import json, urllib.request, time, os, random, sqlite3, collections
|
||||
|
||||
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
||||
DB = "db/research.db"; LOG = "db/collect_fields.log"
|
||||
|
||||
|
||||
def call(a, args, t=90):
|
||||
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
||||
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
||||
return json.loads(urllib.request.urlopen(r, timeout=t).read().decode())
|
||||
|
||||
|
||||
def ev(code, t=90):
|
||||
try:
|
||||
return call("evaluate", {"code": code}, t).get("data", {}).get("value")
|
||||
except Exception as e:
|
||||
return "EXC:" + str(e)[:80]
|
||||
|
||||
|
||||
def log(m):
|
||||
line = time.strftime("%H:%M:%S ") + m
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def db():
|
||||
c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
|
||||
|
||||
|
||||
def ensure():
|
||||
c = db()
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS page_extras(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
|
||||
kind TEXT, label TEXT, ts TEXT)""")
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS field_specs(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
|
||||
field TEXT, field_group TEXT, source TEXT, sort_order INTEGER, ts TEXT)""")
|
||||
c.execute("PRAGMA journal_mode=DELETE")
|
||||
c.commit(); c.close()
|
||||
|
||||
|
||||
def clear_page(nk):
|
||||
c = db()
|
||||
c.execute("DELETE FROM page_extras WHERE node_key=?", (nk,))
|
||||
c.execute("DELETE FROM field_specs WHERE node_key=?", (nk,))
|
||||
c.commit(); c.close()
|
||||
|
||||
|
||||
def save_extras(nk, fp, url, items):
|
||||
c = db(); ts = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
for kind, label in items:
|
||||
c.execute("INSERT INTO page_extras(node_key,full_path,url,kind,label,ts) VALUES(?,?,?,?,?,?)",
|
||||
(nk, fp, url, kind, label, ts))
|
||||
c.commit(); c.close()
|
||||
|
||||
|
||||
def save_fields(nk, fp, url, fields, source):
|
||||
c = db(); ts = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
for i, (grp, name) in enumerate(fields):
|
||||
c.execute("INSERT INTO field_specs(node_key,full_path,url,field,field_group,source,sort_order,ts)"
|
||||
" VALUES(?,?,?,?,?,?,?,?)", (nk, fp, url, name, grp, source, i, ts))
|
||||
c.commit(); c.close()
|
||||
|
||||
|
||||
def human_delay(kind="page"):
|
||||
r = random.random()
|
||||
if kind == "page":
|
||||
d = random.uniform(3.5, 8.0) if r < 0.7 else random.uniform(8.0, 18.0)
|
||||
else:
|
||||
d = random.uniform(1.0, 3.0) if r < 0.75 else random.uniform(3.0, 7.0)
|
||||
d = max(0.7, d + random.gauss(0, 0.4))
|
||||
time.sleep(d); return d
|
||||
|
||||
|
||||
def logged_out():
|
||||
h = ev("location.href")
|
||||
return isinstance(h, str) and "/login" in h
|
||||
|
||||
|
||||
# ---- JS 片段 ----
|
||||
CLICK_MORE = r"""(()=>{const kws=['更多','更多操作','更多设置'];
|
||||
const b=[...document.querySelectorAll('button,a,span,div')].find(x=>x.childElementCount===0&&kws.includes((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);
|
||||
if(!b)return 'no-more';b.click();return 'clicked';})()"""
|
||||
|
||||
# 抓取下拉/浮层菜单里的操作项: 限 dropdown/popper/menu/popover 容器(可见),
|
||||
# 且单容器项数<=15(超过则为筛选/树选择面板, 非操作菜单, 丢弃)
|
||||
MENU_ITEMS = r"""(()=>{const BAD=/搜索|条\/页|进入员工端|进入官网|退出登录|个性化设置|按.*排序|升序|降序|排在前|排在后|帮助/;
|
||||
const boxes=[...document.querySelectorAll('[class*=dropdown],[class*=Dropdown],[class*=popper],[class*=popover],[class*=Popover],[role=menu]')]
|
||||
.filter(m=>{const r=m.getBoundingClientRect();return r.width>0&&r.height>0;});
|
||||
let out=[];
|
||||
for(const box of boxes){
|
||||
const its=[...box.querySelectorAll('li,[role=menuitem],[class*=item],[class*=option]')]
|
||||
.map(e=>(e.innerText||'').replace(/\s+/g,'').trim()).filter(t=>t&&t.length>1&&t.length<16&&!BAD.test(t));
|
||||
const uniq=[...new Set(its)];
|
||||
if(uniq.length>=1&&uniq.length<=15)out=out.concat(uniq); // 超过15视为筛选面板污染, 丢弃
|
||||
}
|
||||
return JSON.stringify([...new Set(out)]);})()"""
|
||||
|
||||
# 点自定义列/设置字段/选择表头 入口(图标或文字)
|
||||
CLICK_COLSET = r"""(()=>{
|
||||
const u=[...document.querySelectorAll('use')].find(u=>{const h=(u.getAttribute('xlink:href')||u.getAttribute('href')||'');return h==='#icon-custom-columns'||h==='#icon-setting'&&false;});
|
||||
if(u){const b=u.closest('button,a,[role=button],span,div');if(b){b.click();return 'icon';}}
|
||||
const kw=/^(自定义列|设置字段|字段设置|选择表头|表头设置|设置花名册字段|列设置|自定义字段)$/;
|
||||
const t=[...document.querySelectorAll('button,a,span,div,li')].find(x=>x.childElementCount===0&&kw.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);
|
||||
if(t){t.click();return 'txt';}
|
||||
return 'none';})()"""
|
||||
|
||||
# 从"选择表头/自定义列"弹窗按顺序抓字段(复选框标签法, 已验证=花名册81字段干净)
|
||||
# 限定弹窗容器(含"选择表头|已选"且checkbox>3且宽180-700高>200), walk遍历childNodes:
|
||||
# text节点匹配GRPS作分组标题, LABEL标签作字段, NOISE集过滤噪音
|
||||
GRAB_FIELDS = r"""(()=>{
|
||||
const pop=[...document.querySelectorAll('div,aside,section')].find(m=>{const t=m.innerText||'';const r=m.getBoundingClientRect();
|
||||
return r.width>180&&r.width<700&&r.height>200&&/选择表头|已选/.test(t)&&m.querySelectorAll('input[type=checkbox]').length>3;});
|
||||
if(!pop)return 'no-modal';
|
||||
const GRPS=['系统字段','自定义字段','基础字段','常用字段','其他字段','个人信息','工作信息','合同信息'];
|
||||
const NOISE=new Set(['选择表头','没有搜索到相关内容','全部','清除','取消','确定','恢复默认','重置','搜索','已选']);
|
||||
let seq=[],cur='';
|
||||
const walk=el=>{for(const c of el.childNodes){
|
||||
if(c.nodeType===3){const t=(c.textContent||'').trim();if(t&&t.length<15){if(GRPS.includes(t))cur=t;else if(!NOISE.has(t)&&!/^已选/.test(t))seq.push([cur,t]);}}
|
||||
else if(c.nodeType===1){if(c.tagName==='LABEL'){const t=(c.innerText||'').trim();if(t&&t.length<15&&!NOISE.has(t))seq.push([cur,t]);}else walk(c);}}};
|
||||
walk(pop);
|
||||
const seen=new Set(),out=[];for(const [g,n] of seq){if(seen.has(n))continue;seen.add(n);out.push([g,n]);}
|
||||
return JSON.stringify(out);})()"""
|
||||
|
||||
# 处理"页面内容已过期, 请点击 立即刷新"横幅(标签闲置后出现)
|
||||
REFRESH = r"""(()=>{const b=[...document.querySelectorAll('a,button,span')].find(x=>x.childElementCount===0&&/立即刷新/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);if(b){b.click();return 'refreshed';}return 'none';})()"""
|
||||
|
||||
CLOSE_MODAL = r"""(()=>{const b=[...document.querySelectorAll('button,a')].find(x=>{const t=(x.innerText||'').trim();return (t==='取消'||t==='关闭')&&x.getBoundingClientRect().width>0;});
|
||||
if(b){b.click();return 'closed';}document.body.click();return 'bodyclick';})()"""
|
||||
|
||||
# 兜底: 默认渲染的可见列头(过滤 操作/序号/全选/日期列等非字段噪音)
|
||||
VISIBLE_COLS = r"""(()=>{const BAD=/^(操作|序号|全选|选择|处理)$|^\d+日|^\d+$/;
|
||||
const heads=[...document.querySelectorAll('[role=columnheader],th,[class*=headerCell],[class*=header_] [class*=cell_],[class*=thead] [class*=cell]')]
|
||||
.map(e=>(e.innerText||'').replace(/\s+/g,'').trim()).filter(t=>t&&t.length<20&&!BAD.test(t));
|
||||
return JSON.stringify([...new Set(heads)]);})()"""
|
||||
|
||||
|
||||
def process(p, idx, total):
|
||||
fp, url, nk = p["full_path"], p["url"], p["node_key"]
|
||||
log("[%d/%d] %s" % (idx, total, fp))
|
||||
call("navigate", {"url": url, "newTab": False})
|
||||
time.sleep(4)
|
||||
for _ in range(4):
|
||||
if not logged_out():
|
||||
break
|
||||
time.sleep(2)
|
||||
if logged_out():
|
||||
log(" !! 登录态失效, 中止"); return -1
|
||||
# 处理"页面内容已过期"横幅(标签闲置后出现)
|
||||
if ev(REFRESH) == "refreshed":
|
||||
time.sleep(3)
|
||||
clear_page(nk)
|
||||
|
||||
# ---- 任务1: 更多菜单隐藏入口(菜单渲染有时滞后, 重试抓取) ----
|
||||
extras = []
|
||||
for attempt in range(3):
|
||||
if ev(CLICK_MORE) != "clicked":
|
||||
break
|
||||
time.sleep(1.2 + attempt * 0.8)
|
||||
try:
|
||||
items = json.loads(ev(MENU_ITEMS))
|
||||
except Exception:
|
||||
items = []
|
||||
if items:
|
||||
for it in items:
|
||||
extras.append(("more_menu", it))
|
||||
ev(CLOSE_MODAL); time.sleep(0.5)
|
||||
break
|
||||
ev(CLOSE_MODAL); time.sleep(0.4)
|
||||
if extras:
|
||||
save_extras(nk, fp, url, extras)
|
||||
log(" 更多菜单项: %d 个 -> %s" % (len(extras), "、".join(x[1] for x in extras[:8])))
|
||||
|
||||
# ---- 任务3: 完整字段 ----
|
||||
fields = []; source = ""
|
||||
if ev(CLICK_COLSET) != "none":
|
||||
time.sleep(1.5)
|
||||
raw = ev(GRAB_FIELDS)
|
||||
if isinstance(raw, str) and raw.startswith("["):
|
||||
try:
|
||||
fields = json.loads(raw)
|
||||
except Exception:
|
||||
fields = []
|
||||
source = "custom_columns"
|
||||
ev(CLOSE_MODAL); time.sleep(0.5)
|
||||
if not fields:
|
||||
# 兜底可见列
|
||||
try:
|
||||
cols = json.loads(ev(VISIBLE_COLS))
|
||||
except Exception:
|
||||
cols = []
|
||||
fields = [("", c) for c in cols]
|
||||
source = "visible_cols"
|
||||
if fields:
|
||||
save_fields(nk, fp, url, fields, source)
|
||||
log(" 字段: %d 个 (来源=%s) -> %s" % (len(fields), source, "、".join(f[1] for f in fields[:10])))
|
||||
else:
|
||||
log(" 无字段入口(可能非表格页)")
|
||||
return len(fields)
|
||||
|
||||
|
||||
def main():
|
||||
ensure()
|
||||
call("find_tab", {"url": "2haohr.com"})
|
||||
c = db()
|
||||
rows = [dict(r) for r in c.execute("SELECT node_key,parent_key,name,full_path,url FROM features")]
|
||||
c.close()
|
||||
kids = collections.defaultdict(int)
|
||||
for r in rows:
|
||||
if r["parent_key"]:
|
||||
kids[r["parent_key"]] += 1
|
||||
leaves = [r for r in rows if kids.get(r["node_key"], 0) == 0 and r["url"]]
|
||||
log("=== 全站字段采集, 共 %d 个叶子页 ===" % len(leaves))
|
||||
total_f = 0
|
||||
for i, p in enumerate(leaves, 1):
|
||||
try:
|
||||
n = process(p, i, len(leaves))
|
||||
if n == -1:
|
||||
log("=== 中止 ==="); break
|
||||
total_f += max(0, n)
|
||||
except Exception as e:
|
||||
log(" 页面异常: %s" % str(e)[:100])
|
||||
human_delay("page")
|
||||
log("=== DONE 累计字段=%d ===" % total_f)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
阶段2: 逐页整页截图爬取。
|
||||
从 SQLite 读取所有 level2 叶子(url + screenshot_path), 依次:
|
||||
随机 5-15s 延时 -> 导航到 url -> 等待加载 -> 整页滚动拼接截图 -> 保存到树形目录
|
||||
支持断点续跑(已存在截图文件则跳过)。进度写 db/crawl_progress.log
|
||||
用法: python3 crawl_shots.py [start_index]
|
||||
"""
|
||||
import os, sys, json, time, random, sqlite3, urllib.request
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from capture_full import capture, ev
|
||||
|
||||
DAEMON = "http://127.0.0.1:10086/command"
|
||||
SESSION = "2haohr"
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
LOG = os.path.join(BASE, "db", "crawl_progress.log")
|
||||
|
||||
def navigate(url):
|
||||
body = json.dumps({"action": "navigate", "args": {"url": url, "newTab": False}, "session": SESSION}).encode()
|
||||
req = urllib.request.Request(DAEMON, data=body, headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=45) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
def wait_ready(max_wait=8):
|
||||
"""等待页面加载完成(readyState + 主内容出现)"""
|
||||
t0 = time.time()
|
||||
while time.time() - t0 < max_wait:
|
||||
try:
|
||||
rs = ev("(()=>document.readyState)()", SESSION)
|
||||
if rs == "complete":
|
||||
time.sleep(1.2)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.6)
|
||||
return False
|
||||
|
||||
def log(msg):
|
||||
line = time.strftime("%H:%M:%S ") + msg
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def main():
|
||||
start = int(sys.argv[1]) if len(sys.argv) > 1 else 0
|
||||
conn = sqlite3.connect(DB); conn.row_factory = sqlite3.Row
|
||||
rows = conn.execute(
|
||||
"SELECT node_key,name,full_path,url,screenshot_path FROM features WHERE level=2 ORDER BY sort_order, id"
|
||||
).fetchall()
|
||||
# 稳定顺序: 按 full_path
|
||||
rows = sorted(rows, key=lambda r: r["full_path"] or "")
|
||||
conn.close()
|
||||
total = len(rows)
|
||||
log(f"=== crawl start, total={total}, start_index={start} ===")
|
||||
done = 0; skipped = 0; failed = 0
|
||||
for i, r in enumerate(rows):
|
||||
if i < start:
|
||||
continue
|
||||
rel = r["screenshot_path"]
|
||||
out = os.path.join(BASE, rel)
|
||||
if os.path.exists(out) and os.path.getsize(out) > 3000:
|
||||
skipped += 1
|
||||
continue
|
||||
delay = random.uniform(5, 15)
|
||||
log(f"[{i+1}/{total}] wait {delay:.1f}s -> {r['full_path']}")
|
||||
time.sleep(delay)
|
||||
try:
|
||||
navigate(r["url"])
|
||||
wait_ready()
|
||||
time.sleep(random.uniform(0.8, 2.0)) # 额外抖动
|
||||
res = capture(SESSION, out, 16)
|
||||
done += 1
|
||||
log(f" OK {r['name']} -> {rel}")
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
log(f" FAIL {r['name']} url={r['url']} err={str(e)[:150]}")
|
||||
log(f"=== DONE done={done} skipped={skipped} failed={failed} total={total} ===")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
2号人事部 功能调研 —— SQLite 数据存取工具
|
||||
|
||||
用法:
|
||||
python3 dbtool.py init 初始化数据库表结构
|
||||
python3 dbtool.py add <json> 新增/更新一个功能节点(传入JSON对象)
|
||||
python3 dbtool.py addmany <json_array> 批量新增/更新(传入JSON数组)
|
||||
python3 dbtool.py setshot <node_key> <path> 设置某节点的截图路径
|
||||
python3 dbtool.py get <node_key> 查询单个节点
|
||||
python3 dbtool.py tree 以树形结构打印(JSON)
|
||||
python3 dbtool.py stats 统计各层级节点数
|
||||
|
||||
节点字段:
|
||||
node_key 唯一键(建议用 URL 或 path 组合, 用于去重更新)
|
||||
parent_key 父节点的 node_key (顶层为空)
|
||||
level 0=顶层模块 1=分组 2=菜单页 3=子页/Tab
|
||||
name 功能名称
|
||||
full_path 全路径名称(面包屑, 如 "人事 / 员工管理 / 员工花名册")
|
||||
url 页面URL
|
||||
description 释义(功能是做什么的)
|
||||
understanding 我理解的信息(交互/字段/价值等)
|
||||
screenshot_path 截图相对路径
|
||||
sort_order 排序
|
||||
keep 是否保留(供管理员勾选, 默认1=保留)
|
||||
"""
|
||||
import sqlite3, sys, json, os
|
||||
|
||||
DB_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "db", "research.db")
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS features (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_key TEXT UNIQUE NOT NULL,
|
||||
parent_key TEXT,
|
||||
level INTEGER NOT NULL DEFAULT 0,
|
||||
name TEXT NOT NULL,
|
||||
full_path TEXT,
|
||||
url TEXT,
|
||||
description TEXT,
|
||||
understanding TEXT,
|
||||
screenshot_path TEXT,
|
||||
sort_order INTEGER DEFAULT 0,
|
||||
keep INTEGER DEFAULT 1,
|
||||
created_at TEXT DEFAULT (datetime('now','localtime')),
|
||||
updated_at TEXT DEFAULT (datetime('now','localtime'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_parent ON features(parent_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_level ON features(level);
|
||||
"""
|
||||
|
||||
FIELDS = ["node_key","parent_key","level","name","full_path","url",
|
||||
"description","understanding","screenshot_path","sort_order","keep"]
|
||||
|
||||
def conn():
|
||||
os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
|
||||
c = sqlite3.connect(DB_PATH)
|
||||
c.row_factory = sqlite3.Row
|
||||
return c
|
||||
|
||||
def init():
|
||||
c = conn()
|
||||
c.executescript(SCHEMA)
|
||||
c.commit(); c.close()
|
||||
print("DB initialized at", DB_PATH)
|
||||
|
||||
def upsert(c, obj):
|
||||
keys = [k for k in FIELDS if k in obj]
|
||||
placeholders = ",".join(["?"]*len(keys))
|
||||
cols = ",".join(keys)
|
||||
updates = ",".join([f"{k}=excluded.{k}" for k in keys if k != "node_key"])
|
||||
sql = f"INSERT INTO features ({cols}) VALUES ({placeholders}) " \
|
||||
f"ON CONFLICT(node_key) DO UPDATE SET {updates}, updated_at=datetime('now','localtime')"
|
||||
c.execute(sql, [obj[k] for k in keys])
|
||||
|
||||
def add(obj):
|
||||
c = conn(); upsert(c, obj); c.commit(); c.close()
|
||||
print("ok:", obj.get("node_key"))
|
||||
|
||||
def addmany(arr):
|
||||
c = conn()
|
||||
for obj in arr:
|
||||
upsert(c, obj)
|
||||
c.commit(); c.close()
|
||||
print("ok addmany:", len(arr))
|
||||
|
||||
def setshot(node_key, path):
|
||||
c = conn()
|
||||
c.execute("UPDATE features SET screenshot_path=?, updated_at=datetime('now','localtime') WHERE node_key=?", (path, node_key))
|
||||
c.commit(); n = c.total_changes; c.close()
|
||||
print("updated" if n else "not found", node_key)
|
||||
|
||||
def get(node_key):
|
||||
c = conn()
|
||||
r = c.execute("SELECT * FROM features WHERE node_key=?", (node_key,)).fetchone()
|
||||
c.close()
|
||||
print(json.dumps(dict(r), ensure_ascii=False) if r else "null")
|
||||
|
||||
def stats():
|
||||
c = conn()
|
||||
rows = c.execute("SELECT level, COUNT(*) n FROM features GROUP BY level ORDER BY level").fetchall()
|
||||
total = c.execute("SELECT COUNT(*) n FROM features").fetchone()["n"]
|
||||
shots = c.execute("SELECT COUNT(*) n FROM features WHERE screenshot_path IS NOT NULL AND screenshot_path<>''").fetchone()["n"]
|
||||
c.close()
|
||||
for r in rows:
|
||||
print(f"level {r['level']}: {r['n']}")
|
||||
print(f"total: {total}, with_screenshot: {shots}")
|
||||
|
||||
def tree():
|
||||
c = conn()
|
||||
rows = [dict(r) for r in c.execute("SELECT * FROM features ORDER BY level, sort_order, id").fetchall()]
|
||||
c.close()
|
||||
by_parent = {}
|
||||
for r in rows:
|
||||
by_parent.setdefault(r["parent_key"] or "", []).append(r)
|
||||
def build(pk):
|
||||
out = []
|
||||
for r in by_parent.get(pk, []):
|
||||
node = {k: r[k] for k in ("node_key","name","level","url","description","understanding","screenshot_path","keep")}
|
||||
node["children"] = build(r["node_key"])
|
||||
out.append(node)
|
||||
return out
|
||||
print(json.dumps(build(""), ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
cmd = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||
if cmd == "init": init()
|
||||
elif cmd == "add": add(json.loads(sys.argv[2]))
|
||||
elif cmd == "addmany": addmany(json.loads(sys.argv[2]))
|
||||
elif cmd == "setshot": setshot(sys.argv[2], sys.argv[3])
|
||||
elif cmd == "get": get(sys.argv[2])
|
||||
elif cmd == "stats": stats()
|
||||
elif cmd == "tree": tree()
|
||||
else:
|
||||
print(__doc__)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""业务数据列表页导出深挖：各模块有真实数据的列表页(花名册/工资表/账单/考勤明细等)
|
||||
触发导出(类型B: #icon-export / 文字“导出”) -> 异步任务中心 -> 统一收割落盘。
|
||||
复用 dl_export 的安全逻辑(防登出授权处理 + 非均匀拟人间隔 + 四类入口)。"""
|
||||
import sys, time
|
||||
sys.path.insert(0, "scripts")
|
||||
import dl_export as X
|
||||
|
||||
# (保存路径 full_path, url, node_key) —— 各模块有真实数据的业务列表页
|
||||
DATA = [
|
||||
("02_人事 / 员工管理 / 员工花名册", "https://2haohr.com/personnel/employee/roster", "biz_roster"),
|
||||
("02_人事 / 员工管理 / 员工档案库", "https://2haohr.com/personnel/employee/archive", "biz_arch"),
|
||||
("02_人事 / 员工管理 / 万能人事表格", "https://2haohr.com/personnel/employee/personnel-form", "biz_form"),
|
||||
("02_人事 / 员工关系 / 入职管理", "https://2haohr.com/personnel/relation/enrollment", "biz_enroll"),
|
||||
("02_人事 / 员工关系 / 离职管理", "https://2haohr.com/personnel/relation/departure", "biz_depart"),
|
||||
("02_人事 / 员工关系 / 合同管理", "https://2haohr.com/personnel/relation/contract", "biz_contract"),
|
||||
("02_人事 / 员工关系 / 奖惩记录", "https://2haohr.com/personnel/relation/award", "biz_award"),
|
||||
("02_人事 / 员工关系 / 人事异动", "https://2haohr.com/personnel/relation/unusual-action", "biz_unusual"),
|
||||
("06_薪酬 / 薪酬管理 / 月工资表", "https://2haohr.com/payroll/salary/monthly", "biz_monthly"),
|
||||
("06_薪酬 / 薪酬管理 / 员工薪资", "https://2haohr.com/payroll/salary/profiles", "biz_profiles"),
|
||||
("06_薪酬 / 薪酬管理 / 工资发放", "https://2haohr.com/payroll/salary/payoff", "biz_payoff"),
|
||||
("05_社保公积金 / 常规 / 参保账单", "https://2haohr.com/social-insurance/records", "biz_sirec"),
|
||||
("05_社保公积金 / 常规 / 账单核对", "https://2haohr.com/social-insurance/bill", "biz_sibill"),
|
||||
("04_考勤假期 / 考勤统计 / 月考勤表", "https://2haohr.com/attendance/stat/monthly", "biz_kqmonth"),
|
||||
("04_考勤假期 / 考勤统计 / 明细记录", "https://2haohr.com/attendance/stat/records", "biz_kqrec"),
|
||||
("04_考勤假期 / 考勤统计 / 每日考勤", "https://2haohr.com/attendance/stat/daily", "biz_kqday"),
|
||||
("04_考勤假期 / 假期管理 / 休假一览", "https://2haohr.com/attendance/holiday/leaveRecords", "biz_leave"),
|
||||
("04_考勤假期 / 假期管理 / 假期余额", "https://2haohr.com/attendance/holiday/balance", "biz_balance"),
|
||||
("03_组织 / 岗位管理 / 员工标签", "https://2haohr.com/org/tag/emp-tags", "biz_tag"),
|
||||
("07_绩效 / 考核结果 / 绩效考核记录", "https://2haohr.com/performance-evaluation/result/assessment", "biz_perfrec"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
X.ensure()
|
||||
X.call("find_tab", {"url": "2haohr.com"})
|
||||
pages = [{"full_path": a, "url": b, "node_key": c, "screenshot_path": None} for a, b, c in DATA]
|
||||
X.log("=== BIZ 业务列表页深挖, 共 %d 页 ===" % len(pages))
|
||||
aborted = False
|
||||
for i, p in enumerate(pages, 1):
|
||||
try:
|
||||
n = X.process_page(p, i, len(pages))
|
||||
if n == -1:
|
||||
X.log("=== 登录态失效, 中止 ===")
|
||||
aborted = True
|
||||
break
|
||||
except Exception as e:
|
||||
X.log(" 页面异常: %s" % str(e)[:100])
|
||||
X.human_delay("page")
|
||||
if not aborted:
|
||||
X.log("=== 收割任务中心 stage2=%d ===" % X.harvest_task_center())
|
||||
X.log("=== BIZ DONE 累计文件=%d ===" % len(X.SAVED))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""第二批业务列表页深挖(补充未深挖模块)：组织/审批/电子合同/电子工资条/
|
||||
社保参保档案/人事关系子表/绩效档案/培训记录/行政公告。
|
||||
复用 dl_export 的四类入口 + 防登出授权 + 非均匀拟人间隔 + 任务中心收割。"""
|
||||
import sys
|
||||
sys.path.insert(0, "scripts")
|
||||
import dl_export as X
|
||||
|
||||
# (保存路径 full_path, url, node_key) —— 大概率有真实数据+导出入口的业务列表页
|
||||
DATA = [
|
||||
("03_组织 / 部门管理 / 组织管理", "https://2haohr.com/org/dept/manage/home", "biz2_dept"),
|
||||
("03_组织 / 岗位管理 / 岗位管理", "https://2haohr.com/org/job/job-management", "biz2_job"),
|
||||
("03_组织 / 岗位管理 / 职务管理", "https://2haohr.com/org/job-system/title", "biz2_title"),
|
||||
("03_组织 / 岗位管理 / 职级管理", "https://2haohr.com/org/rank", "biz2_rank"),
|
||||
("08_审批 / 记录 / 审批记录", "https://2haohr.com/approval/record/list", "biz2_apprec"),
|
||||
("05_社保公积金 / 常规 / 参保档案", "https://2haohr.com/social-insurance/archives", "biz2_siarch"),
|
||||
("12_电子合同 / 合同签署 / 已签署", "https://2haohr.com/econtract/sign/signed", "biz2_signed"),
|
||||
("12_电子合同 / 合同签署 / 合同档案库", "https://2haohr.com/econtract/archive/home", "biz2_econarch"),
|
||||
("20_电子工资条 / 常规 / 发放记录", "https://2haohr.com/payslip/record", "biz2_payslip"),
|
||||
("02_人事 / 员工关系 / 转正管理", "https://2haohr.com/personnel/relation/probation", "biz2_probation"),
|
||||
("02_人事 / 员工关系 / 任职记录", "https://2haohr.com/personnel/relation/office", "biz2_office"),
|
||||
("02_人事 / 员工关系 / 实习生管理", "https://2haohr.com/personnel/relation/internship", "biz2_intern"),
|
||||
("02_人事 / 员工关系 / 培训经历", "https://2haohr.com/personnel/relation/train", "biz2_train"),
|
||||
("02_人事 / 员工关系 / 沟通记录", "https://2haohr.com/personnel/relation/talk", "biz2_talk"),
|
||||
("09_绩效 / 考核结果 / 员工绩效档案", "https://2haohr.com/performance-evaluation/result/archive", "biz2_perfarch"),
|
||||
("09_绩效 / 考核结果 / 绩效面谈记录", "https://2haohr.com/performance-evaluation/result/interview", "biz2_interview"),
|
||||
("13_培训学习 / 培训管理 / 培训记录", "https://2haohr.com/training/manage/record/list", "biz2_trainrec"),
|
||||
("11_行政管理 / 常规 / 企业公告", "https://2haohr.com/oa/notice", "biz2_notice"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
X.ensure()
|
||||
X.call("find_tab", {"url": "2haohr.com"})
|
||||
pages = [{"full_path": a, "url": b, "node_key": c, "screenshot_path": None} for a, b, c in DATA]
|
||||
X.log("=== BIZ2 补充业务列表页深挖, 共 %d 页 ===" % len(pages))
|
||||
aborted = False
|
||||
for i, p in enumerate(pages, 1):
|
||||
try:
|
||||
n = X.process_page(p, i, len(pages))
|
||||
if n == -1:
|
||||
X.log("=== 登录态失效, 中止 ===")
|
||||
aborted = True
|
||||
break
|
||||
except Exception as e:
|
||||
X.log(" 页面异常: %s" % str(e)[:100])
|
||||
X.human_delay("page")
|
||||
if not aborted:
|
||||
X.log("=== 收割任务中心 stage2=%d ===" % X.harvest_task_center())
|
||||
X.log("=== BIZ2 DONE 累计文件=%d ===" % len(X.SAVED))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
报表/文档导出下载器（Excel/Word 为主）。完整链路（已逐环节实测验证）：
|
||||
入口：svg use[xlink:href=#icon-download] 图标（无文字，代码级定位）+ 文字“下载/生成excel”兜底；
|
||||
生成：点图标 -> 弹“下载报表/导出”参数窗 -> 点“确定”(默认参数) -> 异步生成；
|
||||
捕获：hook HTMLAnchorElement.prototype.click 捕获 rpt-cdn/prv-cdn 带 token 直链；
|
||||
落盘：页面内同步 XHR 取字节 -> base64 回传 -> 写 workspace（绕过 macOS ~/Downloads 的 TCC 限制）。
|
||||
阶段2：任务中心(taskCenter)收割兜底。产出真实 .xlsx/.docx，写 SQLite downloads 表；
|
||||
页面间随机 5-15s；断点续跑(同名>2000字节跳过)。
|
||||
"""
|
||||
import json, urllib.request, base64, time, os, random, sqlite3, re, collections
|
||||
|
||||
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
||||
DB = "db/research.db"; LOG = "db/dl_export.log"
|
||||
KW = ['报表', '分析', '仪表盘', '统计', '档案', '模板', '文档', '制度', '公告', '证明']
|
||||
TXT_BTN = ['直接下载', '生成excel', '导出Excel', '导出报表', '批量导出'] # 文字入口(不含裸“下载/导出”避免误点导航)
|
||||
|
||||
def call(a, args, timeout=120):
|
||||
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
||||
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
||||
return json.loads(urllib.request.urlopen(r, timeout=timeout).read().decode())
|
||||
|
||||
def ev(code, timeout=120):
|
||||
try:
|
||||
return call("evaluate", {"code": code}, timeout).get("data", {}).get("value")
|
||||
except Exception as e:
|
||||
return "EXC:" + str(e)[:80]
|
||||
|
||||
def log(m):
|
||||
line = time.strftime("%H:%M:%S ") + m
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
HOOK = r"""(()=>{window.__cap=window.__cap||[];
|
||||
if(!HTMLAnchorElement.prototype.__hk){const ac=HTMLAnchorElement.prototype.click;
|
||||
HTMLAnchorElement.prototype.click=function(){try{const h=this.href||'';
|
||||
if(h&&/-cdn\.2haohr|prv-cdn|rpt-cdn|attname=|\.xlsx|\.xls|\.docx|\.doc|\.zip/i.test(h))window.__cap.push(h);}catch(e){}
|
||||
return ac.apply(this,arguments);};HTMLAnchorElement.prototype.__hk=1;}
|
||||
return 'ok';})()"""
|
||||
|
||||
# 未处理的 #icon-download 数量
|
||||
COUNT_ICON = r"""(()=>{let n=0;document.querySelectorAll('use').forEach(u=>{
|
||||
const h=u.getAttribute('xlink:href')||u.getAttribute('href');
|
||||
if(h==='#icon-download'){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl'))n++;}});return n;})()"""
|
||||
|
||||
# 点击下一个未处理的 #icon-download
|
||||
CLICK_ICON = r"""(()=>{const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-download');
|
||||
for(const u of us){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl')){b.setAttribute('data-dl','1');b.click();return 'ok';}}
|
||||
return 'none';})()"""
|
||||
|
||||
# 文字入口
|
||||
COUNT_TXT = (r"""(()=>{const T=%s;let n=0;document.querySelectorAll('button,a,[role=button]').forEach(x=>{"""
|
||||
r"""const t=(x.innerText||'').trim();if(T.includes(t)&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl'))n++;});return n;})()""") % json.dumps(TXT_BTN)
|
||||
CLICK_TXT = (r"""(()=>{const T=%s;for(const t of T){const b=[...document.querySelectorAll('button,a,[role=button]')]"""
|
||||
r""".find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl'));"""
|
||||
r"""if(b){b.setAttribute('data-dl','1');b.click();return t;}}return 'none';})()""") % json.dumps(TXT_BTN)
|
||||
|
||||
# 参数窗“确定/下载/导出”——关键词精确定位“下载报表/导出”参数窗(避免误点其它弹层)
|
||||
KWWIN = r"下载报表|导出|下载|部门|员工排序|工作性质|字段|范围|格式|参数|排序|在职"
|
||||
CONFIRM = (r"""(()=>{const cand=['确定','确认','下载','导出','生成excel','生成','立即导出','立即下载'];
|
||||
const KW=/%s/;
|
||||
const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')];
|
||||
const vis=ms.filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>80&&getComputedStyle(m).display!=='none'&&getComputedStyle(m).visibility!=='hidden';});
|
||||
// 优先带关键词的参数窗, 再退化到任意可见弹层
|
||||
const ranked=[...vis.filter(m=>KW.test(m.innerText||'')),...vis];
|
||||
for(const m of ranked){
|
||||
for(const t of cand){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0);
|
||||
if(b){b.click();return 'modal:'+t;}}}
|
||||
return vis.length?'modal:noconfirm':'nomodal';})()""") % KWWIN
|
||||
|
||||
CLOSE = r"""(()=>{const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer]')];
|
||||
for(const m of ms){const r=m.getBoundingClientRect();if(r.width>200&&r.height>80&&getComputedStyle(m).display!=='none'){
|
||||
const c=[...m.querySelectorAll('button,a,[class*=close],[class*=Close]')]
|
||||
.find(x=>{const t=(x.innerText||'').trim();return t==='取消'||t==='关闭'||(x.className+'').toLowerCase().includes('close');});
|
||||
if(c){c.click();return 'closed';}}}return 'none';})()"""
|
||||
|
||||
# 数据授权/协议窗：只点“知情并同意”，绝不点“不同意并退出登录”(之前登出的元凶)
|
||||
AGREE = r"""(()=>{const bad=/退出登录|不同意/;const ok=['知情并同意','同意并继续','我已阅读并同意','同意'];
|
||||
const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')]
|
||||
.filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>60&&getComputedStyle(m).display!=='none';});
|
||||
for(const m of ms){const txt=m.innerText||'';if(!/协议|授权|同意|条款|隐私/.test(txt))continue;
|
||||
for(const t of ok){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&!bad.test(x.innerText||''));
|
||||
if(b){b.click();return 'agree:'+t;}}}
|
||||
return 'none';})()"""
|
||||
|
||||
BAD = r"退出登录|不同意|立即购买|预约演示|我要更多功能"
|
||||
# 类型B 数据列表导出入口: #icon-export 或 主内容区文字“导出”(排除导航/任务中心/黑名单)
|
||||
CLICK_EXPORT = (r"""(()=>{const BAD=/%s/;
|
||||
const inNav=e=>!!e.closest('nav,[class*=navBar],[class*=sideMenu],[class*=taskCenter]');
|
||||
const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-export');
|
||||
for(const u of us){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl')&&!inNav(b)){b.setAttribute('data-dl','1');b.click();return 'icon-export';}}
|
||||
const b=[...document.querySelectorAll('button,a,[role=button]')].find(x=>{const t=(x.innerText||'').trim();return t==='导出'&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl')&&!inNav(x)&&!BAD.test(t);});
|
||||
if(b){b.setAttribute('data-dl','1');b.click();return 'txt-导出';}
|
||||
return 'none';})()""") % BAD
|
||||
|
||||
# 类型A 报表中心卡片导出: 生成excel/直接下载/下载(排除导航/任务中心/黑名单)
|
||||
CARD_BTN = ['生成excel', '直接下载', '导出Excel', '导出报表', '批量导出', '下载']
|
||||
CLICK_CARD = (r"""(()=>{const T=%s;const BAD=/%s/;
|
||||
const inNav=e=>!!e.closest('nav,[class*=navBar],[class*=sideMenu],[class*=taskCenter]');
|
||||
for(const t of T){const b=[...document.querySelectorAll('button,a,[role=button]')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl')&&!inNav(x)&&!BAD.test(x.innerText||''));
|
||||
if(b){b.setAttribute('data-dl','1');b.click();return t;}}
|
||||
return 'none';})()""") % (json.dumps(CARD_BTN), BAD)
|
||||
|
||||
FETCH_LAST = r"""(()=>{try{if(!window.__cap||!window.__cap.length)return 'NO_HREF';
|
||||
const u=window.__cap[window.__cap.length-1];const x=new XMLHttpRequest();x.open('GET',u,false);
|
||||
x.overrideMimeType('text/plain;charset=x-user-defined');x.send();if(x.status!==200)return 'HTTP'+x.status;
|
||||
const s=x.responseText;let out='';const CH=8192;for(let i=0;i<s.length;i+=CH){let c='';const e=Math.min(i+CH,s.length);
|
||||
for(let j=i;j<e;j++)c+=String.fromCharCode(s.charCodeAt(j)&0xff);out+=c;}
|
||||
window.__b64=btoa(out);const m=u.match(/attname=([^&]+)/);let name=m?decodeURIComponent(m[1]):('export_'+Date.now()+'.bin');
|
||||
return JSON.stringify({size:s.length,name:name});}catch(e){return 'EX '+e.message;}})()"""
|
||||
|
||||
TC_LIST = r"""(()=>{const tc=document.querySelector('[class*=taskCenter]');if(!tc)return '[]';
|
||||
const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
|
||||
const names=dls.map(a=>{const row=a.closest('[class*=item],li,div');const info=row&&row.querySelector('[class*=info_]');
|
||||
return info?(info.innerText||'').replace(/\n+/g,' ').trim().slice(0,50):'';});
|
||||
return JSON.stringify(names);})()"""
|
||||
TC_CLICK = r"""(function(i){const tc=document.querySelector('[class*=taskCenter]');if(!tc)return 'NO_TC';
|
||||
const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
|
||||
if(i>=dls.length)return 'OOB';dls[i].click();return 'ok';})(%d)"""
|
||||
|
||||
def db():
|
||||
c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
|
||||
|
||||
def ensure():
|
||||
c = db(); c.execute("""CREATE TABLE IF NOT EXISTS downloads(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
|
||||
kind TEXT, file_path TEXT, status TEXT, size INTEGER, note TEXT, ts TEXT)"""); c.commit(); c.close()
|
||||
|
||||
def rec(nk, fp, url, kind, path, status, size, note):
|
||||
c = db(); c.execute("INSERT INTO downloads(node_key,full_path,url,kind,file_path,status,size,note,ts) VALUES(?,?,?,?,?,?,?,?,?)",
|
||||
(nk, fp, url, kind, path, status, size, note, time.strftime("%Y-%m-%d %H:%M:%S"))); c.commit(); c.close()
|
||||
|
||||
def safe(s):
|
||||
return re.sub(r'[\\/:*?"<>|]', '_', (s or "").strip())[:90]
|
||||
|
||||
def out_dir(shot, full_path):
|
||||
if shot and shot.startswith("screenshots/"):
|
||||
return os.path.dirname(shot.replace("screenshots/", "downloads/", 1))
|
||||
parts = [safe(p) for p in (full_path or "").split(" / ") if p.strip()]
|
||||
return os.path.join("downloads", *parts[:-1]) if len(parts) > 1 else "downloads/misc"
|
||||
|
||||
def load_pages():
|
||||
c = db(); rows = [dict(r) for r in c.execute(
|
||||
"SELECT node_key,parent_key,name,full_path,url,screenshot_path FROM features").fetchall()]
|
||||
kids = collections.defaultdict(int)
|
||||
for r in rows:
|
||||
if r["parent_key"]:
|
||||
kids[r["parent_key"]] += 1
|
||||
c.close()
|
||||
leaves = [r for r in rows if kids.get(r["node_key"], 0) == 0]
|
||||
return [r for r in leaves if r["url"] and any(k in (r["full_path"] or "") + (r["name"] or "") for k in KW)]
|
||||
|
||||
SAVED = set()
|
||||
|
||||
def grab_and_save(odir, nk, fp, url, note):
|
||||
meta = ev(FETCH_LAST)
|
||||
if not (isinstance(meta, str) and meta.startswith("{")):
|
||||
return False, str(meta)[:40]
|
||||
info = json.loads(meta); name = safe(info["name"]); ev("window.__cap=[]")
|
||||
if name in SAVED:
|
||||
return False, "dup"
|
||||
try:
|
||||
data = base64.b64decode(ev("window.__b64||''"))
|
||||
except Exception:
|
||||
data = b""
|
||||
if len(data) <= 200:
|
||||
return False, "empty"
|
||||
os.makedirs(odir, exist_ok=True); dst = os.path.join(odir, name)
|
||||
if os.path.exists(dst) and os.path.getsize(dst) > 2000:
|
||||
SAVED.add(name); return True, name + "(exists)"
|
||||
open(dst, "wb").write(data); SAVED.add(name)
|
||||
low = name.lower()
|
||||
kind = "xlsx" if low.endswith((".xlsx", ".xls")) else ("docx" if low.endswith((".docx", ".doc")) else (
|
||||
"pdf" if low.endswith(".pdf") else ("zip" if low.endswith(".zip") else "file")))
|
||||
rec(nk, fp, url, kind, dst, "ok", len(data), note)
|
||||
log(" OK %s (%d bytes) -> %s" % (kind, len(data), dst))
|
||||
return True, name
|
||||
|
||||
def wait_capture(seconds=25):
|
||||
t0 = time.time()
|
||||
while time.time() - t0 < seconds:
|
||||
time.sleep(2)
|
||||
n = ev("(window.__cap&&window.__cap.length)||0")
|
||||
if isinstance(n, int) and n > 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
def human_delay(kind="page"):
|
||||
"""非均匀拟人间隔：大部分短、少量中、偶尔长停顿 + 高斯抖动。"""
|
||||
r = random.random()
|
||||
if kind == "page":
|
||||
if r < 0.60: d = random.uniform(4.0, 9.0)
|
||||
elif r < 0.88: d = random.uniform(9.0, 20.0)
|
||||
else: d = random.uniform(20.0, 42.0)
|
||||
else: # 同页内相邻点击
|
||||
if r < 0.68: d = random.uniform(1.4, 4.0)
|
||||
elif r < 0.93: d = random.uniform(4.0, 9.0)
|
||||
else: d = random.uniform(9.0, 16.0)
|
||||
d = max(0.8, d + random.gauss(0, 0.5))
|
||||
time.sleep(d)
|
||||
return d
|
||||
|
||||
def logged_out():
|
||||
href = ev("location.href")
|
||||
return isinstance(href, str) and "/login" in href
|
||||
|
||||
def done_nodes():
|
||||
c = db()
|
||||
rows = c.execute("SELECT DISTINCT node_key FROM downloads WHERE status='ok' AND node_key<>''").fetchall()
|
||||
c.close()
|
||||
return {r[0] for r in rows}
|
||||
|
||||
def do_click_cycle(clicked_label, odir, nk, fp, url):
|
||||
"""点击入口后：持续处理授权窗/参数窗(可能异步慢加载)并确认, 同时检测直链; 兑底走任务中心。"""
|
||||
ev("window.__cap=[]")
|
||||
got = False
|
||||
# 最多 ~30s: 参数窗可能异步加载, 持续 AGREE+CONFIRM 并检测直链
|
||||
for _ in range(20):
|
||||
time.sleep(1.5)
|
||||
ev(AGREE)
|
||||
ev(CONFIRM)
|
||||
n = ev("(window.__cap&&window.__cap.length)||0")
|
||||
if isinstance(n, int) and n > 0:
|
||||
got = True
|
||||
break
|
||||
if got:
|
||||
ok, name = grab_and_save(odir, nk, fp, url, "阶段1 " + clicked_label)
|
||||
if not ok:
|
||||
log(" [%s] 抓取失败: %s" % (clicked_label, name))
|
||||
result = ok
|
||||
else:
|
||||
log(" [%s] 未产出直链(需必选参数/异步未完成, 待任务中心收割)" % clicked_label)
|
||||
rec(nk, fp, url, "pending", "", "skip", 0, clicked_label)
|
||||
result = False
|
||||
ev(CLOSE)
|
||||
return result
|
||||
|
||||
def process_page(p, idx, total):
|
||||
fp, url, nk = p["full_path"], p["url"], p["node_key"]
|
||||
odir = out_dir(p["screenshot_path"], fp)
|
||||
log("[%d/%d] %s" % (idx, total, fp))
|
||||
call("navigate", {"url": url, "newTab": False})
|
||||
# 等页面加载 + 登录态检测
|
||||
for k in range(6):
|
||||
time.sleep(2)
|
||||
if k == 1 and logged_out():
|
||||
log(" !! 检测到登录态失效, 中止本轮")
|
||||
return -1
|
||||
if logged_out():
|
||||
log(" !! 检测到登录态失效, 中止本轮")
|
||||
return -1
|
||||
ev(HOOK)
|
||||
ev(AGREE) # 若有数据授权协议窗, 先“知情并同意”
|
||||
saved = 0
|
||||
hit = False
|
||||
# 四类入口: 图标下载/文字/类型B导出/类型A报表卡片
|
||||
for label, code in (("icon-download", CLICK_ICON), ("txt", CLICK_TXT),
|
||||
("export", CLICK_EXPORT), ("card", CLICK_CARD)):
|
||||
for _ in range(8):
|
||||
c = ev(code)
|
||||
if c == "none":
|
||||
break
|
||||
hit = True
|
||||
if do_click_cycle(label + "=" + str(c), odir, nk, fp, url):
|
||||
saved += 1
|
||||
human_delay("click")
|
||||
if not hit:
|
||||
log(" 无导出入口, skip")
|
||||
rec(nk, fp, url, "none", "", "skip", 0, "无导出入口")
|
||||
else:
|
||||
log(" 本页触发导出入口, 落盘=%d (其余走任务中心)" % saved)
|
||||
return saved
|
||||
|
||||
def harvest_task_center():
|
||||
log("=== 阶段2: 收割任务中心 ===")
|
||||
ev(HOOK)
|
||||
odir = "downloads/_任务中心"; total = 0
|
||||
try:
|
||||
arr = json.loads(ev(TC_LIST))
|
||||
except Exception:
|
||||
arr = []
|
||||
log("任务中心可下载任务: %d" % len(arr))
|
||||
for i, nm in enumerate(arr):
|
||||
ev("window.__cap=[]")
|
||||
if ev(TC_CLICK % i) != "ok":
|
||||
continue
|
||||
if wait_capture(20):
|
||||
ok, _ = grab_and_save(odir, "", "任务中心 / " + (nm or ""), "", "阶段2 任务=" + (nm or ""))
|
||||
if ok:
|
||||
total += 1
|
||||
time.sleep(random.uniform(2, 4))
|
||||
return total
|
||||
|
||||
def main():
|
||||
ensure()
|
||||
call("find_tab", {"url": "2haohr.com"})
|
||||
pages = load_pages()
|
||||
log("=== 开始导出下载, 共 %d 页 ===" % len(pages))
|
||||
stage1 = 0
|
||||
for i, p in enumerate(pages, 1):
|
||||
try:
|
||||
stage1 += process_page(p, i, len(pages))
|
||||
except Exception as e:
|
||||
log(" 页面异常: %s" % str(e)[:100])
|
||||
rec(p["node_key"], p["full_path"], p["url"], "err", "", "fail", 0, str(e)[:80])
|
||||
time.sleep(random.uniform(5, 15))
|
||||
stage2 = harvest_task_center()
|
||||
log("=== DONE 阶段1=%d 阶段2=%d 累计文件=%d ===" % (stage1, stage2, len(SAVED)))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""招聘报表专用下载器。
|
||||
报表页(/recruitment/stats/report/list)为逐行报表, 每行 hover 出「预览/下载」,
|
||||
点「下载」-> 弹参数窗(默认参数) -> 点「确认」-> 服务端异步生成 -> 进任务中心。
|
||||
本脚本: 触发报表列表区全部「下载」并确认参数窗, 随后按招聘关键词白名单从任务中心
|
||||
收割新生成的 Excel(避免误收历史的电子合同/参保/薪酬任务), 落盘到 04_招聘/统计分析。
|
||||
复用 dl_export 的 hook/同步XHR落盘/AGREE防登出/非均匀拟人间隔。"""
|
||||
import sys, time, json, os, base64
|
||||
sys.path.insert(0, "scripts")
|
||||
import dl_export as X
|
||||
|
||||
REPORT_URL = "https://2haohr.com/recruitment/stats/report"
|
||||
NODE = "招聘::/recruitment/stats/report"
|
||||
FP = "招聘 / 统计分析 / 招聘报表"
|
||||
ODIR = "downloads/04_招聘/统计分析/招聘报表"
|
||||
# 只收割任务名命中这些招聘特征词的新任务(历史的电子合同/参保档案/薪酬档案不含)
|
||||
REPORT_KW = ["招聘", "候选人", "面试", "offer", "渠道", "淘汰", "门户",
|
||||
"趋势", "进度", "转化", "工作量", "时效", "等待期", "贡献"]
|
||||
|
||||
# 报表列表区(排除任务中心浮层)未处理的「下载」计数/点击
|
||||
COUNT_DL = r"""(()=>{return [...document.querySelectorAll('span,a,button')].filter(x=>x.childElementCount===0&&(x.innerText||'').trim()==='下载'&&!x.closest('[class*=taskCenter]')&&!x.hasAttribute('data-dl')).length;})()"""
|
||||
CLICK_DL = r"""(()=>{const d=[...document.querySelectorAll('span,a,button')].find(x=>x.childElementCount===0&&(x.innerText||'').trim()==='下载'&&!x.closest('[class*=taskCenter]')&&!x.hasAttribute('data-dl'));if(!d)return 'none';d.setAttribute('data-dl','1');d.scrollIntoView({block:'center'});d.click();return 'ok';})()"""
|
||||
|
||||
|
||||
def match_kw(name):
|
||||
n = (name or "").lower()
|
||||
return any(k.lower() in n for k in REPORT_KW)
|
||||
|
||||
|
||||
def trigger_all():
|
||||
"""逐个点报表「下载」并确认参数窗, 返回触发数量。"""
|
||||
fired = 0
|
||||
total = X.ev(COUNT_DL)
|
||||
X.log("报表列表区可下载报表: %s" % total)
|
||||
for _ in range(30):
|
||||
if X.ev(COUNT_DL) in (0, "0"):
|
||||
break
|
||||
if X.ev(CLICK_DL) != "ok":
|
||||
break
|
||||
fired += 1
|
||||
# 参数窗异步加载, 循环 AGREE(防登出) + CONFIRM(默认参数确认)
|
||||
confirmed = False
|
||||
for _ in range(12):
|
||||
time.sleep(1.2)
|
||||
X.ev(X.AGREE)
|
||||
r = X.ev(X.CONFIRM)
|
||||
if isinstance(r, str) and r.startswith("modal:") and r != "modal:noconfirm" and r != "modal:nomodal":
|
||||
confirmed = True
|
||||
break
|
||||
X.log(" 触发报表#%d 确认=%s" % (fired, confirmed))
|
||||
X.ev(X.CLOSE)
|
||||
X.human_delay("click")
|
||||
return fired
|
||||
|
||||
|
||||
def harvest_reports():
|
||||
"""逐个点任务中心下载, 用捕获到的真实文件名做招聘白名单过滤(历史的
|
||||
电子合同/参保/薪酬文件名不命中自动跳过), 命中则落盘到招聘目录。"""
|
||||
X.ev(X.HOOK)
|
||||
try:
|
||||
n = len(json.loads(X.ev(X.TC_LIST)))
|
||||
except Exception:
|
||||
n = 0
|
||||
X.log("任务中心任务: %d" % n)
|
||||
got = 0
|
||||
for i in range(n):
|
||||
X.ev("window.__cap=[]")
|
||||
if X.ev(X.TC_CLICK % i) != "ok":
|
||||
continue
|
||||
if not X.wait_capture(20):
|
||||
continue
|
||||
meta = X.ev(X.FETCH_LAST)
|
||||
if not (isinstance(meta, str) and meta.startswith("{")):
|
||||
continue
|
||||
name = X.safe(json.loads(meta)["name"])
|
||||
if not match_kw(name):
|
||||
X.log(" 跳过(非招聘报表): %s" % name)
|
||||
X.ev("window.__cap=[]"); continue
|
||||
if name in X.SAVED:
|
||||
X.ev("window.__cap=[]"); continue
|
||||
try:
|
||||
data = base64.b64decode(X.ev("window.__b64||''"))
|
||||
except Exception:
|
||||
data = b""
|
||||
if len(data) <= 200:
|
||||
continue
|
||||
os.makedirs(ODIR, exist_ok=True)
|
||||
dst = os.path.join(ODIR, name)
|
||||
if os.path.exists(dst) and os.path.getsize(dst) > 2000:
|
||||
X.SAVED.add(name); got += 1; continue
|
||||
open(dst, "wb").write(data); X.SAVED.add(name)
|
||||
kind = "xlsx" if name.lower().endswith((".xlsx", ".xls")) else (
|
||||
"docx" if name.lower().endswith((".docx", ".doc")) else "file")
|
||||
X.rec(NODE, FP, REPORT_URL, kind, dst, "ok", len(data), "招聘报表 " + name)
|
||||
X.log(" 收割: %s (%d bytes)" % (name, len(data)))
|
||||
got += 1
|
||||
time.sleep(2.5)
|
||||
return got
|
||||
|
||||
|
||||
def main():
|
||||
X.ensure()
|
||||
X.call("find_tab", {"url": "2haohr.com"})
|
||||
X.call("navigate", {"url": REPORT_URL, "newTab": False})
|
||||
time.sleep(6)
|
||||
if X.logged_out():
|
||||
X.log("!! 登录态失效, 中止"); return
|
||||
X.ev(X.HOOK)
|
||||
X.ev(X.AGREE)
|
||||
X.log("=== 招聘报表导出: 触发阶段 ===")
|
||||
fired = trigger_all()
|
||||
X.log("=== 已触发 %d 个报表, 等待服务端生成 ===" % fired)
|
||||
time.sleep(25)
|
||||
X.log("=== 招聘报表导出: 收割阶段 ===")
|
||||
got = harvest_reports()
|
||||
# 二次收割(有的报表生成较慢)
|
||||
if fired > got:
|
||||
time.sleep(20)
|
||||
got += harvest_reports()
|
||||
X.log("=== REPORT DONE 触发=%d 收割=%d ===" % (fired, got))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
报表/文档下载器 (独立后台进程)。
|
||||
从 SQLite 选出 35 个报表/文档类页面, 逐个:
|
||||
随机 5-15s 延时 -> 导航 -> 等待加载 -> 隐藏浮层 -> save_as_pdf 按结构归档
|
||||
-> 若存在"明确的导出按钮且点击后未弹出需选参数的弹窗", 顺带把导出的 Excel 归集
|
||||
所有结果写入 SQLite downloads 表; 进度写 db/dl_progress.log。支持断点续跑(PDF已存在则跳过)。
|
||||
用法: python3 scripts/dl_reports.py
|
||||
"""
|
||||
import os, sys, json, time, random, shutil, sqlite3, urllib.request, glob
|
||||
|
||||
D = "http://127.0.0.1:10086/command"
|
||||
S = "2haohr"
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
LOG = os.path.join(BASE, "db", "dl_progress.log")
|
||||
PDF_TMP = "/tmp/kimi-webbridge-pdfs"
|
||||
DOWNLOADS = os.path.expanduser("~/Downloads")
|
||||
KW = ['报表', '分析', '仪表盘', '统计', '档案', '模板', '文档', '制度', '公告', '证明']
|
||||
EXPORT_TEXTS = ('导出', '导出Excel', '导出报表', '下载', '批量导出', '导出全部')
|
||||
|
||||
|
||||
def call(action, args, timeout=90):
|
||||
body = json.dumps({"action": action, "args": args, "session": S}).encode()
|
||||
req = urllib.request.Request(D, data=body, headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
|
||||
def ev(code):
|
||||
return call("evaluate", {"code": code}).get("data", {}).get("value")
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = time.strftime("%H:%M:%S ") + msg
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def reconnect():
|
||||
r = call("find_tab", {"url": "2haohr.com", "active": True})
|
||||
return r.get("ok")
|
||||
|
||||
|
||||
def wait_ready(max_wait=10):
|
||||
t0 = time.time()
|
||||
while time.time() - t0 < max_wait:
|
||||
try:
|
||||
if ev("document.readyState") == "complete":
|
||||
time.sleep(1.5)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.6)
|
||||
return False
|
||||
|
||||
|
||||
HIDE_JS = r"""(()=>{const sels=['[class*="helpBox"]','[class*="aiAssistant"]','[class*="product_"]','[class*="feedback"]','[class*="Feedback"]','[class*="floatBar"]','[class*="robot"]','[class*="advert"]'];let n=0;sels.forEach(s=>document.querySelectorAll(s).forEach(el=>{const r=el.getBoundingClientRect();if(r.width<600&&getComputedStyle(el).position==='fixed'){el.style.setProperty('display','none','important');n++;}}));return n;})()"""
|
||||
|
||||
# 找一个"明确的"导出按钮(可见、文案精确、非下拉触发器)
|
||||
FIND_EXPORT_JS = r"""(()=>{const texts=%s;const els=[...document.querySelectorAll('button,a,[role=button]')];for(const el of els){const t=(el.innerText||'').trim();if(t.length>0&&t.length<=8&&texts.includes(t)){const r=el.getBoundingClientRect();if(r.width>0&&r.height>0&&getComputedStyle(el).display!=='none'){el.setAttribute('data-dlmark','1');return t;}}}return '';})()""" % json.dumps(list(EXPORT_TEXTS))
|
||||
|
||||
# 点击已标记的导出按钮
|
||||
CLICK_EXPORT_JS = r"""(()=>{const el=document.querySelector('[data-dlmark="1"]');if(!el)return false;el.click();el.removeAttribute('data-dlmark');return true;})()"""
|
||||
|
||||
# 检测是否弹出了需要选参数的弹窗(modal/dialog)
|
||||
MODAL_JS = r"""(()=>{const ms=[...document.querySelectorAll('[class*="modal"],[class*="Modal"],[role="dialog"],[class*="dialog"],[class*="Dialog"]')];for(const m of ms){const r=m.getBoundingClientRect();if(r.width>200&&r.height>120&&getComputedStyle(m).display!=='none'&&getComputedStyle(m).visibility!=='hidden')return true;}return false;})()"""
|
||||
|
||||
CLOSE_MODAL_JS = r"""document.activeElement&&document.activeElement.dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}));document.dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}));"""
|
||||
|
||||
|
||||
def snapshot_downloads():
|
||||
"""当前 ~/Downloads 下的文件集合(忽略 .crdownload 临时文件)"""
|
||||
return set(p for p in glob.glob(os.path.join(DOWNLOADS, "*"))
|
||||
if not p.endswith(".crdownload"))
|
||||
|
||||
|
||||
def wait_new_download(before, max_wait=12):
|
||||
"""等待新下载文件出现且完成(无 .crdownload)。返回新文件路径或 None"""
|
||||
t0 = time.time()
|
||||
while time.time() - t0 < max_wait:
|
||||
time.sleep(1.0)
|
||||
# 仍有下载中的临时文件则继续等
|
||||
if glob.glob(os.path.join(DOWNLOADS, "*.crdownload")):
|
||||
continue
|
||||
now = snapshot_downloads()
|
||||
new = now - before
|
||||
if new:
|
||||
# 取最新的一个
|
||||
return max(new, key=lambda p: os.path.getmtime(p))
|
||||
return None
|
||||
|
||||
|
||||
def rec(conn, node_key, full_path, url, kind, file_path, status, size, note):
|
||||
conn.execute(
|
||||
"INSERT INTO downloads(node_key,full_path,url,kind,file_path,status,size,note,ts) "
|
||||
"VALUES(?,?,?,?,?,?,?,?,?)",
|
||||
(node_key, full_path, url, kind, file_path, status, size, note,
|
||||
time.strftime("%Y-%m-%d %H:%M:%S")))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def main():
|
||||
conn = sqlite3.connect(DB)
|
||||
rows = conn.execute(
|
||||
"SELECT node_key,full_path,url,screenshot_path FROM features WHERE level=2").fetchall()
|
||||
targets = [r for r in rows if any(k in (r[1] or "") for k in KW)]
|
||||
targets.sort(key=lambda r: r[3] or "")
|
||||
total = len(targets)
|
||||
log(f"=== 报表/文档下载开始, 目标 {total} 页 ===")
|
||||
if not reconnect():
|
||||
log("!! 无法连接浏览器标签, 退出"); return
|
||||
okp = okx = fail = skip = 0
|
||||
for i, (nk, fp, url, sp) in enumerate(targets):
|
||||
rel_pdf = (sp or "").replace("screenshots/", "downloads/").rsplit(".", 1)[0] + ".pdf"
|
||||
out_pdf = os.path.join(BASE, rel_pdf)
|
||||
if os.path.exists(out_pdf) and os.path.getsize(out_pdf) > 5000:
|
||||
skip += 1
|
||||
continue
|
||||
os.makedirs(os.path.dirname(out_pdf), exist_ok=True)
|
||||
delay = random.uniform(5, 15)
|
||||
log(f"[{i+1}/{total}] wait {delay:.1f}s -> {fp}")
|
||||
time.sleep(delay)
|
||||
try:
|
||||
call("navigate", {"url": url, "newTab": False})
|
||||
wait_ready()
|
||||
time.sleep(random.uniform(1.0, 2.5))
|
||||
try:
|
||||
ev(HIDE_JS)
|
||||
except Exception:
|
||||
pass
|
||||
# --- 存 PDF ---
|
||||
fname = os.path.basename(rel_pdf)[:-4]
|
||||
r = call("save_as_pdf", {"paper_format": "a4", "print_background": True,
|
||||
"file_name": fname}, timeout=120)
|
||||
tmp = r.get("data", {}).get("path")
|
||||
size = r.get("data", {}).get("sizeBytes", 0)
|
||||
if tmp and os.path.exists(tmp):
|
||||
shutil.move(tmp, out_pdf)
|
||||
okp += 1
|
||||
rec(conn, nk, fp, url, "pdf", rel_pdf, "ok", size, "")
|
||||
log(f" PDF ok {size//1024}KB -> {rel_pdf}")
|
||||
else:
|
||||
fail += 1
|
||||
rec(conn, nk, fp, url, "pdf", rel_pdf, "fail", 0, json.dumps(r)[:150])
|
||||
log(f" PDF FAIL -> {json.dumps(r)[:150]}")
|
||||
# --- 尝试导出 Excel(仅当有明确按钮且不弹参数窗) ---
|
||||
try:
|
||||
btn = ev(FIND_EXPORT_JS)
|
||||
if btn:
|
||||
before = snapshot_downloads()
|
||||
ev(CLICK_EXPORT_JS)
|
||||
time.sleep(2.0)
|
||||
if ev(MODAL_JS):
|
||||
ev(CLOSE_MODAL_JS)
|
||||
rec(conn, nk, fp, url, "xlsx", "", "skip", 0, f"按钮[{btn}]触发弹窗,跳过")
|
||||
log(f" Excel skip (弹窗需选参数, 按钮={btn})")
|
||||
else:
|
||||
newf = wait_new_download(before)
|
||||
if newf:
|
||||
ext = os.path.splitext(newf)[1] or ".xlsx"
|
||||
dst = out_pdf[:-4] + ext
|
||||
shutil.move(newf, dst)
|
||||
okx += 1
|
||||
rel_x = os.path.relpath(dst, BASE)
|
||||
rec(conn, nk, fp, url, "xlsx", rel_x, "ok",
|
||||
os.path.getsize(dst), f"按钮={btn}")
|
||||
log(f" Excel ok -> {rel_x}")
|
||||
else:
|
||||
rec(conn, nk, fp, url, "xlsx", "", "skip", 0, f"按钮[{btn}]点击无文件")
|
||||
log(f" Excel none (点击无下载, 按钮={btn})")
|
||||
except Exception as e:
|
||||
log(f" Excel err {str(e)[:120]}")
|
||||
except Exception as e:
|
||||
fail += 1
|
||||
rec(conn, nk, fp, url, "pdf", rel_pdf, "fail", 0, str(e)[:150])
|
||||
log(f" FAIL {fp} err={str(e)[:150]}")
|
||||
try:
|
||||
reconnect()
|
||||
except Exception:
|
||||
pass
|
||||
log(f"=== DONE pdf_ok={okp} xlsx_ok={okx} skip={skip} fail={fail} total={total} ===")
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""生成《2号人事部·功能仿写开发需求文档》(Markdown)。
|
||||
数据来源: SQLite features(172功能页释义/开发提示) + downloads(41个真实导出Excel, 提取表头做字段字典)
|
||||
+ screenshots(整页界面参考)。
|
||||
输出: docs/开发需求文档.md
|
||||
"""
|
||||
import os, sqlite3, collections, re, datetime
|
||||
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
DL_ROOT = os.path.join(BASE, "downloads")
|
||||
OUT = os.path.join(BASE, "docs", "开发需求文档.md")
|
||||
EXCEL_EXT = (".xlsx", ".xls")
|
||||
|
||||
def db():
|
||||
c = sqlite3.connect(DB)
|
||||
c.row_factory = sqlite3.Row
|
||||
return c
|
||||
|
||||
|
||||
def load_tree():
|
||||
c = db()
|
||||
rows = [dict(r) for r in c.execute(
|
||||
"SELECT node_key,parent_key,level,name,full_path,url,description,understanding,"
|
||||
"screenshot_path,keep,sort_order FROM features ORDER BY sort_order,id")]
|
||||
c.close()
|
||||
by = collections.defaultdict(list)
|
||||
for r in rows:
|
||||
by[r["parent_key"] or ""].append(r)
|
||||
return by
|
||||
|
||||
|
||||
def excel_headers(path, max_scan=6):
|
||||
"""读取 Excel(.xlsx 用 openpyxl / .xls 用 xlrd), 返回识别出的表头字段列表
|
||||
(取前若干行中非空单元格最多的一行)。"""
|
||||
try:
|
||||
if path.lower().endswith(".xls"):
|
||||
import xlrd, warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
wb = xlrd.open_workbook(path)
|
||||
ws = wb.sheet_by_index(0)
|
||||
best, best_cnt = [], 0
|
||||
for r in range(min(max_scan, ws.nrows)):
|
||||
vals = [str(ws.cell_value(r, cc)).strip() for cc in range(ws.ncols)
|
||||
if str(ws.cell_value(r, cc)).strip()]
|
||||
if len(vals) > best_cnt:
|
||||
best, best_cnt = vals, len(vals)
|
||||
return [] if best_cnt == 1 else best
|
||||
import openpyxl
|
||||
wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
|
||||
ws = wb.active
|
||||
best, best_cnt = [], 0
|
||||
for row in ws.iter_rows(min_row=1, max_row=max_scan):
|
||||
vals = [str(c.value).strip() for c in row if c.value is not None and str(c.value).strip()]
|
||||
if len(vals) > best_cnt:
|
||||
best, best_cnt = vals, len(vals)
|
||||
wb.close()
|
||||
# 去掉过长(可能是合并标题)的单格行
|
||||
if best_cnt == 1:
|
||||
return []
|
||||
return best
|
||||
except Exception as e:
|
||||
return []
|
||||
|
||||
|
||||
def scan_downloads():
|
||||
"""返回 {module_name: [(dir, filename, headers[])]}。module_name 已去数字前缀。"""
|
||||
out = collections.defaultdict(list)
|
||||
files = []
|
||||
for root, _, fs in os.walk(DL_ROOT):
|
||||
for f in sorted(fs):
|
||||
if f.lower().endswith(EXCEL_EXT):
|
||||
files.append((root, f))
|
||||
for root, f in files:
|
||||
rel = os.path.relpath(root, DL_ROOT)
|
||||
segs = [] if rel == "." else rel.split(os.sep)
|
||||
seg0 = segs[0] if segs else ""
|
||||
module = "任务中心(异步导出)" if seg0.startswith("_任务") else re.sub(r"^\d+_", "", seg0)
|
||||
headers = excel_headers(os.path.join(root, f))
|
||||
out[module].append((rel.replace(os.sep, "/"), f, headers))
|
||||
return out
|
||||
|
||||
|
||||
def slug(s):
|
||||
return re.sub(r"[^\w\u4e00-\u9fa5]+", "-", s or "").strip("-")
|
||||
|
||||
|
||||
def load_approval_templates():
|
||||
"""读取审批模板库清单(若表不存在返回空)。"""
|
||||
try:
|
||||
c = db()
|
||||
rows = [dict(r) for r in c.execute(
|
||||
"SELECT name,category,provider,description FROM approval_templates ORDER BY sort_order,id")]
|
||||
c.close()
|
||||
return rows
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def main():
|
||||
by = load_tree()
|
||||
dl = scan_downloads()
|
||||
total_files = sum(len(v) for v in dl.values())
|
||||
|
||||
modules = by.get("", [])
|
||||
leaves_count = 0
|
||||
|
||||
def count_leaves(pk):
|
||||
n = 0
|
||||
for r in by.get(pk, []):
|
||||
if by.get(r["node_key"]):
|
||||
n += count_leaves(r["node_key"])
|
||||
else:
|
||||
n += 1
|
||||
return n
|
||||
|
||||
L = []
|
||||
w = L.append
|
||||
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
w("# 2号人事部 · 功能仿写开发需求文档\n")
|
||||
w("> **数据来源**：https://2haohr.com 全站功能调研\n>")
|
||||
w("> **调研规模**：%d 个一级模块 / %d 个功能页\n>" % (len(modules), sum(count_leaves(m["node_key"]) for m in modules)))
|
||||
w("> **配套资产**：整页界面截图 `screenshots/`｜真实导出数据样本 `downloads/`（%d 个 Excel）｜交互式审核+样本下载界面 `http://localhost:8777`\n>" % total_files)
|
||||
w("> **生成时间**：%s\n" % now)
|
||||
w("\n本文档用于**仿写开发**：每个功能页含【功能释义】【开发提示】【界面参考截图】，"
|
||||
"并在模块末尾附**真实导出数据的字段清单**（直接来自线上导出的 Excel 表头），可据此设计数据库表结构与列表字段。\n")
|
||||
|
||||
# 概述表
|
||||
w("\n## 一、模块总览\n")
|
||||
w("| # | 模块 | 功能页数 | 抓取的数据样本 |")
|
||||
w("|---|------|:---:|:---:|")
|
||||
for i, m in enumerate(modules, 1):
|
||||
nm = m["name"]
|
||||
cnt = count_leaves(m["node_key"])
|
||||
samp = len(dl.get(nm, []))
|
||||
w("| %d | %s | %d | %s |" % (i, nm, cnt, (str(samp) + " 个" if samp else "—")))
|
||||
|
||||
# 明细
|
||||
w("\n## 二、模块功能明细\n")
|
||||
for mi, m in enumerate(modules, 1):
|
||||
w("\n### %d. %s\n" % (mi, m["name"]))
|
||||
|
||||
def walk(pk, depth):
|
||||
for r in by.get(pk, []):
|
||||
children = by.get(r["node_key"])
|
||||
if children:
|
||||
if r["level"] >= 1:
|
||||
w("\n#### ▸ %s\n" % r["name"])
|
||||
walk(r["node_key"], depth + 1)
|
||||
else:
|
||||
nonlocal leaves_count
|
||||
leaves_count += 1
|
||||
keep = "" if r["keep"] in (None, 1) else " ~~(审核标记删除)~~"
|
||||
w("\n**%s**%s" % (r["name"], keep))
|
||||
if r["url"]:
|
||||
w("- `URL` %s" % r["url"])
|
||||
if r["description"]:
|
||||
w("- **功能释义**：%s" % r["description"])
|
||||
if r["understanding"] and "待补充细化" not in r["understanding"]:
|
||||
w("- **开发提示**：%s" % r["understanding"])
|
||||
if r["screenshot_path"]:
|
||||
w("- **界面参考**：`%s`" % r["screenshot_path"])
|
||||
w("")
|
||||
|
||||
walk(m["node_key"], 0)
|
||||
|
||||
# 模块导出样本字段
|
||||
samples = dl.get(m["name"], [])
|
||||
if samples:
|
||||
w("\n> **📎 本模块真实导出数据样本（%d 个）与字段：**\n" % len(samples))
|
||||
for rel, fn, headers in samples:
|
||||
name = os.path.splitext(fn)[0]
|
||||
if headers:
|
||||
fields = "、".join(headers[:40])
|
||||
more = " …" if len(headers) > 40 else ""
|
||||
w("> - **%s**（%d 字段）：%s%s" % (name, len(headers), fields, more))
|
||||
else:
|
||||
w("> - **%s**（字段未能解析，见文件）" % name)
|
||||
w("")
|
||||
|
||||
# 附: 任务中心异步导出
|
||||
if dl.get("任务中心(异步导出)"):
|
||||
w("\n## 三、异步导出样本（任务中心生成）\n")
|
||||
w("> 以下报表在页面点击导出后由服务端异步生成，经“任务中心”收割：\n")
|
||||
for rel, fn, headers in dl["任务中心(异步导出)"]:
|
||||
name = os.path.splitext(fn)[0]
|
||||
fields = ("：" + "、".join(headers[:40])) if headers else ""
|
||||
w("> - **%s**%s" % (name, fields))
|
||||
w("")
|
||||
|
||||
# 附: 审批模板库清单
|
||||
tpls = load_approval_templates()
|
||||
if tpls:
|
||||
w("\n## 四、审批模板库清单（预置审批流程）\n")
|
||||
w("> 来自 `审批 / 常规 / 审批模板库`（%d 个预置流程模板）。这些是**审批流程模板**"
|
||||
"（点“使用”实例化为审批表单），非数据表，无导出文件；仿写时可据此设计审批流程配置。\n" % len(tpls))
|
||||
bycat = collections.OrderedDict()
|
||||
for t in tpls:
|
||||
bycat.setdefault(t["category"] or "其他", []).append(t)
|
||||
for cat, items in bycat.items():
|
||||
w("\n#### ▸ %s（%d）\n" % (cat, len(items)))
|
||||
w("| 模板 | 提供方 | 说明 |")
|
||||
w("|------|------|------|")
|
||||
for t in items:
|
||||
w("| %s | %s | %s |" % (t["name"], t["provider"] or "—", t["description"] or ""))
|
||||
w("")
|
||||
|
||||
|
||||
w("\n---\n")
|
||||
w("*本文档由调研数据自动汇编。功能释义/开发提示可在 `http://localhost:8777` 审核界面逐条修订后重新生成；"
|
||||
"字段清单来自线上真实导出 Excel 的表头，为一手数据。*\n")
|
||||
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
with open(OUT, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(L))
|
||||
print("written:", OUT)
|
||||
print("modules:", len(modules), "leaves:", leaves_count, "sample files:", total_files)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
从 SQLite 生成"带复选框的交互式功能树" HTML (阶段3 交付物)。
|
||||
管理员可勾选保留/删除, 支持展开折叠、搜索、查看截图、导出勾选结果 JSON。
|
||||
输出: docs/功能树-勾选.html 及 docs/selection-template.json
|
||||
"""
|
||||
import os, json, sqlite3
|
||||
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
OUT = os.path.join(BASE, "docs", "功能树-勾选.html")
|
||||
|
||||
def build_tree():
|
||||
conn = sqlite3.connect(DB); conn.row_factory = sqlite3.Row
|
||||
rows = [dict(r) for r in conn.execute(
|
||||
"SELECT node_key,parent_key,level,name,full_path,url,description,understanding,screenshot_path,sort_order,keep "
|
||||
"FROM features ORDER BY level,sort_order,id").fetchall()]
|
||||
conn.close()
|
||||
by_parent = {}
|
||||
for r in rows:
|
||||
by_parent.setdefault(r["parent_key"] or "", []).append(r)
|
||||
def build(pk):
|
||||
out = []
|
||||
for r in by_parent.get(pk, []):
|
||||
n = dict(r)
|
||||
# 截图相对 docs/ 的路径
|
||||
sp = r.get("screenshot_path")
|
||||
n["shot"] = ("../" + sp) if sp else ""
|
||||
n["children"] = build(r["node_key"])
|
||||
out.append(n)
|
||||
return out
|
||||
return build("")
|
||||
|
||||
HTML = r"""<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>2号人事部 · 功能调研树（第一阶段·管理员勾选）</title>
|
||||
<style>
|
||||
:root{--green:#00b389;--bd:#e5e7eb;--muted:#6b7280;--bg:#f7f8fa;}
|
||||
*{box-sizing:border-box;}
|
||||
body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"PingFang SC","Microsoft YaHei",sans-serif;color:#1f2937;background:var(--bg);}
|
||||
header{position:sticky;top:0;z-index:10;background:#fff;border-bottom:1px solid var(--bd);padding:14px 20px;}
|
||||
header h1{margin:0 0 4px;font-size:18px;}
|
||||
.sub{color:var(--muted);font-size:13px;}
|
||||
.toolbar{margin-top:10px;display:flex;gap:8px;flex-wrap:wrap;align-items:center;}
|
||||
.toolbar input[type=search]{padding:7px 10px;border:1px solid var(--bd);border-radius:6px;width:240px;font-size:13px;}
|
||||
button{cursor:pointer;border:1px solid var(--bd);background:#fff;border-radius:6px;padding:7px 12px;font-size:13px;}
|
||||
button.primary{background:var(--green);color:#fff;border-color:var(--green);}
|
||||
button:hover{filter:brightness(0.97);}
|
||||
.stat{font-size:13px;color:var(--muted);margin-left:auto;}
|
||||
.stat b{color:var(--green);font-size:15px;}
|
||||
#tree{padding:14px 20px 80px;}
|
||||
.node{border-left:2px solid transparent;}
|
||||
.row{display:flex;align-items:flex-start;gap:8px;padding:6px 8px;border-radius:8px;}
|
||||
.row:hover{background:#fff;}
|
||||
.lv0>.row{background:#eef7f4;border:1px solid #d6ede6;margin-top:12px;}
|
||||
.lv0>.row .name{font-size:15px;font-weight:700;}
|
||||
.lv1>.row .name{font-weight:600;color:#0f766e;}
|
||||
.toggle{width:18px;text-align:center;color:var(--muted);cursor:pointer;user-select:none;flex:0 0 auto;padding-top:2px;}
|
||||
.toggle.leaf{visibility:hidden;}
|
||||
input[type=checkbox]{width:16px;height:16px;margin-top:3px;flex:0 0 auto;cursor:pointer;}
|
||||
.meta{flex:1;min-width:0;}
|
||||
.name{font-size:14px;}
|
||||
.name .path{color:#9ca3af;font-weight:400;font-size:11px;margin-left:6px;}
|
||||
.desc{color:#374151;font-size:12.5px;margin-top:2px;line-height:1.5;}
|
||||
.und{color:#8a6d3b;background:#fcf8ee;border:1px solid #f3e9cf;border-radius:4px;padding:2px 6px;font-size:12px;margin-top:3px;display:inline-block;}
|
||||
.url{color:#9ca3af;font-size:11px;margin-top:2px;word-break:break-all;}
|
||||
.thumb{flex:0 0 auto;}
|
||||
.thumb img{width:120px;height:76px;object-fit:cover;object-position:top;border:1px solid var(--bd);border-radius:6px;cursor:zoom-in;background:#fff;}
|
||||
.children{margin-left:26px;border-left:1px dashed #e5e7eb;padding-left:6px;}
|
||||
.hidden{display:none;}
|
||||
.dropped>.row{opacity:.42;}
|
||||
.dropped>.row .name{text-decoration:line-through;}
|
||||
/* lightbox */
|
||||
#lb{position:fixed;inset:0;background:rgba(0,0,0,.85);display:none;z-index:100;overflow:auto;text-align:center;}
|
||||
#lb img{max-width:96%;margin:2% auto;border-radius:6px;}
|
||||
#lb .close{position:fixed;top:14px;right:20px;color:#fff;font-size:30px;cursor:pointer;}
|
||||
.legend{font-size:12px;color:var(--muted);}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>2号人事部 · 功能调研树 <span class="sub">第一阶段 · 管理员勾选（勾选=保留，取消=删除）</span></h1>
|
||||
<div class="sub">共 __MODN__ 个模块 / __LEAFN__ 个功能页。勾选后点【导出勾选结果】保存 JSON，用于生成第二阶段开发需求文档。</div>
|
||||
<div class="toolbar">
|
||||
<input type="search" id="q" placeholder="搜索功能名称/释义…"/>
|
||||
<button onclick="expandAll(true)">展开全部</button>
|
||||
<button onclick="expandAll(false)">折叠全部</button>
|
||||
<button onclick="checkAll(true)">全选</button>
|
||||
<button onclick="checkAll(false)">全不选</button>
|
||||
<button class="primary" onclick="exportSel()">⬇ 导出勾选结果(JSON)</button>
|
||||
<span class="legend">（选择自动保存到浏览器本地）</span>
|
||||
<span class="stat">保留 <b id="cntKeep">0</b> / <span id="cntAll">0</span> 项</span>
|
||||
</div>
|
||||
</header>
|
||||
<div id="tree"></div>
|
||||
<div id="lb" onclick="this.style.display='none'"><span class="close">×</span><img id="lbimg"/></div>
|
||||
<script>
|
||||
const DATA = __DATA__;
|
||||
const STORE = "twohr_sel_v1";
|
||||
let sel = JSON.parse(localStorage.getItem(STORE) || "{}");
|
||||
|
||||
function leafKeys(n, acc){ if(!n.children.length){acc.push(n.node_key);} n.children.forEach(c=>leafKeys(c,acc)); return acc; }
|
||||
const ALL_LEAVES = []; DATA.forEach(n=>leafKeys(n, ALL_LEAVES));
|
||||
function isKeep(k){ return sel[k] !== 0; } // 默认保留
|
||||
|
||||
function render(){
|
||||
const root = document.getElementById('tree'); root.innerHTML='';
|
||||
DATA.forEach(n=>root.appendChild(renderNode(n)));
|
||||
updateStats(); refreshDropped();
|
||||
}
|
||||
function renderNode(n){
|
||||
const wrap=document.createElement('div'); wrap.className='node lv'+n.level; wrap.dataset.key=n.node_key;
|
||||
const row=document.createElement('div'); row.className='row';
|
||||
const hasCh=n.children.length>0;
|
||||
const tg=document.createElement('span'); tg.className='toggle'+(hasCh?'':' leaf'); tg.textContent=hasCh?'▾':'•';
|
||||
tg.onclick=()=>{const c=wrap.querySelector(':scope>.children');if(c){c.classList.toggle('hidden');tg.textContent=c.classList.contains('hidden')?'▸':'▾';}};
|
||||
const cb=document.createElement('input'); cb.type='checkbox'; cb.dataset.key=n.node_key;
|
||||
cb.onchange=()=>{ setSubtree(n, cb.checked); save(); render(); };
|
||||
const meta=document.createElement('div'); meta.className='meta';
|
||||
let html='<div class="name">'+esc(n.name)+'<span class="path">'+esc(n.full_path||'')+'</span></div>';
|
||||
if(n.description) html+='<div class="desc">'+esc(n.description)+'</div>';
|
||||
if(n.understanding) html+='<div class="und">理解：'+esc(n.understanding)+'</div>';
|
||||
if(n.url) html+='<div class="url">'+esc(n.url)+'</div>';
|
||||
meta.innerHTML=html;
|
||||
row.appendChild(tg); row.appendChild(cb); row.appendChild(meta);
|
||||
if(n.shot){ const t=document.createElement('div'); t.className='thumb';
|
||||
t.innerHTML='<img loading="lazy" src="'+n.shot+'" onclick="zoom(this.src)" onerror="this.style.opacity=.2;this.title=\'截图待生成\'"/>'; row.appendChild(t); }
|
||||
wrap.appendChild(row);
|
||||
// checkbox state
|
||||
const st=subtreeState(n);
|
||||
cb.checked = st!=='none'; cb.indeterminate = st==='some';
|
||||
if(hasCh){ const ch=document.createElement('div'); ch.className='children';
|
||||
n.children.forEach(c=>ch.appendChild(renderNode(c))); wrap.appendChild(ch); }
|
||||
return wrap;
|
||||
}
|
||||
function setSubtree(n, val){ if(!n.children.length){ sel[n.node_key]= val?1:0; } n.children.forEach(c=>setSubtree(c,val)); }
|
||||
function subtreeState(n){ const ls=leafKeys(n,[]); const k=ls.filter(isKeep).length;
|
||||
if(k===0)return 'none'; if(k===ls.length)return 'all'; return 'some'; }
|
||||
function refreshDropped(){
|
||||
document.querySelectorAll('.node.lv2').forEach(w=>{ w.classList.toggle('dropped', !isKeep(w.dataset.key)); });
|
||||
}
|
||||
function updateStats(){ const keep=ALL_LEAVES.filter(isKeep).length;
|
||||
document.getElementById('cntKeep').textContent=keep; document.getElementById('cntAll').textContent=ALL_LEAVES.length; }
|
||||
function save(){ localStorage.setItem(STORE, JSON.stringify(sel)); }
|
||||
function checkAll(v){ ALL_LEAVES.forEach(k=>sel[k]=v?1:0); save(); render(); }
|
||||
function expandAll(v){ document.querySelectorAll('.children').forEach(c=>c.classList.toggle('hidden',!v));
|
||||
document.querySelectorAll('.toggle:not(.leaf)').forEach(t=>t.textContent=v?'▾':'▸'); }
|
||||
function zoom(src){ document.getElementById('lbimg').src=src; document.getElementById('lb').style.display='block'; }
|
||||
function esc(s){ return (s||'').replace(/[&<>"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'}[c])); }
|
||||
function exportSel(){
|
||||
const rows=[]; function walk(n,mod){ const m=n.level===0?n.name:mod;
|
||||
if(!n.children.length){ rows.push({module:m,node_key:n.node_key,name:n.name,full_path:n.full_path,url:n.url,keep:isKeep(n.node_key)?1:0}); }
|
||||
n.children.forEach(c=>walk(c,m)); }
|
||||
DATA.forEach(n=>walk(n,n.name));
|
||||
const out={generated_at:new Date().toISOString(),total:rows.length,keep:rows.filter(r=>r.keep).length,items:rows};
|
||||
const blob=new Blob([JSON.stringify(out,null,2)],{type:'application/json'});
|
||||
const a=document.createElement('a'); a.href=URL.createObjectURL(blob); a.download='selection.json'; a.click();
|
||||
}
|
||||
function search(q){ q=q.trim().toLowerCase();
|
||||
document.querySelectorAll('.node').forEach(w=>{ const t=w.querySelector(':scope>.row .meta').innerText.toLowerCase();
|
||||
const hit=!q||t.includes(q); w.style.display = hit||hasVisibleChild(w,q)?'':'none'; });
|
||||
}
|
||||
function hasVisibleChild(w,q){ return [...w.querySelectorAll('.node')].some(c=>c.querySelector(':scope>.row .meta').innerText.toLowerCase().includes(q)); }
|
||||
document.getElementById('q').addEventListener('input',e=>search(e.target.value));
|
||||
render();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
def main():
|
||||
tree = build_tree()
|
||||
modn = len(tree)
|
||||
leafn = sum(1 for _ in _iter_leaves(tree))
|
||||
html = (HTML.replace("__DATA__", json.dumps(tree, ensure_ascii=False))
|
||||
.replace("__MODN__", str(modn)).replace("__LEAFN__", str(leafn)))
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
open(OUT, "w").write(html)
|
||||
print("written:", OUT, "modules:", modn, "leaves:", leafn)
|
||||
|
||||
def _iter_leaves(nodes):
|
||||
for n in nodes:
|
||||
if not n["children"]:
|
||||
yield n
|
||||
else:
|
||||
yield from _iter_leaves(n["children"])
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
任务4: 生成结构化开发规格 docs/spec.json (机器可直接消费)。
|
||||
把审核决策(features.keep/note) + 完整字段字典(field_specs) + 隐藏入口(page_extras)
|
||||
+ 真实导出样本(downloads, 含表头/行列预览) + 审批模板库(approval_templates)
|
||||
汇总为一份"按模块->功能页"组织的结构化规格, 供下一步开发直接读取。
|
||||
|
||||
输出结构:
|
||||
{
|
||||
meta, stats,
|
||||
modules: [ { module, page_count, pages: [ {
|
||||
node_key, name, group, full_path, url, keep,
|
||||
description, understanding, review_note, screenshot,
|
||||
field_count, fields:[{group,name,order}],
|
||||
hidden_entries:[...], # "更多"菜单发现的隐藏功能界面
|
||||
data_samples:[{file,kind,size,sheet,headers,preview_rows,total_rows}]
|
||||
} ] } ],
|
||||
approval_templates: [ {category,name,provider,description} ]
|
||||
}
|
||||
"""
|
||||
import os, json, sqlite3, time, re, collections, warnings
|
||||
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
OUT = os.path.join(BASE, "docs", "spec.json")
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def db():
|
||||
c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
|
||||
|
||||
|
||||
def excel_preview(relpath, max_scan=6, max_head=40):
|
||||
"""读导出文件表头 + 前几行预览。.xls 用 xlrd, .xlsx 用 openpyxl。"""
|
||||
p = os.path.join(BASE, relpath)
|
||||
if not os.path.isfile(p):
|
||||
return None
|
||||
low = p.lower()
|
||||
try:
|
||||
if low.endswith(".xls"):
|
||||
import xlrd
|
||||
wb = xlrd.open_workbook(p); ws = wb.sheet_by_index(0)
|
||||
rows = []
|
||||
for r in range(min(max_scan, ws.nrows)):
|
||||
rows.append([str(ws.cell_value(r, c)).strip() for c in range(min(max_head, ws.ncols))])
|
||||
return {"sheet": ws.name, "total_rows": ws.nrows, "total_cols": ws.ncols, "rows": rows}
|
||||
if low.endswith(".xlsx"):
|
||||
import openpyxl
|
||||
wb = openpyxl.load_workbook(p, read_only=True, data_only=True); ws = wb.active
|
||||
rows = []
|
||||
for i, row in enumerate(ws.iter_rows(values_only=True)):
|
||||
if i >= max_scan:
|
||||
break
|
||||
rows.append([("" if v is None else str(v)).strip() for v in row[:max_head]])
|
||||
r = {"sheet": ws.title, "total_rows": ws.max_row, "total_cols": ws.max_column, "rows": rows}
|
||||
wb.close(); return r
|
||||
except Exception as e:
|
||||
return {"error": str(e)[:120]}
|
||||
return None
|
||||
|
||||
|
||||
def best_header(rows):
|
||||
"""从前几行里挑字段数最多的一行作为表头(过滤单格标题行)。"""
|
||||
if not rows:
|
||||
return []
|
||||
best, cnt = [], 0
|
||||
for r in rows:
|
||||
vals = [v for v in r if v]
|
||||
if len(vals) > cnt:
|
||||
best, cnt = vals, len(vals)
|
||||
return [] if cnt <= 1 else best
|
||||
|
||||
|
||||
def load_leaves():
|
||||
c = db()
|
||||
rows = [dict(r) for r in c.execute(
|
||||
"SELECT node_key,parent_key,level,name,full_path,url,description,"
|
||||
"understanding,screenshot_path,keep,note,sort_order FROM features ORDER BY sort_order,id")]
|
||||
c.close()
|
||||
by_parent = {}
|
||||
for r in rows:
|
||||
by_parent.setdefault(r["parent_key"] or "", []).append(r)
|
||||
leaves = []
|
||||
|
||||
def dfs(pk, module, group):
|
||||
for r in by_parent.get(pk, []):
|
||||
m = r["name"] if r["level"] == 0 else module
|
||||
g = r["name"] if r["level"] == 1 else group
|
||||
kids = by_parent.get(r["node_key"])
|
||||
if kids:
|
||||
dfs(r["node_key"], m, g)
|
||||
else:
|
||||
r["_module"] = m
|
||||
r["_group"] = g if g and g != r["name"] else ""
|
||||
leaves.append(r)
|
||||
|
||||
dfs("", "", "")
|
||||
return leaves
|
||||
|
||||
|
||||
def load_fields():
|
||||
c = db(); out = collections.defaultdict(list)
|
||||
try:
|
||||
for r in c.execute("SELECT node_key,field,field_group,source,sort_order "
|
||||
"FROM field_specs ORDER BY node_key,sort_order"):
|
||||
out[r["node_key"]].append({"group": r["field_group"] or "", "name": r["field"],
|
||||
"order": r["sort_order"], "source": r["source"]})
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
c.close(); return out
|
||||
|
||||
|
||||
def load_extras():
|
||||
c = db(); out = collections.defaultdict(list)
|
||||
try:
|
||||
for r in c.execute("SELECT node_key,label FROM page_extras ORDER BY node_key,id"):
|
||||
out[r["node_key"]].append(r["label"])
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
c.close(); return out
|
||||
|
||||
|
||||
def load_downloads():
|
||||
c = db(); out = collections.defaultdict(list)
|
||||
try:
|
||||
for r in c.execute("SELECT node_key,file_path,kind,size FROM downloads WHERE status='ok' OR status IS NULL"):
|
||||
out[r["node_key"]].append(dict(r))
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
c.close(); return out
|
||||
|
||||
|
||||
def load_deep_shots():
|
||||
"""深层界面截图(非 main): 主操作弹窗/表单(add)、筛选(filter)、自定义列(columns)、行详情(detail)。"""
|
||||
kind_cn = {"add": "主操作弹窗/表单", "filter": "筛选面板",
|
||||
"columns": "自定义列/选择表头", "detail": "行详情/编辑"}
|
||||
c = db(); out = collections.defaultdict(list)
|
||||
try:
|
||||
for r in c.execute("SELECT node_key,kind,label,path FROM page_shots "
|
||||
"WHERE kind<>'main' ORDER BY node_key,id"):
|
||||
out[r["node_key"]].append({"kind": r["kind"], "kind_cn": kind_cn.get(r["kind"], r["kind"]),
|
||||
"label": r["label"] or "", "path": r["path"]})
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
c.close(); return out
|
||||
|
||||
|
||||
def load_templates():
|
||||
c = db(); out = []
|
||||
try:
|
||||
for r in c.execute("SELECT category,name,provider,description,sort_order "
|
||||
"FROM approval_templates ORDER BY sort_order,id"):
|
||||
out.append({"category": r["category"], "name": r["name"],
|
||||
"provider": r["provider"] or "", "description": r["description"] or ""})
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
c.close(); return out
|
||||
|
||||
|
||||
def norm(s):
|
||||
return re.sub(r"^\d+_", "", s or "").strip()
|
||||
|
||||
|
||||
def main():
|
||||
leaves = load_leaves()
|
||||
fields = load_fields()
|
||||
extras = load_extras()
|
||||
downloads = load_downloads()
|
||||
deep_shots = load_deep_shots()
|
||||
templates = load_templates()
|
||||
|
||||
modules = collections.OrderedDict()
|
||||
n_fields_pages = 0
|
||||
for lf in leaves:
|
||||
nk = lf["node_key"]
|
||||
mod = lf["_module"] or "未分类"
|
||||
fl = fields.get(nk, [])
|
||||
if fl:
|
||||
n_fields_pages += 1
|
||||
samples = []
|
||||
for d in downloads.get(nk, []):
|
||||
pv = excel_preview(d["file_path"])
|
||||
item = {"file": d["file_path"], "kind": d["kind"], "size": d["size"]}
|
||||
if pv and "rows" in pv:
|
||||
item.update({"sheet": pv["sheet"], "total_rows": pv["total_rows"],
|
||||
"total_cols": pv["total_cols"], "headers": best_header(pv["rows"])})
|
||||
samples.append(item)
|
||||
page = {
|
||||
"node_key": nk, "name": lf["name"], "group": lf["_group"],
|
||||
"full_path": lf["full_path"], "url": lf["url"] or "",
|
||||
"keep": 1 if (lf["keep"] is None or lf["keep"] == 1) else 0,
|
||||
"description": lf["description"] or "",
|
||||
"understanding": lf["understanding"] or "",
|
||||
"review_note": lf["note"] or "",
|
||||
"screenshot": lf["screenshot_path"] or "",
|
||||
"field_count": len(fl),
|
||||
"fields": [{"group": f["group"], "name": f["name"], "order": f["order"]} for f in fl],
|
||||
"hidden_entries": extras.get(nk, []),
|
||||
"deep_shots": deep_shots.get(nk, []),
|
||||
"data_samples": samples,
|
||||
}
|
||||
modules.setdefault(mod, []).append(page)
|
||||
|
||||
mod_list = [{"module": m, "page_count": len(ps),
|
||||
"field_pages": sum(1 for p in ps if p["field_count"]),
|
||||
"pages": ps} for m, ps in modules.items()]
|
||||
|
||||
total_fields = sum(len(v) for v in fields.values())
|
||||
total_samples = sum(len(v) for v in downloads.values())
|
||||
spec = {
|
||||
"meta": {
|
||||
"product": "2号人事部",
|
||||
"source": "https://2haohr.com 功能调研",
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"schema_version": 1,
|
||||
"notes": "按 模块->功能页 组织; fields 为自定义列/选择表头弹窗采集的完整字段(远超截图);"
|
||||
" hidden_entries 为'更多'菜单发现的隐藏功能界面; data_samples 为真实导出样本(含表头预览);"
|
||||
" screenshot 为去遮挡后重拍的整页主图; deep_shots 为点击按钮抵达的深层界面截图(新增表单/筛选/自定义列/行详情)。",
|
||||
},
|
||||
"stats": {
|
||||
"modules": len(mod_list),
|
||||
"pages": len(leaves),
|
||||
"kept_pages": sum(1 for lf in leaves if (lf["keep"] is None or lf["keep"] == 1)),
|
||||
"pages_with_fields": n_fields_pages,
|
||||
"total_fields": total_fields,
|
||||
"pages_with_hidden_entries": sum(1 for k in extras if extras[k]),
|
||||
"total_hidden_entries": sum(len(v) for v in extras.values()),
|
||||
"data_samples": total_samples,
|
||||
"pages_with_deep_shots": sum(1 for k in deep_shots if deep_shots[k]),
|
||||
"total_deep_shots": sum(len(v) for v in deep_shots.values()),
|
||||
"approval_templates": len(templates),
|
||||
},
|
||||
"modules": mod_list,
|
||||
"approval_templates": templates,
|
||||
}
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
with open(OUT, "w", encoding="utf-8") as f:
|
||||
json.dump(spec, f, ensure_ascii=False, indent=2)
|
||||
size = os.path.getsize(OUT)
|
||||
print("生成 docs/spec.json (%.1f KB)" % (size / 1024))
|
||||
print("统计:", json.dumps(spec["stats"], ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
遍历全部顶导航模块, 映射每个模块的侧边栏功能结构。
|
||||
含随机 5-15s 点击间隔(反爬)。结果写入 db/menu_map.json, 进度写 db/map_progress.log
|
||||
"""
|
||||
import json, time, random, os, urllib.request
|
||||
|
||||
DAEMON = "http://127.0.0.1:10086/command"
|
||||
SESSION = "2haohr"
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
OUT = os.path.join(BASE, "db", "menu_map.json")
|
||||
LOG = os.path.join(BASE, "db", "map_progress.log")
|
||||
|
||||
# (模块名, 是否在"更多"下拉里, 备注)
|
||||
MODULES = [
|
||||
("工作台", 0), ("人事", 0), ("组织", 0), ("招聘", 0), ("考勤假期", 0), ("薪酬", 0),
|
||||
("审批", 1), ("培训学习", 1), ("绩效", 1), ("社保公积金", 1), ("行政管理", 1),
|
||||
("电子合同", 1), ("背景调查", 1), ("人才测评", 1), ("员工福利", 1), ("人事仪表盘", 1),
|
||||
("第三方对接", 1), ("培训商城", 1), ("短信发送", 1), ("电子工资条", 1), ("智能薪酬", 1),
|
||||
("个税对接", 1), ("AI员工助手", 1), ("应用管理", 1),
|
||||
]
|
||||
|
||||
def call(action, args):
|
||||
body = json.dumps({"action": action, "args": args, "session": SESSION}).encode()
|
||||
req = urllib.request.Request(DAEMON, data=body, headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=45) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
def ev(code):
|
||||
try:
|
||||
return call("evaluate", {"code": code}).get("data", {}).get("value")
|
||||
except Exception as e:
|
||||
return "ERR:" + str(e)
|
||||
|
||||
CLICK_TOP = """
|
||||
(()=>{const name=%s;
|
||||
const els=[...document.querySelectorAll("a,span,div,li")].filter(e=>{
|
||||
const t=(e.innerText||"").trim();const r=e.getBoundingClientRect();
|
||||
return t===name && r.top<60 && r.width>8 && r.width<160;});
|
||||
if(els.length){els[0].click();return "top-clicked";}return "not-found";})()
|
||||
"""
|
||||
OPEN_MORE = """
|
||||
(()=>{const els=[...document.querySelectorAll("a,span,div,li")].filter(e=>{
|
||||
const t=(e.innerText||"").trim();const r=e.getBoundingClientRect();
|
||||
return t==="更多" && r.top<60 && r.width>8 && r.width<160;});
|
||||
if(!els.length)return "no-more"; const el=els[0];
|
||||
["pointerover","mouseover","mouseenter","pointerenter"].forEach(tp=>el.dispatchEvent(new MouseEvent(tp,{bubbles:true,cancelable:true,view:window})));
|
||||
el.click(); return "more-hover";})()
|
||||
"""
|
||||
CLICK_ANY = """
|
||||
(()=>{const name=%s;
|
||||
const els=[...document.querySelectorAll("a,span,div,li")].filter(e=>(e.innerText||"").trim()===name && e.getBoundingClientRect().width>=0);
|
||||
if(els.length){els[els.length-1].click();return "clicked";}return "not-found";})()
|
||||
"""
|
||||
EXTRACT = r"""
|
||||
(()=>{
|
||||
const root=document.querySelector("[class*=leftMenu]")||document.querySelector("[class*=menu_]");
|
||||
const groups=[]; let cur={title:"",items:[]}; groups.push(cur); const seen=new Set();
|
||||
if(root){const walk=(el)=>{for(const c of el.children){const cls=(c.className||"").toString();
|
||||
if(/nameTitle/.test(cls)&&!c.closest("a")){const t=(c.innerText||"").trim();if(t){cur={title:t,items:[]};groups.push(cur);}}
|
||||
if(c.tagName==="A"){const h=c.getAttribute("href")||"";const t=(c.innerText||"").trim().replace(/\s+/g," ");
|
||||
if(h&&h.length>1&&t&&!seen.has(h)){seen.add(h);cur.items.push({name:t,href:h});}}
|
||||
walk(c);}}; walk(root);}
|
||||
return JSON.stringify({loc:location.href,groups:groups.filter(g=>g.items.length)});
|
||||
})()
|
||||
"""
|
||||
|
||||
def log(msg):
|
||||
line = time.strftime("%H:%M:%S ") + msg
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def main():
|
||||
results = {}
|
||||
if os.path.exists(OUT):
|
||||
results = json.load(open(OUT))
|
||||
for i, (mod, in_more) in enumerate(MODULES):
|
||||
if mod in results and results[mod].get("groups"):
|
||||
log(f"skip {mod} (already mapped)"); continue
|
||||
delay = random.uniform(5, 15)
|
||||
log(f"[{i+1}/{len(MODULES)}] wait {delay:.1f}s -> {mod}")
|
||||
time.sleep(delay)
|
||||
name_js = json.dumps(mod, ensure_ascii=False)
|
||||
if in_more:
|
||||
ev(OPEN_MORE); time.sleep(1.5); r = ev(CLICK_ANY % name_js)
|
||||
else:
|
||||
r = ev(CLICK_TOP % name_js)
|
||||
if r == "not-found":
|
||||
ev(OPEN_MORE); time.sleep(1.5); r = ev(CLICK_ANY % name_js)
|
||||
time.sleep(3.2)
|
||||
data = ev(EXTRACT)
|
||||
try:
|
||||
parsed = json.loads(data)
|
||||
except Exception:
|
||||
parsed = {"loc": "", "groups": [], "raw": str(data)[:200]}
|
||||
parsed["in_more"] = in_more
|
||||
parsed["click"] = r
|
||||
results[mod] = parsed
|
||||
n = sum(len(g["items"]) for g in parsed.get("groups", []))
|
||||
log(f" {mod}: loc={parsed.get('loc','')} groups={len(parsed.get('groups',[]))} items={n} click={r}")
|
||||
json.dump(results, open(OUT, "w"), ensure_ascii=False, indent=1)
|
||||
log("DONE all modules")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Part0: 把 downloads 表里 node_key 为空的导出文件回填到对应功能页, 实现导出↔页面一一对应。
|
||||
匹配策略:
|
||||
1) 普通导出: 文件目录(如 downloads/02_人事/员工关系) 对应功能页 screenshot 目录(模块/分组);
|
||||
在该分组下的候选页中, 选文件名里能命中的页面名(最长命中优先)。
|
||||
2) _任务中心 收割文件: 无目录线索, 用文件名关键词对全站页面名/路径做全局匹配。
|
||||
仅回填 node_key 为空者; 打印每条匹配结果与未匹配项供人工核对。
|
||||
"""
|
||||
import os, sqlite3, re, collections
|
||||
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
|
||||
|
||||
def db():
|
||||
c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
|
||||
|
||||
|
||||
def strip_num(s):
|
||||
return re.sub(r"^\d+_", "", s or "").strip()
|
||||
|
||||
|
||||
def load_pages():
|
||||
c = db()
|
||||
rows = [dict(r) for r in c.execute(
|
||||
"SELECT node_key,name,full_path,screenshot_path FROM features "
|
||||
"WHERE screenshot_path IS NOT NULL AND screenshot_path<>''")]
|
||||
c.close()
|
||||
for r in rows:
|
||||
# screenshots/02_人事/员工管理/01_员工花名册.jpg -> 目录 02_人事/员工管理
|
||||
d = os.path.dirname(r["screenshot_path"]).replace("screenshots/", "", 1)
|
||||
r["shot_dir"] = d
|
||||
r["name_core"] = strip_num(r["name"]).replace("员工", "").strip()
|
||||
return rows
|
||||
|
||||
|
||||
def best_page_in_group(fname, candidates):
|
||||
"""在同分组候选页里, 选页面名能在文件名中命中且命中最长的。"""
|
||||
best, blen = None, 0
|
||||
for p in candidates:
|
||||
for token in (p["name"], strip_num(p["name"]), p["name_core"]):
|
||||
token = (token or "").strip()
|
||||
if len(token) >= 2 and token in fname and len(token) > blen:
|
||||
best, blen = p, len(token)
|
||||
return best
|
||||
|
||||
|
||||
def global_match(fname, full_path_hint, pages):
|
||||
"""任务中心文件: 按文件名关键词对全站页面做匹配。"""
|
||||
key = fname
|
||||
best, blen = None, 0
|
||||
for p in pages:
|
||||
hay = (p["full_path"] or "") + " " + (p["name"] or "")
|
||||
for token in (strip_num(p["name"]), p["name_core"]):
|
||||
token = (token or "").strip()
|
||||
if len(token) >= 2 and token in key and len(token) > blen:
|
||||
best, blen = p, len(token)
|
||||
return best
|
||||
|
||||
|
||||
# 任务中心文件名 -> 目标页面关键词(人工先验, 提高命中)
|
||||
TC_HINTS = {
|
||||
"参保": ["参保档案", "社保"],
|
||||
"薪酬档案": ["薪酬档案", "薪资档案"],
|
||||
"已签署": ["已签署"],
|
||||
"电子合同": ["已签署", "合同"],
|
||||
}
|
||||
|
||||
# 显式关键词->页面 node_key (第一遍模糊未命中时的人工规则, 按顺序取首个命中)
|
||||
EXPLICIT = [
|
||||
("异动", "人事::/personnel/relation/unusual-action"),
|
||||
("谈话", "人事::/personnel/relation/talk"),
|
||||
("实习生", "人事::/personnel/relation/internship"),
|
||||
("已签订合同", "人事::/personnel/relation/contract"),
|
||||
("合同人员", "人事::/personnel/relation/contract"),
|
||||
("待入职", "人事::/personnel/relation/enrollment"),
|
||||
("待离职", "人事::/personnel/relation/departure"),
|
||||
("试用期", "人事::/personnel/relation/probation"),
|
||||
("考勤明细", "考勤假期::/attendance/stat/records"),
|
||||
("打卡记录", "考勤假期::/attendance/stat/records"),
|
||||
("工资档案", "薪酬::/payroll/salary/profiles"),
|
||||
]
|
||||
# 目录兜底: 该目录剩余未匹配文件统一归属(员工管理各类统计报表 -> 人事报表)
|
||||
DIR_FALLBACK = {
|
||||
"02_人事/员工管理": "人事::/personnel/employee/personnel-report",
|
||||
"02_人事/员工关系": "人事::/personnel/relation/office",
|
||||
}
|
||||
|
||||
|
||||
def map_explicit():
|
||||
"""第二遍: 对仍为空的行用显式关键词规则 + 目录兜底回填。"""
|
||||
c = db()
|
||||
fp = {r["node_key"]: r["full_path"] for r in c.execute("SELECT node_key,full_path FROM features")}
|
||||
rows = [dict(r) for r in c.execute(
|
||||
"SELECT id,file_path FROM downloads WHERE node_key IS NULL OR node_key=''")]
|
||||
done, left = 0, 0
|
||||
for r in rows:
|
||||
rel = r["file_path"].replace("downloads/", "", 1)
|
||||
parts = rel.split("/"); fname = parts[-1]; dir_key = "/".join(parts[:-1])
|
||||
nk = None
|
||||
for sub, node in EXPLICIT:
|
||||
if sub in fname:
|
||||
nk = node; break
|
||||
if not nk:
|
||||
nk = DIR_FALLBACK.get(dir_key)
|
||||
if nk and nk in fp:
|
||||
c.execute("UPDATE downloads SET node_key=?, full_path=? WHERE id=?", (nk, fp[nk], r["id"]))
|
||||
done += 1
|
||||
print(" [OK2] %-40s -> %s" % (fname[:40], fp[nk]))
|
||||
else:
|
||||
left += 1
|
||||
print(" [??2] %-40s (dir=%s)" % (fname[:40], dir_key))
|
||||
c.execute("PRAGMA journal_mode=DELETE")
|
||||
c.commit(); c.close()
|
||||
print("显式回填: 命中 %d, 仍未匹配 %d" % (done, left))
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
pages = load_pages()
|
||||
by_dir = collections.defaultdict(list)
|
||||
for p in pages:
|
||||
by_dir[p["shot_dir"]].append(p)
|
||||
|
||||
c = db()
|
||||
rows = [dict(r) for r in c.execute(
|
||||
"SELECT id,file_path FROM downloads WHERE node_key IS NULL OR node_key=''")]
|
||||
print("待回填导出文件: %d" % len(rows))
|
||||
matched, unmatched = 0, []
|
||||
for r in rows:
|
||||
rel = r["file_path"].replace("downloads/", "", 1)
|
||||
parts = rel.split("/")
|
||||
fname = parts[-1]
|
||||
dir_key = "/".join(parts[:-1])
|
||||
page = None
|
||||
if dir_key.startswith("_任务") or dir_key.startswith("_"):
|
||||
# 任务中心: 关键词优先 + 全局匹配
|
||||
hint_pages = None
|
||||
for kw, targets in TC_HINTS.items():
|
||||
if kw in fname:
|
||||
hint_pages = [p for p in pages if any(t in (p["full_path"] or "") + (p["name"] or "") for t in targets)]
|
||||
break
|
||||
if hint_pages:
|
||||
page = best_page_in_group(fname, hint_pages) or hint_pages[0]
|
||||
if not page:
|
||||
page = global_match(fname, dir_key, pages)
|
||||
else:
|
||||
cands = by_dir.get(dir_key, [])
|
||||
page = best_page_in_group(fname, cands)
|
||||
if not page and len(cands) == 1:
|
||||
page = cands[0]
|
||||
if page:
|
||||
c.execute("UPDATE downloads SET node_key=?, full_path=? WHERE id=?",
|
||||
(page["node_key"], page["full_path"], r["id"]))
|
||||
matched += 1
|
||||
print(" [OK] %-46s -> %s" % (fname[:46], page["full_path"]))
|
||||
else:
|
||||
unmatched.append((fname, dir_key))
|
||||
print(" [??] %-46s (dir=%s) 未匹配" % (fname[:46], dir_key))
|
||||
c.execute("PRAGMA journal_mode=DELETE")
|
||||
c.commit(); c.close()
|
||||
print("\n回填完成: 命中 %d, 未匹配 %d" % (matched, len(unmatched)))
|
||||
if unmatched:
|
||||
print("--- 第二遍: 显式关键词 + 目录兜底 ---")
|
||||
map_explicit()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
映射某个顶导航模块的侧边栏功能结构。
|
||||
用法: python3 map_menu.py "<模块名>" [in_more]
|
||||
in_more: 传 1 表示该模块在"更多"下拉里, 需先展开
|
||||
输出: JSON {module, loc, groups:[{title, items:[{name,href}]}], flat:[...]}
|
||||
"""
|
||||
import sys, json, time, urllib.request
|
||||
|
||||
DAEMON = "http://127.0.0.1:10086/command"
|
||||
SESSION = "2haohr"
|
||||
|
||||
def call(action, args):
|
||||
body = json.dumps({"action": action, "args": args, "session": SESSION}).encode()
|
||||
req = urllib.request.Request(DAEMON, data=body, headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=40) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
def ev(code):
|
||||
return call("evaluate", {"code": code}).get("data", {}).get("value")
|
||||
|
||||
CLICK_TOP = """
|
||||
(()=>{const name=%s;
|
||||
const els=[...document.querySelectorAll("a,span,div,li")].filter(e=>{
|
||||
const t=(e.innerText||"").trim();const r=e.getBoundingClientRect();
|
||||
return t===name && r.top<60 && r.width>8 && r.width<160;});
|
||||
if(els.length){els[0].click();return "top-clicked";}return "not-found";})()
|
||||
"""
|
||||
|
||||
OPEN_MORE = """
|
||||
(()=>{const els=[...document.querySelectorAll("a,span,div,li")].filter(e=>{
|
||||
const t=(e.innerText||"").trim();const r=e.getBoundingClientRect();
|
||||
return t==="更多" && r.top<60 && r.width>8 && r.width<160;});
|
||||
if(!els.length)return "no-more";
|
||||
const el=els[0];
|
||||
["pointerover","mouseover","mouseenter","pointerenter"].forEach(tp=>{
|
||||
el.dispatchEvent(new MouseEvent(tp,{bubbles:true,cancelable:true,view:window}));});
|
||||
el.click();
|
||||
return "more-hover";})()
|
||||
"""
|
||||
|
||||
CLICK_ANY = """
|
||||
(()=>{const name=%s;
|
||||
const els=[...document.querySelectorAll("a,span,div,li")].filter(e=>(e.innerText||"").trim()===name && e.getBoundingClientRect().width>8);
|
||||
if(els.length){els[els.length-1].click();return "clicked";}return "not-found";})()
|
||||
"""
|
||||
|
||||
EXTRACT = r"""
|
||||
(()=>{
|
||||
const root=document.querySelector("[class*=leftMenu]")||document.querySelector("[class*=menu_]");
|
||||
if(!root)return JSON.stringify({loc:location.href,groups:[],flat:[]});
|
||||
const groups=[]; let cur={title:"",items:[]}; groups.push(cur);
|
||||
const seen=new Set();
|
||||
const walk=(el)=>{
|
||||
for(const c of el.children){
|
||||
const cls=(c.className||"").toString();
|
||||
if(/nameTitle/.test(cls) && !c.closest("a")){ const t=(c.innerText||"").trim(); if(t){ cur={title:t,items:[]}; groups.push(cur);} }
|
||||
if(c.tagName==="A"){ const h=c.getAttribute("href")||""; const t=(c.innerText||"").trim().replace(/\s+/g," ");
|
||||
if(h&&h.length>1&&t&&!seen.has(h)){seen.add(h);cur.items.push({name:t,href:h});} }
|
||||
walk(c);
|
||||
}
|
||||
};
|
||||
walk(root);
|
||||
const flat=[]; groups.forEach(g=>g.items.forEach(it=>flat.push({group:g.title,name:it.name,href:it.href})));
|
||||
return JSON.stringify({loc:location.href,groups:groups.filter(g=>g.items.length),flat});
|
||||
})()
|
||||
"""
|
||||
|
||||
def main():
|
||||
module = sys.argv[1]
|
||||
in_more = len(sys.argv) > 2 and sys.argv[2] == "1"
|
||||
name_js = json.dumps(module, ensure_ascii=False)
|
||||
if in_more:
|
||||
print("more:", ev(OPEN_MORE)); time.sleep(1.2)
|
||||
print("click:", ev(CLICK_ANY % name_js))
|
||||
else:
|
||||
r = ev(CLICK_TOP % name_js)
|
||||
print("top:", r)
|
||||
if r == "not-found":
|
||||
print("more:", ev(OPEN_MORE)); time.sleep(1.2)
|
||||
print("click:", ev(CLICK_ANY % name_js))
|
||||
time.sleep(3.0)
|
||||
data = ev(EXTRACT)
|
||||
print(data)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""仅重拍主截图 (不触发任何深层点击)。
|
||||
|
||||
用途: 修复"数据没加载好就截图 -> 只截到首屏空表"的问题。
|
||||
现在 fresh_page 内含 settle_content(等 docH 稳定 + 数据就绪), 重拍后
|
||||
列表/报表页会等到数据渲染完(docH 增长)再整页拼接, 从而截全。
|
||||
|
||||
安全: 只做 导航 -> 去遮挡(仅点"立即刷新"/"跳过引导") -> 截图 -> 覆盖原图,
|
||||
绝不点击 保存/删除/提交/确定/发送 等任何写操作。
|
||||
|
||||
用法:
|
||||
python3 recap_main.py att # 只重拍 18 个有附件的页面
|
||||
python3 recap_main.py all # 重拍全部 level=2 页面主图
|
||||
python3 recap_main.py <node_key> # 重拍指定 node_key
|
||||
"""
|
||||
import os, sys, time
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from capture_full import capture, ev
|
||||
from rewalk import fresh_page, SESSION, BASE, dbc, log
|
||||
|
||||
MODE = sys.argv[1] if len(sys.argv) > 1 else "att"
|
||||
|
||||
|
||||
def targets(conn):
|
||||
if MODE == "all":
|
||||
rows = conn.execute(
|
||||
"SELECT node_key,name,url,full_path,screenshot_path FROM features "
|
||||
"WHERE level=2 AND url IS NOT NULL AND url<>'' ORDER BY sort_order,id").fetchall()
|
||||
elif MODE == "att":
|
||||
rows = conn.execute(
|
||||
"SELECT DISTINCT f.node_key,f.name,f.url,f.full_path,f.screenshot_path FROM features f "
|
||||
"JOIN downloads d ON d.node_key=f.node_key "
|
||||
"WHERE d.node_key IS NOT NULL AND d.node_key<>'' AND f.url IS NOT NULL AND f.url<>'' "
|
||||
"ORDER BY f.name").fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT node_key,name,url,full_path,screenshot_path FROM features WHERE node_key=?",
|
||||
(MODE,)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def main():
|
||||
conn = dbc()
|
||||
conn.execute("PRAGMA journal_mode=DELETE")
|
||||
rows = targets(conn)
|
||||
total = len(rows)
|
||||
log("=== recap_main mode=%s total=%d ===" % (MODE, total))
|
||||
ok = fail = 0
|
||||
for i, r in enumerate(rows):
|
||||
nk = r["node_key"]; url = r["url"]; rel = r["screenshot_path"]
|
||||
if not rel:
|
||||
log("[%d/%d] SKIP no screenshot_path %s" % (i + 1, total, nk)); continue
|
||||
out_abs = os.path.join(BASE, rel)
|
||||
log("[%d/%d] %s %s" % (i + 1, total, r["name"], url))
|
||||
try:
|
||||
fresh_page(url)
|
||||
capture(SESSION, out_abs, 16)
|
||||
conn.execute("DELETE FROM page_shots WHERE node_key=? AND kind='main'", (nk,))
|
||||
conn.execute(
|
||||
"INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) "
|
||||
"VALUES(?,?,?,?,?,?,?)",
|
||||
(nk, r.get("full_path"), url, "main", "主界面", rel,
|
||||
time.strftime("%Y-%m-%d %H:%M:%S")))
|
||||
conn.commit()
|
||||
ok += 1
|
||||
except Exception as e:
|
||||
fail += 1
|
||||
log(" FAIL %s: %s" % (nk, str(e)[:120]))
|
||||
conn.execute("PRAGMA journal_mode=DELETE"); conn.commit(); conn.close()
|
||||
log("=== recap DONE ok=%d fail=%d total=%d ===" % (ok, fail, total))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,431 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
阶段B Part1: 重走全部叶子页 -> 去遮挡 -> 重拍主截图 -> 安全抓取 4 类更深界面。
|
||||
|
||||
安全约束(硬性):
|
||||
浏览器是用户本人登录的真实 Chrome。深层抓取一律"打开 -> 截图 -> 换页重载",
|
||||
绝不点击 保存/删除/提交/确定/发送 等会产生真实数据变更的按钮(见 BAD 黑名单)。
|
||||
为杜绝状态污染: 每抓一个深层界面前都重新导航到干净页面, 只打开一个目标后立即截图,
|
||||
不做任何"确认"操作; 下一个深层界面靠再次重载复位。
|
||||
|
||||
4 类深层界面:
|
||||
add 主操作弹窗(顶部工具栏 新增/添加/新建/创建/录入/发起 表单)
|
||||
detail 行详情/编辑(首个数据行的 详情/查看/编辑)
|
||||
filter 筛选面板(#icon-filter / 文本"筛选")
|
||||
columns 自定义列/表头设置(#icon-custom-columns / 文本 自定义列/列设置/表头设置)
|
||||
menu_* 更多菜单里已发现且安全的配置类入口(page_extras 白名单)
|
||||
|
||||
存储:
|
||||
主截图: 覆盖 features.screenshot_path
|
||||
深层截图: 同目录 <base>__<kind>.jpg, 记录到 page_shots(node_key,full_path,url,kind,label,path,ts)
|
||||
断点续跑: page_shots 已有该 node_key 的 kind=main 记录则整页跳过(除非 --force)。
|
||||
用法: python3 rewalk.py [start_index] [--force]
|
||||
"""
|
||||
import os, sys, json, time, sqlite3, urllib.request
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from capture_full import capture, ev, shot, call, HIDE_JS
|
||||
|
||||
SESSION = "2haohr"
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
LOG = os.path.join(BASE, "db", "rewalk_progress.log")
|
||||
|
||||
# ---- 操作黑名单: 深层抓取时任何匹配的按钮都绝不点击 ----
|
||||
BAD = (r"删除|移除|停用|注销|作废|清空|批量|放弃|提交|保存|确定|确认|发送|禁用|冻结|解除|"
|
||||
r"终止|结束|撤销|重置|清除|导入|上传|同步|生成|发放|归档|封存|开除|辞退|下架|驳回|"
|
||||
r"通过|拒绝|办理|恢复|启用|锁定|解锁|导出|下载|离职|开票|付款|转账|扣款|发起流程")
|
||||
|
||||
# ---- 更多菜单安全白名单: 仅打开明确的配置/查看类界面 ----
|
||||
MENU_SAFE = r"设置|字段|配置|规则|对比|说明|模板|方案|切换视图|调整排序|自定义列|列设置|表头|视图|统计口径"
|
||||
|
||||
|
||||
def dbc():
|
||||
c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = time.strftime("%H:%M:%S ") + msg
|
||||
print(line, flush=True)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def navigate(url):
|
||||
body = json.dumps({"action": "navigate", "args": {"url": url, "newTab": False}, "session": SESSION}).encode()
|
||||
last = None
|
||||
for attempt in range(4):
|
||||
try:
|
||||
req = urllib.request.Request(DAEMON_URL, data=body, headers={"Content-Type": "application/json"})
|
||||
with _OPENER.open(req, timeout=45) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
except Exception as e:
|
||||
last = e
|
||||
time.sleep(1.5 + attempt * 1.5)
|
||||
raise last
|
||||
|
||||
|
||||
DAEMON_URL = "http://127.0.0.1:10086/command"
|
||||
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
|
||||
def wait_ready(max_wait=9):
|
||||
t0 = time.time()
|
||||
while time.time() - t0 < max_wait:
|
||||
try:
|
||||
if ev("(()=>document.readyState)()", SESSION) == "complete":
|
||||
time.sleep(1.2); return True
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.6)
|
||||
return False
|
||||
|
||||
|
||||
# 内容稳定探测: docH(整页高度) + '共N条'数据计数
|
||||
SETTLE_JS = r"""(()=>{const de=document.documentElement;const m=(document.body.innerText||'').match(/共\s*([\d,]+)\s*条/);return JSON.stringify({docH:de.scrollHeight,cnt:m?m[1].replace(/,/g,''):null});})()"""
|
||||
|
||||
|
||||
def settle_content(max_wait=12.0):
|
||||
"""截图前等内容稳定: 轮询 docH 直到连续两次不变, 且(若页面有'共N条')数据>0。
|
||||
列表/报表页的表格数据是异步加载的, 加载完成后 docH 才会增长到真实高度;
|
||||
若在'共0条/加载中'时截图, 会只截到首屏空表(没截全)。"""
|
||||
t0 = time.time(); last = -1; stable = 0
|
||||
while time.time() - t0 < max_wait:
|
||||
try:
|
||||
m = json.loads(ev(SETTLE_JS, SESSION) or "{}")
|
||||
except Exception:
|
||||
m = {}
|
||||
h = int(m.get("docH") or 0)
|
||||
cnt = m.get("cnt")
|
||||
data_ok = (cnt is None) or (str(cnt).isdigit() and int(cnt) > 0)
|
||||
stable = stable + 1 if (h == last and h > 0) else 0
|
||||
last = h
|
||||
if stable >= 2 and data_ok and time.time() - t0 > 1.5:
|
||||
return h
|
||||
time.sleep(0.5)
|
||||
return last
|
||||
|
||||
|
||||
# 主截图去遮挡: 点"立即刷新"(安全重载数据) + 跳过新手引导 + DOM 隐藏遮罩/协议层 + 隐藏浮窗
|
||||
DEOCCLUDE_JS = r"""
|
||||
(()=>{
|
||||
const vis=el=>{const r=el.getBoundingClientRect();return r.width>0&&r.height>0&&getComputedStyle(el).visibility!=='hidden';};
|
||||
let acts=[];
|
||||
const refresh=[...document.querySelectorAll('a,button,span')].find(x=>x.childElementCount===0&&/立即刷新/.test((x.innerText||'').trim())&&vis(x));
|
||||
if(refresh){refresh.click();acts.push('refresh');}
|
||||
const skip=[...document.querySelectorAll('a,button,span,div')].find(x=>x.childElementCount===0&&/^(跳过|我知道了|知道了|跳过引导|不再提示|下次再说|以后再说)$/.test((x.innerText||'').trim())&&vis(x));
|
||||
if(skip){skip.click();acts.push('skip');}
|
||||
// DOM 隐藏引导遮罩/协议弹层(纯客户端, 不产生服务端动作)
|
||||
const masksel=['[class*=guide]','[class*=Guide]','[class*=driver]','[class*=intro]','[class*=newbie]','[class*=Newbie]','[class*=agreement]','[class*=Agreement]','[class*=guideMask]','[class*=coach]'];
|
||||
masksel.forEach(s=>document.querySelectorAll(s).forEach(el=>{const r=el.getBoundingClientRect();const p=getComputedStyle(el).position;if((p==='fixed'||p==='absolute')&&r.width>window.innerWidth*0.4&&r.height>window.innerHeight*0.3){el.style.setProperty('display','none','important');acts.push('mask');}}));
|
||||
return JSON.stringify(acts);
|
||||
})()
|
||||
"""
|
||||
|
||||
|
||||
def deocclude_main():
|
||||
"""返回是否触发了 refresh(需重新等待)。"""
|
||||
try:
|
||||
acts = json.loads(ev(DEOCCLUDE_JS, SESSION) or "[]")
|
||||
except Exception:
|
||||
acts = []
|
||||
ev(HIDE_JS, SESSION)
|
||||
return "refresh" in acts
|
||||
|
||||
|
||||
def save_shot(out_abs):
|
||||
img = shot(SESSION)
|
||||
os.makedirs(os.path.dirname(out_abs), exist_ok=True)
|
||||
img.save(out_abs, "JPEG", quality=80)
|
||||
return img.size
|
||||
|
||||
|
||||
# ---------- 4 类深层触发 (均带 BAD 黑名单过滤, 只打开不确认) ----------
|
||||
OPEN_ADD = r"""
|
||||
(()=>{
|
||||
const bad=/%BAD%/; const add=/(新增|添加|新建|创建|录入|发起)/;
|
||||
const cands=[...document.querySelectorAll('button,a')].filter(x=>{
|
||||
const t=(x.innerText||'').trim(); const r=x.getBoundingClientRect();
|
||||
return t&&t.length<=8&&add.test(t)&&!bad.test(t)&&r.width>0&&r.height>0&&r.top<340;});
|
||||
cands.sort((a,b)=>a.getBoundingClientRect().top-b.getBoundingClientRect().top);
|
||||
if(!cands.length) return '';
|
||||
const t=(cands[0].innerText||'').trim(); cands[0].click(); return t;
|
||||
})()
|
||||
""".replace("%BAD%", BAD)
|
||||
|
||||
OPEN_DETAIL = r"""
|
||||
(()=>{
|
||||
const bad=/%BAD%/; const want=/^(详情|查看|编辑|查看详情|详情信息)$/;
|
||||
const els=[...document.querySelectorAll('span,a,button')].filter(x=>{
|
||||
const t=(x.innerText||'').trim(); const r=x.getBoundingClientRect();
|
||||
return want.test(t)&&!bad.test(t)&&x.childElementCount<=1&&r.width>0&&r.height>0&&r.top>150&&r.top<window.innerHeight;});
|
||||
els.sort((a,b)=>{const ra=a.getBoundingClientRect(),rb=b.getBoundingClientRect();return ra.top-rb.top||ra.left-rb.left;});
|
||||
if(!els.length) return '';
|
||||
const t=(els[0].innerText||'').trim(); els[0].click(); return t;
|
||||
})()
|
||||
""".replace("%BAD%", BAD)
|
||||
|
||||
OPEN_FILTER = r"""
|
||||
(()=>{
|
||||
let target=null;
|
||||
document.querySelectorAll('use').forEach(u=>{const h=u.getAttribute('xlink:href')||u.getAttribute('href')||'';if(/#icon-filter|#icon-shaixuan|filter/i.test(h)&&!target){target=u;}});
|
||||
if(!target){target=[...document.querySelectorAll('button,span,div')].find(x=>x.childElementCount<=2&&/^筛选$/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);}
|
||||
if(!target) return '';
|
||||
let el=target; for(let i=0;i<4&⪙i++){if(el.tagName==='BUTTON'||el.getAttribute('role')==='button'||/btn|button/i.test(el.className||'')){break;}el=el.parentElement;}
|
||||
(el||target).click(); return 'filter';
|
||||
})()
|
||||
"""
|
||||
|
||||
OPEN_COLUMNS = r"""
|
||||
(()=>{
|
||||
let target=null;
|
||||
document.querySelectorAll('use').forEach(u=>{const h=u.getAttribute('xlink:href')||u.getAttribute('href')||'';if(/#icon-custom-columns|#icon-column|#icon-setting-col|columns/i.test(h)&&!target){target=u;}});
|
||||
if(!target){target=[...document.querySelectorAll('button,span,div')].find(x=>x.childElementCount<=2&&/^(自定义列|列设置|表头设置|表头字段|字段设置)$/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);}
|
||||
if(!target) return '';
|
||||
let el=target; for(let i=0;i<4&⪙i++){if(el.tagName==='BUTTON'||el.getAttribute('role')==='button'||/btn|button/i.test(el.className||'')){break;}el=el.parentElement;}
|
||||
(el||target).click(); return 'columns';
|
||||
})()
|
||||
"""
|
||||
|
||||
CLICK_MORE = r"""
|
||||
(()=>{
|
||||
const el=[...document.querySelectorAll('button,a,span,div')].find(x=>x.childElementCount<=2&&/^(更多操作|更多功能|···|\.\.\.|⋯)$/.test((x.innerText||'').replace(/\s/g,''))&&x.getBoundingClientRect().width>0&&x.getBoundingClientRect().top>60&&x.getBoundingClientRect().top<420);
|
||||
if(el){el.click();return 'clicked';}return 'none';
|
||||
})()
|
||||
"""
|
||||
|
||||
CLICK_MENU_ITEM = r"""
|
||||
(()=>{
|
||||
const bad=/%BAD%/; const label=%LABEL%;
|
||||
const el=[...document.querySelectorAll('[class*=dropdown] li,[class*=Dropdown] li,[class*=popper] li,[class*=Popover] li,[role=menu] li,[class*=dropdown] a,[class*=menuItem],li,[role=menuitem]')].find(x=>{
|
||||
const t=(x.innerText||'').trim(); return t===label&&!bad.test(t)&&x.getBoundingClientRect().width>0;});
|
||||
if(el){el.click();return 'ok';}return 'miss';
|
||||
})()
|
||||
"""
|
||||
|
||||
|
||||
def modal_visible():
|
||||
js = ("(()=>{const s=['.el-overlay','.el-dialog__wrapper','.el-drawer','.el-drawer__container',"
|
||||
"'[class*=dialog]','[class*=Dialog]','[class*=drawer]','[class*=Drawer]','[class*=popper]',"
|
||||
"'[class*=Popover]','[role=dialog]'];for(const q of s){for(const el of document.querySelectorAll(q)){"
|
||||
"const r=el.getBoundingClientRect();if(r.width>120&&r.height>120&&getComputedStyle(el).display!=='none')return true;}}return false;})()")
|
||||
try:
|
||||
return bool(ev(js, SESSION))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# 统计当前可见的"浮层面板"(弹窗/抽屉/筛选抑扇/popover)数量, 用于点击前后对比判定是否弹出新面板
|
||||
PANEL_COUNT_JS = r"""
|
||||
(()=>{let n=0;const W=innerWidth,H=innerHeight;
|
||||
document.querySelectorAll('div,aside,section,ul').forEach(el=>{
|
||||
const s=getComputedStyle(el);const r=el.getBoundingClientRect();const z=parseInt(s.zIndex)||0;
|
||||
const floaty=(s.position==='fixed'||s.position==='absolute');
|
||||
if(floaty&&s.display!=='none'&&r.width>=200&&r.width<=W*0.99&&r.height>=160&&r.top<H-30&&r.bottom>60&&
|
||||
(z>=8||/dialog|drawer|popper|popover|modal|mask|panel|Dialog|Drawer/.test(el.className||''))){n++;}
|
||||
});return n;})()
|
||||
"""
|
||||
|
||||
|
||||
def panels():
|
||||
try:
|
||||
return int(ev(PANEL_COUNT_JS, SESSION) or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def fresh_page(url):
|
||||
"""导航到干净页面: 重载 + 去遮挡 + (若刷新)再等待。"""
|
||||
navigate(url); wait_ready()
|
||||
if deocclude_main():
|
||||
wait_ready(); deocclude_main()
|
||||
settle_content()
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
def cur_url():
|
||||
try:
|
||||
return ev("(()=>location.href)()", SESSION) or ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def url_path(u):
|
||||
return (u or "").split("?")[0].split("#")[0]
|
||||
|
||||
|
||||
def deep_capture(url, node_key, full_path, out_base, kind, open_js, label, conn, accept_route=True):
|
||||
"""重载到干净页 -> 执行 open_js -> 判定出现(整页跳转表单 或 弹层) -> 截图 -> 记录。绝不做确认。"""
|
||||
rel_dir = os.path.dirname(out_base)
|
||||
base = os.path.splitext(os.path.basename(out_base))[0]
|
||||
fname = "%s__%s.jpg" % (base, kind)
|
||||
rel_path = os.path.join(rel_dir, fname)
|
||||
out_abs = os.path.join(BASE, rel_path)
|
||||
if os.path.exists(out_abs) and os.path.getsize(out_abs) > 3000:
|
||||
return None
|
||||
fresh_page(url)
|
||||
before = url_path(cur_url())
|
||||
b_panels = panels()
|
||||
clicked = ev(open_js, SESSION)
|
||||
if not clicked:
|
||||
return None
|
||||
time.sleep(1.3)
|
||||
after = url_path(cur_url())
|
||||
routed = after != before
|
||||
if routed and not accept_route:
|
||||
return None # filter/columns 不应发生跳转: 判为误点, 不记录
|
||||
if not routed:
|
||||
opened = modal_visible() or panels() > b_panels
|
||||
if not opened:
|
||||
return None # 点了没反馈, 放弃, 不记录
|
||||
if routed:
|
||||
wait_ready(); time.sleep(0.6); deocclude_main()
|
||||
ev(HIDE_JS, SESSION) # 去掉可能盖在上面的浮窗
|
||||
try:
|
||||
if routed:
|
||||
capture(SESSION, out_abs, 16) # 整页表单: 滚动拼接
|
||||
else:
|
||||
save_shot(out_abs) # 弹层: 单屏视口
|
||||
except Exception as e:
|
||||
log(" shot-fail %s %s: %s" % (kind, clicked, str(e)[:80]))
|
||||
return None
|
||||
conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
|
||||
(node_key, full_path, url, kind, clicked if isinstance(clicked, str) else label, rel_path,
|
||||
time.strftime("%Y-%m-%d %H:%M:%S")))
|
||||
conn.commit()
|
||||
return clicked
|
||||
|
||||
|
||||
def process_page(r, conn, force):
|
||||
url = r["url"]; node_key = r["node_key"]; full_path = r["full_path"]
|
||||
rel_main = r["screenshot_path"]
|
||||
out_main = os.path.join(BASE, rel_main)
|
||||
|
||||
# 断点续跑: 已有 main 记录且未 force 则跳过
|
||||
if not force:
|
||||
done = conn.execute("SELECT 1 FROM page_shots WHERE node_key=? AND kind='main' LIMIT 1",
|
||||
(node_key,)).fetchone()
|
||||
if done:
|
||||
return "skip"
|
||||
|
||||
# ---- 主截图: 去遮挡后整页拼接, 覆盖原图 ----
|
||||
fresh_page(url)
|
||||
try:
|
||||
capture(SESSION, out_main, 16)
|
||||
except Exception as e:
|
||||
log(" main-capture FAIL %s: %s" % (full_path, str(e)[:100]))
|
||||
return "fail"
|
||||
conn.execute("DELETE FROM page_shots WHERE node_key=? AND kind='main'", (node_key,))
|
||||
conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
|
||||
(node_key, full_path, url, "main", "主界面", rel_main, time.strftime("%Y-%m-%d %H:%M:%S")))
|
||||
conn.commit()
|
||||
|
||||
tally = ["main"]
|
||||
# ---- 4 类深层 ----
|
||||
for kind, js, acc in (("filter", OPEN_FILTER, False), ("columns", OPEN_COLUMNS, False),
|
||||
("add", OPEN_ADD, True), ("detail", OPEN_DETAIL, True)):
|
||||
try:
|
||||
c = deep_capture(url, node_key, full_path, rel_main, kind, js, None, conn, accept_route=acc)
|
||||
if c:
|
||||
tally.append("%s(%s)" % (kind, c if isinstance(c, str) else ""))
|
||||
except Exception as e:
|
||||
log(" deep %s err: %s" % (kind, str(e)[:80]))
|
||||
|
||||
# ---- 更多菜单安全入口(最多 3 个) ----
|
||||
import re as _re
|
||||
extras = conn.execute("SELECT label FROM page_extras WHERE node_key=?", (node_key,)).fetchall()
|
||||
safe = []
|
||||
for e in extras:
|
||||
lab = (e["label"] or "").strip()
|
||||
if lab and _re.search(MENU_SAFE, lab) and not _re.search(BAD, lab) and lab not in safe:
|
||||
safe.append(lab)
|
||||
for lab in safe[:3]:
|
||||
item_js = CLICK_MENU_ITEM.replace("%BAD%", BAD).replace("%LABEL%", json.dumps(lab, ensure_ascii=False))
|
||||
try:
|
||||
c = deep_capture_menu(url, node_key, full_path, rel_main, lab, item_js, conn)
|
||||
if c:
|
||||
tally.append("menu(%s)" % lab)
|
||||
except Exception as e:
|
||||
log(" menu %s err: %s" % (lab, str(e)[:80]))
|
||||
|
||||
log(" OK %s [%s]" % (full_path, ", ".join(tally)))
|
||||
return "done"
|
||||
|
||||
|
||||
def deep_capture_menu(url, node_key, full_path, out_base, label, item_js, conn):
|
||||
"""更多菜单类: 重载 -> 打开更多 -> 点安全项 -> 截图。"""
|
||||
rel_dir = os.path.dirname(out_base)
|
||||
base = os.path.splitext(os.path.basename(out_base))[0]
|
||||
safe_label = label.replace("/", "_").replace(" ", "")[:16]
|
||||
rel_path = os.path.join(rel_dir, "%s__menu_%s.jpg" % (base, safe_label))
|
||||
out_abs = os.path.join(BASE, rel_path)
|
||||
if os.path.exists(out_abs) and os.path.getsize(out_abs) > 3000:
|
||||
return None
|
||||
fresh_page(url)
|
||||
b_panels = panels()
|
||||
ok = False
|
||||
for attempt in range(3):
|
||||
if ev(CLICK_MORE, SESSION) != "clicked":
|
||||
break
|
||||
time.sleep(1.0 + attempt * 0.6)
|
||||
if ev(item_js, SESSION) == "ok":
|
||||
ok = True; break
|
||||
ev("(()=>{document.body.click();return 1;})()", SESSION)
|
||||
time.sleep(0.4)
|
||||
if not ok:
|
||||
return None
|
||||
time.sleep(1.3)
|
||||
after = url_path(cur_url())
|
||||
routed = after != url_path(url)
|
||||
if not routed and not (modal_visible() or panels() > b_panels):
|
||||
return None
|
||||
if routed:
|
||||
wait_ready(); time.sleep(0.6); deocclude_main()
|
||||
ev(HIDE_JS, SESSION)
|
||||
try:
|
||||
if routed:
|
||||
capture(SESSION, out_abs, 16)
|
||||
else:
|
||||
save_shot(out_abs)
|
||||
except Exception:
|
||||
return None
|
||||
conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
|
||||
(node_key, full_path, url, "menu", label, rel_path, time.strftime("%Y-%m-%d %H:%M:%S")))
|
||||
conn.commit()
|
||||
return label
|
||||
|
||||
|
||||
def main():
|
||||
args = [a for a in sys.argv[1:] if a != "--force"]
|
||||
force = "--force" in sys.argv
|
||||
start = int(args[0]) if args else 0
|
||||
|
||||
conn = dbc()
|
||||
conn.execute("PRAGMA journal_mode=DELETE")
|
||||
rows = [dict(r) for r in conn.execute(
|
||||
"SELECT node_key,name,full_path,url,screenshot_path FROM features "
|
||||
"WHERE level=2 AND url IS NOT NULL AND url<>'' ORDER BY sort_order,id")]
|
||||
rows = sorted(rows, key=lambda r: r["full_path"] or "")
|
||||
total = len(rows)
|
||||
log("=== rewalk start total=%d start=%d force=%s ===" % (total, start, force))
|
||||
done = skip = fail = 0
|
||||
for i, r in enumerate(rows):
|
||||
if i < start:
|
||||
continue
|
||||
log("[%d/%d] %s" % (i + 1, total, r["full_path"]))
|
||||
try:
|
||||
st = process_page(r, conn, force)
|
||||
except Exception as e:
|
||||
st = "fail"; log(" PAGE-FAIL %s: %s" % (r["full_path"], str(e)[:120]))
|
||||
if st == "done":
|
||||
done += 1
|
||||
elif st == "skip":
|
||||
skip += 1
|
||||
else:
|
||||
fail += 1
|
||||
conn.execute("PRAGMA journal_mode=DELETE"); conn.commit(); conn.close()
|
||||
log("=== DONE done=%d skip=%d fail=%d total=%d ===" % (done, skip, fail, total))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
阶段3(改进版) 审核后端服务。
|
||||
- 打开 http://localhost:8777/ 直接进入"一页一个功能"的审核界面
|
||||
- 每次点击 保留/删除 立即写回 SQLite(features.keep)，供第二阶段开发直接使用
|
||||
- 提供 API: 功能列表 / 更新保留状态 / 导出 selection.json
|
||||
依赖: Flask (已确认可用)
|
||||
用法: python3 scripts/serve.py 然后浏览器打开 http://localhost:8777
|
||||
"""
|
||||
import os, json, sqlite3, time, re
|
||||
from flask import Flask, request, jsonify, send_from_directory, Response
|
||||
|
||||
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DB = os.path.join(BASE, "db", "research.db")
|
||||
DOCS = os.path.join(BASE, "docs")
|
||||
DL_ROOT = os.path.join(BASE, "downloads")
|
||||
REVIEW_HTML = os.path.join(DOCS, "审核.html")
|
||||
EXPORT_JSON = os.path.join(DOCS, "selection.json")
|
||||
EXCEL_EXT = (".xlsx", ".xls", ".docx", ".doc")
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
def db():
|
||||
conn = sqlite3.connect(DB)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def load_features():
|
||||
"""按菜单树先序遍历返回叶子(待审核功能)列表，同一模块的功能连续排列。"""
|
||||
conn = db()
|
||||
rows = [dict(r) for r in conn.execute(
|
||||
"SELECT node_key,parent_key,level,name,full_path,url,description,"
|
||||
"understanding,screenshot_path,sort_order,keep,note FROM features "
|
||||
"ORDER BY sort_order,id").fetchall()]
|
||||
# 深层界面截图(非 main): node_key -> [{kind,kind_cn,label,shot}]
|
||||
kind_cn = {"add": "主操作弹窗/表单", "filter": "筛选面板",
|
||||
"columns": "自定义列/选择表头", "detail": "行详情/编辑"}
|
||||
deep = {}
|
||||
try:
|
||||
for r in conn.execute("SELECT node_key,kind,label,path FROM page_shots "
|
||||
"WHERE kind<>'main' ORDER BY node_key,id").fetchall():
|
||||
deep.setdefault(r["node_key"], []).append({
|
||||
"kind": r["kind"], "kind_cn": kind_cn.get(r["kind"], r["kind"]),
|
||||
"label": r["label"] or "", "shot": "/shot/" + r["path"]})
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
# 导出文件按 node_key 精确挂到页面: node_key -> [{name,rel,size,kind}]
|
||||
dls = {}
|
||||
try:
|
||||
for r in conn.execute("SELECT node_key,file_path,size,kind FROM downloads "
|
||||
"WHERE node_key IS NOT NULL AND node_key<>'' ORDER BY file_path").fetchall():
|
||||
fp = r["file_path"] or ""
|
||||
name = fp.rsplit("/", 1)[-1]
|
||||
sz = r["size"]
|
||||
if not sz:
|
||||
ap = os.path.join(BASE, fp)
|
||||
sz = os.path.getsize(ap) if os.path.isfile(ap) else 0
|
||||
dls.setdefault(r["node_key"], []).append(
|
||||
{"name": name, "rel": fp, "size": sz or 0, "kind": r["kind"] or ""})
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
conn.close()
|
||||
# 按父分组(已按 sort_order,id 排序，同父下保持菜单顺序)
|
||||
by_parent = {}
|
||||
for r in rows:
|
||||
by_parent.setdefault(r["parent_key"] or "", []).append(r)
|
||||
|
||||
leaves = []
|
||||
|
||||
def dfs(pk, module, group):
|
||||
for r in by_parent.get(pk, []):
|
||||
m = r["name"] if r["level"] == 0 else module
|
||||
g = r["name"] if r["level"] == 1 else group
|
||||
children = by_parent.get(r["node_key"])
|
||||
if children:
|
||||
dfs(r["node_key"], m, g)
|
||||
else: # 叶子
|
||||
shot = ("/shot/" + r["screenshot_path"]) if r.get("screenshot_path") else ""
|
||||
leaves.append({
|
||||
"node_key": r["node_key"],
|
||||
"name": r["name"],
|
||||
"module": m,
|
||||
"group": g if g and g != r["name"] else "",
|
||||
"full_path": r["full_path"],
|
||||
"url": r["url"] or "",
|
||||
"description": r["description"] or "",
|
||||
"understanding": r["understanding"] or "",
|
||||
"shot": shot,
|
||||
"deep_shots": deep.get(r["node_key"], []),
|
||||
"downloads": dls.get(r["node_key"], []),
|
||||
"keep": 1 if (r["keep"] is None or r["keep"] == 1) else 0,
|
||||
"note": r.get("note") or "",
|
||||
})
|
||||
|
||||
dfs("", "", "")
|
||||
return leaves
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
with open(REVIEW_HTML, "r", encoding="utf-8") as f:
|
||||
return Response(f.read(), mimetype="text/html")
|
||||
|
||||
|
||||
@app.route("/api/features")
|
||||
def api_features():
|
||||
feats = load_features()
|
||||
keep = sum(1 for f in feats if f["keep"])
|
||||
return jsonify({"total": len(feats), "keep": keep, "items": feats})
|
||||
|
||||
|
||||
@app.route("/api/keep", methods=["POST"])
|
||||
def api_keep():
|
||||
data = request.get_json(force=True)
|
||||
node_key = data.get("node_key")
|
||||
keep = 1 if data.get("keep") else 0
|
||||
if not node_key:
|
||||
return jsonify({"ok": False, "err": "missing node_key"}), 400
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
conn = db()
|
||||
conn.execute("UPDATE features SET keep=?, updated_at=? WHERE node_key=?",
|
||||
(keep, now, node_key))
|
||||
conn.commit()
|
||||
# 若为叶子节点，同步父节点(分组/模块)保留状态: 只要有一个子叶子保留，父就保留
|
||||
conn.close()
|
||||
return jsonify({"ok": True, "node_key": node_key, "keep": keep})
|
||||
|
||||
|
||||
@app.route("/api/keep_batch", methods=["POST"])
|
||||
def api_keep_batch():
|
||||
"""批量设置(全选/全不选/按模块)。body: {node_keys:[...], keep:0/1}"""
|
||||
data = request.get_json(force=True)
|
||||
keys = data.get("node_keys") or []
|
||||
keep = 1 if data.get("keep") else 0
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
conn = db()
|
||||
conn.executemany("UPDATE features SET keep=?, updated_at=? WHERE node_key=?",
|
||||
[(keep, now, k) for k in keys])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return jsonify({"ok": True, "updated": len(keys), "keep": keep})
|
||||
|
||||
|
||||
@app.route("/api/note", methods=["POST"])
|
||||
def api_note():
|
||||
"""保存某功能的审核备注(建议/说明)到 SQLite。"""
|
||||
data = request.get_json(force=True)
|
||||
node_key = data.get("node_key")
|
||||
note = data.get("note") or ""
|
||||
if not node_key:
|
||||
return jsonify({"ok": False, "err": "missing node_key"}), 400
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
conn = db()
|
||||
conn.execute("UPDATE features SET note=?, updated_at=? WHERE node_key=?",
|
||||
(note, now, node_key))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return jsonify({"ok": True, "node_key": node_key})
|
||||
|
||||
|
||||
@app.route("/api/export", methods=["POST", "GET"])
|
||||
def api_export():
|
||||
"""把当前 SQLite 中的审核决策导出为 docs/selection.json(包含开发所需全部信息)"""
|
||||
feats = load_features()
|
||||
kept = sum(1 for f in feats if f["keep"])
|
||||
# 按模块汇总
|
||||
mods = {}
|
||||
for f in feats:
|
||||
m = mods.setdefault(f["module"], {"module": f["module"], "total": 0, "keep": 0})
|
||||
m["total"] += 1
|
||||
m["keep"] += f["keep"]
|
||||
|
||||
def item(f):
|
||||
return {
|
||||
"module": f["module"], "group": f["group"], "name": f["name"],
|
||||
"node_key": f["node_key"], "full_path": f["full_path"], "url": f["url"],
|
||||
"keep": f["keep"],
|
||||
"description": f["description"], # 功能释义
|
||||
"understanding": f["understanding"], # 我的理解/开发提示
|
||||
"note": f["note"], # 审核备注/开发建议
|
||||
"screenshot": f["shot"].replace("/shot/", "") if f["shot"] else "", # 界面参考图(相对项目根)
|
||||
}
|
||||
|
||||
out = {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"source": "2号人事部 https://2haohr.com 功能调研",
|
||||
"total": len(feats),
|
||||
"keep": kept,
|
||||
"dropped": len(feats) - kept,
|
||||
"module_summary": sorted(mods.values(), key=lambda x: -x["keep"]),
|
||||
"items": [item(f) for f in feats], # 全量(含删除项, 便于回溯)
|
||||
"kept_items": [item(f) for f in feats if f["keep"]], # 仅保留项(开发直接用)
|
||||
}
|
||||
os.makedirs(DOCS, exist_ok=True)
|
||||
with open(EXPORT_JSON, "w", encoding="utf-8") as f:
|
||||
json.dump(out, f, ensure_ascii=False, indent=2)
|
||||
return jsonify({"ok": True, "path": "docs/selection.json",
|
||||
"total": out["total"], "keep": out["keep"], "dropped": out["dropped"]})
|
||||
|
||||
|
||||
@app.route("/shot/<path:relpath>")
|
||||
def shot(relpath):
|
||||
return send_from_directory(BASE, relpath)
|
||||
|
||||
|
||||
def _strip_num(s):
|
||||
"""去掉目录名前缀数字, 使 '02_人事' -> '人事', 便于与功能树模块名匹配。"""
|
||||
return re.sub(r"^\d+_", "", s or "").strip()
|
||||
|
||||
|
||||
def scan_attachments():
|
||||
"""实时扫描 downloads/ 目录, 按目录分组返回真实导出的 Excel/Word 附件清单。
|
||||
dl_biz/dl_biz2 新增文件会自动出现(无需重启)。"""
|
||||
groups = []
|
||||
if not os.path.isdir(DL_ROOT):
|
||||
return groups
|
||||
for root, _, files in os.walk(DL_ROOT):
|
||||
fs = [f for f in files if f.lower().endswith(EXCEL_EXT)]
|
||||
if not fs:
|
||||
continue
|
||||
rel = os.path.relpath(root, DL_ROOT)
|
||||
segs = [] if rel == "." else rel.split(os.sep)
|
||||
seg0 = segs[0] if segs else ""
|
||||
module = "任务中心" if seg0.startswith("_任务") else _strip_num(seg0)
|
||||
group = segs[1] if len(segs) > 1 else ""
|
||||
items = []
|
||||
for f in sorted(fs):
|
||||
p = os.path.join(root, f)
|
||||
items.append({"name": f, "size": os.path.getsize(p),
|
||||
"rel": os.path.relpath(p, BASE)})
|
||||
groups.append({"dir": rel.replace(os.sep, " / "), "module": module,
|
||||
"group": group, "files": items})
|
||||
groups.sort(key=lambda g: g["dir"])
|
||||
return groups
|
||||
|
||||
|
||||
@app.route("/api/attachments")
|
||||
def api_attachments():
|
||||
"""返回全部真实导出附件(按目录分组), 供审核界面展示与下载。"""
|
||||
gs = scan_attachments()
|
||||
total = sum(len(g["files"]) for g in gs)
|
||||
return jsonify({"total": total, "groups": gs})
|
||||
|
||||
|
||||
@app.route("/dl/<path:relpath>")
|
||||
def dl(relpath):
|
||||
"""下载真实导出文件(强制作为附件下载)。"""
|
||||
d = os.path.dirname(relpath)
|
||||
fn = os.path.basename(relpath)
|
||||
return send_from_directory(os.path.join(BASE, d), fn, as_attachment=True)
|
||||
|
||||
|
||||
SHEET_EXT = (".xlsx", ".xls")
|
||||
MAX_ROWS = 500
|
||||
MAX_COLS = 60
|
||||
|
||||
|
||||
def _parse_xlsx(path):
|
||||
import openpyxl
|
||||
wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
|
||||
sheets = []
|
||||
for ws in wb.worksheets:
|
||||
rows = []
|
||||
for i, row in enumerate(ws.iter_rows(values_only=True)):
|
||||
if i >= MAX_ROWS:
|
||||
break
|
||||
cells = [("" if v is None else str(v)) for v in row[:MAX_COLS]]
|
||||
rows.append(cells)
|
||||
sheets.append({"name": ws.title, "rows": rows,
|
||||
"total_rows": ws.max_row, "total_cols": ws.max_column})
|
||||
wb.close()
|
||||
return sheets
|
||||
|
||||
|
||||
def _parse_xls(path):
|
||||
import xlrd, warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
wb = xlrd.open_workbook(path)
|
||||
sheets = []
|
||||
for ws in wb.sheets():
|
||||
rows = []
|
||||
for r in range(min(MAX_ROWS, ws.nrows)):
|
||||
cells = []
|
||||
for cc in range(min(MAX_COLS, ws.ncols)):
|
||||
v = ws.cell_value(r, cc)
|
||||
if isinstance(v, float) and v == int(v):
|
||||
v = int(v)
|
||||
cells.append("" if v == "" else str(v))
|
||||
rows.append(cells)
|
||||
sheets.append({"name": ws.name, "rows": rows,
|
||||
"total_rows": ws.nrows, "total_cols": ws.ncols})
|
||||
return sheets
|
||||
|
||||
|
||||
@app.route("/api/preview")
|
||||
def api_preview():
|
||||
"""后端解析 Excel(.xlsx/openpyxl, .xls/xlrd) 为行列 JSON, 供前端表格预览。
|
||||
参数 rel=相对项目根的文件路径(限 downloads/ 内, 防穿越)。"""
|
||||
rel = request.args.get("rel", "")
|
||||
abspath = os.path.abspath(os.path.join(BASE, rel))
|
||||
if not abspath.startswith(DL_ROOT + os.sep) or not os.path.isfile(abspath):
|
||||
return jsonify({"ok": False, "err": "file not found or forbidden"}), 404
|
||||
low = abspath.lower()
|
||||
if not low.endswith(SHEET_EXT):
|
||||
return jsonify({"ok": False, "err": "not a spreadsheet", "kind": "other"}), 200
|
||||
try:
|
||||
sheets = _parse_xls(abspath) if low.endswith(".xls") else _parse_xlsx(abspath)
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "err": str(e)[:200]}), 200
|
||||
return jsonify({"ok": True, "name": os.path.basename(abspath),
|
||||
"rel": rel, "sheets": sheets,
|
||||
"max_rows": MAX_ROWS, "max_cols": MAX_COLS})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host="127.0.0.1", port=8777, debug=False, threaded=True)
|
||||
Reference in New Issue
Block a user