314 lines
16 KiB
Python
314 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
报表/文档导出下载器（Excel/Word 为主）。完整链路（已逐环节实测验证）：
|
|
入口：svg use[xlink:href=#icon-download] 图标（无文字，代码级定位）+ 文字“下载/生成excel”兜底；
|
|
生成：点图标 -> 弹“下载报表/导出”参数窗 -> 点“确定”(默认参数) -> 异步生成；
|
|
捕获：hook HTMLAnchorElement.prototype.click 捕获 rpt-cdn/prv-cdn 带 token 直链；
|
|
落盘：页面内同步 XHR 取字节 -> base64 回传 -> 写 workspace（绕过 macOS ~/Downloads 的 TCC 限制）。
|
|
阶段2：任务中心(taskCenter)收割兜底。产出真实 .xlsx/.docx，写 SQLite downloads 表；
|
|
页面间随机 5-15s；断点续跑(同名>2000字节跳过)。
|
|
"""
|
|
import json, urllib.request, base64, time, os, random, sqlite3, re, collections
|
|
|
|
D = "http://127.0.0.1:10086/command"; S = "2haohr"
|
|
DB = "db/research.db"; LOG = "db/dl_export.log"
|
|
KW = ['报表', '分析', '仪表盘', '统计', '档案', '模板', '文档', '制度', '公告', '证明']
|
|
TXT_BTN = ['直接下载', '生成excel', '导出Excel', '导出报表', '批量导出'] # 文字入口(不含裸“下载/导出”避免误点导航)
|
|
|
|
def call(a, args, timeout=120):
|
|
b = json.dumps({"action": a, "args": args, "session": S}).encode()
|
|
r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
|
|
return json.loads(urllib.request.urlopen(r, timeout=timeout).read().decode())
|
|
|
|
def ev(code, timeout=120):
|
|
try:
|
|
return call("evaluate", {"code": code}, timeout).get("data", {}).get("value")
|
|
except Exception as e:
|
|
return "EXC:" + str(e)[:80]
|
|
|
|
def log(m):
|
|
line = time.strftime("%H:%M:%S ") + m
|
|
print(line, flush=True)
|
|
with open(LOG, "a") as f:
|
|
f.write(line + "\n")
|
|
|
|
HOOK = r"""(()=>{window.__cap=window.__cap||[];
|
|
if(!HTMLAnchorElement.prototype.__hk){const ac=HTMLAnchorElement.prototype.click;
|
|
HTMLAnchorElement.prototype.click=function(){try{const h=this.href||'';
|
|
if(h&&/-cdn\.2haohr|prv-cdn|rpt-cdn|attname=|\.xlsx|\.xls|\.docx|\.doc|\.zip/i.test(h))window.__cap.push(h);}catch(e){}
|
|
return ac.apply(this,arguments);};HTMLAnchorElement.prototype.__hk=1;}
|
|
return 'ok';})()"""
|
|
|
|
# 未处理的 #icon-download 数量
|
|
COUNT_ICON = r"""(()=>{let n=0;document.querySelectorAll('use').forEach(u=>{
|
|
const h=u.getAttribute('xlink:href')||u.getAttribute('href');
|
|
if(h==='#icon-download'){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl'))n++;}});return n;})()"""
|
|
|
|
# 点击下一个未处理的 #icon-download
|
|
CLICK_ICON = r"""(()=>{const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-download');
|
|
for(const u of us){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl')){b.setAttribute('data-dl','1');b.click();return 'ok';}}
|
|
return 'none';})()"""
|
|
|
|
# 文字入口
|
|
COUNT_TXT = (r"""(()=>{const T=%s;let n=0;document.querySelectorAll('button,a,[role=button]').forEach(x=>{"""
|
|
r"""const t=(x.innerText||'').trim();if(T.includes(t)&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl'))n++;});return n;})()""") % json.dumps(TXT_BTN)
|
|
CLICK_TXT = (r"""(()=>{const T=%s;for(const t of T){const b=[...document.querySelectorAll('button,a,[role=button]')]"""
|
|
r""".find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl'));"""
|
|
r"""if(b){b.setAttribute('data-dl','1');b.click();return t;}}return 'none';})()""") % json.dumps(TXT_BTN)
|
|
|
|
# 参数窗“确定/下载/导出”——关键词精确定位“下载报表/导出”参数窗(避免误点其它弹层)
|
|
KWWIN = r"下载报表|导出|下载|部门|员工排序|工作性质|字段|范围|格式|参数|排序|在职"
|
|
CONFIRM = (r"""(()=>{const cand=['确定','确认','下载','导出','生成excel','生成','立即导出','立即下载'];
|
|
const KW=/%s/;
|
|
const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')];
|
|
const vis=ms.filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>80&&getComputedStyle(m).display!=='none'&&getComputedStyle(m).visibility!=='hidden';});
|
|
// 优先带关键词的参数窗, 再退化到任意可见弹层
|
|
const ranked=[...vis.filter(m=>KW.test(m.innerText||'')),...vis];
|
|
for(const m of ranked){
|
|
for(const t of cand){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0);
|
|
if(b){b.click();return 'modal:'+t;}}}
|
|
return vis.length?'modal:noconfirm':'nomodal';})()""") % KWWIN
|
|
|
|
CLOSE = r"""(()=>{const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer]')];
|
|
for(const m of ms){const r=m.getBoundingClientRect();if(r.width>200&&r.height>80&&getComputedStyle(m).display!=='none'){
|
|
const c=[...m.querySelectorAll('button,a,[class*=close],[class*=Close]')]
|
|
.find(x=>{const t=(x.innerText||'').trim();return t==='取消'||t==='关闭'||(x.className+'').toLowerCase().includes('close');});
|
|
if(c){c.click();return 'closed';}}}return 'none';})()"""
|
|
|
|
# 数据授权/协议窗：只点“知情并同意”，绝不点“不同意并退出登录”(之前登出的元凶)
|
|
AGREE = r"""(()=>{const bad=/退出登录|不同意/;const ok=['知情并同意','同意并继续','我已阅读并同意','同意'];
|
|
const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')]
|
|
.filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>60&&getComputedStyle(m).display!=='none';});
|
|
for(const m of ms){const txt=m.innerText||'';if(!/协议|授权|同意|条款|隐私/.test(txt))continue;
|
|
for(const t of ok){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&!bad.test(x.innerText||''));
|
|
if(b){b.click();return 'agree:'+t;}}}
|
|
return 'none';})()"""
|
|
|
|
BAD = r"退出登录|不同意|立即购买|预约演示|我要更多功能"
|
|
# 类型B 数据列表导出入口: #icon-export 或 主内容区文字“导出”(排除导航/任务中心/黑名单)
|
|
CLICK_EXPORT = (r"""(()=>{const BAD=/%s/;
|
|
const inNav=e=>!!e.closest('nav,[class*=navBar],[class*=sideMenu],[class*=taskCenter]');
|
|
const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-export');
|
|
for(const u of us){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl')&&!inNav(b)){b.setAttribute('data-dl','1');b.click();return 'icon-export';}}
|
|
const b=[...document.querySelectorAll('button,a,[role=button]')].find(x=>{const t=(x.innerText||'').trim();return t==='导出'&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl')&&!inNav(x)&&!BAD.test(t);});
|
|
if(b){b.setAttribute('data-dl','1');b.click();return 'txt-导出';}
|
|
return 'none';})()""") % BAD
|
|
|
|
# 类型A 报表中心卡片导出: 生成excel/直接下载/下载(排除导航/任务中心/黑名单)
|
|
CARD_BTN = ['生成excel', '直接下载', '导出Excel', '导出报表', '批量导出', '下载']
|
|
CLICK_CARD = (r"""(()=>{const T=%s;const BAD=/%s/;
|
|
const inNav=e=>!!e.closest('nav,[class*=navBar],[class*=sideMenu],[class*=taskCenter]');
|
|
for(const t of T){const b=[...document.querySelectorAll('button,a,[role=button]')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl')&&!inNav(x)&&!BAD.test(x.innerText||''));
|
|
if(b){b.setAttribute('data-dl','1');b.click();return t;}}
|
|
return 'none';})()""") % (json.dumps(CARD_BTN), BAD)
|
|
|
|
FETCH_LAST = r"""(()=>{try{if(!window.__cap||!window.__cap.length)return 'NO_HREF';
|
|
const u=window.__cap[window.__cap.length-1];const x=new XMLHttpRequest();x.open('GET',u,false);
|
|
x.overrideMimeType('text/plain;charset=x-user-defined');x.send();if(x.status!==200)return 'HTTP'+x.status;
|
|
const s=x.responseText;let out='';const CH=8192;for(let i=0;i<s.length;i+=CH){let c='';const e=Math.min(i+CH,s.length);
|
|
for(let j=i;j<e;j++)c+=String.fromCharCode(s.charCodeAt(j)&0xff);out+=c;}
|
|
window.__b64=btoa(out);const m=u.match(/attname=([^&]+)/);let name=m?decodeURIComponent(m[1]):('export_'+Date.now()+'.bin');
|
|
return JSON.stringify({size:s.length,name:name});}catch(e){return 'EX '+e.message;}})()"""
|
|
|
|
TC_LIST = r"""(()=>{const tc=document.querySelector('[class*=taskCenter]');if(!tc)return '[]';
|
|
const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
|
|
const names=dls.map(a=>{const row=a.closest('[class*=item],li,div');const info=row&&row.querySelector('[class*=info_]');
|
|
return info?(info.innerText||'').replace(/\n+/g,' ').trim().slice(0,50):'';});
|
|
return JSON.stringify(names);})()"""
|
|
TC_CLICK = r"""(function(i){const tc=document.querySelector('[class*=taskCenter]');if(!tc)return 'NO_TC';
|
|
const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
|
|
if(i>=dls.length)return 'OOB';dls[i].click();return 'ok';})(%d)"""
|
|
|
|
def db():
|
|
c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
|
|
|
|
def ensure():
|
|
c = db(); c.execute("""CREATE TABLE IF NOT EXISTS downloads(
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
|
|
kind TEXT, file_path TEXT, status TEXT, size INTEGER, note TEXT, ts TEXT)"""); c.commit(); c.close()
|
|
|
|
def rec(nk, fp, url, kind, path, status, size, note):
|
|
c = db(); c.execute("INSERT INTO downloads(node_key,full_path,url,kind,file_path,status,size,note,ts) VALUES(?,?,?,?,?,?,?,?,?)",
|
|
(nk, fp, url, kind, path, status, size, note, time.strftime("%Y-%m-%d %H:%M:%S"))); c.commit(); c.close()
|
|
|
|
def safe(s):
|
|
return re.sub(r'[\\/:*?"<>|]', '_', (s or "").strip())[:90]
|
|
|
|
def out_dir(shot, full_path):
|
|
if shot and shot.startswith("screenshots/"):
|
|
return os.path.dirname(shot.replace("screenshots/", "downloads/", 1))
|
|
parts = [safe(p) for p in (full_path or "").split(" / ") if p.strip()]
|
|
return os.path.join("downloads", *parts[:-1]) if len(parts) > 1 else "downloads/misc"
|
|
|
|
def load_pages():
|
|
c = db(); rows = [dict(r) for r in c.execute(
|
|
"SELECT node_key,parent_key,name,full_path,url,screenshot_path FROM features").fetchall()]
|
|
kids = collections.defaultdict(int)
|
|
for r in rows:
|
|
if r["parent_key"]:
|
|
kids[r["parent_key"]] += 1
|
|
c.close()
|
|
leaves = [r for r in rows if kids.get(r["node_key"], 0) == 0]
|
|
return [r for r in leaves if r["url"] and any(k in (r["full_path"] or "") + (r["name"] or "") for k in KW)]
|
|
|
|
SAVED = set()
|
|
|
|
def grab_and_save(odir, nk, fp, url, note):
|
|
meta = ev(FETCH_LAST)
|
|
if not (isinstance(meta, str) and meta.startswith("{")):
|
|
return False, str(meta)[:40]
|
|
info = json.loads(meta); name = safe(info["name"]); ev("window.__cap=[]")
|
|
if name in SAVED:
|
|
return False, "dup"
|
|
try:
|
|
data = base64.b64decode(ev("window.__b64||''"))
|
|
except Exception:
|
|
data = b""
|
|
if len(data) <= 200:
|
|
return False, "empty"
|
|
os.makedirs(odir, exist_ok=True); dst = os.path.join(odir, name)
|
|
if os.path.exists(dst) and os.path.getsize(dst) > 2000:
|
|
SAVED.add(name); return True, name + "(exists)"
|
|
open(dst, "wb").write(data); SAVED.add(name)
|
|
low = name.lower()
|
|
kind = "xlsx" if low.endswith((".xlsx", ".xls")) else ("docx" if low.endswith((".docx", ".doc")) else (
|
|
"pdf" if low.endswith(".pdf") else ("zip" if low.endswith(".zip") else "file")))
|
|
rec(nk, fp, url, kind, dst, "ok", len(data), note)
|
|
log(" OK %s (%d bytes) -> %s" % (kind, len(data), dst))
|
|
return True, name
|
|
|
|
def wait_capture(seconds=25):
|
|
t0 = time.time()
|
|
while time.time() - t0 < seconds:
|
|
time.sleep(2)
|
|
n = ev("(window.__cap&&window.__cap.length)||0")
|
|
if isinstance(n, int) and n > 0:
|
|
return True
|
|
return False
|
|
|
|
def human_delay(kind="page"):
|
|
"""非均匀拟人间隔：大部分短、少量中、偶尔长停顿 + 高斯抖动。"""
|
|
r = random.random()
|
|
if kind == "page":
|
|
if r < 0.60: d = random.uniform(4.0, 9.0)
|
|
elif r < 0.88: d = random.uniform(9.0, 20.0)
|
|
else: d = random.uniform(20.0, 42.0)
|
|
else: # 同页内相邻点击
|
|
if r < 0.68: d = random.uniform(1.4, 4.0)
|
|
elif r < 0.93: d = random.uniform(4.0, 9.0)
|
|
else: d = random.uniform(9.0, 16.0)
|
|
d = max(0.8, d + random.gauss(0, 0.5))
|
|
time.sleep(d)
|
|
return d
|
|
|
|
def logged_out():
|
|
href = ev("location.href")
|
|
return isinstance(href, str) and "/login" in href
|
|
|
|
def done_nodes():
|
|
c = db()
|
|
rows = c.execute("SELECT DISTINCT node_key FROM downloads WHERE status='ok' AND node_key<>''").fetchall()
|
|
c.close()
|
|
return {r[0] for r in rows}
|
|
|
|
def do_click_cycle(clicked_label, odir, nk, fp, url):
|
|
"""点击入口后：持续处理授权窗/参数窗(可能异步慢加载)并确认, 同时检测直链; 兑底走任务中心。"""
|
|
ev("window.__cap=[]")
|
|
got = False
|
|
# 最多 ~30s: 参数窗可能异步加载, 持续 AGREE+CONFIRM 并检测直链
|
|
for _ in range(20):
|
|
time.sleep(1.5)
|
|
ev(AGREE)
|
|
ev(CONFIRM)
|
|
n = ev("(window.__cap&&window.__cap.length)||0")
|
|
if isinstance(n, int) and n > 0:
|
|
got = True
|
|
break
|
|
if got:
|
|
ok, name = grab_and_save(odir, nk, fp, url, "阶段1 " + clicked_label)
|
|
if not ok:
|
|
log(" [%s] 抓取失败: %s" % (clicked_label, name))
|
|
result = ok
|
|
else:
|
|
log(" [%s] 未产出直链(需必选参数/异步未完成, 待任务中心收割)" % clicked_label)
|
|
rec(nk, fp, url, "pending", "", "skip", 0, clicked_label)
|
|
result = False
|
|
ev(CLOSE)
|
|
return result
|
|
|
|
def process_page(p, idx, total):
|
|
fp, url, nk = p["full_path"], p["url"], p["node_key"]
|
|
odir = out_dir(p["screenshot_path"], fp)
|
|
log("[%d/%d] %s" % (idx, total, fp))
|
|
call("navigate", {"url": url, "newTab": False})
|
|
# 等页面加载 + 登录态检测
|
|
for k in range(6):
|
|
time.sleep(2)
|
|
if k == 1 and logged_out():
|
|
log(" !! 检测到登录态失效, 中止本轮")
|
|
return -1
|
|
if logged_out():
|
|
log(" !! 检测到登录态失效, 中止本轮")
|
|
return -1
|
|
ev(HOOK)
|
|
ev(AGREE) # 若有数据授权协议窗, 先“知情并同意”
|
|
saved = 0
|
|
hit = False
|
|
# 四类入口: 图标下载/文字/类型B导出/类型A报表卡片
|
|
for label, code in (("icon-download", CLICK_ICON), ("txt", CLICK_TXT),
|
|
("export", CLICK_EXPORT), ("card", CLICK_CARD)):
|
|
for _ in range(8):
|
|
c = ev(code)
|
|
if c == "none":
|
|
break
|
|
hit = True
|
|
if do_click_cycle(label + "=" + str(c), odir, nk, fp, url):
|
|
saved += 1
|
|
human_delay("click")
|
|
if not hit:
|
|
log(" 无导出入口, skip")
|
|
rec(nk, fp, url, "none", "", "skip", 0, "无导出入口")
|
|
else:
|
|
log(" 本页触发导出入口, 落盘=%d (其余走任务中心)" % saved)
|
|
return saved
|
|
|
|
def harvest_task_center():
|
|
log("=== 阶段2: 收割任务中心 ===")
|
|
ev(HOOK)
|
|
odir = "downloads/_任务中心"; total = 0
|
|
try:
|
|
arr = json.loads(ev(TC_LIST))
|
|
except Exception:
|
|
arr = []
|
|
log("任务中心可下载任务: %d" % len(arr))
|
|
for i, nm in enumerate(arr):
|
|
ev("window.__cap=[]")
|
|
if ev(TC_CLICK % i) != "ok":
|
|
continue
|
|
if wait_capture(20):
|
|
ok, _ = grab_and_save(odir, "", "任务中心 / " + (nm or ""), "", "阶段2 任务=" + (nm or ""))
|
|
if ok:
|
|
total += 1
|
|
time.sleep(random.uniform(2, 4))
|
|
return total
|
|
|
|
def main():
|
|
ensure()
|
|
call("find_tab", {"url": "2haohr.com"})
|
|
pages = load_pages()
|
|
log("=== 开始导出下载, 共 %d 页 ===" % len(pages))
|
|
stage1 = 0
|
|
for i, p in enumerate(pages, 1):
|
|
try:
|
|
stage1 += process_page(p, i, len(pages))
|
|
except Exception as e:
|
|
log(" 页面异常: %s" % str(e)[:100])
|
|
rec(p["node_key"], p["full_path"], p["url"], "err", "", "fail", 0, str(e)[:80])
|
|
time.sleep(random.uniform(5, 15))
|
|
stage2 = harvest_task_center()
|
|
log("=== DONE 阶段1=%d 阶段2=%d 累计文件=%d ===" % (stage1, stage2, len(SAVED)))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|