first commit

2026-09-03 15:35:23 +08:00
commit 9b26161bcf
510 changed files with 29402 additions and 0 deletions
@@ -0,0 +1,313 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+报表/文档导出下载器（Excel/Word 为主）。完整链路（已逐环节实测验证）：
+  入口：svg use[xlink:href=#icon-download] 图标（无文字，代码级定位）+ 文字“下载/生成excel”兜底；
+  生成：点图标 -> 弹“下载报表/导出”参数窗 -> 点“确定”(默认参数) -> 异步生成；
+  捕获：hook HTMLAnchorElement.prototype.click 捕获 rpt-cdn/prv-cdn 带 token 直链；
+  落盘：页面内同步 XHR 取字节 -> base64 回传 -> 写 workspace（绕过 macOS ~/Downloads 的 TCC 限制）。
+阶段2：任务中心(taskCenter)收割兜底。产出真实 .xlsx/.docx，写 SQLite downloads 表；
+页面间随机 5-15s；断点续跑(同名>2000字节跳过)。
+"""
+import json, urllib.request, base64, time, os, random, sqlite3, re, collections
+
+D = "http://127.0.0.1:10086/command"; S = "2haohr"
+DB = "db/research.db"; LOG = "db/dl_export.log"
+KW = ['报表', '分析', '仪表盘', '统计', '档案', '模板', '文档', '制度', '公告', '证明']
+TXT_BTN = ['直接下载', '生成excel', '导出Excel', '导出报表', '批量导出']  # 文字入口(不含裸“下载/导出”避免误点导航)
+
+def call(a, args, timeout=120):
+    b = json.dumps({"action": a, "args": args, "session": S}).encode()
+    r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
+    return json.loads(urllib.request.urlopen(r, timeout=timeout).read().decode())
+
+def ev(code, timeout=120):
+    try:
+        return call("evaluate", {"code": code}, timeout).get("data", {}).get("value")
+    except Exception as e:
+        return "EXC:" + str(e)[:80]
+
+def log(m):
+    line = time.strftime("%H:%M:%S ") + m
+    print(line, flush=True)
+    with open(LOG, "a") as f:
+        f.write(line + "\n")
+
+HOOK = r"""(()=>{window.__cap=window.__cap||[];
+  if(!HTMLAnchorElement.prototype.__hk){const ac=HTMLAnchorElement.prototype.click;
+    HTMLAnchorElement.prototype.click=function(){try{const h=this.href||'';
+      if(h&&/-cdn\.2haohr|prv-cdn|rpt-cdn|attname=|\.xlsx|\.xls|\.docx|\.doc|\.zip/i.test(h))window.__cap.push(h);}catch(e){}
+      return ac.apply(this,arguments);};HTMLAnchorElement.prototype.__hk=1;}
+  return 'ok';})()"""
+
+# 未处理的 #icon-download 数量
+COUNT_ICON = r"""(()=>{let n=0;document.querySelectorAll('use').forEach(u=>{
+  const h=u.getAttribute('xlink:href')||u.getAttribute('href');
+  if(h==='#icon-download'){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl'))n++;}});return n;})()"""
+
+# 点击下一个未处理的 #icon-download
+CLICK_ICON = r"""(()=>{const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-download');
+  for(const u of us){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl')){b.setAttribute('data-dl','1');b.click();return 'ok';}}
+  return 'none';})()"""
+
+# 文字入口
+COUNT_TXT = (r"""(()=>{const T=%s;let n=0;document.querySelectorAll('button,a,[role=button]').forEach(x=>{"""
+    r"""const t=(x.innerText||'').trim();if(T.includes(t)&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl'))n++;});return n;})()""") % json.dumps(TXT_BTN)
+CLICK_TXT = (r"""(()=>{const T=%s;for(const t of T){const b=[...document.querySelectorAll('button,a,[role=button]')]"""
+    r""".find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl'));"""
+    r"""if(b){b.setAttribute('data-dl','1');b.click();return t;}}return 'none';})()""") % json.dumps(TXT_BTN)
+
+# 参数窗“确定/下载/导出”——关键词精确定位“下载报表/导出”参数窗(避免误点其它弹层)
+KWWIN = r"下载报表|导出|下载|部门|员工排序|工作性质|字段|范围|格式|参数|排序|在职"
+CONFIRM = (r"""(()=>{const cand=['确定','确认','下载','导出','生成excel','生成','立即导出','立即下载'];
+  const KW=/%s/;
+  const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')];
+  const vis=ms.filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>80&&getComputedStyle(m).display!=='none'&&getComputedStyle(m).visibility!=='hidden';});
+  // 优先带关键词的参数窗, 再退化到任意可见弹层
+  const ranked=[...vis.filter(m=>KW.test(m.innerText||'')),...vis];
+  for(const m of ranked){
+    for(const t of cand){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0);
+      if(b){b.click();return 'modal:'+t;}}}
+  return vis.length?'modal:noconfirm':'nomodal';})()""") % KWWIN
+
+CLOSE = r"""(()=>{const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer]')];
+  for(const m of ms){const r=m.getBoundingClientRect();if(r.width>200&&r.height>80&&getComputedStyle(m).display!=='none'){
+    const c=[...m.querySelectorAll('button,a,[class*=close],[class*=Close]')]
+      .find(x=>{const t=(x.innerText||'').trim();return t==='取消'||t==='关闭'||(x.className+'').toLowerCase().includes('close');});
+    if(c){c.click();return 'closed';}}}return 'none';})()"""
+
+# 数据授权/协议窗：只点“知情并同意”，绝不点“不同意并退出登录”(之前登出的元凶)
+AGREE = r"""(()=>{const bad=/退出登录|不同意/;const ok=['知情并同意','同意并继续','我已阅读并同意','同意'];
+  const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')]
+    .filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>60&&getComputedStyle(m).display!=='none';});
+  for(const m of ms){const txt=m.innerText||'';if(!/协议|授权|同意|条款|隐私/.test(txt))continue;
+    for(const t of ok){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&!bad.test(x.innerText||''));
+      if(b){b.click();return 'agree:'+t;}}}
+  return 'none';})()"""
+
+BAD = r"退出登录|不同意|立即购买|预约演示|我要更多功能"
+# 类型B 数据列表导出入口: #icon-export 或 主内容区文字“导出”(排除导航/任务中心/黑名单)
+CLICK_EXPORT = (r"""(()=>{const BAD=/%s/;
+  const inNav=e=>!!e.closest('nav,[class*=navBar],[class*=sideMenu],[class*=taskCenter]');
+  const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-export');
+  for(const u of us){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl')&&!inNav(b)){b.setAttribute('data-dl','1');b.click();return 'icon-export';}}
+  const b=[...document.querySelectorAll('button,a,[role=button]')].find(x=>{const t=(x.innerText||'').trim();return t==='导出'&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl')&&!inNav(x)&&!BAD.test(t);});
+  if(b){b.setAttribute('data-dl','1');b.click();return 'txt-导出';}
+  return 'none';})()""") % BAD
+
+# 类型A 报表中心卡片导出: 生成excel/直接下载/下载(排除导航/任务中心/黑名单)
+CARD_BTN = ['生成excel', '直接下载', '导出Excel', '导出报表', '批量导出', '下载']
+CLICK_CARD = (r"""(()=>{const T=%s;const BAD=/%s/;
+  const inNav=e=>!!e.closest('nav,[class*=navBar],[class*=sideMenu],[class*=taskCenter]');
+  for(const t of T){const b=[...document.querySelectorAll('button,a,[role=button]')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl')&&!inNav(x)&&!BAD.test(x.innerText||''));
+    if(b){b.setAttribute('data-dl','1');b.click();return t;}}
+  return 'none';})()""") % (json.dumps(CARD_BTN), BAD)
+
+FETCH_LAST = r"""(()=>{try{if(!window.__cap||!window.__cap.length)return 'NO_HREF';
+  const u=window.__cap[window.__cap.length-1];const x=new XMLHttpRequest();x.open('GET',u,false);
+  x.overrideMimeType('text/plain;charset=x-user-defined');x.send();if(x.status!==200)return 'HTTP'+x.status;
+  const s=x.responseText;let out='';const CH=8192;for(let i=0;i<s.length;i+=CH){let c='';const e=Math.min(i+CH,s.length);
+    for(let j=i;j<e;j++)c+=String.fromCharCode(s.charCodeAt(j)&0xff);out+=c;}
+  window.__b64=btoa(out);const m=u.match(/attname=([^&]+)/);let name=m?decodeURIComponent(m[1]):('export_'+Date.now()+'.bin');
+  return JSON.stringify({size:s.length,name:name});}catch(e){return 'EX '+e.message;}})()"""
+
+TC_LIST = r"""(()=>{const tc=document.querySelector('[class*=taskCenter]');if(!tc)return '[]';
+  const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
+  const names=dls.map(a=>{const row=a.closest('[class*=item],li,div');const info=row&&row.querySelector('[class*=info_]');
+    return info?(info.innerText||'').replace(/\n+/g,' ').trim().slice(0,50):'';});
+  return JSON.stringify(names);})()"""
+TC_CLICK = r"""(function(i){const tc=document.querySelector('[class*=taskCenter]');if(!tc)return 'NO_TC';
+  const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
+  if(i>=dls.length)return 'OOB';dls[i].click();return 'ok';})(%d)"""
+
+def db():
+    c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c
+
+def ensure():
+    c = db(); c.execute("""CREATE TABLE IF NOT EXISTS downloads(
+        id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
+        kind TEXT, file_path TEXT, status TEXT, size INTEGER, note TEXT, ts TEXT)"""); c.commit(); c.close()
+
+def rec(nk, fp, url, kind, path, status, size, note):
+    c = db(); c.execute("INSERT INTO downloads(node_key,full_path,url,kind,file_path,status,size,note,ts) VALUES(?,?,?,?,?,?,?,?,?)",
+        (nk, fp, url, kind, path, status, size, note, time.strftime("%Y-%m-%d %H:%M:%S"))); c.commit(); c.close()
+
+def safe(s):
+    return re.sub(r'[\\/:*?"<>|]', '_', (s or "").strip())[:90]
+
+def out_dir(shot, full_path):
+    if shot and shot.startswith("screenshots/"):
+        return os.path.dirname(shot.replace("screenshots/", "downloads/", 1))
+    parts = [safe(p) for p in (full_path or "").split(" / ") if p.strip()]
+    return os.path.join("downloads", *parts[:-1]) if len(parts) > 1 else "downloads/misc"
+
+def load_pages():
+    c = db(); rows = [dict(r) for r in c.execute(
+        "SELECT node_key,parent_key,name,full_path,url,screenshot_path FROM features").fetchall()]
+    kids = collections.defaultdict(int)
+    for r in rows:
+        if r["parent_key"]:
+            kids[r["parent_key"]] += 1
+    c.close()
+    leaves = [r for r in rows if kids.get(r["node_key"], 0) == 0]
+    return [r for r in leaves if r["url"] and any(k in (r["full_path"] or "") + (r["name"] or "") for k in KW)]
+
+SAVED = set()
+
+def grab_and_save(odir, nk, fp, url, note):
+    meta = ev(FETCH_LAST)
+    if not (isinstance(meta, str) and meta.startswith("{")):
+        return False, str(meta)[:40]
+    info = json.loads(meta); name = safe(info["name"]); ev("window.__cap=[]")
+    if name in SAVED:
+        return False, "dup"
+    try:
+        data = base64.b64decode(ev("window.__b64||''"))
+    except Exception:
+        data = b""
+    if len(data) <= 200:
+        return False, "empty"
+    os.makedirs(odir, exist_ok=True); dst = os.path.join(odir, name)
+    if os.path.exists(dst) and os.path.getsize(dst) > 2000:
+        SAVED.add(name); return True, name + "(exists)"
+    open(dst, "wb").write(data); SAVED.add(name)
+    low = name.lower()
+    kind = "xlsx" if low.endswith((".xlsx", ".xls")) else ("docx" if low.endswith((".docx", ".doc")) else (
+        "pdf" if low.endswith(".pdf") else ("zip" if low.endswith(".zip") else "file")))
+    rec(nk, fp, url, kind, dst, "ok", len(data), note)
+    log("    OK %s (%d bytes) -> %s" % (kind, len(data), dst))
+    return True, name
+
+def wait_capture(seconds=25):
+    t0 = time.time()
+    while time.time() - t0 < seconds:
+        time.sleep(2)
+        n = ev("(window.__cap&&window.__cap.length)||0")
+        if isinstance(n, int) and n > 0:
+            return True
+    return False
+
+def human_delay(kind="page"):
+    """非均匀拟人间隔：大部分短、少量中、偶尔长停顿 + 高斯抖动。"""
+    r = random.random()
+    if kind == "page":
+        if r < 0.60: d = random.uniform(4.0, 9.0)
+        elif r < 0.88: d = random.uniform(9.0, 20.0)
+        else: d = random.uniform(20.0, 42.0)
+    else:  # 同页内相邻点击
+        if r < 0.68: d = random.uniform(1.4, 4.0)
+        elif r < 0.93: d = random.uniform(4.0, 9.0)
+        else: d = random.uniform(9.0, 16.0)
+    d = max(0.8, d + random.gauss(0, 0.5))
+    time.sleep(d)
+    return d
+
+def logged_out():
+    href = ev("location.href")
+    return isinstance(href, str) and "/login" in href
+
+def done_nodes():
+    c = db()
+    rows = c.execute("SELECT DISTINCT node_key FROM downloads WHERE status='ok' AND node_key<>''").fetchall()
+    c.close()
+    return {r[0] for r in rows}
+
+def do_click_cycle(clicked_label, odir, nk, fp, url):
+    """点击入口后：持续处理授权窗/参数窗(可能异步慢加载)并确认, 同时检测直链; 兑底走任务中心。"""
+    ev("window.__cap=[]")
+    got = False
+    # 最多 ~30s: 参数窗可能异步加载, 持续 AGREE+CONFIRM 并检测直链
+    for _ in range(20):
+        time.sleep(1.5)
+        ev(AGREE)
+        ev(CONFIRM)
+        n = ev("(window.__cap&&window.__cap.length)||0")
+        if isinstance(n, int) and n > 0:
+            got = True
+            break
+    if got:
+        ok, name = grab_and_save(odir, nk, fp, url, "阶段1 " + clicked_label)
+        if not ok:
+            log("    [%s] 抓取失败: %s" % (clicked_label, name))
+        result = ok
+    else:
+        log("    [%s] 未产出直链(需必选参数/异步未完成, 待任务中心收割)" % clicked_label)
+        rec(nk, fp, url, "pending", "", "skip", 0, clicked_label)
+        result = False
+    ev(CLOSE)
+    return result
+
+def process_page(p, idx, total):
+    fp, url, nk = p["full_path"], p["url"], p["node_key"]
+    odir = out_dir(p["screenshot_path"], fp)
+    log("[%d/%d] %s" % (idx, total, fp))
+    call("navigate", {"url": url, "newTab": False})
+    # 等页面加载 + 登录态检测
+    for k in range(6):
+        time.sleep(2)
+        if k == 1 and logged_out():
+            log("    !! 检测到登录态失效, 中止本轮")
+            return -1
+    if logged_out():
+        log("    !! 检测到登录态失效, 中止本轮")
+        return -1
+    ev(HOOK)
+    ev(AGREE)  # 若有数据授权协议窗, 先“知情并同意”
+    saved = 0
+    hit = False
+    # 四类入口: 图标下载/文字/类型B导出/类型A报表卡片
+    for label, code in (("icon-download", CLICK_ICON), ("txt", CLICK_TXT),
+                         ("export", CLICK_EXPORT), ("card", CLICK_CARD)):
+        for _ in range(8):
+            c = ev(code)
+            if c == "none":
+                break
+            hit = True
+            if do_click_cycle(label + "=" + str(c), odir, nk, fp, url):
+                saved += 1
+            human_delay("click")
+    if not hit:
+        log("    无导出入口, skip")
+        rec(nk, fp, url, "none", "", "skip", 0, "无导出入口")
+    else:
+        log("    本页触发导出入口, 落盘=%d (其余走任务中心)" % saved)
+    return saved
+
+def harvest_task_center():
+    log("=== 阶段2: 收割任务中心 ===")
+    ev(HOOK)
+    odir = "downloads/_任务中心"; total = 0
+    try:
+        arr = json.loads(ev(TC_LIST))
+    except Exception:
+        arr = []
+    log("任务中心可下载任务: %d" % len(arr))
+    for i, nm in enumerate(arr):
+        ev("window.__cap=[]")
+        if ev(TC_CLICK % i) != "ok":
+            continue
+        if wait_capture(20):
+            ok, _ = grab_and_save(odir, "", "任务中心 / " + (nm or ""), "", "阶段2 任务=" + (nm or ""))
+            if ok:
+                total += 1
+        time.sleep(random.uniform(2, 4))
+    return total
+
+def main():
+    ensure()
+    call("find_tab", {"url": "2haohr.com"})
+    pages = load_pages()
+    log("=== 开始导出下载, 共 %d 页 ===" % len(pages))
+    stage1 = 0
+    for i, p in enumerate(pages, 1):
+        try:
+            stage1 += process_page(p, i, len(pages))
+        except Exception as e:
+            log("    页面异常: %s" % str(e)[:100])
+            rec(p["node_key"], p["full_path"], p["url"], "err", "", "fail", 0, str(e)[:80])
+        time.sleep(random.uniform(5, 15))
+    stage2 = harvest_task_center()
+    log("=== DONE 阶段1=%d 阶段2=%d 累计文件=%d ===" % (stage1, stage2, len(SAVED)))
+
+if __name__ == "__main__":
+    main()