#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
报表/文档导出下载器（Excel/Word 为主）。完整链路（已逐环节实测验证）：
  入口：svg use[xlink:href=#icon-download] 图标（无文字，代码级定位）+ 文字“下载/生成excel”兜底；
  生成：点图标 -> 弹“下载报表/导出”参数窗 -> 点“确定”(默认参数) -> 异步生成；
  捕获：hook HTMLAnchorElement.prototype.click 捕获 rpt-cdn/prv-cdn 带 token 直链；
  落盘：页面内同步 XHR 取字节 -> base64 回传 -> 写 workspace（绕过 macOS ~/Downloads 的 TCC 限制）。
阶段2：任务中心(taskCenter)收割兜底。产出真实 .xlsx/.docx，写 SQLite downloads 表；
页面间随机 5-15s；断点续跑(同名>2000字节跳过)。
"""
import json, urllib.request, base64, time, os, random, sqlite3, re, collections

D = "http://127.0.0.1:10086/command"; S = "2haohr"
DB = "db/research.db"; LOG = "db/dl_export.log"
KW = ['报表', '分析', '仪表盘', '统计', '档案', '模板', '文档', '制度', '公告', '证明']
TXT_BTN = ['直接下载', '生成excel', '导出Excel', '导出报表', '批量导出']  # 文字入口(不含裸“下载/导出”避免误点导航)

def call(a, args, timeout=120):
    b = json.dumps({"action": a, "args": args, "session": S}).encode()
    r = urllib.request.Request(D, data=b, headers={"Content-Type": "application/json"})
    return json.loads(urllib.request.urlopen(r, timeout=timeout).read().decode())

def ev(code, timeout=120):
    try:
        return call("evaluate", {"code": code}, timeout).get("data", {}).get("value")
    except Exception as e:
        return "EXC:" + str(e)[:80]

def log(m):
    line = time.strftime("%H:%M:%S ") + m
    print(line, flush=True)
    with open(LOG, "a") as f:
        f.write(line + "\n")

HOOK = r"""(()=>{window.__cap=window.__cap||[];
  if(!HTMLAnchorElement.prototype.__hk){const ac=HTMLAnchorElement.prototype.click;
    HTMLAnchorElement.prototype.click=function(){try{const h=this.href||'';
      if(h&&/-cdn\.2haohr|prv-cdn|rpt-cdn|attname=|\.xlsx|\.xls|\.docx|\.doc|\.zip/i.test(h))window.__cap.push(h);}catch(e){}
      return ac.apply(this,arguments);};HTMLAnchorElement.prototype.__hk=1;}
  return 'ok';})()"""

# 未处理的 #icon-download 数量
COUNT_ICON = r"""(()=>{let n=0;document.querySelectorAll('use').forEach(u=>{
  const h=u.getAttribute('xlink:href')||u.getAttribute('href');
  if(h==='#icon-download'){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl'))n++;}});return n;})()"""

# 点击下一个未处理的 #icon-download
CLICK_ICON = r"""(()=>{const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-download');
  for(const u of us){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl')){b.setAttribute('data-dl','1');b.click();return 'ok';}}
  return 'none';})()"""

# 文字入口
COUNT_TXT = (r"""(()=>{const T=%s;let n=0;document.querySelectorAll('button,a,[role=button]').forEach(x=>{"""
    r"""const t=(x.innerText||'').trim();if(T.includes(t)&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl'))n++;});return n;})()""") % json.dumps(TXT_BTN)
CLICK_TXT = (r"""(()=>{const T=%s;for(const t of T){const b=[...document.querySelectorAll('button,a,[role=button]')]"""
    r""".find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl'));"""
    r"""if(b){b.setAttribute('data-dl','1');b.click();return t;}}return 'none';})()""") % json.dumps(TXT_BTN)

# 参数窗“确定/下载/导出”——关键词精确定位“下载报表/导出”参数窗(避免误点其它弹层)
KWWIN = r"下载报表|导出|下载|部门|员工排序|工作性质|字段|范围|格式|参数|排序|在职"
CONFIRM = (r"""(()=>{const cand=['确定','确认','下载','导出','生成excel','生成','立即导出','立即下载'];
  const KW=/%s/;
  const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')];
  const vis=ms.filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>80&&getComputedStyle(m).display!=='none'&&getComputedStyle(m).visibility!=='hidden';});
  // 优先带关键词的参数窗, 再退化到任意可见弹层
  const ranked=[...vis.filter(m=>KW.test(m.innerText||'')),...vis];
  for(const m of ranked){
    for(const t of cand){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0);
      if(b){b.click();return 'modal:'+t;}}}
  return vis.length?'modal:noconfirm':'nomodal';})()""") % KWWIN

CLOSE = r"""(()=>{const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer]')];
  for(const m of ms){const r=m.getBoundingClientRect();if(r.width>200&&r.height>80&&getComputedStyle(m).display!=='none'){
    const c=[...m.querySelectorAll('button,a,[class*=close],[class*=Close]')]
      .find(x=>{const t=(x.innerText||'').trim();return t==='取消'||t==='关闭'||(x.className+'').toLowerCase().includes('close');});
    if(c){c.click();return 'closed';}}}return 'none';})()"""

# 数据授权/协议窗：只点“知情并同意”，绝不点“不同意并退出登录”(之前登出的元凶)
AGREE = r"""(()=>{const bad=/退出登录|不同意/;const ok=['知情并同意','同意并继续','我已阅读并同意','同意'];
  const ms=[...document.querySelectorAll('[class*=modal],[class*=Modal],[role=dialog],[class*=dialog],[class*=drawer],[class*=Drawer]')]
    .filter(m=>{const r=m.getBoundingClientRect();return r.width>200&&r.height>60&&getComputedStyle(m).display!=='none';});
  for(const m of ms){const txt=m.innerText||'';if(!/协议|授权|同意|条款|隐私/.test(txt))continue;
    for(const t of ok){const b=[...m.querySelectorAll('button,a')].find(x=>(x.innerText||'').trim()===t&&!bad.test(x.innerText||''));
      if(b){b.click();return 'agree:'+t;}}}
  return 'none';})()"""

BAD = r"退出登录|不同意|立即购买|预约演示|我要更多功能"
# 类型B 数据列表导出入口: #icon-export 或 主内容区文字“导出”(排除导航/任务中心/黑名单)
CLICK_EXPORT = (r"""(()=>{const BAD=/%s/;
  const inNav=e=>!!e.closest('nav,[class*=navBar],[class*=sideMenu],[class*=taskCenter]');
  const us=[...document.querySelectorAll('use')].filter(u=>(u.getAttribute('xlink:href')||u.getAttribute('href'))==='#icon-export');
  for(const u of us){const b=u.closest('button,a,[role=button]');if(b&&!b.hasAttribute('data-dl')&&!inNav(b)){b.setAttribute('data-dl','1');b.click();return 'icon-export';}}
  const b=[...document.querySelectorAll('button,a,[role=button]')].find(x=>{const t=(x.innerText||'').trim();return t==='导出'&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl')&&!inNav(x)&&!BAD.test(t);});
  if(b){b.setAttribute('data-dl','1');b.click();return 'txt-导出';}
  return 'none';})()""") % BAD

# 类型A 报表中心卡片导出: 生成excel/直接下载/下载(排除导航/任务中心/黑名单)
CARD_BTN = ['生成excel', '直接下载', '导出Excel', '导出报表', '批量导出', '下载']
CLICK_CARD = (r"""(()=>{const T=%s;const BAD=/%s/;
  const inNav=e=>!!e.closest('nav,[class*=navBar],[class*=sideMenu],[class*=taskCenter]');
  for(const t of T){const b=[...document.querySelectorAll('button,a,[role=button]')].find(x=>(x.innerText||'').trim()===t&&x.getBoundingClientRect().width>0&&!x.hasAttribute('data-dl')&&!inNav(x)&&!BAD.test(x.innerText||''));
    if(b){b.setAttribute('data-dl','1');b.click();return t;}}
  return 'none';})()""") % (json.dumps(CARD_BTN), BAD)

FETCH_LAST = r"""(()=>{try{if(!window.__cap||!window.__cap.length)return 'NO_HREF';
  const u=window.__cap[window.__cap.length-1];const x=new XMLHttpRequest();x.open('GET',u,false);
  x.overrideMimeType('text/plain;charset=x-user-defined');x.send();if(x.status!==200)return 'HTTP'+x.status;
  const s=x.responseText;let out='';const CH=8192;for(let i=0;i<s.length;i+=CH){let c='';const e=Math.min(i+CH,s.length);
    for(let j=i;j<e;j++)c+=String.fromCharCode(s.charCodeAt(j)&0xff);out+=c;}
  window.__b64=btoa(out);const m=u.match(/attname=([^&]+)/);let name=m?decodeURIComponent(m[1]):('export_'+Date.now()+'.bin');
  return JSON.stringify({size:s.length,name:name});}catch(e){return 'EX '+e.message;}})()"""

TC_LIST = r"""(()=>{const tc=document.querySelector('[class*=taskCenter]');if(!tc)return '[]';
  const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
  const names=dls.map(a=>{const row=a.closest('[class*=item],li,div');const info=row&&row.querySelector('[class*=info_]');
    return info?(info.innerText||'').replace(/\n+/g,' ').trim().slice(0,50):'';});
  return JSON.stringify(names);})()"""
TC_CLICK = r"""(function(i){const tc=document.querySelector('[class*=taskCenter]');if(!tc)return 'NO_TC';
  const dls=[...tc.querySelectorAll('a')].filter(a=>(a.innerText||'').trim()==='下载');
  if(i>=dls.length)return 'OOB';dls[i].click();return 'ok';})(%d)"""

def db():
    c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c

def ensure():
    c = db(); c.execute("""CREATE TABLE IF NOT EXISTS downloads(
        id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
        kind TEXT, file_path TEXT, status TEXT, size INTEGER, note TEXT, ts TEXT)"""); c.commit(); c.close()

def rec(nk, fp, url, kind, path, status, size, note):
    c = db(); c.execute("INSERT INTO downloads(node_key,full_path,url,kind,file_path,status,size,note,ts) VALUES(?,?,?,?,?,?,?,?,?)",
        (nk, fp, url, kind, path, status, size, note, time.strftime("%Y-%m-%d %H:%M:%S"))); c.commit(); c.close()

def safe(s):
    return re.sub(r'[\\/:*?"<>|]', '_', (s or "").strip())[:90]

def out_dir(shot, full_path):
    if shot and shot.startswith("screenshots/"):
        return os.path.dirname(shot.replace("screenshots/", "downloads/", 1))
    parts = [safe(p) for p in (full_path or "").split(" / ") if p.strip()]
    return os.path.join("downloads", *parts[:-1]) if len(parts) > 1 else "downloads/misc"

def load_pages():
    c = db(); rows = [dict(r) for r in c.execute(
        "SELECT node_key,parent_key,name,full_path,url,screenshot_path FROM features").fetchall()]
    kids = collections.defaultdict(int)
    for r in rows:
        if r["parent_key"]:
            kids[r["parent_key"]] += 1
    c.close()
    leaves = [r for r in rows if kids.get(r["node_key"], 0) == 0]
    return [r for r in leaves if r["url"] and any(k in (r["full_path"] or "") + (r["name"] or "") for k in KW)]

SAVED = set()

def grab_and_save(odir, nk, fp, url, note):
    meta = ev(FETCH_LAST)
    if not (isinstance(meta, str) and meta.startswith("{")):
        return False, str(meta)[:40]
    info = json.loads(meta); name = safe(info["name"]); ev("window.__cap=[]")
    if name in SAVED:
        return False, "dup"
    try:
        data = base64.b64decode(ev("window.__b64||''"))
    except Exception:
        data = b""
    if len(data) <= 200:
        return False, "empty"
    os.makedirs(odir, exist_ok=True); dst = os.path.join(odir, name)
    if os.path.exists(dst) and os.path.getsize(dst) > 2000:
        SAVED.add(name); return True, name + "(exists)"
    open(dst, "wb").write(data); SAVED.add(name)
    low = name.lower()
    kind = "xlsx" if low.endswith((".xlsx", ".xls")) else ("docx" if low.endswith((".docx", ".doc")) else (
        "pdf" if low.endswith(".pdf") else ("zip" if low.endswith(".zip") else "file")))
    rec(nk, fp, url, kind, dst, "ok", len(data), note)
    log("    OK %s (%d bytes) -> %s" % (kind, len(data), dst))
    return True, name

def wait_capture(seconds=25):
    t0 = time.time()
    while time.time() - t0 < seconds:
        time.sleep(2)
        n = ev("(window.__cap&&window.__cap.length)||0")
        if isinstance(n, int) and n > 0:
            return True
    return False

def human_delay(kind="page"):
    """非均匀拟人间隔：大部分短、少量中、偶尔长停顿 + 高斯抖动。"""
    r = random.random()
    if kind == "page":
        if r < 0.60: d = random.uniform(4.0, 9.0)
        elif r < 0.88: d = random.uniform(9.0, 20.0)
        else: d = random.uniform(20.0, 42.0)
    else:  # 同页内相邻点击
        if r < 0.68: d = random.uniform(1.4, 4.0)
        elif r < 0.93: d = random.uniform(4.0, 9.0)
        else: d = random.uniform(9.0, 16.0)
    d = max(0.8, d + random.gauss(0, 0.5))
    time.sleep(d)
    return d

def logged_out():
    href = ev("location.href")
    return isinstance(href, str) and "/login" in href

def done_nodes():
    c = db()
    rows = c.execute("SELECT DISTINCT node_key FROM downloads WHERE status='ok' AND node_key<>''").fetchall()
    c.close()
    return {r[0] for r in rows}

def do_click_cycle(clicked_label, odir, nk, fp, url):
    """点击入口后：持续处理授权窗/参数窗(可能异步慢加载)并确认, 同时检测直链; 兑底走任务中心。"""
    ev("window.__cap=[]")
    got = False
    # 最多 ~30s: 参数窗可能异步加载, 持续 AGREE+CONFIRM 并检测直链
    for _ in range(20):
        time.sleep(1.5)
        ev(AGREE)
        ev(CONFIRM)
        n = ev("(window.__cap&&window.__cap.length)||0")
        if isinstance(n, int) and n > 0:
            got = True
            break
    if got:
        ok, name = grab_and_save(odir, nk, fp, url, "阶段1 " + clicked_label)
        if not ok:
            log("    [%s] 抓取失败: %s" % (clicked_label, name))
        result = ok
    else:
        log("    [%s] 未产出直链(需必选参数/异步未完成, 待任务中心收割)" % clicked_label)
        rec(nk, fp, url, "pending", "", "skip", 0, clicked_label)
        result = False
    ev(CLOSE)
    return result

def process_page(p, idx, total):
    fp, url, nk = p["full_path"], p["url"], p["node_key"]
    odir = out_dir(p["screenshot_path"], fp)
    log("[%d/%d] %s" % (idx, total, fp))
    call("navigate", {"url": url, "newTab": False})
    # 等页面加载 + 登录态检测
    for k in range(6):
        time.sleep(2)
        if k == 1 and logged_out():
            log("    !! 检测到登录态失效, 中止本轮")
            return -1
    if logged_out():
        log("    !! 检测到登录态失效, 中止本轮")
        return -1
    ev(HOOK)
    ev(AGREE)  # 若有数据授权协议窗, 先“知情并同意”
    saved = 0
    hit = False
    # 四类入口: 图标下载/文字/类型B导出/类型A报表卡片
    for label, code in (("icon-download", CLICK_ICON), ("txt", CLICK_TXT),
                         ("export", CLICK_EXPORT), ("card", CLICK_CARD)):
        for _ in range(8):
            c = ev(code)
            if c == "none":
                break
            hit = True
            if do_click_cycle(label + "=" + str(c), odir, nk, fp, url):
                saved += 1
            human_delay("click")
    if not hit:
        log("    无导出入口, skip")
        rec(nk, fp, url, "none", "", "skip", 0, "无导出入口")
    else:
        log("    本页触发导出入口, 落盘=%d (其余走任务中心)" % saved)
    return saved

def harvest_task_center():
    log("=== 阶段2: 收割任务中心 ===")
    ev(HOOK)
    odir = "downloads/_任务中心"; total = 0
    try:
        arr = json.loads(ev(TC_LIST))
    except Exception:
        arr = []
    log("任务中心可下载任务: %d" % len(arr))
    for i, nm in enumerate(arr):
        ev("window.__cap=[]")
        if ev(TC_CLICK % i) != "ok":
            continue
        if wait_capture(20):
            ok, _ = grab_and_save(odir, "", "任务中心 / " + (nm or ""), "", "阶段2 任务=" + (nm or ""))
            if ok:
                total += 1
        time.sleep(random.uniform(2, 4))
    return total

def main():
    ensure()
    call("find_tab", {"url": "2haohr.com"})
    pages = load_pages()
    log("=== 开始导出下载, 共 %d 页 ===" % len(pages))
    stage1 = 0
    for i, p in enumerate(pages, 1):
        try:
            stage1 += process_page(p, i, len(pages))
        except Exception as e:
            log("    页面异常: %s" % str(e)[:100])
            rec(p["node_key"], p["full_path"], p["url"], "err", "", "fail", 0, str(e)[:80])
        time.sleep(random.uniform(5, 15))
    stage2 = harvest_task_center()
    log("=== DONE 阶段1=%d 阶段2=%d 累计文件=%d ===" % (stage1, stage2, len(SAVED)))

if __name__ == "__main__":
    main()