#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
阶段B Part1: 重走全部叶子页 -> 去遮挡 -> 重拍主截图 -> 安全抓取 4 类更深界面。

安全约束(硬性):
  浏览器是用户本人登录的真实 Chrome。深层抓取一律"打开 -> 截图 -> 换页重载",
  绝不点击 保存/删除/提交/确定/发送 等会产生真实数据变更的按钮(见 BAD 黑名单)。
  为杜绝状态污染: 每抓一个深层界面前都重新导航到干净页面, 只打开一个目标后立即截图,
  不做任何"确认"操作; 下一个深层界面靠再次重载复位。

4 类深层界面:
  add     主操作弹窗(顶部工具栏 新增/添加/新建/创建/录入/发起 表单)
  detail  行详情/编辑(首个数据行的 详情/查看/编辑)
  filter  筛选面板(#icon-filter / 文本"筛选")
  columns 自定义列/表头设置(#icon-custom-columns / 文本 自定义列/列设置/表头设置)
  menu_*  更多菜单里已发现且安全的配置类入口(page_extras 白名单)

存储:
  主截图: 覆盖 features.screenshot_path
  深层截图: 同目录 <base>__<kind>.jpg, 记录到 page_shots(node_key,full_path,url,kind,label,path,ts)
断点续跑: page_shots 已有该 node_key 的 kind=main 记录则整页跳过(除非 --force)。
用法: python3 rewalk.py [start_index] [--force]
"""
import os, sys, json, time, sqlite3, urllib.request
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from capture_full import capture, ev, shot, call, HIDE_JS

SESSION = "2haohr"
BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DB = os.path.join(BASE, "db", "research.db")
LOG = os.path.join(BASE, "db", "rewalk_progress.log")

# ---- 操作黑名单: 深层抓取时任何匹配的按钮都绝不点击 ----
BAD = (r"删除|移除|停用|注销|作废|清空|批量|放弃|提交|保存|确定|确认|发送|禁用|冻结|解除|"
       r"终止|结束|撤销|重置|清除|导入|上传|同步|生成|发放|归档|封存|开除|辞退|下架|驳回|"
       r"通过|拒绝|办理|恢复|启用|锁定|解锁|导出|下载|离职|开票|付款|转账|扣款|发起流程")

# ---- 更多菜单安全白名单: 仅打开明确的配置/查看类界面 ----
MENU_SAFE = r"设置|字段|配置|规则|对比|说明|模板|方案|切换视图|调整排序|自定义列|列设置|表头|视图|统计口径"


def dbc():
    c = sqlite3.connect(DB); c.row_factory = sqlite3.Row; return c


def log(msg):
    line = time.strftime("%H:%M:%S ") + msg
    print(line, flush=True)
    with open(LOG, "a") as f:
        f.write(line + "\n")


def navigate(url):
    body = json.dumps({"action": "navigate", "args": {"url": url, "newTab": False}, "session": SESSION}).encode()
    last = None
    for attempt in range(4):
        try:
            req = urllib.request.Request(DAEMON_URL, data=body, headers={"Content-Type": "application/json"})
            with _OPENER.open(req, timeout=45) as resp:
                return json.loads(resp.read().decode())
        except Exception as e:
            last = e
            time.sleep(1.5 + attempt * 1.5)
    raise last


DAEMON_URL = "http://127.0.0.1:10086/command"
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))


def wait_ready(max_wait=9):
    t0 = time.time()
    while time.time() - t0 < max_wait:
        try:
            if ev("(()=>document.readyState)()", SESSION) == "complete":
                time.sleep(1.2); return True
        except Exception:
            pass
        time.sleep(0.6)
    return False


# 内容稳定探测: docH(整页高度) + '共N条'数据计数
SETTLE_JS = r"""(()=>{const de=document.documentElement;const m=(document.body.innerText||'').match(/共\s*([\d,]+)\s*条/);return JSON.stringify({docH:de.scrollHeight,cnt:m?m[1].replace(/,/g,''):null});})()"""


def settle_content(max_wait=12.0):
    """截图前等内容稳定: 轮询 docH 直到连续两次不变, 且(若页面有'共N条')数据>0。
    列表/报表页的表格数据是异步加载的, 加载完成后 docH 才会增长到真实高度;
    若在'共0条/加载中'时截图, 会只截到首屏空表(没截全)。"""
    t0 = time.time(); last = -1; stable = 0
    while time.time() - t0 < max_wait:
        try:
            m = json.loads(ev(SETTLE_JS, SESSION) or "{}")
        except Exception:
            m = {}
        h = int(m.get("docH") or 0)
        cnt = m.get("cnt")
        data_ok = (cnt is None) or (str(cnt).isdigit() and int(cnt) > 0)
        stable = stable + 1 if (h == last and h > 0) else 0
        last = h
        if stable >= 2 and data_ok and time.time() - t0 > 1.5:
            return h
        time.sleep(0.5)
    return last


# 主截图去遮挡: 点"立即刷新"(安全重载数据) + 跳过新手引导 + DOM 隐藏遮罩/协议层 + 隐藏浮窗
DEOCCLUDE_JS = r"""
(()=>{
  const vis=el=>{const r=el.getBoundingClientRect();return r.width>0&&r.height>0&&getComputedStyle(el).visibility!=='hidden';};
  let acts=[];
  const refresh=[...document.querySelectorAll('a,button,span')].find(x=>x.childElementCount===0&&/立即刷新/.test((x.innerText||'').trim())&&vis(x));
  if(refresh){refresh.click();acts.push('refresh');}
  const skip=[...document.querySelectorAll('a,button,span,div')].find(x=>x.childElementCount===0&&/^(跳过|我知道了|知道了|跳过引导|不再提示|下次再说|以后再说)$/.test((x.innerText||'').trim())&&vis(x));
  if(skip){skip.click();acts.push('skip');}
  // DOM 隐藏引导遮罩/协议弹层(纯客户端, 不产生服务端动作)
  const masksel=['[class*=guide]','[class*=Guide]','[class*=driver]','[class*=intro]','[class*=newbie]','[class*=Newbie]','[class*=agreement]','[class*=Agreement]','[class*=guideMask]','[class*=coach]'];
  masksel.forEach(s=>document.querySelectorAll(s).forEach(el=>{const r=el.getBoundingClientRect();const p=getComputedStyle(el).position;if((p==='fixed'||p==='absolute')&&r.width>window.innerWidth*0.4&&r.height>window.innerHeight*0.3){el.style.setProperty('display','none','important');acts.push('mask');}}));
  return JSON.stringify(acts);
})()
"""


def deocclude_main():
    """返回是否触发了 refresh(需重新等待)。"""
    try:
        acts = json.loads(ev(DEOCCLUDE_JS, SESSION) or "[]")
    except Exception:
        acts = []
    ev(HIDE_JS, SESSION)
    return "refresh" in acts


def save_shot(out_abs):
    img = shot(SESSION)
    os.makedirs(os.path.dirname(out_abs), exist_ok=True)
    img.save(out_abs, "JPEG", quality=80)
    return img.size


# ---------- 4 类深层触发 (均带 BAD 黑名单过滤, 只打开不确认) ----------
OPEN_ADD = r"""
(()=>{
  const bad=/%BAD%/; const add=/(新增|添加|新建|创建|录入|发起)/;
  const cands=[...document.querySelectorAll('button,a')].filter(x=>{
     const t=(x.innerText||'').trim(); const r=x.getBoundingClientRect();
     return t&&t.length<=8&&add.test(t)&&!bad.test(t)&&r.width>0&&r.height>0&&r.top<340;});
  cands.sort((a,b)=>a.getBoundingClientRect().top-b.getBoundingClientRect().top);
  if(!cands.length) return '';
  const t=(cands[0].innerText||'').trim(); cands[0].click(); return t;
})()
""".replace("%BAD%", BAD)

OPEN_DETAIL = r"""
(()=>{
  const bad=/%BAD%/; const want=/^(详情|查看|编辑|查看详情|详情信息)$/;
  const els=[...document.querySelectorAll('span,a,button')].filter(x=>{
     const t=(x.innerText||'').trim(); const r=x.getBoundingClientRect();
     return want.test(t)&&!bad.test(t)&&x.childElementCount<=1&&r.width>0&&r.height>0&&r.top>150&&r.top<window.innerHeight;});
  els.sort((a,b)=>{const ra=a.getBoundingClientRect(),rb=b.getBoundingClientRect();return ra.top-rb.top||ra.left-rb.left;});
  if(!els.length) return '';
  const t=(els[0].innerText||'').trim(); els[0].click(); return t;
})()
""".replace("%BAD%", BAD)

OPEN_FILTER = r"""
(()=>{
  let target=null;
  document.querySelectorAll('use').forEach(u=>{const h=u.getAttribute('xlink:href')||u.getAttribute('href')||'';if(/#icon-filter|#icon-shaixuan|filter/i.test(h)&&!target){target=u;}});
  if(!target){target=[...document.querySelectorAll('button,span,div')].find(x=>x.childElementCount<=2&&/^筛选$/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);}
  if(!target) return '';
  let el=target; for(let i=0;i<4&&el;i++){if(el.tagName==='BUTTON'||el.getAttribute('role')==='button'||/btn|button/i.test(el.className||'')){break;}el=el.parentElement;}
  (el||target).click(); return 'filter';
})()
"""

OPEN_COLUMNS = r"""
(()=>{
  let target=null;
  document.querySelectorAll('use').forEach(u=>{const h=u.getAttribute('xlink:href')||u.getAttribute('href')||'';if(/#icon-custom-columns|#icon-column|#icon-setting-col|columns/i.test(h)&&!target){target=u;}});
  if(!target){target=[...document.querySelectorAll('button,span,div')].find(x=>x.childElementCount<=2&&/^(自定义列|列设置|表头设置|表头字段|字段设置)$/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);}
  if(!target) return '';
  let el=target; for(let i=0;i<4&&el;i++){if(el.tagName==='BUTTON'||el.getAttribute('role')==='button'||/btn|button/i.test(el.className||'')){break;}el=el.parentElement;}
  (el||target).click(); return 'columns';
})()
"""

CLICK_MORE = r"""
(()=>{
  const el=[...document.querySelectorAll('button,a,span,div')].find(x=>x.childElementCount<=2&&/^(更多操作|更多功能|···|\.\.\.|⋯)$/.test((x.innerText||'').replace(/\s/g,''))&&x.getBoundingClientRect().width>0&&x.getBoundingClientRect().top>60&&x.getBoundingClientRect().top<420);
  if(el){el.click();return 'clicked';}return 'none';
})()
"""

CLICK_MENU_ITEM = r"""
(()=>{
  const bad=/%BAD%/; const label=%LABEL%;
  const el=[...document.querySelectorAll('[class*=dropdown] li,[class*=Dropdown] li,[class*=popper] li,[class*=Popover] li,[role=menu] li,[class*=dropdown] a,[class*=menuItem],li,[role=menuitem]')].find(x=>{
     const t=(x.innerText||'').trim(); return t===label&&!bad.test(t)&&x.getBoundingClientRect().width>0;});
  if(el){el.click();return 'ok';}return 'miss';
})()
"""


def modal_visible():
    js = ("(()=>{const s=['.el-overlay','.el-dialog__wrapper','.el-drawer','.el-drawer__container',"
          "'[class*=dialog]','[class*=Dialog]','[class*=drawer]','[class*=Drawer]','[class*=popper]',"
          "'[class*=Popover]','[role=dialog]'];for(const q of s){for(const el of document.querySelectorAll(q)){"
          "const r=el.getBoundingClientRect();if(r.width>120&&r.height>120&&getComputedStyle(el).display!=='none')return true;}}return false;})()")
    try:
        return bool(ev(js, SESSION))
    except Exception:
        return False


# 统计当前可见的"浮层面板"(弹窗/抽屉/筛选抑扇/popover)数量, 用于点击前后对比判定是否弹出新面板
PANEL_COUNT_JS = r"""
(()=>{let n=0;const W=innerWidth,H=innerHeight;
  document.querySelectorAll('div,aside,section,ul').forEach(el=>{
    const s=getComputedStyle(el);const r=el.getBoundingClientRect();const z=parseInt(s.zIndex)||0;
    const floaty=(s.position==='fixed'||s.position==='absolute');
    if(floaty&&s.display!=='none'&&r.width>=200&&r.width<=W*0.99&&r.height>=160&&r.top<H-30&&r.bottom>60&&
       (z>=8||/dialog|drawer|popper|popover|modal|mask|panel|Dialog|Drawer/.test(el.className||''))){n++;}
  });return n;})()
"""


def panels():
    try:
        return int(ev(PANEL_COUNT_JS, SESSION) or 0)
    except Exception:
        return 0


def fresh_page(url):
    """导航到干净页面: 重载 + 去遮挡 + (若刷新)再等待。"""
    navigate(url); wait_ready()
    if deocclude_main():
        wait_ready(); deocclude_main()
    settle_content()
    time.sleep(0.3)


def cur_url():
    try:
        return ev("(()=>location.href)()", SESSION) or ""
    except Exception:
        return ""


def url_path(u):
    return (u or "").split("?")[0].split("#")[0]


def deep_capture(url, node_key, full_path, out_base, kind, open_js, label, conn, accept_route=True):
    """重载到干净页 -> 执行 open_js -> 判定出现(整页跳转表单 或 弹层) -> 截图 -> 记录。绝不做确认。"""
    rel_dir = os.path.dirname(out_base)
    base = os.path.splitext(os.path.basename(out_base))[0]
    fname = "%s__%s.jpg" % (base, kind)
    rel_path = os.path.join(rel_dir, fname)
    out_abs = os.path.join(BASE, rel_path)
    if os.path.exists(out_abs) and os.path.getsize(out_abs) > 3000:
        return None
    fresh_page(url)
    before = url_path(cur_url())
    b_panels = panels()
    clicked = ev(open_js, SESSION)
    if not clicked:
        return None
    time.sleep(1.3)
    after = url_path(cur_url())
    routed = after != before
    if routed and not accept_route:
        return None  # filter/columns 不应发生跳转: 判为误点, 不记录
    if not routed:
        opened = modal_visible() or panels() > b_panels
        if not opened:
            return None  # 点了没反馈, 放弃, 不记录
    if routed:
        wait_ready(); time.sleep(0.6); deocclude_main()
    ev(HIDE_JS, SESSION)  # 去掉可能盖在上面的浮窗
    try:
        if routed:
            capture(SESSION, out_abs, 16)      # 整页表单: 滚动拼接
        else:
            save_shot(out_abs)                 # 弹层: 单屏视口
    except Exception as e:
        log("      shot-fail %s %s: %s" % (kind, clicked, str(e)[:80]))
        return None
    conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
                 (node_key, full_path, url, kind, clicked if isinstance(clicked, str) else label, rel_path,
                  time.strftime("%Y-%m-%d %H:%M:%S")))
    conn.commit()
    return clicked


def process_page(r, conn, force):
    url = r["url"]; node_key = r["node_key"]; full_path = r["full_path"]
    rel_main = r["screenshot_path"]
    out_main = os.path.join(BASE, rel_main)

    # 断点续跑: 已有 main 记录且未 force 则跳过
    if not force:
        done = conn.execute("SELECT 1 FROM page_shots WHERE node_key=? AND kind='main' LIMIT 1",
                            (node_key,)).fetchone()
        if done:
            return "skip"

    # ---- 主截图: 去遮挡后整页拼接, 覆盖原图 ----
    fresh_page(url)
    try:
        capture(SESSION, out_main, 16)
    except Exception as e:
        log("    main-capture FAIL %s: %s" % (full_path, str(e)[:100]))
        return "fail"
    conn.execute("DELETE FROM page_shots WHERE node_key=? AND kind='main'", (node_key,))
    conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
                 (node_key, full_path, url, "main", "主界面", rel_main, time.strftime("%Y-%m-%d %H:%M:%S")))
    conn.commit()

    tally = ["main"]
    # ---- 4 类深层 ----
    for kind, js, acc in (("filter", OPEN_FILTER, False), ("columns", OPEN_COLUMNS, False),
                          ("add", OPEN_ADD, True), ("detail", OPEN_DETAIL, True)):
        try:
            c = deep_capture(url, node_key, full_path, rel_main, kind, js, None, conn, accept_route=acc)
            if c:
                tally.append("%s(%s)" % (kind, c if isinstance(c, str) else ""))
        except Exception as e:
            log("      deep %s err: %s" % (kind, str(e)[:80]))

    # ---- 更多菜单安全入口(最多 3 个) ----
    import re as _re
    extras = conn.execute("SELECT label FROM page_extras WHERE node_key=?", (node_key,)).fetchall()
    safe = []
    for e in extras:
        lab = (e["label"] or "").strip()
        if lab and _re.search(MENU_SAFE, lab) and not _re.search(BAD, lab) and lab not in safe:
            safe.append(lab)
    for lab in safe[:3]:
        item_js = CLICK_MENU_ITEM.replace("%BAD%", BAD).replace("%LABEL%", json.dumps(lab, ensure_ascii=False))
        try:
            c = deep_capture_menu(url, node_key, full_path, rel_main, lab, item_js, conn)
            if c:
                tally.append("menu(%s)" % lab)
        except Exception as e:
            log("      menu %s err: %s" % (lab, str(e)[:80]))

    log("    OK %s  [%s]" % (full_path, ", ".join(tally)))
    return "done"


def deep_capture_menu(url, node_key, full_path, out_base, label, item_js, conn):
    """更多菜单类: 重载 -> 打开更多 -> 点安全项 -> 截图。"""
    rel_dir = os.path.dirname(out_base)
    base = os.path.splitext(os.path.basename(out_base))[0]
    safe_label = label.replace("/", "_").replace(" ", "")[:16]
    rel_path = os.path.join(rel_dir, "%s__menu_%s.jpg" % (base, safe_label))
    out_abs = os.path.join(BASE, rel_path)
    if os.path.exists(out_abs) and os.path.getsize(out_abs) > 3000:
        return None
    fresh_page(url)
    b_panels = panels()
    ok = False
    for attempt in range(3):
        if ev(CLICK_MORE, SESSION) != "clicked":
            break
        time.sleep(1.0 + attempt * 0.6)
        if ev(item_js, SESSION) == "ok":
            ok = True; break
        ev("(()=>{document.body.click();return 1;})()", SESSION)
        time.sleep(0.4)
    if not ok:
        return None
    time.sleep(1.3)
    after = url_path(cur_url())
    routed = after != url_path(url)
    if not routed and not (modal_visible() or panels() > b_panels):
        return None
    if routed:
        wait_ready(); time.sleep(0.6); deocclude_main()
    ev(HIDE_JS, SESSION)
    try:
        if routed:
            capture(SESSION, out_abs, 16)
        else:
            save_shot(out_abs)
    except Exception:
        return None
    conn.execute("INSERT INTO page_shots(node_key,full_path,url,kind,label,path,ts) VALUES(?,?,?,?,?,?,?)",
                 (node_key, full_path, url, "menu", label, rel_path, time.strftime("%Y-%m-%d %H:%M:%S")))
    conn.commit()
    return label


def main():
    args = [a for a in sys.argv[1:] if a != "--force"]
    force = "--force" in sys.argv
    start = int(args[0]) if args else 0

    conn = dbc()
    conn.execute("PRAGMA journal_mode=DELETE")
    rows = [dict(r) for r in conn.execute(
        "SELECT node_key,name,full_path,url,screenshot_path FROM features "
        "WHERE level=2 AND url IS NOT NULL AND url<>'' ORDER BY sort_order,id")]
    rows = sorted(rows, key=lambda r: r["full_path"] or "")
    total = len(rows)
    log("=== rewalk start total=%d start=%d force=%s ===" % (total, start, force))
    done = skip = fail = 0
    for i, r in enumerate(rows):
        if i < start:
            continue
        log("[%d/%d] %s" % (i + 1, total, r["full_path"]))
        try:
            st = process_page(r, conn, force)
        except Exception as e:
            st = "fail"; log("    PAGE-FAIL %s: %s" % (r["full_path"], str(e)[:120]))
        if st == "done":
            done += 1
        elif st == "skip":
            skip += 1
        else:
            fail += 1
    conn.execute("PRAGMA journal_mode=DELETE"); conn.commit(); conn.close()
    log("=== DONE done=%d skip=%d fail=%d total=%d ===" % (done, skip, fail, total))


if __name__ == "__main__":
    main()