102 lines
15 KiB
Plaintext
102 lines
15 KiB
Plaintext
|
ó
|
|||
|
|
…KZj”* ã óÜ • S r SSKrSSKrSSKrSSKrSSKrSSKrSSKrSr Sr
|
|||
|
|
Sr Sr SS jr
|
|||
|
|
SS jrS rS
|
|||
|
|
rS rS rS
|
|||
|
|
rS rS S jrS rSrSrSrSrSrSr Sr S r S r \ S:X a \ " 5 gg) uY 全站列表页深度采集器 (任务1 + 任务3)。
|
|||
|
|
对æ¯�个有 URL çš„å�¶å�功能页:
|
|||
|
|
1) 点「更多/更多æ“�作ã€�ç‰å…¥å�£, 抓å�–下拉è�œå�•é‡Œçš„éš�è—�功能项(å�‘现之å‰�未扫æ��到的界é�¢);
|
|||
|
|
2) 点「自定义列/设置å—段/选择表头ã€�(#icon-custom-columns 或文å—)弹窗, 抓å�–全部å�¯é€‰å—段
|
|||
|
|
(å�«"系统å—段/自定义å—段"分组), 远超截图与导出Excelçš„å�¯è§�å—段;
|
|||
|
|
3) 兜底: 抓�默认渲染出的��列头。
|
|||
|
|
结果写 SQLite: page_extras(éš�è—�å…¥å�£) + field_specs(完整å—段å—å…¸)。
|
|||
|
|
å¤�用é�žå�‡åŒ€æ‹Ÿäººé—´éš”与登录æ€�检测。é Nz http://127.0.0.1:10086/commandÚ2haohrzdb/research.dbzdb/collect_fields.logc ó< • [ R " X[ S.5 R 5 n[ R
|
|||
|
|
R
|
|||
|
|
[ USS0S9n[ R " [ R
|
|||
|
|
R XBS9R 5 R 5 5 $ )N)ÚactionÚargsÚsessionz Content-Typezapplication/json)ÚdataÚheaders)Útimeout) ÚjsonÚdumpsÚSÚencodeÚurllibÚrequestÚRequestÚDÚloadsÚurlopenÚreadÚdecode)Úar ÚtÚbÚrs ÚA/Users/lucky/Documents/WorkSapce/TwoJob/scripts/collect_fields.pyÚcallr sr € Ü �
|
|||
|
|
Š
|
|||
|
|
˜a¼!Ñ<Ó=×DÑDÓF€AÜ�‰× Ñ œq q°>ÐCUÐ2VÐ ÐW€AÜ �:Š:”f—n‘n×,Ñ,¨QÐ,Ð:×?Ñ?ÓA×HÑHÓJÓ KÐKó c ó° • [ SSU 0U5 R S0 5 R S5 $ ! [ a nS[ U5 S S - s S nA$ S nAff = f)NÚevaluateÚcoder ÚvaluezEXC:éP )r ÚgetÚ ExceptionÚstr)r r Úes r Úevr' sW € ð$Ü�J ¨ °Ó2×6Ñ6°v¸rÓB×FÑFÀwÓOÐOøÜ ó $Øœ˜A›˜s ˜ Ñ#Õ#ûð$ús ‚-0 °
|
|||
|
|
AºAÁ
|
|||
|
|
AÁAc óÆ • [ R " S5 U - n[ USS9 [ [ S5 nUR US- 5 S S S 5 g ! , ( d f g = f)Nz %H:%M:%S T)Úflushr Ú
|
|||
|
|
)ÚtimeÚstrftimeÚprintÚopenÚLOGÚwrite)ÚmÚlineÚfs r Úlogr4 sD € Ü �=Š=˜Ó %¨Ñ )€DÜ ˆ$�dÒÜ
|
|||
|
|
Œc�3Œ ˜1Ø �‰��t‘ Ô ÷
|
|||
|
|
� Ž ús ´AÁ
|
|||
|
|
A c ód • [ R " [ 5 n [ R U l U $ )N)Úsqlite3ÚconnectÚDBÚRowÚ row_factory©Úcs r Údbr= % s € Ü�ŠœÓ€A¬W¯[©[˜Qœ]À¸(r c ó¾ • [ 5 n U R S5 U R S5 U R S5 U R 5 U R 5 g )Nz¦CREATE TABLE IF NOT EXISTS page_extras(
|
|||
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
|
|||
|
|
kind TEXT, label TEXT, ts TEXT)zÎCREATE TABLE IF NOT EXISTS field_specs(
|
|||
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, node_key TEXT, full_path TEXT, url TEXT,
|
|||
|
|
field TEXT, field_group TEXT, source TEXT, sort_order INTEGER, ts TEXT)zPRAGMA journal_mode=DELETE©r= ÚexecuteÚcommitÚcloser; s r ÚensurerC ) sO € Ü
|
|||
|
|
‹€A؇I�Ið +ô ,ð ‡I�Ið Sô Tð ‡I�IÐ*Ô+؇H�H„J�—‘• r c ó¤ • [ 5 nUR SU 45 UR SU 45 UR 5 UR 5 g )Nz(DELETE FROM page_extras WHERE node_key=?z(DELETE FROM field_specs WHERE node_key=?r? )Únkr< s r Ú
|
|||
|
|
clear_pagerF 5 s< € Ü
|
|||
|
|
‹€A؇I�IÐ8¸2¸%Ô@؇I�IÐ8¸2¸%Ô@؇H�H„J�—‘• r c
|
|||
|
|
óÄ • [ 5 n[ R " S5 nU H u pgUR SXX&Xu45 M UR 5 UR 5 g )Nú%Y-%m-%d %H:%M:%SzQINSERT INTO page_extras(node_key,full_path,url,kind,label,ts) VALUES(?,?,?,?,?,?))r= r+ r, r@ rA rB )rE ÚfpÚurlÚitemsr< ÚtsÚkindÚlabels r Ú save_extrasrO < sO € Ü
|
|||
|
|
‹€A”4—=’=Ð!4Ó5ˆbÛ ‰ ˆØ � ‰ Ðeؘ3 eÐ0ö 2ñ ð ‡H�H„J�—‘• r c óÞ • [ 5 n[ R " S5 n[ U5 H u nu p‰UR SXX)X„Xv45 M UR 5 UR
|
|||
|
|
5 g )NrH znINSERT INTO field_specs(node_key,full_path,url,field,field_group,source,sort_order,ts) VALUES(?,?,?,?,?,?,?,?))r= r+ r, Ú enumerater@ rA rB )
|
|||
|
|
rE rI rJ ÚfieldsÚsourcer< rL ÚiÚgrpÚnames
|
|||
|
|
r Ú save_fieldsrW D s] € Ü
|
|||
|
|
‹€A”4—=’=Ð!4Ó5ˆbÜ# FÖ+‰ˆ‰;ˆCØ � ‰ ð -Ø/1°sÀ#ÈqÐ.Uö Wñ ,ð ‡H�H„J�—‘• r c ó‚ • [ R " 5 nU S:X a5 US: a [ R " SS5 O[ R " SS5 nO4US: a [ R " SS5 O[ R " SS 5 n[ SU[ R " S
|
|||
|
|
S 5 - 5 n[ R
|
|||
|
|
" U5 U$ ) NÚpagegffffffæ?g @g @g 2@g è?g ð?g @g @r çš™™™™™Ù?)ÚrandomÚuniformÚmaxÚgaussr+ Úsleep)rM r Úds r Ú human_delayra L s… € Ü�
|
|||
|
|
Š
|
|||
|
|
‹€AØ ˆvƒ~Ø()¨C«ŒF�NŠN˜3 Ô $´V·^²^ÀCÈÓ5N‰à()¨D«ŒF�NŠN˜3 Ô $´f·n²nÀSÈ#Ó6NˆÜ ˆC�”V—\’\ ! SÓ)Ñ)Ó*€A܇J‚Jˆq„M˜!�8r c óP • [ S5 n [ U [ 5 =( a SU ; $ )Nz
|
|||
|
|
location.hrefz/login)r' Ú
|
|||
|
|
isinstancer% )Úhs r Ú
|
|||
|
|
logged_outre V s" € Ü
|
|||
|
|
ˆ?Ó€AÜ �aœÓ × / (¨a¡-Ð/r u (()=>{const kws=['更多','更多æ“�作','更多设置'];
|
|||
|
|
const b=[...document.querySelectorAll('button,a,span,div')].find(x=>x.childElementCount===0&&kws.includes((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);
|
|||
|
|
if(!b)return 'no-more';b.click();return 'clicked';})()u (()=>{const BAD=/�索|�\/页|进入员工端|进入官网|退出登录|个性化设置|按.*排�|��|��|帮助/;
|
|||
|
|
const boxes=[...document.querySelectorAll('[class*=dropdown],[class*=Dropdown],[class*=popper],[class*=popover],[class*=Popover],[role=menu]')]
|
|||
|
|
.filter(m=>{const r=m.getBoundingClientRect();return r.width>0&&r.height>0;});
|
|||
|
|
let out=[];
|
|||
|
|
for(const box of boxes){
|
|||
|
|
const its=[...box.querySelectorAll('li,[role=menuitem],[class*=item],[class*=option]')]
|
|||
|
|
.map(e=>(e.innerText||'').replace(/\s+/g,'').trim()).filter(t=>t&&t.length>1&&t.length<16&&!BAD.test(t));
|
|||
|
|
const uniq=[...new Set(its)];
|
|||
|
|
if(uniq.length>=1&&uniq.length<=15)out=out.concat(uniq); // 超过15视为ç›é€‰é�¢æ�¿æ±¡æŸ“, 丢弃
|
|||
|
|
}
|
|||
|
|
return JSON.stringify([...new Set(out)]);})()u„ (()=>{
|
|||
|
|
const u=[...document.querySelectorAll('use')].find(u=>{const h=(u.getAttribute('xlink:href')||u.getAttribute('href')||'');return h==='#icon-custom-columns'||h==='#icon-setting'&&false;});
|
|||
|
|
if(u){const b=u.closest('button,a,[role=button],span,div');if(b){b.click();return 'icon';}}
|
|||
|
|
const kw=/^(自定义列|设置å—段|å—段设置|选择表头|表头设置|设置花å��册å—段|列设置|自定义å—段)$/;
|
|||
|
|
const t=[...document.querySelectorAll('button,a,span,div,li')].find(x=>x.childElementCount===0&&kw.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);
|
|||
|
|
if(t){t.click();return 'txt';}
|
|||
|
|
return 'none';})()us (()=>{
|
|||
|
|
const pop=[...document.querySelectorAll('div,aside,section')].find(m=>{const t=m.innerText||'';const r=m.getBoundingClientRect();
|
|||
|
|
return r.width>180&&r.width<700&&r.height>200&&/选择表头|已选/.test(t)&&m.querySelectorAll('input[type=checkbox]').length>3;});
|
|||
|
|
if(!pop)return 'no-modal';
|
|||
|
|
const GRPS=['系统å—段','自定义å—段','基础å—段','常用å—段','其他å—段','个人信æ�¯','工作信æ�¯','å�ˆå�Œä¿¡æ�¯'];
|
|||
|
|
const NOISE=new Set(['选择表头','没有�索到相关内容','全部','清除','�消','确定','��默认','�置','�索','已选']);
|
|||
|
|
let seq=[],cur='';
|
|||
|
|
const walk=el=>{for(const c of el.childNodes){
|
|||
|
|
if(c.nodeType===3){const t=(c.textContent||'').trim();if(t&&t.length<15){if(GRPS.includes(t))cur=t;else if(!NOISE.has(t)&&!/^已选/.test(t))seq.push([cur,t]);}}
|
|||
|
|
else if(c.nodeType===1){if(c.tagName==='LABEL'){const t=(c.innerText||'').trim();if(t&&t.length<15&&!NOISE.has(t))seq.push([cur,t]);}else walk(c);}}};
|
|||
|
|
walk(pop);
|
|||
|
|
const seen=new Set(),out=[];for(const [g,n] of seq){if(seen.has(n))continue;seen.add(n);out.push([g,n]);}
|
|||
|
|
return JSON.stringify(out);})()uç (()=>{const b=[...document.querySelectorAll('a,button,span')].find(x=>x.childElementCount===0&&/ç«‹å�³åˆ·æ–°/.test((x.innerText||'').trim())&&x.getBoundingClientRect().width>0);if(b){b.click();return 'refreshed';}return 'none';})()uþ (()=>{const b=[...document.querySelectorAll('button,a')].find(x=>{const t=(x.innerText||'').trim();return (t==='å�–消'||t==='å…³é—')&&x.getBoundingClientRect().width>0;});
|
|||
|
|
if(b){b.click();return 'closed';}document.body.click();return 'bodyclick';})()uq (()=>{const BAD=/^(�作|��|全选|选择|处�)$|^\d+日|^\d+$/;
|
|||
|
|
const heads=[...document.querySelectorAll('[role=columnheader],th,[class*=headerCell],[class*=header_] [class*=cell_],[class*=thead] [class*=cell]')]
|
|||
|
|
.map(e=>(e.innerText||'').replace(/\s+/g,'').trim()).filter(t=>t&&t.length<20&&!BAD.test(t));
|
|||
|
|
return JSON.stringify([...new Set(heads)]);})()c óÒ • U S U S U S pTn[ SXU4- 5 [ SUSS.5 [ R " S5 [ S5 H* n[ 5 ( d O[ R " S 5 M, [ 5 ( a [ S
|
|||
|
|
5 g [
|
|||
|
|
[ 5 S :X a [ R " S
|
|||
|
|
5 [ U5 / n[ S
|
|||
|
|
5 HÇ n[
|
|||
|
|
[ 5 S:w a O³[ R " SUS- - 5 [ R " [
|
|||
|
|
[ 5 5 n U ( aC U H n
|
|||
|
|
UR SU
|
|||
|
|
45 M [
|
|||
|
|
[ 5 [ R " S5 O)[
|
|||
|
|
[ 5 [ R " S5 MÉ U( a> [! XSXG5 [ S[# U5 SR% S US S 5 5 4- 5 / n Sn [
|
|||
|
|
[& 5 S:w aŽ [ R " S5 [
|
|||
|
|
[( 5 n
|
|||
|
|
[+ U
|
|||
|
|
[, 5 ( a- U
|
|||
|
|
R/ S5 ( a [ R " U
|
|||
|
|
5 n S n [
|
|||
|
|
[ 5 [ R " S5 U ( d: [ R " [
|
|||
|
|
[0 5 5 nU Vs/ s H nSU4PM n nS n U ( aA [3 XSXKU 5 [ S [# U 5 U SR% S U S S 5 5 4- 5 O [ S!5 [# U 5 $ ! [ a / n GN f = f! [ a / n Nçf = f! [ a / n N§f = fs snf )"NÚ full_pathrJ Únode_keyz
|
|||
|
|
[%d/%d] %sÚnavigateF)rJ ÚnewTabé é u !! 登录æ€�失效, ä¸æ¢éÿÿÿÿÚ refreshedé Úclickedg333333ó?gš™™™™™é?Ú more_menug à?rZ u! 更多è�œå�•é¡¹: %d 个 -> %su ã€�c 3 ó* # • U H oS v • M g7f©é N© )Ú.0Úxs r Ú <genexpr>Úprocess.<locals>.<genexpr>· s é € ÐJdÒYcÐTUÈQÎ4ÒYcùó ‚é Ú Únoneg ø?Ú[Úcustom_columnsÚ visible_colsu$ å—段: %d 个 (æ�¥æº�=%s) -> %sc 3 ó* # • U H oS v • M g7frs ru )rv r3 s r rx ry Ï s é € ÐUpÒdoÐ_`ÐXYÖVZÒdoùrz é
|
|||
|
|
u' æ— å—段入å�£(å�¯èƒ½é�žè¡¨æ ¼é¡µ))r4 r r+ r_ Úrangere r' ÚREFRESHrF Ú
|
|||
|
|
CLICK_MOREr r Ú
|
|||
|
|
MENU_ITEMSr$ ÚappendÚ CLOSE_MODALrO ÚlenÚjoinÚ CLICK_COLSETÚ GRAB_FIELDSrc r% Ú
|
|||
|
|
startswithÚ VISIBLE_COLSrW )ÚpÚidxÚtotalrI rJ rE Ú_ÚextrasÚattemptrK ÚitrR rS ÚrawÚcolsr< s r Úprocessr˜ • s° € Ø�K‘. ! E¡(¨A¨j©MˆR€B܈ ˜ BÐ'Ñ'Ô(܈˜S¨EÑ2Ô3܇J‚Jˆq„MÜ
|