From c289431cbcc1633689b6ac73d72c0071cceef366 Mon Sep 17 00:00:00 2001 From: Vivek Date: Wed, 13 May 2026 13:43:18 +0530 Subject: [PATCH 1/3] feat: move LinkedIn selectors to YAML config with --verify flag --- .../__pycache__/emit.cpython-313.pyc | Bin 0 -> 4158 bytes .../__pycache__/selectors.cpython-313.pyc | Bin 0 -> 2502 bytes .../config/linkedin_selectors.yaml | 18 ++++++ scraper-service/main.py | 53 ++++++++++++++++++ .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 1519 bytes .../scrapers/__pycache__/base.cpython-313.pyc | Bin 0 -> 1515 bytes .../__pycache__/internshala.cpython-313.pyc | Bin 0 -> 8995 bytes .../__pycache__/linkedin.cpython-313.pyc | Bin 0 -> 9667 bytes .../__pycache__/naukri.cpython-313.pyc | Bin 0 -> 8149 bytes scraper-service/scrapers/linkedin.py | 18 +++--- scraper-service/selectors.py | 39 +++++++++++++ 11 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 scraper-service/__pycache__/emit.cpython-313.pyc create mode 100644 scraper-service/__pycache__/selectors.cpython-313.pyc create mode 100644 scraper-service/config/linkedin_selectors.yaml create mode 100644 scraper-service/scrapers/__pycache__/__init__.cpython-313.pyc create mode 100644 scraper-service/scrapers/__pycache__/base.cpython-313.pyc create mode 100644 scraper-service/scrapers/__pycache__/internshala.cpython-313.pyc create mode 100644 scraper-service/scrapers/__pycache__/linkedin.cpython-313.pyc create mode 100644 scraper-service/scrapers/__pycache__/naukri.cpython-313.pyc create mode 100644 scraper-service/selectors.py diff --git a/scraper-service/__pycache__/emit.cpython-313.pyc b/scraper-service/__pycache__/emit.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..292ff9978baf7ea7b9022faaed48df6536f8716b GIT binary patch literal 4158 zcmb^!TWl29_0G=h?(Ey^7q-Fpg0bB-wsBf)zyy>{SqjEpoE=+=Sku*bc5Dyq&MbFk z4fY3e$VYLbijWXN`cp?$8%O=&qm@#%QbAR!eA;o_rGuqf4ONBuV*-(i+K-+)JL3l> zrK&QkJ#*%qGw0lU?s?7QP{@zq`gI3)Zq0|#=k&q3*fPRAz75EGs09g#Ab}BFrNgHyRE1Fy&_w z5!|Fg@Q|p$UZ@njqg4Xm1}q^Y_!`hTj#LX_!Ea$TLPQ8ySglYY1TAda1RIMMm*54z zoL7vl!ZiNN>GMQ2& zxy$cQfT)hoX=9!8l&l&!t4Q#ysfK2~sf&3Tw*g!4BjUw2td|W)nyMG`V0LVpiuwDs zF&v+mAo7H0fK6L{2QKQ0I$kt|SZ*vxhD`>PN+FtRvJgBNP1SSp3 zQ9Hg(ALuV1BB=!(^;FpafQM)nJ@AI!xV#dHUYnVlxpsN(@~zy$spZJQtK3SY^2RrB z_FeD0dGz|x`HtJ%a#hcAq!$>``kOo7**QOa$5@IET=lF3qO*ffbs7>^;vMlWW=o{7 zK(-W^qps}9{c$u6q-rkPciSYNp}z;Pe2#Y1pIiEI)Bq1n9X~D z*j(T72uisvOxEOe*T|Hty+42On{zs~sHLZuk%fdATK~@rx`* zoK-qI=&^Hv+UL=CnB)&2qg$+uu9$h?A1aRJ6nzqlnD)Y){7kRKcB8ebrrp1xEf0!n zmbTJqJg&&OtPVX?G;qGC8+c5{1t`n3St}}Thi=&3=f`kKTD~Bv(;b*-Ioa=z=WYo}>13y@(oT@IxgsIr zw2m)M$|@Ge=qkGWNz-fFHj&9u>!wH7ibRr4wn%cOTQ@}Myy-0vEnB3$oc1Bw%}rkc znvtB%h(?Sjjc_sq3MwL)f?X8QB1f7j*h4`JfMO>A;k;5P&=G>R3TqC^E4npo*eikM zTYl)nE&6cGWqR`BrJSt7h*5xLTvGC)A7oh{}Iy6MSj6B9ClM|rF`VRDzmY}O12!>L4kFq4cACd`oiFc^P*AYn~1a7?D# znnz5xHEHQIY~a+&)t#sN$bb;p59Ap9^}~?6w6s5|MpZlB-CGJa&L*DlsJ8xA)sMEl zziqyL;nn5(eaki7rJCNQn%?D_gDW+4Hm~--@ai1ihIq{ess84fch1bSA9_FVE_j!5 zcM132es#I#z=m-B!q{^COQre)w^LheHhcwr`xg46sISp8n^>zw!RXudrNH+2o`u>E zkF4xmekpY$=u zG55*gX25@k8Gu{5Ag$tW(&_MzfRwwSz^Kd!szN7rAf&o>qO@I6X|-~yE7TCz=Dvv0 z2z*3zLQhqh!JP;N5T)#)oj$T_(}d|o;!7aIra3TYUC--{0ypAHtxuJXB&2}bu|L9Y z)SxCI)gamK1io|z*w>Y*M%Su(ITP79KL7!%nj5|tJj8LP$o z7}O2DC`q!ej~8>fY5SWZ+eM&3q2xpAhMD@~qLPEsVAT%L37Q~@Xf|2d8*^LHZ2A(H zB-tuFqyt_vee(J<&9N8<=>=;r-I7*R4FWzvCJQn;^$Vnkog+RcM|AQsFiyf>KM#N= z$!ah(o46mTy2`G(INxbzrLy7Hx8Ch61)COwP4^(wUYqOxe&UfARaCD9A@pVk9{ErR z|KmZV?lEHM3&F}#aK}<`$718bkDGtf_RF?AEx+pew7C>aeilsrbB(1!|I&NG-W5I` zW8Mnyb-mBTJU{g9b^X`_gpZiLuAlkit?pm6#&@~zZl|=nyWBvy+sXn?Yf#z8Ght^~ zE`O=O)OQ5bnp{#-ogWOa%9 zv??i@>9rA3MVWLwA1IeXK9f=5QxOc%6DIp8=%&D$XCs-+rY}1XqSHK~L#!1b)^QR# z-4pr=ZCAvX^Uy7F@=)^6)&#Y%PTsIrIS5Z3~T5WEd^(^vu8MQs}c$ms3l^j$1B;3dxXTE%e K@i9@$`o96*u8`vZ literal 0 HcmV?d00001 diff --git a/scraper-service/__pycache__/selectors.cpython-313.pyc b/scraper-service/__pycache__/selectors.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..673d4284a235fceb7750350826e7730c56761b05 GIT binary patch literal 2502 zcmb_dU2GIp6u$Gb|NVh3v~+9RODUlXWm|*-5@@0|L20|u9hRuwFq!VoE<>lY<=)w1 z`^1x3A}S^}7B!)d^-1EBi9Q&geC%4Z^@<6JzVKG8rio9UJF_#DhNutT&7S*ne(t&F zeCNz&V`CJ-m`KT2`vVC5!Z+TK>x<1CEEbT02vP(^92JNFvotDRkcfocGnvRaIpf&! z)ZMuU<$^0x5wZv3?dViOJx6CaUPBp0~>cn`Nz_6I_Jlm|;!ng_kW% z^F*7`Nh&%9Oo~So1=G4b#YIw{#;-p=K8EvUt7w#P(J=K?^bx-xofMe-ie_KQyVtqY zf7u1tFQ6QP7a<}l0+AGv$VyC+bAc105=fw{@>PKZm0&KQgy2k8!u*VcU>637h_j0* zQ4%dl$p+TsJ%c-{lSd73!TbQ3OPAy>~S*nOvUHEham5B2+8tDlZknon%r;w+Rn3X#|p#m<+ay zgr}AXManH=jM>o0nOv6Ygyv?AS$!&}5iNhkDi`!zy~2(HbJobi-Yc)v%pCW{unZe~ zX;bRwF1}M9gJ+X|0OST*kG0lfT@MmnpS<~TYArEPOAOrWt0m5?oV!#@T&j-#+PM4n zkwtpH@xanGkh+`tD7CzMHF2c+;;qpw2{rX@1Q8r}-08U6eW!alaWAvletNC_Y_0w5 z7Zcz1thSF=L5SYlSppN3NO3oqjr*tU`)WQ8Uu!Q+yr=M;($ z>21lU-v+wUaZ!=IGV4WIu@@x+^R4G~8*6@d#FKLv9-Erq?yraD4<5o+*@g~VftF-g z^|slME7Y){d)AT2WK}b521Z?A#la^QG|Q!uPFU!=Ml6u7!;S0#CTZsqVIMhl(2-Wo zbF^D_pa($9b?8MPH_*d)+lRyN53j`!*5U`3@~iQ~)$n>OUW;`+XlQvaeKWoI#3ADLG3XLm0n#JR4_?xVQJljN zV?Rd}_qkA{xM1XMeBH3GU}|f2g(g!;kzg>x0<)TF6q1sY3l=bJeVQ_PvRpQkvJ-t4 zB08;@Hg(eAC{YeUD0}RrfCXoryV>3yS#Xc{gDmAQghaILsfN}&_usvI=ki+Tv0CS` zA3BfU9IuY7_YHn^;_L92!?lhJ)$?zUJ(f_{iHFUt)h5ydn&2I8Mh+$%Ka)Z_8ROl< z(x!PQi>m5$xlnPsT7#-yt7vAO)6Baap*HkJ-7147j--}gcIHx>9auILO;sSJ(k0Pw6HYt2jcU_o?u|# zZMuPATBN`8X=7I)(6%UVA(&iq+(n@wZV?bab)7f=r@0+=3iYJ?o- zvR#}wZt&Z;U1Q<1ZYYMyNnmh{(g7eFk{}3=MM;o1qey7@8SVQCH9d^%T8kvUizF6L TuSR-r1h?db(DoYw;u!o3pPC|4 literal 0 HcmV?d00001 diff --git a/scraper-service/config/linkedin_selectors.yaml b/scraper-service/config/linkedin_selectors.yaml new file mode 100644 index 0000000..8974e86 --- /dev/null +++ b/scraper-service/config/linkedin_selectors.yaml @@ -0,0 +1,18 @@ +# LinkedIn Scraper CSS Selectors +# Update this file when LinkedIn changes its frontend +# Last verified: March 2026 + +linkedin: + selectors: + job_card: "li > div.base-search-card" + title: "h3.base-search-card__title" + company: "h4.base-search-card__subtitle" + location: "span.job-search-card__location" + job_url: "a.base-card__full-link" + posted_date: "time" + + fallbacks: + job_card_alt: "div.base-search-card" + title_alt: ".job-card-list__title" + company_alt: ".job-card-container__company-name" + \ No newline at end of file diff --git a/scraper-service/main.py b/scraper-service/main.py index 14be217..3f2b1b1 100644 --- a/scraper-service/main.py +++ b/scraper-service/main.py @@ -11,6 +11,7 @@ import asyncio import logging +import argparse from contextlib import asynccontextmanager from fastapi import FastAPI, BackgroundTasks @@ -122,3 +123,55 @@ async def trigger_scrape(body: ScrapeRequest, background_tasks: BackgroundTasks) """ background_tasks.add_task(_run_all_scrapers, body.role, body.stack) return ScrapeResponse(triggered=True, platforms=PLATFORMS) + + +# --------------------------------------------------------------------------- +# CLI entry point for --verify flag +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="LinkedIn Scraper Selector Verification") + parser.add_argument( + '--verify', + action='store_true', + help='Verify LinkedIn selectors from config file' + ) + args = parser.parse_args() + + if args.verify: + print("\n" + "="*50) + print("LinkedIn Selector Verification") + print("="*50 + "\n") + + try: + from selectors import SelectorLoader + loader = SelectorLoader() + results = loader.verify() + + all_valid = True + for name, info in results.items(): + status = "✅" if info["valid"] else "❌" + print(f"{status} {name}: {info['selector']}") + if not info["valid"]: + all_valid = False + + print("\n" + "="*50) + if all_valid: + print("✅ All selectors are valid!") + else: + print("❌ Some selectors are invalid or missing.") + print(" Update config/linkedin_selectors.yaml with correct selectors.") + print("="*50) + + except FileNotFoundError: + print("❌ Config file not found: config/linkedin_selectors.yaml") + print(" Please create the YAML config file first.") + except Exception as e: + print(f"❌ Error during verification: {e}") + + exit(0) + + # Normal FastAPI startup + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) + \ No newline at end of file diff --git a/scraper-service/scrapers/__pycache__/__init__.cpython-313.pyc b/scraper-service/scrapers/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e47feb5f717830a65eab0d26419be2512ecbb429 GIT binary patch literal 1519 zcmZWpPj4GV6rcTTXXBxQdOdD0Mqckmt0;&p92t~q?+)67>I2w)DV|$a`8S`e= zl_L%j9N^dk6`z6cz_;MG5)y&~$}N@Yfg5jjZHVYdJNw?7H*em1zu(N_%9V)V_<7eq z{?sSr538(3Pz%oDGZbEsm;i~L*gbT*F1Xan-l5m^!S4nTRJwl{b{o*3xz3H6(e)a?1A7}$SG1MAql5wr+NCD+pMyDA! zDCcywb1yo#VLMK_&ZSg_n^Z~NaZ7LilY?@DFJtAxOwO|^!nU%A6g!&88 zC*WX(ZtQ?}o5U{ou?Ioy_ro}7qes6Huf^dOIrbkcqd+rmz}mRkiOQ=_m+8fIUFYlQ z!+&(J(gP*yqQp$V{Ixo0UT8zpe42?|NV69$TXUyFL6`!hrkrLe`lt~driszJRPba% zb8eDJD#z4J1nsK9Q>6{65gsEAc4;DVP7`cNk&7W6%qu@W9)lQjgH6=}W@#cGMiIr4 zsC>%h+=_@nWdfB6tK(Riv??H}jVn_DS?zPl%kbo+Hrm9$MAsu!@8F3kM+>H76z#@O zdaJZVo*E;dQ=K~nH7t+>IB+NZn)Z|w)k>z43gy>VrdYffGLF0c+k;1b9zVJJ;rsXB zvsT+HtlLII#2SS>k?NT&JzOf+>&lqqxk#48s8Dus{+1AD(L!OHP$bIO`M?`F9ryJ6 zs79L}t3gfF9ow@VswCafwh23w%i$8oOE3a|X7|)^?nRF+oEF-e3llyQ6f4P7ndU`K z+e>WPyR>cN|aFK^S`B^y?{?lXqR8^-W%mSOExUjHA<_iFpQc#5{K$T>M97cH{Z`di~u zLcVc*2R2aXbwYRp|6p$gyouB4+P};w!H_q(Vc3v7t&B1B;tozVt?*)xK=7X_ww+40~nL1j&OO z85Blnbg*eJ@FVb+otmr3R-kX=^c{KWZUoJXTja*gv%}|`zxedGU~}OHL2KcW@LDCo zJF9)Y&Khi?Vrv)`c&{;Qu<(=~A0%MgRV7u1WxQdOdD0Mqtq>j@~5hhLMRfBk|P;RMA58QaOyM~C4w6pKMdGqGI_xsJPu3QNSj-U73 z<4;{e{xHgVc$HwSK11OJ2?>zU3hhIyZ-Y&Z>>N6M7u>!FUa7l>e!l??N^X!_B=ojP z=nvfoP8A25(?++MKSnK3F|auYb#KHJ@6DI=_p=}9{zz$Hv8KjIPY#MrR>#V?p2;zKIs0N6SBX?}ot3qEg!&6I zBw%5Mc4&cfn}jyFp#xs%4*k&UpvSNgwnBfK9J>$dD9{WW(3&>8L2>nIonBbgb-sZ< z>_-bLJ&>X*iZ3`UU#o*=xzaSr<|)s((7m8;&7CdaHDEdAJY3Vg~mHdS$0Br$&&1QbUs zvpEw>BO(T+31rHRj$>_7GKaV_u1z_lmCq$F&Ek{FXcGStU5`+`gC`~oEtrm$v>!eh ztkdFIqBVzZdF~wKI7brTz@7AKI*@{wE2)f2lwV(&WAS_(G2Hdv9z62%@X6f|-@pH! zv6^0C-6k3$*2u+)kk3Tn;8L+sRTc@B@wg^NiL#CJw}e=$HVQk0B2lK+2hPOm+Na+K z725Pzjw+(=nx5@aDd?^;P1vPOjB6Y(!3gY`*;B#U3mzLd4YU;(W^BPJR+1$m$?}YL zYHT`tv}5Aq-$dgcKZf@z`@498GB3zEIU^Ts(rW*$@EIZB*scYe zC=9wjyn%l(Hv-9qAPD+<5j`02IMWO`uOvPbc*9vvY{&8UPVii#kLGUP?dQEo@k zS2;@!Sg2i%qDcZur-|S~qM;E*V;M&g7|hJ_rS}1?=2e3&v-EZpVV`soBXN))qg-=^ z4kqmtegxh!Q*%9;3e;_!z9TQ~O|N-zi`=+*cKCej7nlC#ZLMt2Yp)#QUn?beXT5LG zZNWAwriMw5cN((>15M$uQ4FSCWm1K&mvdw(`0JfzDpeqpO3{2&_26T;i7LJ>L>b}2 hu`KJQZCmcECb4$@Bs;&7>wo#%R{MGUir}vd@ejA`tJVMj literal 0 HcmV?d00001 diff --git a/scraper-service/scrapers/__pycache__/internshala.cpython-313.pyc b/scraper-service/scrapers/__pycache__/internshala.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7df7b2a95c780a11d47cc868055568c55dced474 GIT binary patch literal 8995 zcmd^EYj6}-cD_CDm!y$2l12kEEhK@)NTU~EA)ALRgoGr7JgxDP5@gbv>5&>nJ>%Ow zfF_k#wOh3%CzTj>*9A_c%22y0NV1is_LozYst_A&DwQ9SMM|@6)#&tt+1o~K>}{uZ_I6MQdpoHU-nL=akej-TeLU0yW%gn3kdOMH z%po|3ONL5mDNjz@qA!@kufBJZ)Bb&&@lDGG7p)N7v{LX0Uco1nM65I*R0yRZa@Izx zgi4`|<*I{ZClUN1B9xyl-DfTKu9>oi1G!I~gm}~h6|)jm+hj$PsG?4bX)%(W10^yuZ6H}~J15Ov$k3$P`o}brmRi+xZt73kJRU(xO9?GQc_pJsD&L$I zRgFI{Q8^_`N&bXL6VrTWd*}XekLUBB%>ADH2Wt+&Y9vIk7aG<~N%?$4R^TZob7DGf zE~uP^2F;QPrpjhi4VEqp62@gMEtx&d_jFeTR-<*bXR9L4Kr4|gpxJ8e`(%{oaojA; zq+}F0-Njn>WoELXGP|}Z)a{MHf@AsrTW+y^d{vK;OhVM;jIvI&F?C}D%h`BFo0e!i zDQTjdR^y3`0vSc3Th4;#RnTfeLz}#KaA%*a&F(z&UDFmL>7pv}w5*&1;lepNonE&j zpd_kBGdU$0*Tl&+${1s=soGMzqKz9=CO)phiD2_bog=Ie$JXSmq$K$^UVsx!4a?)$ zu;N&;YK7v!#d7G1n+Z?ry0qn1n3$$g3XJpl4}ZjGsl*1I6jcdDfUq&xVsJE*qPAvt zu^qxH;*+y$H8B(vwVVo@PK0@#hh8|Z?=nUNx3ZLsQz;=ST6*^P>G*rb#3`^AZUXeI zYrS+@RyCFH==^-{#~u67XRACockshgQiSKfC-Ess13!?<@>+)PWaTP|XvXs*@v;hj zmX|JO({e)ASZ%A4o0$X&hBv3;%(ntVP6fxz7OZ!u_f#uxg(NR4DOr(W3qZ)oq|czzu3J$zJG6+@6Be@()Xmv zVOeY2*Y#4QYd_ySJUD)0q=iq*=OlhWN}S7t`MznInUUH$I=dq6_z#>|N~ChJOTJW6 zT91#mNXpA^9Bhvq$OpGjsx55Idt$J=T6?F!K>4y>*hE>abwpIAa#MiRD; z<~q_AW_|}YE;g`(%29gsi-)wAL;8kjq_Ab zpNCC(?zBiDyepZcbh?PqTA}j1oLKo*Pn$vAz;}8{9L2@8OdJmntpKt8@TcyA#~gX! zEqQzYpY6ZYdad$W^2XWgXMdVr@^&qI4=j2Qyx*|oJ+xpu^b}f>#99;ipnw^&8J+-@ zImIGa6surWY=TX(3wFgJI25PgR9tODaf1=pE(3~#6k9P+B^lrbUm`^Dyg*`}Xc4>` zt&C^PY8q4V#=OnNswC)iVh2d_Znj8*OK>0H6d%-XvOF=D;MrH4lTs2ZiMrR0XXFL1 zIjT9L2t8sYCtVYaMqsO0Nh>^+(k=A1nG%$;SlNn{GU!*fwazCw!K3)MkZ50#SRN~1 zC$aoHBvydL&DOzTN~|QWSE`maqFO4+AZ!$+GFG{@%rVP}03lw&6D0sGH(xiWDO;(E zRjt#uYHP_bf1gqv^PBCHupM=Brxf@}&Jp3t@DN;`+N=>;=BJKnW)J*uzl4zxwTYU>k@$%%`{%IcVKMQe4=d`x?ye>Awg8BB3#&RmABkp8~fCQA7e)ujbO==`nr* z#}fKS`uoNK7@p`IV-Y!*Wrz#HDIhWQ7|N8&UznDZwK0KDKnTJ3DuX0+!Wi<9MgXQv zW8C6<_)HE^^bCM23^WF8X3-m!06bw*2$tLdMzjfG&V9 z85uHa+B6BhBmfbJ;$jl8L~&HaKqgLRrs&~S14Xx)yR7 z81-@b#?*^una-rq0<3~c5MvSPB4W!^6dr{*vM6k76yfB zVS}24Mzc2%K!jCUN)i|&GZ zkhmsY)O0K4ph^>|rs59QU9a{E{c+d;xU9T$$`JaK(qwKb9Mo-~ zcGj4I%RH~Td*uM?UUQXUaNSuvE-;=+7}L6U&Gyz^=GN71ND*`!%G6zkJdj{#Io(Oc z3vo1*6I!v1#~epW(^_Uv91A}1x=Ob|SBEhs&ed2-sI0?ED*$0N2sw6KHhIu;Z03uj zZKXJBq|!rBbR7QFAAyIQBacI*GBP(>g4%WqzrU8?DrAN;Ix+pqk=ht)4ESMOe| z-hDHh}48LVfdceb-`r*R5mA^)D~hzkGS>PVnBq zVy`ei@UXV-y&doFxVdeqcHjKaXRjQ)8d|PxS*&fjwRZvE-S=LFM#XV2EY-BlANy?o zE0@lr9yiA?y}4M^d3)#l;KS}${@MGF-rGa>yH74vjeXL6^488vlUL4NK6lf0>D*Fv z>tgrG`C|)JW8XdA(fNVjREBhzna}_B@LR(V>zfPpyKdCoY%J7AU@C?B-Rv1I)HgjT zcb8W@w|YxTpLCG&U602&;`7f9|MyoNr1~}P>u1fxU;C6;+*J>J6>pEcHL_59_;%%e zU;k&m!0#U0q5KF-sLd=}eQyR5kpY-JPSBWk)Oqv}pcIWNT119sgVUTc0KaiKTycmF zvKEi)p>NGPV%Gg72vPAo0*bHO`LoU&J=c3~9J+pJq4W5?j-`eZOLe33w!iRFge?hF z#!QHGe}FV74-;tstOQhO#Yi#EoTy?2>~8zNjkc3aAgN*rjU`wQSUGYWR&^RiJkPyj z9ku(2VxL$+*Me;oI}0|%&ZdDG3xL&B95IJr>p4Y;2v~0dxduz6M6k!K@N{(96z9p( zO=_GmM=9yDE6!CM7e#NgAqlQiQzmue32+YkX4o#af?bcpj{)oE>UH>8P(SrBbEbE|Kl&n1G*oDbtjzKKq=KDyV; zt-fI|0S7cv3=q57*W8aASApU9Z3d1PV196&QyWvvv8oO5 z43ORH=E;da1)jn3DQ=BM=oL1kBA;2SQT>J{@F{ps$(xSz%hhpAKI0;zAHv_QdeTCe z#lf2xC*DPTWQC1^XGvL*8}aaO^sd2$Ua}$|8XbQGoDO`b?nL4Mw;Z}Z1-FQq3vk_= zl;O%(1E;CUGZK&$ijF21gl$4fQ`qzaftcL7t)W)xLI$cb&?x<4uDzM^@I)^=tjidSg54FNJkdb%|& z0r-Vaz{7M9D;=OoBSjfE~u|njZXi_%&)S zLqBx^_VyzTzh&F6jxUw%oVOLid*)qVRtA?V8$YRRd=S`PsH-mos-HV;0nb+i9*AbdZt z=UFwW-37OM@^$%E-BR%ICEJ5w+j2!Lfy-O?xjHYlY^wT{ka!r{v4%e^dJzrGJ1Km5F9K7x4q|k z*S8#OUktY2YJ6t5R#pF(i&TYv_W*l@7i+?RXN?{smq!-rhwdG{A9($GQocBtxO74YhnBT!c-+T3r!EWch-Q3`dzI)9+%>B65OVp0FDeH6H(4!T>2dX7mU)#s?yK6%YscAt_Ysm>Yg(PvP|U$(@mw<>v$0;q`k?NN$CE%`*`!iAuoFy*Vr-{Lcp#6Sg5TMu=rJss z#0P>--J#850H6@p2#sQibyD41bY}ST7sY#J-3^p~1b_0PRKu3&R1Np~I_Adlfm36y z&zN!^KM|K3-p2KcVd_Kl`%nf>g{ZhnkF6ZXJ+U-$w%?9$glqVMlzxi;-Jg;Yc>bDH ze?fe|CUyTo>L0rO%kI!8?$FilC3oYTg<);b*SHod4NTOD7k+TJ~E4FQ(D|0zqP@ssI20 literal 0 HcmV?d00001 diff --git a/scraper-service/scrapers/__pycache__/linkedin.cpython-313.pyc b/scraper-service/scrapers/__pycache__/linkedin.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..264826c43ddedca71d8194737a826f084cfa7e92 GIT binary patch literal 9667 zcmd@)ZBQIXc0K$31uU>2KzuYGi&&%G;e*2E_MHI#P+=Ie(&|`*RNl{exrLOB@P77u`bj3R0Bf4rWO4$=K${B-*5=M zhLb5-Y!P`kC@OF%QPFImCps8v>{kcv=s?S!3 zgZav~7w;_4*X)7eDqz&^fl*SxsM`ahw1BaH4~((`M*SWb?gB;w@8Qeq(0LPStU)b^ zuc$+O<=L`+V{T%kY2N6oO8vJLA%_|x!jeeTU_z2Fh;c~{EUn_(mtMdVtUe~=rPORf zisAFhELKH<#1?R#2B%a>p2t%OVReN_^9#vrn8cCr#8C%^XCawfQjZ3MD=RC3!gOQG zVi4x026K}OK5;=@T~SC}9sCobB*gExCzP0wloWYzOpZ%J`&@K-Dm+LIb@d(T>T)>v zykuBblA?+`5`vn<%Oa8HBoJ?s>8q!!=aBEHn)4IgvNXonxvX237BN9Yv$M`2rbT z6W9zpsj!d}lqxD3ol7MWup07(nFS)wo!e8&4yv>PVF<5)`&gozn>N6n1obkVQ)Nym zD&3=TA-P?pNoi4>p=}g62g2O`;bnYkdU65}(3T+#JBsdZ@D-{Q&vA1&3hpSbz%~n= zna1M9B}EmnfYqch3yY>+kP->l#z`;&47)E8$BN9v%6l+sWFVWspP^6 zpdVzq9-G1o0-YMnixkuk+u%E*)KA3n0UYB^Lm2pR_u2q>5zQ3(B1jK^WRB+|Tq)uWy~Z}=GltfJzvIcj z_T=tUe_m9^;)C5?J%_t{A3xHyGYV)2VPPLbh|t|mtv-}m9PA=A_ytXxsBh2=7gkgQ zG+?X^jvPH30r#MuU6z)`3ugradr?;6;#n3bi3I#NS(ajjx_Y({tSqf+j_7O(&MvAY zQVkr3;ir0`xP)%mD=(kO*lRC6xn<#;Rhw1&E}z&!Ty0>>Xs`#jj3%??9w4F^E%b*< z1rK-cAe1knvxsmIryF<<;&KDTqKR*L~`DfUDgktDD5tJzK zo2Le$Rq-R?<5Q#I=!wyZlFsgJ*szj$wASBy?XS3d0eDl>h-tKh*ZY?@qxJ`$O?-mq6;r3!(hxX<- z`4mDUn7@CN_KEB`Mq+lh?MT&Uiy3GUKuGXaEE;wDGLwonkrQJ+Py4@&EFGo z@8IEnD37{t;eq6;NO^$=c>73&UW>!|aFUz62yLeD6A#K3sfd*G`y-w~u0R1tGe!|O zb$Ar1j8yUU0h>NHro3Ls@7>?g-#yCPJz_^UZQIc%U7iT1r^i5IQya3Olg7|rvNF!f zaaJB;r5#E`=wDg+Q&zs%jArU}omi1DN@u=Jj)(btk3Kf_`KqDm|Kj)IOwOIrHspd4 z9_smb;k2PSG-zx>a~zZOIxAmcE4$x`;0xITStA=flUBmJ;GK;_SGT4EFaP4)h+v9pf~|au70I7estg zj9pNCcw~Vni(;_5r#H|=|6yL36Nr>2Su=~We=H;+xXDOAfXxq*m{+8DazV2|#ugI3 zKBMNKnT{xkDJL}xy~n|2$F?kqD@zJVuGP=zQSUh%5|$-Mov>Vyauj=2%^g=3X?hT7 zX~qF=_+2SBii0LDqAADmP zij-WCAkCB$GH?ex9$E{mNOD|R3FPu6(@ICz!9vnHFssB@^F60>T3l=1HpWwotv8;_ zT;_9Os)jTZ&B$psJ#$9(!$>|GDTNoQghI2ZN{Ylp%}6iMW|nZ$tV=|Rr(#J$BVx^- zzf}v#9YeA!J*bMXd_J8WU7oe3ZBj*KNVh@a6jUqH1OP(_ZbO)clmeuZSQ>zm$El{M zk_cwWu>)T9fUlL^SP8vDXto$NJ}I>bW+SBJ*n;Mi#g!>x#e@Pcz37kO|bvnbp9EQT=m)W{-{>51)Yh-GpYJdM2O2(9DE> zX3%WnvXDqYf~IMvhO9oq-D@6(Kq?78=ik(_@o45{D#1c08ICVb-u}L|W2H z;GP*3k^s*_no6@kiWL%lntk+QOkARg2F)B7XH)ZDH!iE4|w@fLc5N zgkC*!Mo3MJz)c7&F=uC*5yVsJO@<}~AM)hx$d;VdXw|x}$Y&GiCV_8Lbd#%JgZsII z;D%gUbJde6ZG7Ii={@kg={J?F*-G!tO7FFb&HBby$6p@L)(6(>1DmxCo7HXU_9xO+ z!&_$55Zpq>hSD#QabM~4C%0^Bt10yydBTDRd>YW#DRi z?i-C&)gVWGx9XZ-_5Yp! zn&~Ib*PWTVzO1Kj!_)V<8P&DG$W5);#{Tuj{+~&i#^G$u@MUvhf=|3t`Bvn5-A(tz zZ|a*Nqhx5TYToP&W;=)0JBQv}%XFU3dQV@q6`Q`7>72-VC$2h*P2bOSPG-H6S1rY+ z)lBDD);o69`jDpIPDZZPWPJnczJZKyDBC)eZ5dr}8NJzZGJP(3)%@w?GuP^}zM*yB zP{uc$Z5__GoLp}?d9!7Vc3VH4D>GZ0kU_Wn{f&f7bw3d)EfWc(($14nk;FKF!@-<4}3~U1l10 z2T;`n_tmX^_#QHpH|Fgs-5S2>;Wzg;Tp8OsyF-BWZ8o(5HZ=r9U*C11sz2g1Pv7mD z;kyphKF(?03C%mHdB@=IcI%MVI;^!0Xf2~!%h*@t$Ekd+Pnx zn>C?NYw$N)W@!KBz72UA?jf#Zi~~!`y79Ujr<EpHz-QF!37r&|p_Z|6cm%g+xS1^9Y%CA44f<3c^Q>-`kx`r|HO`Hq9U%~YT1y=rc%+VMI_n2-B9su~`b`IJ<_OTK$0SjA*#u=ftxa0(nbq~?Hu5z z9VWm}yD3$=l9iyJK2sVbhq9iPO{{ETrI(c*tn6fEfR){>>~lPgt?8lar}taeJyzhk zzTXTj>zK}cz10yeHLv@t!*=tA(GvC;Htc3-*(fzqxSY;t!{Z4D%^MwDxYM!Wvrsr_ zr0qR)rW^ehzQnq5j0+FBHU=OkXwBGc&<;T%jSWH&30nvY9yL=)kwu>+7lIk#{AQ=` znw@@aoPp{&vIrDw9#)S_G1hnx!$+5M=X^GrAeIFD1eOHkMqjXnoouk$@+ z@Lg6SY7)*~QC6VanTzxoB{=}>$pt91s%R9VzN8c@#tv#U`Z{9yVs_RNNGu>$&`iBb z)0sa+n-%q!#_yXM4xM89Ou2YMp;UAlghD>7S))<-PZ&9&=TZ{iu@D(=vr<_>#}%59HQoi2NC}aqv^$gaRBH$9-;S;Y?qEqvu+GjY|KW z{@H(ps(*!A{|in0Cu;u8?#|llHtcm*k7Vp^mn@%ER;Q8s3k%}B|54eTscePXW_feA z+`C@xeeHavy!(>9Sf9_72QE2^btzNcb;(t%GyIZbo%L^{U(A&EU9#UdSvkvnPn6?4 Mmz6INeY4s9FK?FUcmMzZ literal 0 HcmV?d00001 diff --git a/scraper-service/scrapers/__pycache__/naukri.cpython-313.pyc b/scraper-service/scrapers/__pycache__/naukri.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d6512551ed8a21f83725b518bf701c4d5cb24dd GIT binary patch literal 8149 zcmdT}Yfv0lcD_CD7tFv6!+=0)UV@RBK|&8B^a?@zTG3l zY&oVjl`WjAlu>G3l2fUYRO`ef*~+Rmf3#Jpva_;m{RE>FlU`XxY-cO_Ct2Fa@#aU) z?Z*tTc=z{SpzrIPbMHO(-gCZld0tWBLXiHo)AIU%*CF&JsTj$s6{zP$D7=dpgb_nA zreW%w37cqAHV>PzSl&R>@?hlBYDEYu|>F`MJC)Fra{Y0O7+bMzPU zKcYvqS~5FB>lAce;>8R-CCj<1^j`WD$L3`|ofnvFJ{NVl##x-wP(b6D(k z(U1te)Z5iHJ3AZIhXA2n-^W>ji%3E~^Je7r?4)!u$7ZlrD*k-Xr$^4KuwVx zbVX9^#`PJC&~&ipc(cadOte9^gdE~>(lt3Tg}LrzJX_#V#^bn#V9snwwQQq$o|F z)#i|6=LkDJ$-x&k&HiDPkc>f|&x2XjpwS54HLyEho7C&YY)DaFl-V42k8D~&?k z+$YXKW_=?M_6iCWn~dWVVzr^wJ0`F)tbc~lOzsn6jSIo;4P=2kpDn-+uw93u`{<5~ z`}ZHEM|d%RmA=|@IC1!3nC{EL{eO*{9OmV&LkC`o9ym;Q43ll?ggu<*=s_+yoek6d zQyBEvwZHp7bRYSl88*#gUS}y(a$@A{XeTEgedGAP=#j!nnY$`?P02GtCkq4c+9mC} zN?P__-P$@MXl$K(y7s80o*)$x3a&Vuph#ba6Xk-pFPY?Wa%2Q-cplD-V$&KF zdyGrNfx}7N*AbN?8~lN`sOJI{-bE9nO-(VAQ7Y%)@4#EfQB3#aGnyjIl>vu1=qx%u#v*P$dSu? z6)F<8C??D)W+^|ZScU8?hZU>FNBes(5#>miuE2GhzQkfSIVEOO z+$G)lM#$aclcl464S;oa~B2tAx6Coqc{oIFL?f{pD7Rm~-HOr9q#9(=bS#hZ)5ku?nK~ z%_&|PcOR!p)Ucy8ic<{iXuO)Ki2LG!t-S;cM;*r8Xd=B&TT9&03hSrgPpnF#CNxoL zjAch`Gw7sd!e_L^{6?vqGv0WOai(ar3o#X9O-{tVkU$PlPoE0RMIU%0c?k*< z-vjiL3I$b`ULB9f+zj~r@{CjxGEE8S3F^}a*j7H7elvlAfOBlG8q)!p+$>rE~5r`LUf>+Wmr z8$)Zpwgt;m({5)?v3AGxx30Z)Yi_l6|H9B?|BnCgh1P>R*MjXU!SywtrKj4lio{q<}9wiSQdt@N5dvf__Cs&Bsa+Dd)rqJ6!t@#e1gb}j8# ztvj@E?(wlRH|p2wI#=pC?;KnvdC!AW3xn(Wu(PYRT?=O(A3nBtg-l~UT!o9$BvIx!dy4bKW=4rPvY>Rk9>tvVf&XYY?#4-%~tCD6%O%CM5L5Ljs_R|Urq zr--%*0=d;xi5P3#3}0K^R*4Q+MSB^l#E6aGk{J8Nj1kY!2tc-VJNObW0k*U7qPc0N z4gg<7VOx?BE7548#pnXcR-;_bH-K%|M7z;`@wTx_(fxZ?;EvlS!UkWw1kB5;id$lP zjs9|8rUY2%-V75jTn3pXFy9MKPYo!34p0-t$Q9g@CZa||Y@bmo=MYoEJFBrCEijnq zHu_@+jZ!&>Q}T*Yi+h+#gV#7|TX#W)stygRGO$BT>5QB;ZQd`~j}mm<_B?nu=&}^1 zz!}%?1L(@qiW~rqN5|E#3BYjKL=e}tB>_N&wTR9Q5zTnjGDHPK$US``%Z08js;g8%&|h&v4nij7ik0P+f(JP0!zj#tF^vU8c`)auQzGGFqh2rlPDoIcOz{ zq4DuCfL;(z7&L0UAOYOZ>5ROnzEVXCZ6&T4_eP$Bw>M6Ob7fwF;tFo^&J$I!3C@#! zE3u7{hrPCx`tAW4flwLfhqV5oUU+&^FMM;d5GgT^!%LZ<{}X{MLgOV5fi!uypxfJ| z_HcvxY@t}y*F44DC(Vh;eu0MwMR6pC2Kr79#2Cej$%DV*Jl}UQG1fOYz~FWm9`AvV zV&O%QsQMQlW_f^CP)z{bl=G6}C7NQ?vI=4q`z(t|WTQAB?#g9FiBsGfZ6XP;E{aE2 zl16P+RfUPsS1bg~s(?)~3mgDdr~-DygQU|26HH}i6nkFe$wQxFXURhNEFf%w<8q35 zhP@iDz(Zu(V380eM;qKAyxS-aRuB>@GPNbOm0QTuBW_el$UO0J!a{79VpV}7fydxU zB@#Jr#F(_^O>NrakrLSKOSRU&03PXI;5a`azSP@rV|>-yykIGY_bxcT@`u*^t)KZ@ z9|anUjZMWs@TJ`naD9#7^VESV1CK(DH`w>_@eodFGwKja>LQP;}3mi_YswH^BwPZtBBwLo|!5Plfg z`yz&y?0N4-0~53Z_h*D`Jyj$-FwY@%lgo_ z`&YFM{~4%T_t*Yy_}~34#RmH3@O#5Q+5OY#KSu8c?ni&q-?J9zSq=0&dj-|)d+CM+z9g%BahgI-?>p|d-OGXgkC_LddzY8TFFtg9 zujs8^@P0i=5p`6fhW+1sRok#u+qqKP`5alR>K_H_ua90EEr#HC$4$?Bp0&`vmC(LB ztuL(Rn&AI9P)+@B9ubc4N^KZmr^YdIZDhG==t1kl!1&`p9f6%y_1`|*iE56#>?0$o zQ&K-q$>jb|`g_>CA9N0NQ2$JwG1-1wq9wjXgOjc-lV7HL6hXURQmA0Wa`-1>ARO@+A<00iHJG4VsO0LFe*aWwO zlk-?qETa$^hHY9fXNCt2^%fMI+1(%4DE8~f$U5XS;WB<~7< z5nYMI3*pE!VPpDG_(3+B!`ObEHauq7K%NY1%)52`UFmf}}iro|-9&dSm`AhQa z{3WV@@8>A^IokOp@_ddO{|hy(JAG@;`p=y8H+oi`t@E~ZPsO~m=x&*JJaw6zVK6qK oj$-A7qPGL$IFD)G@giiRPQGxRpnNY*nIcqZ@xreWWa_m3A95yQa{vGU literal 0 HcmV?d00001 diff --git a/scraper-service/scrapers/linkedin.py b/scraper-service/scrapers/linkedin.py index b8d8e8e..1438f3a 100644 --- a/scraper-service/scrapers/linkedin.py +++ b/scraper-service/scrapers/linkedin.py @@ -50,14 +50,16 @@ MIN_SCROLL_DELAY = 4.0 # seconds — do NOT lower this -# ── Selectors — update here when LinkedIn changes its markup ───────────────── -_CARD_SEL = "li > div.base-search-card" -_TITLE_SEL = "h3.base-search-card__title" -_COMPANY_SEL = "h4.base-search-card__subtitle" -_LOCATION_SEL = "span.job-search-card__location" -_LINK_SEL = "a.base-card__full-link" -_TIME_SEL = "time" - +# ── Selectors ── loaded from YAML config +from selectors import SelectorLoader + +_selector_loader = SelectorLoader() +_CARD_SEL = _selector_loader.get("job_card") +_TITLE_SEL = _selector_loader.get("title") +_COMPANY_SEL = _selector_loader.get("company") +_LOCATION_SEL = _selector_loader.get("location") +_LINK_SEL = _selector_loader.get("job_url") +_TIME_SEL = _selector_loader.get("posted_date") def _build_url(role: str) -> str: kw = quote_plus(role) diff --git a/scraper-service/selectors.py b/scraper-service/selectors.py new file mode 100644 index 0000000..aa2c27a --- /dev/null +++ b/scraper-service/selectors.py @@ -0,0 +1,39 @@ +""" +Selector loader for LinkedIn scraper. +Loads CSS selectors from YAML config file. +""" + +import yaml +from pathlib import Path +import logging + +logger = logging.getLogger(__name__) + +class SelectorLoader: + """Load and manage CSS selectors from YAML config.""" + + def __init__(self, config_path: str = None): + if config_path is None: + config_path = Path(__file__).parent / "config" / "linkedin_selectors.yaml" + + with open(config_path, 'r') as f: + self.config = yaml.safe_load(f) + + self.selectors = self.config.get("linkedin", {}).get("selectors", {}) + self.fallbacks = self.config.get("linkedin", {}).get("fallbacks", {}) + + def get(self, key: str) -> str: + """Get a selector by key.""" + selector = self.selectors.get(key) + if selector is None: + logger.warning(f"Selector '{key}' not found in config, using fallback") + return self.fallbacks.get(f"{key}_alt", "") + return selector + + def verify(self) -> dict: + """Verify all selectors (returns dict with status).""" + results = {} + for key, selector in self.selectors.items(): + results[key] = {"selector": selector, "valid": bool(selector)} + return results + \ No newline at end of file From 4639dd3a23b1e91aa264365aeaf4b2dbea972c76 Mon Sep 17 00:00:00 2001 From: Vivek Date: Wed, 13 May 2026 20:48:05 +0530 Subject: [PATCH 2/3] fix: simplify verify script to check config loading only --- .gitignore | 2 + .../__pycache__/emit.cpython-313.pyc | Bin 4158 -> 0 bytes .../__pycache__/selectors.cpython-313.pyc | Bin 2502 -> 0 bytes scraper-service/main.py | 51 ----------- .../__pycache__/__init__.cpython-313.pyc | Bin 1519 -> 0 bytes .../scrapers/__pycache__/base.cpython-313.pyc | Bin 1515 -> 0 bytes .../__pycache__/internshala.cpython-313.pyc | Bin 8995 -> 0 bytes .../__pycache__/linkedin.cpython-313.pyc | Bin 9667 -> 0 bytes .../__pycache__/naukri.cpython-313.pyc | Bin 8149 -> 0 bytes scraper-service/scrapers/linkedin.py | 2 +- scraper-service/selector_loader.py | 80 ++++++++++++++++++ scraper-service/selectors.py | 39 --------- scraper-service/verify_selectors.py | 38 +++++++++ 13 files changed, 121 insertions(+), 91 deletions(-) delete mode 100644 scraper-service/__pycache__/emit.cpython-313.pyc delete mode 100644 scraper-service/__pycache__/selectors.cpython-313.pyc delete mode 100644 scraper-service/scrapers/__pycache__/__init__.cpython-313.pyc delete mode 100644 scraper-service/scrapers/__pycache__/base.cpython-313.pyc delete mode 100644 scraper-service/scrapers/__pycache__/internshala.cpython-313.pyc delete mode 100644 scraper-service/scrapers/__pycache__/linkedin.cpython-313.pyc delete mode 100644 scraper-service/scrapers/__pycache__/naukri.cpython-313.pyc create mode 100644 scraper-service/selector_loader.py delete mode 100644 scraper-service/selectors.py create mode 100644 scraper-service/verify_selectors.py diff --git a/.gitignore b/.gitignore index f10862a..ffe5d54 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /.env +__pycache__/ +*.pyc diff --git a/scraper-service/__pycache__/emit.cpython-313.pyc b/scraper-service/__pycache__/emit.cpython-313.pyc deleted file mode 100644 index 292ff9978baf7ea7b9022faaed48df6536f8716b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4158 zcmb^!TWl29_0G=h?(Ey^7q-Fpg0bB-wsBf)zyy>{SqjEpoE=+=Sku*bc5Dyq&MbFk z4fY3e$VYLbijWXN`cp?$8%O=&qm@#%QbAR!eA;o_rGuqf4ONBuV*-(i+K-+)JL3l> zrK&QkJ#*%qGw0lU?s?7QP{@zq`gI3)Zq0|#=k&q3*fPRAz75EGs09g#Ab}BFrNgHyRE1Fy&_w z5!|Fg@Q|p$UZ@njqg4Xm1}q^Y_!`hTj#LX_!Ea$TLPQ8ySglYY1TAda1RIMMm*54z zoL7vl!ZiNN>GMQ2& zxy$cQfT)hoX=9!8l&l&!t4Q#ysfK2~sf&3Tw*g!4BjUw2td|W)nyMG`V0LVpiuwDs zF&v+mAo7H0fK6L{2QKQ0I$kt|SZ*vxhD`>PN+FtRvJgBNP1SSp3 zQ9Hg(ALuV1BB=!(^;FpafQM)nJ@AI!xV#dHUYnVlxpsN(@~zy$spZJQtK3SY^2RrB z_FeD0dGz|x`HtJ%a#hcAq!$>``kOo7**QOa$5@IET=lF3qO*ffbs7>^;vMlWW=o{7 zK(-W^qps}9{c$u6q-rkPciSYNp}z;Pe2#Y1pIiEI)Bq1n9X~D z*j(T72uisvOxEOe*T|Hty+42On{zs~sHLZuk%fdATK~@rx`* zoK-qI=&^Hv+UL=CnB)&2qg$+uu9$h?A1aRJ6nzqlnD)Y){7kRKcB8ebrrp1xEf0!n zmbTJqJg&&OtPVX?G;qGC8+c5{1t`n3St}}Thi=&3=f`kKTD~Bv(;b*-Ioa=z=WYo}>13y@(oT@IxgsIr zw2m)M$|@Ge=qkGWNz-fFHj&9u>!wH7ibRr4wn%cOTQ@}Myy-0vEnB3$oc1Bw%}rkc znvtB%h(?Sjjc_sq3MwL)f?X8QB1f7j*h4`JfMO>A;k;5P&=G>R3TqC^E4npo*eikM zTYl)nE&6cGWqR`BrJSt7h*5xLTvGC)A7oh{}Iy6MSj6B9ClM|rF`VRDzmY}O12!>L4kFq4cACd`oiFc^P*AYn~1a7?D# znnz5xHEHQIY~a+&)t#sN$bb;p59Ap9^}~?6w6s5|MpZlB-CGJa&L*DlsJ8xA)sMEl zziqyL;nn5(eaki7rJCNQn%?D_gDW+4Hm~--@ai1ihIq{ess84fch1bSA9_FVE_j!5 zcM132es#I#z=m-B!q{^COQre)w^LheHhcwr`xg46sISp8n^>zw!RXudrNH+2o`u>E zkF4xmekpY$=u zG55*gX25@k8Gu{5Ag$tW(&_MzfRwwSz^Kd!szN7rAf&o>qO@I6X|-~yE7TCz=Dvv0 z2z*3zLQhqh!JP;N5T)#)oj$T_(}d|o;!7aIra3TYUC--{0ypAHtxuJXB&2}bu|L9Y z)SxCI)gamK1io|z*w>Y*M%Su(ITP79KL7!%nj5|tJj8LP$o z7}O2DC`q!ej~8>fY5SWZ+eM&3q2xpAhMD@~qLPEsVAT%L37Q~@Xf|2d8*^LHZ2A(H zB-tuFqyt_vee(J<&9N8<=>=;r-I7*R4FWzvCJQn;^$Vnkog+RcM|AQsFiyf>KM#N= z$!ah(o46mTy2`G(INxbzrLy7Hx8Ch61)COwP4^(wUYqOxe&UfARaCD9A@pVk9{ErR z|KmZV?lEHM3&F}#aK}<`$718bkDGtf_RF?AEx+pew7C>aeilsrbB(1!|I&NG-W5I` zW8Mnyb-mBTJU{g9b^X`_gpZiLuAlkit?pm6#&@~zZl|=nyWBvy+sXn?Yf#z8Ght^~ zE`O=O)OQ5bnp{#-ogWOa%9 zv??i@>9rA3MVWLwA1IeXK9f=5QxOc%6DIp8=%&D$XCs-+rY}1XqSHK~L#!1b)^QR# z-4pr=ZCAvX^Uy7F@=)^6)&#Y%PTsIrIS5Z3~T5WEd^(^vu8MQs}c$ms3l^j$1B;3dxXTE%e K@i9@$`o96*u8`vZ diff --git a/scraper-service/__pycache__/selectors.cpython-313.pyc b/scraper-service/__pycache__/selectors.cpython-313.pyc deleted file mode 100644 index 673d4284a235fceb7750350826e7730c56761b05..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2502 zcmb_dU2GIp6u$Gb|NVh3v~+9RODUlXWm|*-5@@0|L20|u9hRuwFq!VoE<>lY<=)w1 z`^1x3A}S^}7B!)d^-1EBi9Q&geC%4Z^@<6JzVKG8rio9UJF_#DhNutT&7S*ne(t&F zeCNz&V`CJ-m`KT2`vVC5!Z+TK>x<1CEEbT02vP(^92JNFvotDRkcfocGnvRaIpf&! z)ZMuU<$^0x5wZv3?dViOJx6CaUPBp0~>cn`Nz_6I_Jlm|;!ng_kW% z^F*7`Nh&%9Oo~So1=G4b#YIw{#;-p=K8EvUt7w#P(J=K?^bx-xofMe-ie_KQyVtqY zf7u1tFQ6QP7a<}l0+AGv$VyC+bAc105=fw{@>PKZm0&KQgy2k8!u*VcU>637h_j0* zQ4%dl$p+TsJ%c-{lSd73!TbQ3OPAy>~S*nOvUHEham5B2+8tDlZknon%r;w+Rn3X#|p#m<+ay zgr}AXManH=jM>o0nOv6Ygyv?AS$!&}5iNhkDi`!zy~2(HbJobi-Yc)v%pCW{unZe~ zX;bRwF1}M9gJ+X|0OST*kG0lfT@MmnpS<~TYArEPOAOrWt0m5?oV!#@T&j-#+PM4n zkwtpH@xanGkh+`tD7CzMHF2c+;;qpw2{rX@1Q8r}-08U6eW!alaWAvletNC_Y_0w5 z7Zcz1thSF=L5SYlSppN3NO3oqjr*tU`)WQ8Uu!Q+yr=M;($ z>21lU-v+wUaZ!=IGV4WIu@@x+^R4G~8*6@d#FKLv9-Erq?yraD4<5o+*@g~VftF-g z^|slME7Y){d)AT2WK}b521Z?A#la^QG|Q!uPFU!=Ml6u7!;S0#CTZsqVIMhl(2-Wo zbF^D_pa($9b?8MPH_*d)+lRyN53j`!*5U`3@~iQ~)$n>OUW;`+XlQvaeKWoI#3ADLG3XLm0n#JR4_?xVQJljN zV?Rd}_qkA{xM1XMeBH3GU}|f2g(g!;kzg>x0<)TF6q1sY3l=bJeVQ_PvRpQkvJ-t4 zB08;@Hg(eAC{YeUD0}RrfCXoryV>3yS#Xc{gDmAQghaILsfN}&_usvI=ki+Tv0CS` zA3BfU9IuY7_YHn^;_L92!?lhJ)$?zUJ(f_{iHFUt)h5ydn&2I8Mh+$%Ka)Z_8ROl< z(x!PQi>m5$xlnPsT7#-yt7vAO)6Baap*HkJ-7147j--}gcIHx>9auILO;sSJ(k0Pw6HYt2jcU_o?u|# zZMuPATBN`8X=7I)(6%UVA(&iq+(n@wZV?bab)7f=r@0+=3iYJ?o- zvR#}wZt&Z;U1Q<1ZYYMyNnmh{(g7eFk{}3=MM;o1qey7@8SVQCH9d^%T8kvUizF6L TuSR-r1h?db(DoYw;u!o3pPC|4 diff --git a/scraper-service/main.py b/scraper-service/main.py index 3f2b1b1..c37f1e5 100644 --- a/scraper-service/main.py +++ b/scraper-service/main.py @@ -124,54 +124,3 @@ async def trigger_scrape(body: ScrapeRequest, background_tasks: BackgroundTasks) background_tasks.add_task(_run_all_scrapers, body.role, body.stack) return ScrapeResponse(triggered=True, platforms=PLATFORMS) - -# --------------------------------------------------------------------------- -# CLI entry point for --verify flag -# --------------------------------------------------------------------------- - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="LinkedIn Scraper Selector Verification") - parser.add_argument( - '--verify', - action='store_true', - help='Verify LinkedIn selectors from config file' - ) - args = parser.parse_args() - - if args.verify: - print("\n" + "="*50) - print("LinkedIn Selector Verification") - print("="*50 + "\n") - - try: - from selectors import SelectorLoader - loader = SelectorLoader() - results = loader.verify() - - all_valid = True - for name, info in results.items(): - status = "✅" if info["valid"] else "❌" - print(f"{status} {name}: {info['selector']}") - if not info["valid"]: - all_valid = False - - print("\n" + "="*50) - if all_valid: - print("✅ All selectors are valid!") - else: - print("❌ Some selectors are invalid or missing.") - print(" Update config/linkedin_selectors.yaml with correct selectors.") - print("="*50) - - except FileNotFoundError: - print("❌ Config file not found: config/linkedin_selectors.yaml") - print(" Please create the YAML config file first.") - except Exception as e: - print(f"❌ Error during verification: {e}") - - exit(0) - - # Normal FastAPI startup - import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8000) - \ No newline at end of file diff --git a/scraper-service/scrapers/__pycache__/__init__.cpython-313.pyc b/scraper-service/scrapers/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index e47feb5f717830a65eab0d26419be2512ecbb429..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1519 zcmZWpPj4GV6rcTTXXBxQdOdD0Mqckmt0;&p92t~q?+)67>I2w)DV|$a`8S`e= zl_L%j9N^dk6`z6cz_;MG5)y&~$}N@Yfg5jjZHVYdJNw?7H*em1zu(N_%9V)V_<7eq z{?sSr538(3Pz%oDGZbEsm;i~L*gbT*F1Xan-l5m^!S4nTRJwl{b{o*3xz3H6(e)a?1A7}$SG1MAql5wr+NCD+pMyDA! zDCcywb1yo#VLMK_&ZSg_n^Z~NaZ7LilY?@DFJtAxOwO|^!nU%A6g!&88 zC*WX(ZtQ?}o5U{ou?Ioy_ro}7qes6Huf^dOIrbkcqd+rmz}mRkiOQ=_m+8fIUFYlQ z!+&(J(gP*yqQp$V{Ixo0UT8zpe42?|NV69$TXUyFL6`!hrkrLe`lt~driszJRPba% zb8eDJD#z4J1nsK9Q>6{65gsEAc4;DVP7`cNk&7W6%qu@W9)lQjgH6=}W@#cGMiIr4 zsC>%h+=_@nWdfB6tK(Riv??H}jVn_DS?zPl%kbo+Hrm9$MAsu!@8F3kM+>H76z#@O zdaJZVo*E;dQ=K~nH7t+>IB+NZn)Z|w)k>z43gy>VrdYffGLF0c+k;1b9zVJJ;rsXB zvsT+HtlLII#2SS>k?NT&JzOf+>&lqqxk#48s8Dus{+1AD(L!OHP$bIO`M?`F9ryJ6 zs79L}t3gfF9ow@VswCafwh23w%i$8oOE3a|X7|)^?nRF+oEF-e3llyQ6f4P7ndU`K z+e>WPyR>cN|aFK^S`B^y?{?lXqR8^-W%mSOExUjHA<_iFpQc#5{K$T>M97cH{Z`di~u zLcVc*2R2aXbwYRp|6p$gyouB4+P};w!H_q(Vc3v7t&B1B;tozVt?*)xK=7X_ww+40~nL1j&OO z85Blnbg*eJ@FVb+otmr3R-kX=^c{KWZUoJXTja*gv%}|`zxedGU~}OHL2KcW@LDCo zJF9)Y&Khi?Vrv)`c&{;Qu<(=~A0%MgRV7u1WxQdOdD0Mqtq>j@~5hhLMRfBk|P;RMA58QaOyM~C4w6pKMdGqGI_xsJPu3QNSj-U73 z<4;{e{xHgVc$HwSK11OJ2?>zU3hhIyZ-Y&Z>>N6M7u>!FUa7l>e!l??N^X!_B=ojP z=nvfoP8A25(?++MKSnK3F|auYb#KHJ@6DI=_p=}9{zz$Hv8KjIPY#MrR>#V?p2;zKIs0N6SBX?}ot3qEg!&6I zBw%5Mc4&cfn}jyFp#xs%4*k&UpvSNgwnBfK9J>$dD9{WW(3&>8L2>nIonBbgb-sZ< z>_-bLJ&>X*iZ3`UU#o*=xzaSr<|)s((7m8;&7CdaHDEdAJY3Vg~mHdS$0Br$&&1QbUs zvpEw>BO(T+31rHRj$>_7GKaV_u1z_lmCq$F&Ek{FXcGStU5`+`gC`~oEtrm$v>!eh ztkdFIqBVzZdF~wKI7brTz@7AKI*@{wE2)f2lwV(&WAS_(G2Hdv9z62%@X6f|-@pH! zv6^0C-6k3$*2u+)kk3Tn;8L+sRTc@B@wg^NiL#CJw}e=$HVQk0B2lK+2hPOm+Na+K z725Pzjw+(=nx5@aDd?^;P1vPOjB6Y(!3gY`*;B#U3mzLd4YU;(W^BPJR+1$m$?}YL zYHT`tv}5Aq-$dgcKZf@z`@498GB3zEIU^Ts(rW*$@EIZB*scYe zC=9wjyn%l(Hv-9qAPD+<5j`02IMWO`uOvPbc*9vvY{&8UPVii#kLGUP?dQEo@k zS2;@!Sg2i%qDcZur-|S~qM;E*V;M&g7|hJ_rS}1?=2e3&v-EZpVV`soBXN))qg-=^ z4kqmtegxh!Q*%9;3e;_!z9TQ~O|N-zi`=+*cKCej7nlC#ZLMt2Yp)#QUn?beXT5LG zZNWAwriMw5cN((>15M$uQ4FSCWm1K&mvdw(`0JfzDpeqpO3{2&_26T;i7LJ>L>b}2 hu`KJQZCmcECb4$@Bs;&7>wo#%R{MGUir}vd@ejA`tJVMj diff --git a/scraper-service/scrapers/__pycache__/internshala.cpython-313.pyc b/scraper-service/scrapers/__pycache__/internshala.cpython-313.pyc deleted file mode 100644 index 7df7b2a95c780a11d47cc868055568c55dced474..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8995 zcmd^EYj6}-cD_CDm!y$2l12kEEhK@)NTU~EA)ALRgoGr7JgxDP5@gbv>5&>nJ>%Ow zfF_k#wOh3%CzTj>*9A_c%22y0NV1is_LozYst_A&DwQ9SMM|@6)#&tt+1o~K>}{uZ_I6MQdpoHU-nL=akej-TeLU0yW%gn3kdOMH z%po|3ONL5mDNjz@qA!@kufBJZ)Bb&&@lDGG7p)N7v{LX0Uco1nM65I*R0yRZa@Izx zgi4`|<*I{ZClUN1B9xyl-DfTKu9>oi1G!I~gm}~h6|)jm+hj$PsG?4bX)%(W10^yuZ6H}~J15Ov$k3$P`o}brmRi+xZt73kJRU(xO9?GQc_pJsD&L$I zRgFI{Q8^_`N&bXL6VrTWd*}XekLUBB%>ADH2Wt+&Y9vIk7aG<~N%?$4R^TZob7DGf zE~uP^2F;QPrpjhi4VEqp62@gMEtx&d_jFeTR-<*bXR9L4Kr4|gpxJ8e`(%{oaojA; zq+}F0-Njn>WoELXGP|}Z)a{MHf@AsrTW+y^d{vK;OhVM;jIvI&F?C}D%h`BFo0e!i zDQTjdR^y3`0vSc3Th4;#RnTfeLz}#KaA%*a&F(z&UDFmL>7pv}w5*&1;lepNonE&j zpd_kBGdU$0*Tl&+${1s=soGMzqKz9=CO)phiD2_bog=Ie$JXSmq$K$^UVsx!4a?)$ zu;N&;YK7v!#d7G1n+Z?ry0qn1n3$$g3XJpl4}ZjGsl*1I6jcdDfUq&xVsJE*qPAvt zu^qxH;*+y$H8B(vwVVo@PK0@#hh8|Z?=nUNx3ZLsQz;=ST6*^P>G*rb#3`^AZUXeI zYrS+@RyCFH==^-{#~u67XRACockshgQiSKfC-Ess13!?<@>+)PWaTP|XvXs*@v;hj zmX|JO({e)ASZ%A4o0$X&hBv3;%(ntVP6fxz7OZ!u_f#uxg(NR4DOr(W3qZ)oq|czzu3J$zJG6+@6Be@()Xmv zVOeY2*Y#4QYd_ySJUD)0q=iq*=OlhWN}S7t`MznInUUH$I=dq6_z#>|N~ChJOTJW6 zT91#mNXpA^9Bhvq$OpGjsx55Idt$J=T6?F!K>4y>*hE>abwpIAa#MiRD; z<~q_AW_|}YE;g`(%29gsi-)wAL;8kjq_Ab zpNCC(?zBiDyepZcbh?PqTA}j1oLKo*Pn$vAz;}8{9L2@8OdJmntpKt8@TcyA#~gX! zEqQzYpY6ZYdad$W^2XWgXMdVr@^&qI4=j2Qyx*|oJ+xpu^b}f>#99;ipnw^&8J+-@ zImIGa6surWY=TX(3wFgJI25PgR9tODaf1=pE(3~#6k9P+B^lrbUm`^Dyg*`}Xc4>` zt&C^PY8q4V#=OnNswC)iVh2d_Znj8*OK>0H6d%-XvOF=D;MrH4lTs2ZiMrR0XXFL1 zIjT9L2t8sYCtVYaMqsO0Nh>^+(k=A1nG%$;SlNn{GU!*fwazCw!K3)MkZ50#SRN~1 zC$aoHBvydL&DOzTN~|QWSE`maqFO4+AZ!$+GFG{@%rVP}03lw&6D0sGH(xiWDO;(E zRjt#uYHP_bf1gqv^PBCHupM=Brxf@}&Jp3t@DN;`+N=>;=BJKnW)J*uzl4zxwTYU>k@$%%`{%IcVKMQe4=d`x?ye>Awg8BB3#&RmABkp8~fCQA7e)ujbO==`nr* z#}fKS`uoNK7@p`IV-Y!*Wrz#HDIhWQ7|N8&UznDZwK0KDKnTJ3DuX0+!Wi<9MgXQv zW8C6<_)HE^^bCM23^WF8X3-m!06bw*2$tLdMzjfG&V9 z85uHa+B6BhBmfbJ;$jl8L~&HaKqgLRrs&~S14Xx)yR7 z81-@b#?*^una-rq0<3~c5MvSPB4W!^6dr{*vM6k76yfB zVS}24Mzc2%K!jCUN)i|&GZ zkhmsY)O0K4ph^>|rs59QU9a{E{c+d;xU9T$$`JaK(qwKb9Mo-~ zcGj4I%RH~Td*uM?UUQXUaNSuvE-;=+7}L6U&Gyz^=GN71ND*`!%G6zkJdj{#Io(Oc z3vo1*6I!v1#~epW(^_Uv91A}1x=Ob|SBEhs&ed2-sI0?ED*$0N2sw6KHhIu;Z03uj zZKXJBq|!rBbR7QFAAyIQBacI*GBP(>g4%WqzrU8?DrAN;Ix+pqk=ht)4ESMOe| z-hDHh}48LVfdceb-`r*R5mA^)D~hzkGS>PVnBq zVy`ei@UXV-y&doFxVdeqcHjKaXRjQ)8d|PxS*&fjwRZvE-S=LFM#XV2EY-BlANy?o zE0@lr9yiA?y}4M^d3)#l;KS}${@MGF-rGa>yH74vjeXL6^488vlUL4NK6lf0>D*Fv z>tgrG`C|)JW8XdA(fNVjREBhzna}_B@LR(V>zfPpyKdCoY%J7AU@C?B-Rv1I)HgjT zcb8W@w|YxTpLCG&U602&;`7f9|MyoNr1~}P>u1fxU;C6;+*J>J6>pEcHL_59_;%%e zU;k&m!0#U0q5KF-sLd=}eQyR5kpY-JPSBWk)Oqv}pcIWNT119sgVUTc0KaiKTycmF zvKEi)p>NGPV%Gg72vPAo0*bHO`LoU&J=c3~9J+pJq4W5?j-`eZOLe33w!iRFge?hF z#!QHGe}FV74-;tstOQhO#Yi#EoTy?2>~8zNjkc3aAgN*rjU`wQSUGYWR&^RiJkPyj z9ku(2VxL$+*Me;oI}0|%&ZdDG3xL&B95IJr>p4Y;2v~0dxduz6M6k!K@N{(96z9p( zO=_GmM=9yDE6!CM7e#NgAqlQiQzmue32+YkX4o#af?bcpj{)oE>UH>8P(SrBbEbE|Kl&n1G*oDbtjzKKq=KDyV; zt-fI|0S7cv3=q57*W8aASApU9Z3d1PV196&QyWvvv8oO5 z43ORH=E;da1)jn3DQ=BM=oL1kBA;2SQT>J{@F{ps$(xSz%hhpAKI0;zAHv_QdeTCe z#lf2xC*DPTWQC1^XGvL*8}aaO^sd2$Ua}$|8XbQGoDO`b?nL4Mw;Z}Z1-FQq3vk_= zl;O%(1E;CUGZK&$ijF21gl$4fQ`qzaftcL7t)W)xLI$cb&?x<4uDzM^@I)^=tjidSg54FNJkdb%|& z0r-Vaz{7M9D;=OoBSjfE~u|njZXi_%&)S zLqBx^_VyzTzh&F6jxUw%oVOLid*)qVRtA?V8$YRRd=S`PsH-mos-HV;0nb+i9*AbdZt z=UFwW-37OM@^$%E-BR%ICEJ5w+j2!Lfy-O?xjHYlY^wT{ka!r{v4%e^dJzrGJ1Km5F9K7x4q|k z*S8#OUktY2YJ6t5R#pF(i&TYv_W*l@7i+?RXN?{smq!-rhwdG{A9($GQocBtxO74YhnBT!c-+T3r!EWch-Q3`dzI)9+%>B65OVp0FDeH6H(4!T>2dX7mU)#s?yK6%YscAt_Ysm>Yg(PvP|U$(@mw<>v$0;q`k?NN$CE%`*`!iAuoFy*Vr-{Lcp#6Sg5TMu=rJss z#0P>--J#850H6@p2#sQibyD41bY}ST7sY#J-3^p~1b_0PRKu3&R1Np~I_Adlfm36y z&zN!^KM|K3-p2KcVd_Kl`%nf>g{ZhnkF6ZXJ+U-$w%?9$glqVMlzxi;-Jg;Yc>bDH ze?fe|CUyTo>L0rO%kI!8?$FilC3oYTg<);b*SHod4NTOD7k+TJ~E4FQ(D|0zqP@ssI20 diff --git a/scraper-service/scrapers/__pycache__/linkedin.cpython-313.pyc b/scraper-service/scrapers/__pycache__/linkedin.cpython-313.pyc deleted file mode 100644 index 264826c43ddedca71d8194737a826f084cfa7e92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9667 zcmd@)ZBQIXc0K$31uU>2KzuYGi&&%G;e*2E_MHI#P+=Ie(&|`*RNl{exrLOB@P77u`bj3R0Bf4rWO4$=K${B-*5=M zhLb5-Y!P`kC@OF%QPFImCps8v>{kcv=s?S!3 zgZav~7w;_4*X)7eDqz&^fl*SxsM`ahw1BaH4~((`M*SWb?gB;w@8Qeq(0LPStU)b^ zuc$+O<=L`+V{T%kY2N6oO8vJLA%_|x!jeeTU_z2Fh;c~{EUn_(mtMdVtUe~=rPORf zisAFhELKH<#1?R#2B%a>p2t%OVReN_^9#vrn8cCr#8C%^XCawfQjZ3MD=RC3!gOQG zVi4x026K}OK5;=@T~SC}9sCobB*gExCzP0wloWYzOpZ%J`&@K-Dm+LIb@d(T>T)>v zykuBblA?+`5`vn<%Oa8HBoJ?s>8q!!=aBEHn)4IgvNXonxvX237BN9Yv$M`2rbT z6W9zpsj!d}lqxD3ol7MWup07(nFS)wo!e8&4yv>PVF<5)`&gozn>N6n1obkVQ)Nym zD&3=TA-P?pNoi4>p=}g62g2O`;bnYkdU65}(3T+#JBsdZ@D-{Q&vA1&3hpSbz%~n= zna1M9B}EmnfYqch3yY>+kP->l#z`;&47)E8$BN9v%6l+sWFVWspP^6 zpdVzq9-G1o0-YMnixkuk+u%E*)KA3n0UYB^Lm2pR_u2q>5zQ3(B1jK^WRB+|Tq)uWy~Z}=GltfJzvIcj z_T=tUe_m9^;)C5?J%_t{A3xHyGYV)2VPPLbh|t|mtv-}m9PA=A_ytXxsBh2=7gkgQ zG+?X^jvPH30r#MuU6z)`3ugradr?;6;#n3bi3I#NS(ajjx_Y({tSqf+j_7O(&MvAY zQVkr3;ir0`xP)%mD=(kO*lRC6xn<#;Rhw1&E}z&!Ty0>>Xs`#jj3%??9w4F^E%b*< z1rK-cAe1knvxsmIryF<<;&KDTqKR*L~`DfUDgktDD5tJzK zo2Le$Rq-R?<5Q#I=!wyZlFsgJ*szj$wASBy?XS3d0eDl>h-tKh*ZY?@qxJ`$O?-mq6;r3!(hxX<- z`4mDUn7@CN_KEB`Mq+lh?MT&Uiy3GUKuGXaEE;wDGLwonkrQJ+Py4@&EFGo z@8IEnD37{t;eq6;NO^$=c>73&UW>!|aFUz62yLeD6A#K3sfd*G`y-w~u0R1tGe!|O zb$Ar1j8yUU0h>NHro3Ls@7>?g-#yCPJz_^UZQIc%U7iT1r^i5IQya3Olg7|rvNF!f zaaJB;r5#E`=wDg+Q&zs%jArU}omi1DN@u=Jj)(btk3Kf_`KqDm|Kj)IOwOIrHspd4 z9_smb;k2PSG-zx>a~zZOIxAmcE4$x`;0xITStA=flUBmJ;GK;_SGT4EFaP4)h+v9pf~|au70I7estg zj9pNCcw~Vni(;_5r#H|=|6yL36Nr>2Su=~We=H;+xXDOAfXxq*m{+8DazV2|#ugI3 zKBMNKnT{xkDJL}xy~n|2$F?kqD@zJVuGP=zQSUh%5|$-Mov>Vyauj=2%^g=3X?hT7 zX~qF=_+2SBii0LDqAADmP zij-WCAkCB$GH?ex9$E{mNOD|R3FPu6(@ICz!9vnHFssB@^F60>T3l=1HpWwotv8;_ zT;_9Os)jTZ&B$psJ#$9(!$>|GDTNoQghI2ZN{Ylp%}6iMW|nZ$tV=|Rr(#J$BVx^- zzf}v#9YeA!J*bMXd_J8WU7oe3ZBj*KNVh@a6jUqH1OP(_ZbO)clmeuZSQ>zm$El{M zk_cwWu>)T9fUlL^SP8vDXto$NJ}I>bW+SBJ*n;Mi#g!>x#e@Pcz37kO|bvnbp9EQT=m)W{-{>51)Yh-GpYJdM2O2(9DE> zX3%WnvXDqYf~IMvhO9oq-D@6(Kq?78=ik(_@o45{D#1c08ICVb-u}L|W2H z;GP*3k^s*_no6@kiWL%lntk+QOkARg2F)B7XH)ZDH!iE4|w@fLc5N zgkC*!Mo3MJz)c7&F=uC*5yVsJO@<}~AM)hx$d;VdXw|x}$Y&GiCV_8Lbd#%JgZsII z;D%gUbJde6ZG7Ii={@kg={J?F*-G!tO7FFb&HBby$6p@L)(6(>1DmxCo7HXU_9xO+ z!&_$55Zpq>hSD#QabM~4C%0^Bt10yydBTDRd>YW#DRi z?i-C&)gVWGx9XZ-_5Yp! zn&~Ib*PWTVzO1Kj!_)V<8P&DG$W5);#{Tuj{+~&i#^G$u@MUvhf=|3t`Bvn5-A(tz zZ|a*Nqhx5TYToP&W;=)0JBQv}%XFU3dQV@q6`Q`7>72-VC$2h*P2bOSPG-H6S1rY+ z)lBDD);o69`jDpIPDZZPWPJnczJZKyDBC)eZ5dr}8NJzZGJP(3)%@w?GuP^}zM*yB zP{uc$Z5__GoLp}?d9!7Vc3VH4D>GZ0kU_Wn{f&f7bw3d)EfWc(($14nk;FKF!@-<4}3~U1l10 z2T;`n_tmX^_#QHpH|Fgs-5S2>;Wzg;Tp8OsyF-BWZ8o(5HZ=r9U*C11sz2g1Pv7mD z;kyphKF(?03C%mHdB@=IcI%MVI;^!0Xf2~!%h*@t$Ekd+Pnx zn>C?NYw$N)W@!KBz72UA?jf#Zi~~!`y79Ujr<EpHz-QF!37r&|p_Z|6cm%g+xS1^9Y%CA44f<3c^Q>-`kx`r|HO`Hq9U%~YT1y=rc%+VMI_n2-B9su~`b`IJ<_OTK$0SjA*#u=ftxa0(nbq~?Hu5z z9VWm}yD3$=l9iyJK2sVbhq9iPO{{ETrI(c*tn6fEfR){>>~lPgt?8lar}taeJyzhk zzTXTj>zK}cz10yeHLv@t!*=tA(GvC;Htc3-*(fzqxSY;t!{Z4D%^MwDxYM!Wvrsr_ zr0qR)rW^ehzQnq5j0+FBHU=OkXwBGc&<;T%jSWH&30nvY9yL=)kwu>+7lIk#{AQ=` znw@@aoPp{&vIrDw9#)S_G1hnx!$+5M=X^GrAeIFD1eOHkMqjXnoouk$@+ z@Lg6SY7)*~QC6VanTzxoB{=}>$pt91s%R9VzN8c@#tv#U`Z{9yVs_RNNGu>$&`iBb z)0sa+n-%q!#_yXM4xM89Ou2YMp;UAlghD>7S))<-PZ&9&=TZ{iu@D(=vr<_>#}%59HQoi2NC}aqv^$gaRBH$9-;S;Y?qEqvu+GjY|KW z{@H(ps(*!A{|in0Cu;u8?#|llHtcm*k7Vp^mn@%ER;Q8s3k%}B|54eTscePXW_feA z+`C@xeeHavy!(>9Sf9_72QE2^btzNcb;(t%GyIZbo%L^{U(A&EU9#UdSvkvnPn6?4 Mmz6INeY4s9FK?FUcmMzZ diff --git a/scraper-service/scrapers/__pycache__/naukri.cpython-313.pyc b/scraper-service/scrapers/__pycache__/naukri.cpython-313.pyc deleted file mode 100644 index 6d6512551ed8a21f83725b518bf701c4d5cb24dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8149 zcmdT}Yfv0lcD_CD7tFv6!+=0)UV@RBK|&8B^a?@zTG3l zY&oVjl`WjAlu>G3l2fUYRO`ef*~+Rmf3#Jpva_;m{RE>FlU`XxY-cO_Ct2Fa@#aU) z?Z*tTc=z{SpzrIPbMHO(-gCZld0tWBLXiHo)AIU%*CF&JsTj$s6{zP$D7=dpgb_nA zreW%w37cqAHV>PzSl&R>@?hlBYDEYu|>F`MJC)Fra{Y0O7+bMzPU zKcYvqS~5FB>lAce;>8R-CCj<1^j`WD$L3`|ofnvFJ{NVl##x-wP(b6D(k z(U1te)Z5iHJ3AZIhXA2n-^W>ji%3E~^Je7r?4)!u$7ZlrD*k-Xr$^4KuwVx zbVX9^#`PJC&~&ipc(cadOte9^gdE~>(lt3Tg}LrzJX_#V#^bn#V9snwwQQq$o|F z)#i|6=LkDJ$-x&k&HiDPkc>f|&x2XjpwS54HLyEho7C&YY)DaFl-V42k8D~&?k z+$YXKW_=?M_6iCWn~dWVVzr^wJ0`F)tbc~lOzsn6jSIo;4P=2kpDn-+uw93u`{<5~ z`}ZHEM|d%RmA=|@IC1!3nC{EL{eO*{9OmV&LkC`o9ym;Q43ll?ggu<*=s_+yoek6d zQyBEvwZHp7bRYSl88*#gUS}y(a$@A{XeTEgedGAP=#j!nnY$`?P02GtCkq4c+9mC} zN?P__-P$@MXl$K(y7s80o*)$x3a&Vuph#ba6Xk-pFPY?Wa%2Q-cplD-V$&KF zdyGrNfx}7N*AbN?8~lN`sOJI{-bE9nO-(VAQ7Y%)@4#EfQB3#aGnyjIl>vu1=qx%u#v*P$dSu? z6)F<8C??D)W+^|ZScU8?hZU>FNBes(5#>miuE2GhzQkfSIVEOO z+$G)lM#$aclcl464S;oa~B2tAx6Coqc{oIFL?f{pD7Rm~-HOr9q#9(=bS#hZ)5ku?nK~ z%_&|PcOR!p)Ucy8ic<{iXuO)Ki2LG!t-S;cM;*r8Xd=B&TT9&03hSrgPpnF#CNxoL zjAch`Gw7sd!e_L^{6?vqGv0WOai(ar3o#X9O-{tVkU$PlPoE0RMIU%0c?k*< z-vjiL3I$b`ULB9f+zj~r@{CjxGEE8S3F^}a*j7H7elvlAfOBlG8q)!p+$>rE~5r`LUf>+Wmr z8$)Zpwgt;m({5)?v3AGxx30Z)Yi_l6|H9B?|BnCgh1P>R*MjXU!SywtrKj4lio{q<}9wiSQdt@N5dvf__Cs&Bsa+Dd)rqJ6!t@#e1gb}j8# ztvj@E?(wlRH|p2wI#=pC?;KnvdC!AW3xn(Wu(PYRT?=O(A3nBtg-l~UT!o9$BvIx!dy4bKW=4rPvY>Rk9>tvVf&XYY?#4-%~tCD6%O%CM5L5Ljs_R|Urq zr--%*0=d;xi5P3#3}0K^R*4Q+MSB^l#E6aGk{J8Nj1kY!2tc-VJNObW0k*U7qPc0N z4gg<7VOx?BE7548#pnXcR-;_bH-K%|M7z;`@wTx_(fxZ?;EvlS!UkWw1kB5;id$lP zjs9|8rUY2%-V75jTn3pXFy9MKPYo!34p0-t$Q9g@CZa||Y@bmo=MYoEJFBrCEijnq zHu_@+jZ!&>Q}T*Yi+h+#gV#7|TX#W)stygRGO$BT>5QB;ZQd`~j}mm<_B?nu=&}^1 zz!}%?1L(@qiW~rqN5|E#3BYjKL=e}tB>_N&wTR9Q5zTnjGDHPK$US``%Z08js;g8%&|h&v4nij7ik0P+f(JP0!zj#tF^vU8c`)auQzGGFqh2rlPDoIcOz{ zq4DuCfL;(z7&L0UAOYOZ>5ROnzEVXCZ6&T4_eP$Bw>M6Ob7fwF;tFo^&J$I!3C@#! zE3u7{hrPCx`tAW4flwLfhqV5oUU+&^FMM;d5GgT^!%LZ<{}X{MLgOV5fi!uypxfJ| z_HcvxY@t}y*F44DC(Vh;eu0MwMR6pC2Kr79#2Cej$%DV*Jl}UQG1fOYz~FWm9`AvV zV&O%QsQMQlW_f^CP)z{bl=G6}C7NQ?vI=4q`z(t|WTQAB?#g9FiBsGfZ6XP;E{aE2 zl16P+RfUPsS1bg~s(?)~3mgDdr~-DygQU|26HH}i6nkFe$wQxFXURhNEFf%w<8q35 zhP@iDz(Zu(V380eM;qKAyxS-aRuB>@GPNbOm0QTuBW_el$UO0J!a{79VpV}7fydxU zB@#Jr#F(_^O>NrakrLSKOSRU&03PXI;5a`azSP@rV|>-yykIGY_bxcT@`u*^t)KZ@ z9|anUjZMWs@TJ`naD9#7^VESV1CK(DH`w>_@eodFGwKja>LQP;}3mi_YswH^BwPZtBBwLo|!5Plfg z`yz&y?0N4-0~53Z_h*D`Jyj$-FwY@%lgo_ z`&YFM{~4%T_t*Yy_}~34#RmH3@O#5Q+5OY#KSu8c?ni&q-?J9zSq=0&dj-|)d+CM+z9g%BahgI-?>p|d-OGXgkC_LddzY8TFFtg9 zujs8^@P0i=5p`6fhW+1sRok#u+qqKP`5alR>K_H_ua90EEr#HC$4$?Bp0&`vmC(LB ztuL(Rn&AI9P)+@B9ubc4N^KZmr^YdIZDhG==t1kl!1&`p9f6%y_1`|*iE56#>?0$o zQ&K-q$>jb|`g_>CA9N0NQ2$JwG1-1wq9wjXgOjc-lV7HL6hXURQmA0Wa`-1>ARO@+A<00iHJG4VsO0LFe*aWwO zlk-?qETa$^hHY9fXNCt2^%fMI+1(%4DE8~f$U5XS;WB<~7< z5nYMI3*pE!VPpDG_(3+B!`ObEHauq7K%NY1%)52`UFmf}}iro|-9&dSm`AhQa z{3WV@@8>A^IokOp@_ddO{|hy(JAG@;`p=y8H+oi`t@E~ZPsO~m=x&*JJaw6zVK6qK oj$-A7qPGL$IFD)G@giiRPQGxRpnNY*nIcqZ@xreWWa_m3A95yQa{vGU diff --git a/scraper-service/scrapers/linkedin.py b/scraper-service/scrapers/linkedin.py index 1438f3a..7a32ad7 100644 --- a/scraper-service/scrapers/linkedin.py +++ b/scraper-service/scrapers/linkedin.py @@ -51,7 +51,7 @@ MIN_SCROLL_DELAY = 4.0 # seconds — do NOT lower this # ── Selectors ── loaded from YAML config -from selectors import SelectorLoader +from selector_loader import SelectorLoader _selector_loader = SelectorLoader() _CARD_SEL = _selector_loader.get("job_card") diff --git a/scraper-service/selector_loader.py b/scraper-service/selector_loader.py new file mode 100644 index 0000000..9e8e25e --- /dev/null +++ b/scraper-service/selector_loader.py @@ -0,0 +1,80 @@ +""" +Selector loader for LinkedIn scraper. +Loads CSS selectors from YAML config file. +""" + +import yaml +from pathlib import Path +import logging +import asyncio +from playwright.async_api import async_playwright + +logger = logging.getLogger(__name__) + +class SelectorLoader: + """Load and manage CSS selectors from YAML config.""" + + def __init__(self, config_path: str = None): + if config_path is None: + config_path = Path(__file__).parent / "config" / "linkedin_selectors.yaml" + + with open(config_path, 'r') as f: + self.config = yaml.safe_load(f) + + self.selectors = self.config.get("linkedin", {}).get("selectors", {}) + self.fallbacks = self.config.get("linkedin", {}).get("fallbacks", {}) + + def get(self, key: str) -> str: + """Get a selector by key, fallback to alt if available.""" + selector = self.selectors.get(key) + if selector: + return selector + # Try fallback + fallback_key = f"{key}_alt" + if fallback_key in self.fallbacks: + logger.warning(f"Using fallback for '{key}': {self.fallbacks[fallback_key]}") + return self.fallbacks[fallback_key] + logger.warning(f"Selector '{key}' not found") + return "" + + async def verify_async(self, url: str = "https://www.linkedin.com/jobs") -> dict: + """Actually test selectors against a real LinkedIn page.""" + results = {} + + async with async_playwright() as pw: + browser = await pw.chromium.launch(headless=True) + page = await browser.new_page() + + try: + await page.goto(url, timeout=30000, wait_until="domcontentloaded") + + for name, selector in self.selectors.items(): + try: + element = await page.query_selector(selector) + results[name] = { + "selector": selector, + "valid": element is not None, + "found": element is not None + } + except Exception as e: + results[name] = { + "selector": selector, + "valid": False, + "error": str(e) + } + finally: + await browser.close() + + return results + + def verify(self) -> dict: + """Sync wrapper for basic check (no network).""" + results = {} + for name, selector in self.selectors.items(): + results[name] = { + "selector": selector, + "valid": bool(selector and len(selector) > 0), + "found": None + } + return results + \ No newline at end of file diff --git a/scraper-service/selectors.py b/scraper-service/selectors.py deleted file mode 100644 index aa2c27a..0000000 --- a/scraper-service/selectors.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Selector loader for LinkedIn scraper. -Loads CSS selectors from YAML config file. -""" - -import yaml -from pathlib import Path -import logging - -logger = logging.getLogger(__name__) - -class SelectorLoader: - """Load and manage CSS selectors from YAML config.""" - - def __init__(self, config_path: str = None): - if config_path is None: - config_path = Path(__file__).parent / "config" / "linkedin_selectors.yaml" - - with open(config_path, 'r') as f: - self.config = yaml.safe_load(f) - - self.selectors = self.config.get("linkedin", {}).get("selectors", {}) - self.fallbacks = self.config.get("linkedin", {}).get("fallbacks", {}) - - def get(self, key: str) -> str: - """Get a selector by key.""" - selector = self.selectors.get(key) - if selector is None: - logger.warning(f"Selector '{key}' not found in config, using fallback") - return self.fallbacks.get(f"{key}_alt", "") - return selector - - def verify(self) -> dict: - """Verify all selectors (returns dict with status).""" - results = {} - for key, selector in self.selectors.items(): - results[key] = {"selector": selector, "valid": bool(selector)} - return results - \ No newline at end of file diff --git a/scraper-service/verify_selectors.py b/scraper-service/verify_selectors.py new file mode 100644 index 0000000..655e6c6 --- /dev/null +++ b/scraper-service/verify_selectors.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Standalone selector verification tool - checks config loading only.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from selector_loader import SelectorLoader + +def main(): + print("\n" + "="*50) + print("LinkedIn Selector Verification") + print("="*50 + "\n") + + loader = SelectorLoader() + + print("Selectors loaded from config:\n") + + all_valid = True + for name, info in loader.verify().items(): + status = "✅" if info["valid"] else "❌" + print(f"{status} {name}: {info['selector']}") + if not info["valid"]: + all_valid = False + + print("\n" + "="*50) + if all_valid: + print("✅ All selectors are valid in config file!") + print(" (Live LinkedIn test blocked by anti-bot measures)") + else: + print("❌ Some selectors are empty or missing.") + print(" Update config/linkedin_selectors.yaml with correct selectors.") + print("="*50) + +if __name__ == "__main__": + main() + \ No newline at end of file From cd0e4d2d992a849d608bd15711013a51d1aa722f Mon Sep 17 00:00:00 2001 From: Vivek Date: Fri, 15 May 2026 07:48:32 +0530 Subject: [PATCH 3/3] feat: add verify_async for live LinkedIn selector testing --- scraper-service/verify_selectors.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scraper-service/verify_selectors.py b/scraper-service/verify_selectors.py index 655e6c6..223a49f 100644 --- a/scraper-service/verify_selectors.py +++ b/scraper-service/verify_selectors.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -"""Standalone selector verification tool - checks config loading only.""" +"""Standalone selector verification tool - tests selectors against LinkedIn.""" +import asyncio import sys from pathlib import Path @@ -8,31 +9,33 @@ from selector_loader import SelectorLoader -def main(): - print("\n" + "="*50) - print("LinkedIn Selector Verification") - print("="*50 + "\n") +async def main(): + print("\n" + "="*60) + print("LinkedIn Selector Verification (Live Test)") + print("="*60 + "\n") loader = SelectorLoader() - print("Selectors loaded from config:\n") + print("Testing selectors against LinkedIn...\n") + results = await loader.verify_async("https://www.linkedin.com/jobs") all_valid = True - for name, info in loader.verify().items(): + for name, info in results.items(): status = "✅" if info["valid"] else "❌" print(f"{status} {name}: {info['selector']}") if not info["valid"]: all_valid = False + if "error" in info: + print(f" Error: {info['error']}") - print("\n" + "="*50) + print("\n" + "="*60) if all_valid: - print("✅ All selectors are valid in config file!") - print(" (Live LinkedIn test blocked by anti-bot measures)") + print("✅ All selectors resolved successfully against LinkedIn!") else: - print("❌ Some selectors are empty or missing.") + print("❌ Some selectors failed to resolve.") print(" Update config/linkedin_selectors.yaml with correct selectors.") - print("="*50) + print("="*60) if __name__ == "__main__": - main() + asyncio.run(main()) \ No newline at end of file