From f3b96bf2d41eec74d6aaf44c20ed5e1c0dda67c0 Mon Sep 17 00:00:00 2001 From: spacewaterbear <38250076+spacewaterbear@users.noreply.github.com> Date: Wed, 8 Jul 2026 01:36:04 +0200 Subject: [PATCH] feat: add crawlab (#759) * feat: add crawlab * feat: add crawlab - re order * feat: add crawlab - fix compose * feat: split crawlab into worker and master --- blueprints/crawlab-master/crawlab.jpeg | Bin 0 -> 8314 bytes blueprints/crawlab-master/docker-compose.yml | 41 +++++++++++++++++++ blueprints/crawlab-master/template.toml | 12 ++++++ blueprints/crawlab-worker/crawlab.jpeg | Bin 0 -> 8314 bytes blueprints/crawlab-worker/docker-compose.yml | 20 +++++++++ blueprints/crawlab-worker/template.toml | 7 ++++ meta.json | 36 ++++++++++++++++ 7 files changed, 116 insertions(+) create mode 100644 blueprints/crawlab-master/crawlab.jpeg create mode 100644 blueprints/crawlab-master/docker-compose.yml create mode 100644 blueprints/crawlab-master/template.toml create mode 100644 blueprints/crawlab-worker/crawlab.jpeg create mode 100644 blueprints/crawlab-worker/docker-compose.yml create mode 100644 blueprints/crawlab-worker/template.toml diff --git a/blueprints/crawlab-master/crawlab.jpeg b/blueprints/crawlab-master/crawlab.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..1aef3cb99f599e539b586ba2a56c261c0445b969 GIT binary patch literal 8314 zcmbVxcT`jTwrvokNs%T+iUJA}K1*8)ol+ZzHRHRCe4bcchN;uYq*#VyFg&dx7&Pw;_=xVShMpNxXEsJyV4 zxF`_?1qIbLDi&&L7SY@6w?+S-5Bx6x9XSyT5eqR9H{c2#5iuPRzJ&+{01#az`s)Dv z^B}rHOmdZ!jGTh<+I7GcB4XkzB*a&*l8_L78AP}TAfdZTe_KSElmTQ*#_i51`X(}s zoJXa&k;$NcmsiZr<1Gc{4Q3WrwmWzE`0ojbOGrvd%g8=@tg5E|L_^ci2yAR(Y6h`? z>fq?)?BeR_bO zkdP3Qkp00!bmbKxi0MeK-WDOHR|b*Ux-)Q#z9DB+iA*bQq~H-V*k!Ww=%>8FD?WDz z`v=lri2iqg-u{0f`Ww*Sc<|!@Dq_y>y!Jn% z_oaS&i!U(qIp-B?Y6@)DXh)gOsqD$3A~GWYg|r6PgT9!-f)*$qpm1jqN1S>IMb=`3 z4=Dn>YC@4UFXU-cSAm1+NaN&^W6+9m_Zs?%tG~{1nAd0d-0l$?r4MwK!9hUMjd?0G z_dqLie`X-=9anE-Ls$eOWEn%I<#|%S1biVGGg0aah7PNc$iqH(;24^ zOGv|p)|nsr`xPKojIwjQ=7fG@7RwzcIfaL3c`omFb4J}#y7V6_J9#HMYdOYqR;q1s zAcOO6?YG$fHMdf0vgk%->HM4rlOO1iVPy+(tbl2JAX^!zuuqWrB5wf9S?%0j+}rIDs?8G=75BK;`7@DD8x^gQ_dpj7MV+jD z`REOvtRq>iMI48on0S(4Et*Omad+Gd%e&VuF)xvs0gX!#Ki``1a@uK`@oxBaJ`(x4 zb3w^J0#We@)9ff3T4Q1qaFr=89XjmaQ-5OqELYL@>qQriXSxjdEAbA6M~7MRBp+*` zb4@A=E};4Ed?)b#7nb=`YJa+j`llD8fjP;{Qw>cB2H#KOP-|iCf>*arna|VD+iL4w zGAiSjm0vFDh3?Bw`bdohe3{wf@_wao*lc0ZIiNXZiV+7{Yp`VJ z@iacK@L>e0id-dhJbH-q^kv@3pUI}q;4}sUMs4LtNr^^q1!D?`< zh3?2mcs0_tk;Ddmux644wxU@{GKg8L(J|HJN0p2H$>QCz>g3Kjr*dRE(qD8W^Te}d zcSGc(LyHMqMtc{xfn!?2oMc8(L^&r#J(e*xd`s;Sgl0-Z{l=~cz^VIu8uhtJ0D1nR;Ds#LagZW%dv;&-zDM~b9?*>~HuFR@XuUvFr~-jN zCL8dZMiLL|I|1rZgkk|`K|LjsFs~jccG$}pS{0{mh(IC!-Ier3T>>!ghNfD0X{myh z8XBMj)wu!qzkOYoe?36iLUYJvcbjfzo2)Q@G z^(5&~S(Q$ojNus+AKP9;w>R+Pu$xMJY+InlXtmKo=oH3sSd-r{6QK!}ERm1RWmY&*K?pB1vHr_ZoB)0s~nYPA^2Uu1A z5_!mis@E-Z8UB`dte6naGJp_Nf%E5w6HcjD;8_8=qiY@2+so2*MDi)A8x~6=h)sK5 zb4ZV*TroNAzC@Xz1f+rFg}ou2W(u082vxi7_L1!!!{Yb^Aw5u`OH~s>&2O0YX}O&r z!P=^a!JmSdr#@yWSjTBtg47{379O8Jml(_O+_~e`1n19qK%f*c z5~~*yZJ<_LqF>=E+9|7r2aFsSNjp5X%cuMyeJet_$i0xKEts9EAlEFbS0;(M5_fzu zQWCq62uTO6{oZpwk;^i^c_^*PkE(x-y+U{oG=sm!`y&i9wP%1)zoYy)H!fkaJ9#bK z$XVo_aCrNB(I4yNMK=9qj6+_PF9!3dq;VTGsCrL{W)Hn_w9{$ww)w|p#Lw7`z5D&D zOpD2>CPWKcT0t#r9$c{^%NmCYWJ@W%H62e)ZT;Biz*-EOz&7t|Xp98K_Tu?K)iZyZ z=3ScIP40qYDy)^Y! zORlLSyv0JE9|Y;~PI8=3D-rAUK)(<3wg^GfHSeVbQL< zpsuYfA~yLmmVJA#%8w&yBC6X%M-mPk(nKc^YVpAz)nnpFf4SOf>cOf@URb1#t((dd zguIrzelV?gMvw&$FJp-x2OB^gwt)hJ=vvk-Nt2`T@Sl^A&Yx;iX#3S5YO^A?K?(D^ zyNi0zyGpg8+dmGt>$Fsdx61OAWEsjvKJh3R5#B*v;ItUApxiP9eKvXgtRrz+X}=mOEzlhZ4Gzk6GMdfRAkFZ z)?*wR^Xb!*vTw_Xi6K~@Yg2)cxn=$EQ_|&!Jch_mq>Qj|$B_}+bOZ6IysEvS$vdFx zfUArk^Z2VioL4YmCM7Q0!5SY9@Bm-$OC_ZHp+XWSR_e#p<6z&!2ggT+fncuZYWkAR zLq}l7J}4^v?bFJ)GU+nv`kg@^do--shY-2CXNh<~SW!KCZVjx<>A}tQXykyjw!B8~ zqm)@?Nh#TOPGsoFn~QLF?}JmYqPA>YkX~}PtEUO{2SG{J*oiE;-ibE`6?{r`lmjhu z`QW(LVU#Pt!9WHZAupyx#I!}RM^H7 zqpq;%7?M@G-l1wslKJ<#BxuUKe0B1vKf-=T-_?I8BKpQg&f8*EQ?%fWvX%Y!lXv<9 z1$QgTVgx#sJ6QPxA`kP|zQ@iR$GWg--26jDU-yoQ?9-H*ljPbGJ$H|yfL&Q?r#7ka zFV4^Rp>DX2)6syG-9isIa4b_LiDj?RbT1d2WYQz`l2DX}e>CWiQzjm~UWUj8IG|Y0P7N!Oq5*IziS2hs&BOSUjwq3Grp^-jQxz4lpxP z7<)pn&R9JQf#f{@YM(fFubI6b3S!I=SF*s8J05UdILzOn?1%KegJ4h-0?S>|Y;aU& zPqQoyPwtKR?ss0J#q(s-Y&6y`AX*0nKWEGjOa-Yj{`yaL{6AeA+s$95r5{V&-PFh3 zLae|Kp9R3N1)|nG;F7rWXdc)?QynvoUivccSI>{V9I&PW!bzw_ElG-ba$jbqyAp{9 z)Gp#mJ82=nY0XY_1VO#N^Q7(?1zhM6;pXLF3mGuTp}_wAD+CYJrnK=Hrg<=anmC3B zlm#6x6xLKnU)zVa-g|&Jn{DVGNk;s z_*}WfJ;gz)LCX7SfA#Y}J^lY4S8@azL-job+{*PX_w_*rbF42ud^4ww_tE2qdADD> zAYYjxI)-=BS4nmd-*pT!b@r7-Iv_d(aWK~S?JJ;@%=ZRXAJK5KrZiBeSsU6M?Z?So z9JnAFf(O)B;>tGCBdw*rwGBmYBLaZPHHl>*MvxU~EyQmGF$c^u*BP6eIAy>|td^lK z9+utzK4I}DMZh3y^Z_BazzX4%yLT1+*?mv;^@G)u31xg3+Cy0Q;r})$i5vmmd`$Pp z9C33w;-j^f1_SqZx<8*rT7}{P#Bsr&)aVa6bW39j@F-uw#V2(?rVwQ#~ zqe)-#ELaHA>dhfM|9j#1qe}Ex4ObtN58OTavyK-dDkH%uv|9W{`)pp$?KhjMlGS}T zSC@zJfO8ez%kCB0H^@{%KzG1@3y8nZyKL9k z5(~&<4sPj2X~CeYPF;*&H2XEf+2Jocn@Nn+^6&659;!-f+__t7Pr&(8emXj`znqj!_;Bpivxn zB6_ISLFWLo)Z+rtL~W!&;QU3-dn(mRSJze0kUd^$_~NrOsa-q(bTT5NncqH#`O@h@ zz6ljLKtf*7=41*)Bw|J$O*-e-5rZwQHQ@_yZk;+1m(VmUoD63MKAWB6V|{_hdykpV zC?|{KSM!3V~6LY)NeH&axP5QBEW-u1NWzg-4i_!VQ=#W9Bgz;5kKW#e$_BR&NM*=^KTD6OD}u#@}tPM zo=4DneNNiwI(Do2Ju~q&?%l2fiA>eN&KwD0uS3O9*mXMjnDWk*@^a*X>H3_!~2K4w6-`O z3ZqUbnmTK9E}*h?mAYaGam)TiP5O6BQ6+@_nEwR@Pv{-inrMHHV)e7&#{<|5qTn@b zIyF24@(gy*#jheiaL#q7geqT@oQ!CICYoZ8u+1z@YH3CRMDVD3;WBlD^l$4y4s5}_ zlpJp>X}Mh59S|j+02n*=J_!9uv@IPo7x3% z$moOqueapw+_kbb4WuVwqJ?pOeBIJ-CMt0|Cnld0Y)<3K-;NZWCYM{nmx4RZz2yA) zQMJSqe+&W8+SJ@tvd7JvzT>8kg7P6*aUq%MuFh0?X?5@mn+*cW3uympVWOgNA;&pF z7cCJeB-=F&IhL0}Ms~{nW60M$(z@hsHn6Z9N8iRBJ!rTD4W-Vq=Y$eYBVFG!3yM^s-2lmVQFE2SROa3k-7m3! z4Hj-Rx?%qd@HAmv97Zy2CE5l1<}AB~ljPE!K`dDgeF(SxP6e4zXT=?fqOtNrXZs?o zrOO2x=6yM<`dvKC{t-*&)Up#g<)%V*6f6hpdN~T~LyRV0CSGhF(u0f%^QQre0X-wV zKTT;_t8Z6}o%nOnUcKoDE!#@(VXxK$h6yubEr#3Ps?@o$SB(r_*02?%^-NeNUr7+Q z#kpll*jutzi*Zp_k>gofH$rMub^Y05a(-xHJu4*FTOX(IzNaF|caAPvR7&yywe59A zwq+)@gF%>Rt47pXPNs|gxG)7MPBGsM>j21JEzo5Gd*`J^+Ff=)$z|0_1e$(AL9Yj z*4U!7PHZ6dNObD?s;^nZ%_ys3IWb9sl-cYxAx-m04IUAHQC?)z1G7n(<6?9JuS z*L>0`(HZF%(A**#wrllUu79Bo9c~-~EO|un5!tWNiMrmch&4A-%e;e?$)6#@QvCrT7pv9U)xD_L{>59T8L;i56O68V z7W2Jl@b?yr=^SJIzBg6(9c44R_HF~o75aQGWMOA{4cP?+^JNtg1{*X-cw6g|Bori0fSvdLu0agUV%y;mzde7kis% z;ZP#4yFLsB<_Kk}M76zaN-F|kl|l{8%J*%F+<3Td<4$mEMIy2HZXxH9lutsb@*gM{ z5)fi>?%|AS(Tt9Qx1tbhQ& zuuWB%JcGIb*Ss2)l4hJl%TR^vpq>rk=~zH&&OHD|BTtJ*Zmdy$_Tgk$Ie-$Ru>qSL z?UJiWW8pJT&Aib1(EOEUlxx`0Spl8T?f|^@?vYR+^udTND0vLj+eV#p? z6EP{2?^Mm+u!3Xisi}>y9E(TX`MQ{J`115|a+zfQ|k`E2|}9!M4ggdkJo0wzZwYWZ6YK_5$R@uN~04Ri!r>BZVtD04+MGNvj> z3T4Nq&1=4H&8qCRzeLqpKkT>b_rhLp);@8uK1W|%HQa3oWqwk=U&tu2_-TQsTl(!kZ4+6m z_z%Ss2j7YF_7<*z(lo4a8QCPpQE6wcAsH$F@Ab;*n}<8JWi0HLN5GYCLN$&vKM}Me z45n?=u$%0P2TYa`B;4r&`W02|D=2qPKC;e3pzo!J-f*;u-L>w4(d`QOn=RHT*G2k0DjOw z<7>siE#j^?=EmYNI;RkNGStKTu(aRy%AFYzF8OX5w_VPm0&AJ@ZU^(ptPGggRO2l8 zez1Sn-e36{_&YcMnkRUG?~62Nr|_rYZLFqG*<#_63?}`FG-b0+Dvkr69M)|-+wlM| z!ldp#(8^0+YN*!Vo<*ov)qo%n#w&osKq&4d@J#Xn!RKt)mY6k9<>g-JiP6QjK?eJA zWa0gmv6-_sKmI%moF_ELWRHu2(*vbUj|v|tWBRX!slF-wq1I_TR#WU?wiyxxcOhpc z-wvvdAY!Dq|JyLdq)>qDIj9Hc(`(}3Rp^m{m<&|m=)m)_>xT1ijSSOyH9wX)u$o3; zh3+m>cV&=@J0%|Ap%mYs`0lWFgu`X34s}j|JZZSaDa3Fg653X88-wRx+H}EnJ@-ziCR(!ehFi90F#!l8LGnI7vWGE%&*8U`t|h zzC|Ei;^hCR`2YM8B_Lx9_qr$Oyy~Av_09nFP{K2*e%C76S4;h-xy%m5uL}fI2*GG9 zc|5;w{{`j)2A+kDz?9c4y|AP30IeO&QqJ2ng?T@D9K&!bpQI`}|Jlv6ux+BER<jR4dPUrRh1^gr9?S=G zmHXx;iN(9d(aw)FZcbM20Nqb%lw@Z2}&;)+SbVYJ3Z~F{pQo8;& RCQ$TWzH|HwPnGav{{<%Q8IS+~ literal 0 HcmV?d00001 diff --git a/blueprints/crawlab-master/docker-compose.yml b/blueprints/crawlab-master/docker-compose.yml new file mode 100644 index 00000000..8b4f58af --- /dev/null +++ b/blueprints/crawlab-master/docker-compose.yml @@ -0,0 +1,41 @@ +version: "3.8" +services: + crawlab-master: + image: crawlabteam/crawlab:latest + restart: unless-stopped + environment: + - CRAWLAB_NODE_MASTER=Y + - CRAWLAB_MONGO_HOST=mongo + - CRAWLAB_MONGO_PORT=27017 + - CRAWLAB_MONGO_DB=crawlab + - CRAWLAB_MONGO_USERNAME=${MONGO_USERNAME} + - CRAWLAB_MONGO_PASSWORD=${MONGO_PASSWORD} + - CRAWLAB_MONGO_AUTHSOURCE=admin + volumes: + - crawlab-master-data:/root/.crawlab + - crawlab-app-data:/data + - crawlab-logs:/var/log/crawlab + expose: + - "8080" + - "9666" + depends_on: + - mongo + deploy: + resources: + limits: + cpus: "${CPU_LIMIT:-0.8}" # 0.8 will limit to 80% of CPU if one core. If 2 cores, you need to put 0.8*2=1.6 + + mongo: + image: mongo:4.2 + restart: unless-stopped + environment: + - MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME} + - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD} + volumes: + - mongo-data:/data/db + +volumes: + crawlab-master-data: + crawlab-app-data: + crawlab-logs: + mongo-data: diff --git a/blueprints/crawlab-master/template.toml b/blueprints/crawlab-master/template.toml new file mode 100644 index 00000000..e1bf5524 --- /dev/null +++ b/blueprints/crawlab-master/template.toml @@ -0,0 +1,12 @@ +[variables] +MONGO_USERNAME = "crawlab" +MONGO_PASSWORD = "${password:24}" +CPU_LIMIT = "0.8" + +[config] +env = ["MONGO_USERNAME=${MONGO_USERNAME}", "MONGO_PASSWORD=${MONGO_PASSWORD}", "CPU_LIMIT=${CPU_LIMIT}"] + +[[config.domains]] +serviceName = "crawlab-master" +port = 8080 +host = "${domain}" diff --git a/blueprints/crawlab-worker/crawlab.jpeg b/blueprints/crawlab-worker/crawlab.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..1aef3cb99f599e539b586ba2a56c261c0445b969 GIT binary patch literal 8314 zcmbVxcT`jTwrvokNs%T+iUJA}K1*8)ol+ZzHRHRCe4bcchN;uYq*#VyFg&dx7&Pw;_=xVShMpNxXEsJyV4 zxF`_?1qIbLDi&&L7SY@6w?+S-5Bx6x9XSyT5eqR9H{c2#5iuPRzJ&+{01#az`s)Dv z^B}rHOmdZ!jGTh<+I7GcB4XkzB*a&*l8_L78AP}TAfdZTe_KSElmTQ*#_i51`X(}s zoJXa&k;$NcmsiZr<1Gc{4Q3WrwmWzE`0ojbOGrvd%g8=@tg5E|L_^ci2yAR(Y6h`? z>fq?)?BeR_bO zkdP3Qkp00!bmbKxi0MeK-WDOHR|b*Ux-)Q#z9DB+iA*bQq~H-V*k!Ww=%>8FD?WDz z`v=lri2iqg-u{0f`Ww*Sc<|!@Dq_y>y!Jn% z_oaS&i!U(qIp-B?Y6@)DXh)gOsqD$3A~GWYg|r6PgT9!-f)*$qpm1jqN1S>IMb=`3 z4=Dn>YC@4UFXU-cSAm1+NaN&^W6+9m_Zs?%tG~{1nAd0d-0l$?r4MwK!9hUMjd?0G z_dqLie`X-=9anE-Ls$eOWEn%I<#|%S1biVGGg0aah7PNc$iqH(;24^ zOGv|p)|nsr`xPKojIwjQ=7fG@7RwzcIfaL3c`omFb4J}#y7V6_J9#HMYdOYqR;q1s zAcOO6?YG$fHMdf0vgk%->HM4rlOO1iVPy+(tbl2JAX^!zuuqWrB5wf9S?%0j+}rIDs?8G=75BK;`7@DD8x^gQ_dpj7MV+jD z`REOvtRq>iMI48on0S(4Et*Omad+Gd%e&VuF)xvs0gX!#Ki``1a@uK`@oxBaJ`(x4 zb3w^J0#We@)9ff3T4Q1qaFr=89XjmaQ-5OqELYL@>qQriXSxjdEAbA6M~7MRBp+*` zb4@A=E};4Ed?)b#7nb=`YJa+j`llD8fjP;{Qw>cB2H#KOP-|iCf>*arna|VD+iL4w zGAiSjm0vFDh3?Bw`bdohe3{wf@_wao*lc0ZIiNXZiV+7{Yp`VJ z@iacK@L>e0id-dhJbH-q^kv@3pUI}q;4}sUMs4LtNr^^q1!D?`< zh3?2mcs0_tk;Ddmux644wxU@{GKg8L(J|HJN0p2H$>QCz>g3Kjr*dRE(qD8W^Te}d zcSGc(LyHMqMtc{xfn!?2oMc8(L^&r#J(e*xd`s;Sgl0-Z{l=~cz^VIu8uhtJ0D1nR;Ds#LagZW%dv;&-zDM~b9?*>~HuFR@XuUvFr~-jN zCL8dZMiLL|I|1rZgkk|`K|LjsFs~jccG$}pS{0{mh(IC!-Ier3T>>!ghNfD0X{myh z8XBMj)wu!qzkOYoe?36iLUYJvcbjfzo2)Q@G z^(5&~S(Q$ojNus+AKP9;w>R+Pu$xMJY+InlXtmKo=oH3sSd-r{6QK!}ERm1RWmY&*K?pB1vHr_ZoB)0s~nYPA^2Uu1A z5_!mis@E-Z8UB`dte6naGJp_Nf%E5w6HcjD;8_8=qiY@2+so2*MDi)A8x~6=h)sK5 zb4ZV*TroNAzC@Xz1f+rFg}ou2W(u082vxi7_L1!!!{Yb^Aw5u`OH~s>&2O0YX}O&r z!P=^a!JmSdr#@yWSjTBtg47{379O8Jml(_O+_~e`1n19qK%f*c z5~~*yZJ<_LqF>=E+9|7r2aFsSNjp5X%cuMyeJet_$i0xKEts9EAlEFbS0;(M5_fzu zQWCq62uTO6{oZpwk;^i^c_^*PkE(x-y+U{oG=sm!`y&i9wP%1)zoYy)H!fkaJ9#bK z$XVo_aCrNB(I4yNMK=9qj6+_PF9!3dq;VTGsCrL{W)Hn_w9{$ww)w|p#Lw7`z5D&D zOpD2>CPWKcT0t#r9$c{^%NmCYWJ@W%H62e)ZT;Biz*-EOz&7t|Xp98K_Tu?K)iZyZ z=3ScIP40qYDy)^Y! zORlLSyv0JE9|Y;~PI8=3D-rAUK)(<3wg^GfHSeVbQL< zpsuYfA~yLmmVJA#%8w&yBC6X%M-mPk(nKc^YVpAz)nnpFf4SOf>cOf@URb1#t((dd zguIrzelV?gMvw&$FJp-x2OB^gwt)hJ=vvk-Nt2`T@Sl^A&Yx;iX#3S5YO^A?K?(D^ zyNi0zyGpg8+dmGt>$Fsdx61OAWEsjvKJh3R5#B*v;ItUApxiP9eKvXgtRrz+X}=mOEzlhZ4Gzk6GMdfRAkFZ z)?*wR^Xb!*vTw_Xi6K~@Yg2)cxn=$EQ_|&!Jch_mq>Qj|$B_}+bOZ6IysEvS$vdFx zfUArk^Z2VioL4YmCM7Q0!5SY9@Bm-$OC_ZHp+XWSR_e#p<6z&!2ggT+fncuZYWkAR zLq}l7J}4^v?bFJ)GU+nv`kg@^do--shY-2CXNh<~SW!KCZVjx<>A}tQXykyjw!B8~ zqm)@?Nh#TOPGsoFn~QLF?}JmYqPA>YkX~}PtEUO{2SG{J*oiE;-ibE`6?{r`lmjhu z`QW(LVU#Pt!9WHZAupyx#I!}RM^H7 zqpq;%7?M@G-l1wslKJ<#BxuUKe0B1vKf-=T-_?I8BKpQg&f8*EQ?%fWvX%Y!lXv<9 z1$QgTVgx#sJ6QPxA`kP|zQ@iR$GWg--26jDU-yoQ?9-H*ljPbGJ$H|yfL&Q?r#7ka zFV4^Rp>DX2)6syG-9isIa4b_LiDj?RbT1d2WYQz`l2DX}e>CWiQzjm~UWUj8IG|Y0P7N!Oq5*IziS2hs&BOSUjwq3Grp^-jQxz4lpxP z7<)pn&R9JQf#f{@YM(fFubI6b3S!I=SF*s8J05UdILzOn?1%KegJ4h-0?S>|Y;aU& zPqQoyPwtKR?ss0J#q(s-Y&6y`AX*0nKWEGjOa-Yj{`yaL{6AeA+s$95r5{V&-PFh3 zLae|Kp9R3N1)|nG;F7rWXdc)?QynvoUivccSI>{V9I&PW!bzw_ElG-ba$jbqyAp{9 z)Gp#mJ82=nY0XY_1VO#N^Q7(?1zhM6;pXLF3mGuTp}_wAD+CYJrnK=Hrg<=anmC3B zlm#6x6xLKnU)zVa-g|&Jn{DVGNk;s z_*}WfJ;gz)LCX7SfA#Y}J^lY4S8@azL-job+{*PX_w_*rbF42ud^4ww_tE2qdADD> zAYYjxI)-=BS4nmd-*pT!b@r7-Iv_d(aWK~S?JJ;@%=ZRXAJK5KrZiBeSsU6M?Z?So z9JnAFf(O)B;>tGCBdw*rwGBmYBLaZPHHl>*MvxU~EyQmGF$c^u*BP6eIAy>|td^lK z9+utzK4I}DMZh3y^Z_BazzX4%yLT1+*?mv;^@G)u31xg3+Cy0Q;r})$i5vmmd`$Pp z9C33w;-j^f1_SqZx<8*rT7}{P#Bsr&)aVa6bW39j@F-uw#V2(?rVwQ#~ zqe)-#ELaHA>dhfM|9j#1qe}Ex4ObtN58OTavyK-dDkH%uv|9W{`)pp$?KhjMlGS}T zSC@zJfO8ez%kCB0H^@{%KzG1@3y8nZyKL9k z5(~&<4sPj2X~CeYPF;*&H2XEf+2Jocn@Nn+^6&659;!-f+__t7Pr&(8emXj`znqj!_;Bpivxn zB6_ISLFWLo)Z+rtL~W!&;QU3-dn(mRSJze0kUd^$_~NrOsa-q(bTT5NncqH#`O@h@ zz6ljLKtf*7=41*)Bw|J$O*-e-5rZwQHQ@_yZk;+1m(VmUoD63MKAWB6V|{_hdykpV zC?|{KSM!3V~6LY)NeH&axP5QBEW-u1NWzg-4i_!VQ=#W9Bgz;5kKW#e$_BR&NM*=^KTD6OD}u#@}tPM zo=4DneNNiwI(Do2Ju~q&?%l2fiA>eN&KwD0uS3O9*mXMjnDWk*@^a*X>H3_!~2K4w6-`O z3ZqUbnmTK9E}*h?mAYaGam)TiP5O6BQ6+@_nEwR@Pv{-inrMHHV)e7&#{<|5qTn@b zIyF24@(gy*#jheiaL#q7geqT@oQ!CICYoZ8u+1z@YH3CRMDVD3;WBlD^l$4y4s5}_ zlpJp>X}Mh59S|j+02n*=J_!9uv@IPo7x3% z$moOqueapw+_kbb4WuVwqJ?pOeBIJ-CMt0|Cnld0Y)<3K-;NZWCYM{nmx4RZz2yA) zQMJSqe+&W8+SJ@tvd7JvzT>8kg7P6*aUq%MuFh0?X?5@mn+*cW3uympVWOgNA;&pF z7cCJeB-=F&IhL0}Ms~{nW60M$(z@hsHn6Z9N8iRBJ!rTD4W-Vq=Y$eYBVFG!3yM^s-2lmVQFE2SROa3k-7m3! z4Hj-Rx?%qd@HAmv97Zy2CE5l1<}AB~ljPE!K`dDgeF(SxP6e4zXT=?fqOtNrXZs?o zrOO2x=6yM<`dvKC{t-*&)Up#g<)%V*6f6hpdN~T~LyRV0CSGhF(u0f%^QQre0X-wV zKTT;_t8Z6}o%nOnUcKoDE!#@(VXxK$h6yubEr#3Ps?@o$SB(r_*02?%^-NeNUr7+Q z#kpll*jutzi*Zp_k>gofH$rMub^Y05a(-xHJu4*FTOX(IzNaF|caAPvR7&yywe59A zwq+)@gF%>Rt47pXPNs|gxG)7MPBGsM>j21JEzo5Gd*`J^+Ff=)$z|0_1e$(AL9Yj z*4U!7PHZ6dNObD?s;^nZ%_ys3IWb9sl-cYxAx-m04IUAHQC?)z1G7n(<6?9JuS z*L>0`(HZF%(A**#wrllUu79Bo9c~-~EO|un5!tWNiMrmch&4A-%e;e?$)6#@QvCrT7pv9U)xD_L{>59T8L;i56O68V z7W2Jl@b?yr=^SJIzBg6(9c44R_HF~o75aQGWMOA{4cP?+^JNtg1{*X-cw6g|Bori0fSvdLu0agUV%y;mzde7kis% z;ZP#4yFLsB<_Kk}M76zaN-F|kl|l{8%J*%F+<3Td<4$mEMIy2HZXxH9lutsb@*gM{ z5)fi>?%|AS(Tt9Qx1tbhQ& zuuWB%JcGIb*Ss2)l4hJl%TR^vpq>rk=~zH&&OHD|BTtJ*Zmdy$_Tgk$Ie-$Ru>qSL z?UJiWW8pJT&Aib1(EOEUlxx`0Spl8T?f|^@?vYR+^udTND0vLj+eV#p? z6EP{2?^Mm+u!3Xisi}>y9E(TX`MQ{J`115|a+zfQ|k`E2|}9!M4ggdkJo0wzZwYWZ6YK_5$R@uN~04Ri!r>BZVtD04+MGNvj> z3T4Nq&1=4H&8qCRzeLqpKkT>b_rhLp);@8uK1W|%HQa3oWqwk=U&tu2_-TQsTl(!kZ4+6m z_z%Ss2j7YF_7<*z(lo4a8QCPpQE6wcAsH$F@Ab;*n}<8JWi0HLN5GYCLN$&vKM}Me z45n?=u$%0P2TYa`B;4r&`W02|D=2qPKC;e3pzo!J-f*;u-L>w4(d`QOn=RHT*G2k0DjOw z<7>siE#j^?=EmYNI;RkNGStKTu(aRy%AFYzF8OX5w_VPm0&AJ@ZU^(ptPGggRO2l8 zez1Sn-e36{_&YcMnkRUG?~62Nr|_rYZLFqG*<#_63?}`FG-b0+Dvkr69M)|-+wlM| z!ldp#(8^0+YN*!Vo<*ov)qo%n#w&osKq&4d@J#Xn!RKt)mY6k9<>g-JiP6QjK?eJA zWa0gmv6-_sKmI%moF_ELWRHu2(*vbUj|v|tWBRX!slF-wq1I_TR#WU?wiyxxcOhpc z-wvvdAY!Dq|JyLdq)>qDIj9Hc(`(}3Rp^m{m<&|m=)m)_>xT1ijSSOyH9wX)u$o3; zh3+m>cV&=@J0%|Ap%mYs`0lWFgu`X34s}j|JZZSaDa3Fg653X88-wRx+H}EnJ@-ziCR(!ehFi90F#!l8LGnI7vWGE%&*8U`t|h zzC|Ei;^hCR`2YM8B_Lx9_qr$Oyy~Av_09nFP{K2*e%C76S4;h-xy%m5uL}fI2*GG9 zc|5;w{{`j)2A+kDz?9c4y|AP30IeO&QqJ2ng?T@D9K&!bpQI`}|Jlv6ux+BER<jR4dPUrRh1^gr9?S=G zmHXx;iN(9d(aw)FZcbM20Nqb%lw@Z2}&;)+SbVYJ3Z~F{pQo8;& RCQ$TWzH|HwPnGav{{<%Q8IS+~ literal 0 HcmV?d00001 diff --git a/blueprints/crawlab-worker/docker-compose.yml b/blueprints/crawlab-worker/docker-compose.yml new file mode 100644 index 00000000..7f147afa --- /dev/null +++ b/blueprints/crawlab-worker/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3.8" +services: + crawlab-worker: + image: crawlabteam/crawlab:latest + restart: unless-stopped + environment: + - CRAWLAB_NODE_MASTER=N + - CRAWLAB_GRPC_ADDRESS=${MASTER_GRPC_ADDRESS} + - CRAWLAB_FS_FILER_URL=${MASTER_FILER_URL} + volumes: + - crawlab-worker-data:/root/.crawlab + - crawlab-worker-app:/data + deploy: + resources: + limits: + cpus: "${CPU_LIMIT:-0.8}" # 0.8 will limit to 80% of CPU if one core. If 2 cores, you need to put 0.8*2=1.6 + +volumes: + crawlab-worker-data: + crawlab-worker-app: diff --git a/blueprints/crawlab-worker/template.toml b/blueprints/crawlab-worker/template.toml new file mode 100644 index 00000000..ba9e53d1 --- /dev/null +++ b/blueprints/crawlab-worker/template.toml @@ -0,0 +1,7 @@ +[variables] +MASTER_GRPC_ADDRESS = ":9666" +MASTER_FILER_URL = "http://:8080/api/filer" +CPU_LIMIT = "0.8" + +[config] +env = ["MASTER_GRPC_ADDRESS=${MASTER_GRPC_ADDRESS}", "MASTER_FILER_URL=${MASTER_FILER_URL}", "CPU_LIMIT=${CPU_LIMIT}"] diff --git a/meta.json b/meta.json index 089aa9d6..56c98349 100644 --- a/meta.json +++ b/meta.json @@ -1752,6 +1752,42 @@ "API" ] }, + { + "id": "crawlab-master", + "name": "Crawlab (Master)", + "version": "latest", + "description": "Crawlab is a distributed web crawler admin platform supporting multiple programming languages and frameworks. This template deploys the master node with MongoDB.", + "logo": "crawlab.jpeg", + "links": { + "github": "https://github.com/crawlab-team/crawlab", + "website": "https://crawlab.cn", + "docs": "https://docs.crawlab.cn" + }, + "tags": [ + "crawler", + "scraping", + "distributed", + "self-hosted" + ] + }, + { + "id": "crawlab-worker", + "name": "Crawlab (Worker)", + "version": "latest", + "description": "Crawlab worker node for distributed web crawling. Connects to an existing Crawlab master node via gRPC.", + "logo": "crawlab.jpeg", + "links": { + "github": "https://github.com/crawlab-team/crawlab", + "website": "https://crawlab.cn", + "docs": "https://docs.crawlab.cn" + }, + "tags": [ + "crawler", + "scraping", + "distributed", + "self-hosted" + ] + }, { "id": "crowdsec", "name": "Crowdsec",