From fe94f4ce0b30f049a329736fcb4cc971e71c0861 Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Tue, 14 Jul 2026 16:15:50 -0600 Subject: [PATCH] feat: add n8n queue mode template n8n running in queue mode (EXECUTIONS_MODE=queue): main instance for the editor/webhooks, a scalable pool of workers (deploy.replicas driven by the N8N_WORKER_REPLICAS env var), Redis (Bull) as the queue broker and PostgreSQL as the database. Pinned to the current stable n8nio/n8n:2.30.4. Closes #455 Co-Authored-By: Claude Fable 5 --- blueprints/n8n-queue/docker-compose.yml | 124 ++++++++++++++++++++++++ blueprints/n8n-queue/instructions.md | 24 +++++ blueprints/n8n-queue/meta.json | 18 ++++ blueprints/n8n-queue/n8n.png | Bin 0 -> 6592 bytes blueprints/n8n-queue/template.toml | 33 +++++++ 5 files changed, 199 insertions(+) create mode 100644 blueprints/n8n-queue/docker-compose.yml create mode 100644 blueprints/n8n-queue/instructions.md create mode 100644 blueprints/n8n-queue/meta.json create mode 100644 blueprints/n8n-queue/n8n.png create mode 100644 blueprints/n8n-queue/template.toml diff --git a/blueprints/n8n-queue/docker-compose.yml b/blueprints/n8n-queue/docker-compose.yml new file mode 100644 index 00000000..f0b601d1 --- /dev/null +++ b/blueprints/n8n-queue/docker-compose.yml @@ -0,0 +1,124 @@ +services: + postgres: + image: postgres:17-alpine + restart: unless-stopped + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + start_period: 30s + interval: 10s + timeout: 5s + retries: 5 + + redis: + image: redis:7-alpine + restart: unless-stopped + command: redis-server --requirepass ${REDIS_PASSWORD} + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD} + volumes: + - redis_data:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_PASSWORD\" ping | grep PONG"] + start_period: 20s + interval: 10s + timeout: 5s + retries: 5 + + # Main instance: serves the editor UI and webhooks, publishes executions + # to the Redis (Bull) queue instead of running them itself. + n8n: + image: n8nio/n8n:2.30.4 + restart: unless-stopped + environment: + # PostgreSQL + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PORT=5432 + - DB_POSTGRESDB_DATABASE=${POSTGRES_DB} + - DB_POSTGRESDB_USER=${POSTGRES_USER} + - DB_POSTGRESDB_PASSWORD=${POSTGRES_PASSWORD} + + # Queue mode (Redis / Bull) + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - QUEUE_BULL_REDIS_PORT=6379 + - QUEUE_BULL_REDIS_PASSWORD=${REDIS_PASSWORD} + - QUEUE_HEALTH_CHECK_ACTIVE=true + - OFFLOAD_MANUAL_EXECUTIONS_TO_WORKERS=true + + # Encryption key shared between the main instance and the workers + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} + + # Networking + - N8N_HOST=${N8N_HOST} + - N8N_PORT=5678 + - N8N_PROTOCOL=http + - N8N_PROXY_HOPS=1 + - NODE_ENV=production + - N8N_WEBHOOK_URL=https://${N8N_HOST}/ + - N8N_EDITOR_BASE_URL=https://${N8N_HOST}/ + - GENERIC_TIMEZONE=${GENERIC_TIMEZONE} + - N8N_SECURE_COOKIE=false + volumes: + - n8n_data:/home/node/.n8n + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget --spider -q http://127.0.0.1:5678/healthz"] + start_period: 60s + interval: 10s + timeout: 5s + retries: 10 + + # Queue workers: pull executions from Redis and run them. Scale them by + # changing the N8N_WORKER_REPLICAS environment variable and redeploying. + n8n-worker: + image: n8nio/n8n:2.30.4 + restart: unless-stopped + command: worker + deploy: + replicas: ${N8N_WORKER_REPLICAS:-2} + environment: + # PostgreSQL + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PORT=5432 + - DB_POSTGRESDB_DATABASE=${POSTGRES_DB} + - DB_POSTGRESDB_USER=${POSTGRES_USER} + - DB_POSTGRESDB_PASSWORD=${POSTGRES_PASSWORD} + + # Queue mode (Redis / Bull) + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - QUEUE_BULL_REDIS_PORT=6379 + - QUEUE_BULL_REDIS_PASSWORD=${REDIS_PASSWORD} + - QUEUE_HEALTH_CHECK_ACTIVE=true + + # Must be the same key as the main instance + - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY} + + - NODE_ENV=production + - GENERIC_TIMEZONE=${GENERIC_TIMEZONE} + depends_on: + n8n: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget --spider -q http://127.0.0.1:5678/healthz"] + start_period: 60s + interval: 10s + timeout: 5s + retries: 10 + +volumes: + postgres_data: + redis_data: + n8n_data: diff --git a/blueprints/n8n-queue/instructions.md b/blueprints/n8n-queue/instructions.md new file mode 100644 index 00000000..782571b8 --- /dev/null +++ b/blueprints/n8n-queue/instructions.md @@ -0,0 +1,24 @@ +# n8n Queue Mode + +This template runs n8n in [queue mode](https://docs.n8n.io/hosting/scaling/queue-mode/): + +- **n8n** (main): serves the editor UI and receives webhooks/triggers, then publishes executions to the queue. +- **n8n-worker** (x2 by default): pulls executions from the Redis (Bull) queue and runs them. +- **redis**: the message broker for the queue (password protected). +- **postgres**: stores workflows, credentials and execution data. + +All n8n containers share the same `N8N_ENCRYPTION_KEY` (generated automatically), which is required for workers to decrypt credentials. + +## Scaling workers + +Worker replicas are controlled by the `N8N_WORKER_REPLICAS` environment variable (default `2`): + +1. Open the compose service in Dokploy and go to the **Environment** tab. +2. Change `N8N_WORKER_REPLICAS` to the number of workers you want. +3. Redeploy. Docker Compose will scale the `n8n-worker` service to the requested number of replicas. + +Each worker runs up to 10 concurrent executions by default. You can tune this by adding `N8N_CONCURRENCY_PRODUCTION_LIMIT` to the worker environment. + +## Optional: dedicated webhook processors + +For very high webhook volumes, n8n supports dedicated webhook processor instances (`command: webhook`). To use them, add another service to the compose file with the same environment as `n8n-worker` but with `command: webhook`, and route the `/webhook/` path of your domain to it. See the [n8n queue mode docs](https://docs.n8n.io/hosting/scaling/queue-mode/#webhook-processors) for details. diff --git a/blueprints/n8n-queue/meta.json b/blueprints/n8n-queue/meta.json new file mode 100644 index 00000000..e0ada259 --- /dev/null +++ b/blueprints/n8n-queue/meta.json @@ -0,0 +1,18 @@ +{ + "id": "n8n-queue", + "name": "n8n Queue Mode", + "version": "2.30.4", + "description": "n8n running in queue mode for high-throughput production workloads: the main instance handles the editor and webhooks while a scalable pool of workers executes workflows from a Redis (Bull) queue, backed by PostgreSQL.", + "logo": "n8n.png", + "links": { + "github": "https://github.com/n8n-io/n8n", + "website": "https://n8n.io/", + "docs": "https://docs.n8n.io/hosting/scaling/queue-mode/" + }, + "tags": [ + "automation", + "workflow", + "low-code", + "queue" + ] +} diff --git a/blueprints/n8n-queue/n8n.png b/blueprints/n8n-queue/n8n.png new file mode 100644 index 0000000000000000000000000000000000000000..0e9a607e2494055b97bfd12878f496a6aa6e749f GIT binary patch literal 6592 zcmeI1*H=?fxAsvGK|pMDDX%mELy;ymqSAYn7LX=lC{jZ2)zBe=LMYOtgY*)Lw8Ve} z>BRs-fFLbE=p}r4zyIKjaW2orzFBMRwf30%nR7kAxu6Dm>I}EoZ&6TCFlcJLGNPck z#`@of<~n(VW-Ig$1qEB2<|`Ew|D3J)ApZxeoi}&q+qOes4B+b#tduko_e8%LKFE6< zX~d)gsE#j*3ly2el}|9&s)G{-0vpH9ep_y<0GKLQ|CP5a-zd4+CbZg~7Rgua(yGCp zz9j&k&z;Ru(!ZytC6Setd+PQU77WzCdcP2I445ZiwiF2GL2by&%o&Ow50M+!DCnta z?onK$q7A2rq1nD^-r^=htinIN30b`=*Ty&NVh<6^}V<8t5a zdhM(^6j|IUUDqO`9i+VZd6X@JM+LYncYKkBvB+>$$%}X`$jIqp)g}}0Rhg1kl1?Qu zv^PoY2+$V6Vfv;%f2JLLQg)|(1-907_9NDXIfm}}Kn7HMi#@9uFO6JW z3s~71e$A;M1Khjjf`+WqA3z-Fq9T|US202b8^0B)XrB%dJzlrf!c`UkxZ@SluJ6&W zr4=QPC>|B+wl%Z1Zh6U(Iot~v`XRKzm5WL$k8$hXK%VMrrvt#un;Jrp>@4dh z5IL1XK`od^T##6-!AYcPzBo0iABxa(aZxzzHerXC z7`UM-33SGS{>;NmYO9rgz5bw@HdD^p4zT<0Z-1w5ugi&bhtr-&vqO#iXDg1JZ4>~i zS7VM=kRg!1nr*NGlXjWkTJ~!vTy=*qkH&NTFN%5%x~MtE0k_Gy5KVM)6ZfmvwZz@x}u^7q9r8K&J5{9A7f<}B|LTMeXw&&-Sjp2ctnCSW152f-&fYk4x2H-wZ5X-6 zQ9iXh@;Qa3;=+_xeb~pHI#TM{tWr&mQ$hnkHJ1hLt#&}Cu)^_aP9)xsa}z31Cw6N1Oc z;SJkB^VNu21H-`-vCOe2`uH44-H<3)EAhD{=U8v`{2N}AOi$IhCR}j{M35P{(7c?n z5)s(@VAi$S1PnhZM;eb}SN&)*oX*ZyMnn!56km7h+T%noIn!UJQgh%FZwf)~MI46^1Q6?-9_?+&8gg|5xz1pq9( zpB42PB0$wM)wc{a;S=Ttr{9+0W8|#+&gC;Q^Cdigd(Nr4|9-Ro3-{!o zVOQ33wu``&XVF?oHz3gJ0@5kqE{K?1(Z>ZxNJUW33IoQ`I5%>rEU)A~tVQc>ABn{a@ivl#>cKY3@vS z2oDEkh-tYps7`l?IPZrGHy8g#yh}DnJ33lROa`2suL@VPt>A`-Tb>N{pC2XLd2w}u zz7&GGdlGl+P21lC9nCXs(xL~S5|%gVfp9--b5WB|$i?&4%kD*pXa^?{I}w zvn%c@ckx0Q|sWu6FE z!NQLf8y?C}naxw@uLn_Z$UC^zYuZNVfib(4CBfFkKPbR9>}-Hlr0m!5hDTX@VCHPMSR$a8S3*#I_1&C?JF0X#VX73a;fG9Msu@mHJM&J1oAG?R_2p0<#e2eI}Xk zy82UfW1~etf9@w&t33oc>;fLdWe(p)>}hms2>1NYcaGYrigdL+S6#pi(BDhO`6_SS zK^N?_ZRWqa(@aq*3y*O0Vfp7AmX?@b2l_9tX7e2s<87WxiSI zus#N!Y~naj#R;_2SxYY3=%MyjY8eLG?jag&29f~#SaA`DD0wKu+SyJHw*@JEiR)p5 z@Fa&%N7v{F{HNmL)@np;sDgb+B!nvM+9O<*AkR#-a3fYOt`+cS_I%H z>}YEJVs_iA-5|(Sw|YrEZ`0dV#!46~8oFQJ*bAuWM+OT*j2x?%6XvJ z1oY=c?y6XT!^*@>9bApd)JAUXnvAk6t_`VG zazpuba_?MyJx=MG$JrAp$<}b*S#v#5+c3Ub7svNV962_*NPpiNVeLfcX%Bw9KSM_? zzb2T74!O|tjvAi|o`z8~dmfL>1iqT?fV6VJw=OSjaO^3N@FU2;%!;IZ&cb8B z(KN@OTmLbg*1E~@RJ_oJT(19V;NK{U{~F{e7pphv-PXLn`sALKB%)opE+A!TI^OEs zAx%X%lSa(SX{l!EKLdk264&wvZDG>ix|(5-tji#KPmhLd>%BbZ?>IQ6 zZrQX9+Gb?>O6H2_R=>5W(UjRUS6h7^%wG7w<|wk&JM8V^4ZSo#f?0%tmYI=Rid|XV zQ;l--7pt{8KfvF-5BVk6>e}8Pmu!5Y=RSAxlk?&_69B4A6Cd-~Ii*ezx+Y|Ayt6kQ zxe`wqZ+X!4I*aQM+Ku;E)N!9OE0lNrty5}x#pZ*o@O%#NMc>93`JP?y;T@d=f=209 zlby~Fg6$atM-<|i^k%iYKDgB|#P2g0BjupvjY~Lf7?F|}1w}(0qzAW;A^<`w+9QKM zJ_h!DW=5E_yry8%{!F8(=^55J$n1_!jGt+*b`fh`kt1*tC*h6QOXJfS+Cv#q3Ekw- zRDegfKydd0C!3%|qTe25L${yvoL9eU=TP6b^aqvqY5hHN{%&k02KqDq(z^&yN8)MY z6cbtu#Y0oR)<=J`i$>#$H|83ZZ$>a_PrP;;Lcxo~x}0F?8&`D*s<@4`GqTqQl;Bok zjQ!ou>WO1wx4L%&8y0VSinz7NjJ(s^Ujwt8AkH=vf)CkwL@~1V>`z`A?n7S zdvsxMqZ0-%k3(Kw>Q!0o|Gs+}bAuK+WvV%Ic_?yurJTR#GEMuQ%sBJ9)5{pjixc~# z`_%HwV>KgV8_Z+rH&hSU1PHT1v%QiBjk?hE##+&U?<&eujxZ*~Vjok=9gLWR&vd<3 z?V5bwO7(?uC5{iRSN~+EQhTSCz{bJxYgLVf3~2I0rTxH}g`)$JK$Pz-#3Jk%ODW9s zCY>P(uo#McKGH5u55w6JiCfpW$4z&iFzG+mwdP;hDP|>Na z%~$W5ah5Fv*{>5gZu(p%r7>`SY$q!A9LQv=jBGg(>GA-HnB?>^Q*Q_7Ga9nCz&Mn~ z$o@;(Ssr)%GfW|Kr~B%88k5RK$LL~^!DA6L_LN)3F#h61b@sO>{a7A**TECt{_phk zl?jhyK3^eMpql08y`H15z7d$hx4vU%INpL^J?--fUIhIGNyTvns9Ayqwyuxl8!u-| z&I^d3&qi)8P6~miQZ&%J4q|0u>`(Q~dWUQ@T8{dFLRu?zremdXAJr#y&3g5#GLP>3 zqAK&~vA?zd#ZK<+ncuG}`YP*+25RHlt?L8}DM63Pm)Gan9$GxIs?liPiMb6w-gQX$ zSgT$5fHW%H0QXM!vLAORZAn7^{}9@vEO~G_{ub3_h0d}qse-lAZbW>+3Ln5SY$(R? z6$bj7D=FMsrwRoGxy9Re^|_|%lH-uVg5&V3>v;&%?hg8ig7>8;>U&FQ4({n9jME-l zDOxvRifZbln_|Ou!7hx$giH)$*~oTPcHtP0MNq#+ghK3|7-+rY;<|Cv`E~adHZmG6 z7tKfUYR@w9Sxtw0BHIP`cPk^?-l7_%WMF*+3^kOisrx2cUB}NjAxP?gab^zXmUt@W zwUf=$BcSCZ80yexRKJy1+5F1X2gI*z+)z6o#I8m$jKRjOj z>f(WUeXYG@Momy4Ww*7gb6e~^)Qp)I+J&KL z@fq@>PhN{<+QXPUkDZOmWSZTzh1oarmpu7nFhiyC-}J{=t)`Q5S_3?S`;rsQl+hu9 z(6c2LfFH0rjKWpZ?r%9E^+MEW`fR@3c;JYduY8`o0h1htRfX z34fuKjJ9+f?ELEOl4Ly0L4*rKt;GKm zu}a$F0>ydyz?e?`8@vjeh{jC2-n@}AW`vyY*oiqWeAwZiXs~o!eWMhw!Qo;H+6*HJ@Adeen!^Bl zweVb6J@!KfQU7c$sdbbWx7mCfd$Y-c_?%foLvxtc9l5zL1}a-X$~71fy^0$fo0Lzq zhoPu`_4X;*r#37=o9Vp8Ei&iJkUvuD&xa!QwI^0s{EM4;ez_$xFcbnC~Ee|ux-g-n7sw9h5rmLKt3(&pE7uX zf)J#(MW*EiQW_2~&y&!5UPbCh>UIb8&r_)j`qdHlBr)R+wyhkvV{v@))M6O9kHV0s zC3iQudU2SBExdLT*ow>>6Z)7>A zv-4mp;X#GO#oS5{^F4fGL-Ue!4JT@0iEU1U%r8=@ zB|Y(^skl7;E)-0M-}}3}w0bJF+<*Vdm~(L_l7P~4@XKz0Hv^dtVmB8D%bo}&0je1< zBE`6I*;Y<(i~MCJgG9HvSSkZ$K4BFNz@KfFwCJ5DCnE@6MxuC3HvXJrUg7PhQ3J)-4`&Dha$3+-hc7AeST zgMi6zzzc&t1v7lsKL)1G^h$Ol)VvQPK`sdnez&^#*_~;~ZqHCBhsgu*Yp&r^f}LDf5c5eUw#-6>KVbdi?4!KQUJt%?kk2q8nK&ec znS9_+*q(0f4lHlQg*OsXeZDGBj%B7Z3H-$&ndLBI-a7CyB2UVC_BHG8*Hj{v(GYbq z-77fjPr9P1F362dj0G}5L#u$jqJ7w`r<(JiJY*8iHkYs`_DaZp-M^_Sl1C-qXw=+s zR(SsqS4s=4vvhGquUj2ZQqfNEfp5p7P4wN*EiWfMV`XRk=-7- z6r37?RAY_nUXf3Ba#bcvZtbQs0~Kwe!#*)O-Y7I!LP0p6cJ2GMD|C^yqpAz^XA%G% z#b95iHg`Gu1~B=XQsuglUJs_8@~oKGK^%OuczHP-czf!OqWb?e=l|C(B_giqC@8M3 ZQeyfU3uE)l$Tu?-nyPxQYF@mF{C_OaY>@x} literal 0 HcmV?d00001 diff --git a/blueprints/n8n-queue/template.toml b/blueprints/n8n-queue/template.toml new file mode 100644 index 00000000..46ce3fbe --- /dev/null +++ b/blueprints/n8n-queue/template.toml @@ -0,0 +1,33 @@ +[variables] +main_domain = "${domain}" +postgres_user = "${username}" +postgres_password = "${password:24}" +postgres_db = "n8n" +redis_password = "${password:24}" +n8n_encryption_key = "${base64:32}" + +[config] +mounts = [] + +[[config.domains]] +serviceName = "n8n" +port = 5_678 +host = "${main_domain}" + +[config.env] +N8N_HOST = "${main_domain}" +GENERIC_TIMEZONE = "Europe/Berlin" + +# Number of queue workers. Change it and redeploy to scale. +N8N_WORKER_REPLICAS = "2" + +# PostgreSQL +POSTGRES_USER = "${postgres_user}" +POSTGRES_PASSWORD = "${postgres_password}" +POSTGRES_DB = "${postgres_db}" + +# Redis (Bull queue) +REDIS_PASSWORD = "${redis_password}" + +# Encryption key shared by the main instance and the workers (IMPORTANT) +N8N_ENCRYPTION_KEY = "${n8n_encryption_key}"