feat: add crawlab (#759)

* feat: add crawlab

* feat: add crawlab - re order

* feat: add crawlab - fix compose

* feat: split crawlab into worker and master

blueprints/crawlab-master/docker-compose.yml

@@ -0,0 +1,41 @@
+version: "3.8"
+services:
+  crawlab-master:
+    image: crawlabteam/crawlab:latest
+    restart: unless-stopped
+    environment:
+      - CRAWLAB_NODE_MASTER=Y
+      - CRAWLAB_MONGO_HOST=mongo
+      - CRAWLAB_MONGO_PORT=27017
+      - CRAWLAB_MONGO_DB=crawlab
+      - CRAWLAB_MONGO_USERNAME=${MONGO_USERNAME}
+      - CRAWLAB_MONGO_PASSWORD=${MONGO_PASSWORD}
+      - CRAWLAB_MONGO_AUTHSOURCE=admin
+    volumes:
+      - crawlab-master-data:/root/.crawlab
+      - crawlab-app-data:/data
+      - crawlab-logs:/var/log/crawlab
+    expose:
+      - "8080"
+      - "9666"
+    depends_on:
+      - mongo
+    deploy:
+      resources:
+        limits:
+          cpus: "${CPU_LIMIT:-0.8}" # 0.8 will limit to 80% of CPU if one core. If 2 cores, you need to put 0.8*2=1.6
+
+  mongo:
+    image: mongo:4.2
+    restart: unless-stopped
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME}
+      - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD}
+    volumes:
+      - mongo-data:/data/db
+
+volumes:
+  crawlab-master-data:
+  crawlab-app-data:
+  crawlab-logs:
+  mongo-data:

blueprints/crawlab-master/template.toml

@@ -0,0 +1,12 @@
+[variables]
+MONGO_USERNAME = "crawlab"
+MONGO_PASSWORD = "${password:24}"
+CPU_LIMIT = "0.8"
+
+[config]
+env = ["MONGO_USERNAME=${MONGO_USERNAME}", "MONGO_PASSWORD=${MONGO_PASSWORD}", "CPU_LIMIT=${CPU_LIMIT}"]
+
+[[config.domains]]
+serviceName = "crawlab-master"
+port = 8080
+host = "${domain}"

blueprints/crawlab-worker/docker-compose.yml

@@ -0,0 +1,20 @@
+version: "3.8"
+services:
+  crawlab-worker:
+    image: crawlabteam/crawlab:latest
+    restart: unless-stopped
+    environment:
+      - CRAWLAB_NODE_MASTER=N
+      - CRAWLAB_GRPC_ADDRESS=${MASTER_GRPC_ADDRESS}
+      - CRAWLAB_FS_FILER_URL=${MASTER_FILER_URL}
+    volumes:
+      - crawlab-worker-data:/root/.crawlab
+      - crawlab-worker-app:/data
+    deploy:
+      resources:
+        limits:
+          cpus: "${CPU_LIMIT:-0.8}" # 0.8 will limit to 80% of CPU if one core. If 2 cores, you need to put 0.8*2=1.6
+
+volumes:
+  crawlab-worker-data:
+  crawlab-worker-app:

blueprints/crawlab-worker/template.toml

@@ -0,0 +1,7 @@
+[variables]
+MASTER_GRPC_ADDRESS = "<master_node_ip>:9666"
+MASTER_FILER_URL = "http://<master_node_ip>:8080/api/filer"
+CPU_LIMIT = "0.8"
+
+[config]
+env = ["MASTER_GRPC_ADDRESS=${MASTER_GRPC_ADDRESS}", "MASTER_FILER_URL=${MASTER_FILER_URL}", "CPU_LIMIT=${CPU_LIMIT}"]

meta.json

@@ -1752,6 +1752,42 @@
       "API"
     ]
   },
+  {
+    "id": "crawlab-master",
+    "name": "Crawlab (Master)",
+    "version": "latest",
+    "description": "Crawlab is a distributed web crawler admin platform supporting multiple programming languages and frameworks. This template deploys the master node with MongoDB.",
+    "logo": "crawlab.jpeg",
+    "links": {
+      "github": "https://github.com/crawlab-team/crawlab",
+      "website": "https://crawlab.cn",
+      "docs": "https://docs.crawlab.cn"
+    },
+    "tags": [
+      "crawler",
+      "scraping",
+      "distributed",
+      "self-hosted"
+    ]
+  },
+  {
+    "id": "crawlab-worker",
+    "name": "Crawlab (Worker)",
+    "version": "latest",
+    "description": "Crawlab worker node for distributed web crawling. Connects to an existing Crawlab master node via gRPC.",
+    "logo": "crawlab.jpeg",
+    "links": {
+      "github": "https://github.com/crawlab-team/crawlab",
+      "website": "https://crawlab.cn",
+      "docs": "https://docs.crawlab.cn"
+    },
+    "tags": [
+      "crawler",
+      "scraping",
+      "distributed",
+      "self-hosted"
+    ]
+  },
   {
     "id": "crowdsec",
     "name": "Crowdsec",