diff --git a/blueprints/crawlab-master/crawlab.jpeg b/blueprints/crawlab-master/crawlab.jpeg new file mode 100644 index 00000000..1aef3cb9 Binary files /dev/null and b/blueprints/crawlab-master/crawlab.jpeg differ diff --git a/blueprints/crawlab-master/docker-compose.yml b/blueprints/crawlab-master/docker-compose.yml new file mode 100644 index 00000000..8b4f58af --- /dev/null +++ b/blueprints/crawlab-master/docker-compose.yml @@ -0,0 +1,41 @@ +version: "3.8" +services: + crawlab-master: + image: crawlabteam/crawlab:latest + restart: unless-stopped + environment: + - CRAWLAB_NODE_MASTER=Y + - CRAWLAB_MONGO_HOST=mongo + - CRAWLAB_MONGO_PORT=27017 + - CRAWLAB_MONGO_DB=crawlab + - CRAWLAB_MONGO_USERNAME=${MONGO_USERNAME} + - CRAWLAB_MONGO_PASSWORD=${MONGO_PASSWORD} + - CRAWLAB_MONGO_AUTHSOURCE=admin + volumes: + - crawlab-master-data:/root/.crawlab + - crawlab-app-data:/data + - crawlab-logs:/var/log/crawlab + expose: + - "8080" + - "9666" + depends_on: + - mongo + deploy: + resources: + limits: + cpus: "${CPU_LIMIT:-0.8}" # 0.8 will limit to 80% of CPU if one core. If 2 cores, you need to put 0.8*2=1.6 + + mongo: + image: mongo:4.2 + restart: unless-stopped + environment: + - MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME} + - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD} + volumes: + - mongo-data:/data/db + +volumes: + crawlab-master-data: + crawlab-app-data: + crawlab-logs: + mongo-data: diff --git a/blueprints/crawlab-master/template.toml b/blueprints/crawlab-master/template.toml new file mode 100644 index 00000000..e1bf5524 --- /dev/null +++ b/blueprints/crawlab-master/template.toml @@ -0,0 +1,12 @@ +[variables] +MONGO_USERNAME = "crawlab" +MONGO_PASSWORD = "${password:24}" +CPU_LIMIT = "0.8" + +[config] +env = ["MONGO_USERNAME=${MONGO_USERNAME}", "MONGO_PASSWORD=${MONGO_PASSWORD}", "CPU_LIMIT=${CPU_LIMIT}"] + +[[config.domains]] +serviceName = "crawlab-master" +port = 8080 +host = "${domain}" diff --git a/blueprints/crawlab-worker/crawlab.jpeg b/blueprints/crawlab-worker/crawlab.jpeg new file mode 100644 index 00000000..1aef3cb9 Binary files /dev/null and b/blueprints/crawlab-worker/crawlab.jpeg differ diff --git a/blueprints/crawlab-worker/docker-compose.yml b/blueprints/crawlab-worker/docker-compose.yml new file mode 100644 index 00000000..7f147afa --- /dev/null +++ b/blueprints/crawlab-worker/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3.8" +services: + crawlab-worker: + image: crawlabteam/crawlab:latest + restart: unless-stopped + environment: + - CRAWLAB_NODE_MASTER=N + - CRAWLAB_GRPC_ADDRESS=${MASTER_GRPC_ADDRESS} + - CRAWLAB_FS_FILER_URL=${MASTER_FILER_URL} + volumes: + - crawlab-worker-data:/root/.crawlab + - crawlab-worker-app:/data + deploy: + resources: + limits: + cpus: "${CPU_LIMIT:-0.8}" # 0.8 will limit to 80% of CPU if one core. If 2 cores, you need to put 0.8*2=1.6 + +volumes: + crawlab-worker-data: + crawlab-worker-app: diff --git a/blueprints/crawlab-worker/template.toml b/blueprints/crawlab-worker/template.toml new file mode 100644 index 00000000..ba9e53d1 --- /dev/null +++ b/blueprints/crawlab-worker/template.toml @@ -0,0 +1,7 @@ +[variables] +MASTER_GRPC_ADDRESS = ":9666" +MASTER_FILER_URL = "http://:8080/api/filer" +CPU_LIMIT = "0.8" + +[config] +env = ["MASTER_GRPC_ADDRESS=${MASTER_GRPC_ADDRESS}", "MASTER_FILER_URL=${MASTER_FILER_URL}", "CPU_LIMIT=${CPU_LIMIT}"] diff --git a/meta.json b/meta.json index 089aa9d6..56c98349 100644 --- a/meta.json +++ b/meta.json @@ -1752,6 +1752,42 @@ "API" ] }, + { + "id": "crawlab-master", + "name": "Crawlab (Master)", + "version": "latest", + "description": "Crawlab is a distributed web crawler admin platform supporting multiple programming languages and frameworks. This template deploys the master node with MongoDB.", + "logo": "crawlab.jpeg", + "links": { + "github": "https://github.com/crawlab-team/crawlab", + "website": "https://crawlab.cn", + "docs": "https://docs.crawlab.cn" + }, + "tags": [ + "crawler", + "scraping", + "distributed", + "self-hosted" + ] + }, + { + "id": "crawlab-worker", + "name": "Crawlab (Worker)", + "version": "latest", + "description": "Crawlab worker node for distributed web crawling. Connects to an existing Crawlab master node via gRPC.", + "logo": "crawlab.jpeg", + "links": { + "github": "https://github.com/crawlab-team/crawlab", + "website": "https://crawlab.cn", + "docs": "https://docs.crawlab.cn" + }, + "tags": [ + "crawler", + "scraping", + "distributed", + "self-hosted" + ] + }, { "id": "crowdsec", "name": "Crowdsec",