diff --git a/Dockerfile b/Dockerfile
index 5d43b8396..b88ddf892 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux
# install backend
RUN pip install scrapy pymongo bs4 requests
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500
# add files
COPY ./backend/conf /app/backend/conf
diff --git a/Dockerfile.cn b/Dockerfile.cn
index 13dfecadf..225880bc3 100644
--- a/Dockerfile.cn
+++ b/Dockerfile.cn
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux
# install backend
RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500
# add files
COPY ./backend/conf /app/backend/conf
diff --git a/Dockerfile.local b/Dockerfile.local
index c101b77f6..4a1956b7e 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux
# install backend
RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500
# add files
COPY ./backend/conf /app/backend/conf
diff --git a/backend/cmd/root.go b/backend/cmd/root.go
index 8c6c322f5..9d0ef3eb4 100644
--- a/backend/cmd/root.go
+++ b/backend/cmd/root.go
@@ -2,6 +2,7 @@ package cmd
import (
"fmt"
+ "github.com/apex/log"
"strings"
"github.com/spf13/cobra"
@@ -54,4 +55,17 @@ func initConfig() {
if err := viper.ReadInConfig(); err == nil {
fmt.Println("Using config file:", viper.ConfigFileUsed())
}
+
+ // initialize log level
+ initLogLevel()
+}
+
+func initLogLevel() {
+ // set log level
+ logLevel := viper.GetString("log.level")
+ l, err := log.ParseLevel(logLevel)
+ if err != nil {
+ l = log.InfoLevel
+ }
+ log.SetLevel(l)
}
diff --git a/backend/go.mod b/backend/go.mod
index ef8367fc0..178b73305 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -4,7 +4,7 @@ go 1.16
require (
github.com/apex/log v1.9.0
- github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134
+ github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718
github.com/crawlab-team/go-trace v0.1.1
github.com/gin-gonic/gin v1.7.1
github.com/spf13/cobra v1.1.3
diff --git a/backend/go.mod.local b/backend/go.mod.local
index 129fe1943..41c139328 100644
--- a/backend/go.mod.local
+++ b/backend/go.mod.local
@@ -11,11 +11,10 @@ replace (
require (
github.com/apex/log v1.9.0
- github.com/crawlab-team/crawlab-core v0.6.0-beta.20211213.2006
- github.com/crawlab-team/go-trace v0.1.0
+ github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134
+ github.com/crawlab-team/go-trace v0.1.1
github.com/gin-gonic/gin v1.7.1
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.1
- go.mongodb.org/mongo-driver v1.6.0 // indirect
go.uber.org/dig v1.10.0
)
diff --git a/backend/go.sum b/backend/go.sum
index a47cfbe3e..be239ac93 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -63,8 +63,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134 h1:J2NKuIhkXZPlfx9hHT2L4VLS9FMHzEaDGgHP/8ZJKkI=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718 h1:nNmFoR/o+xEZ6ywe9mwx0QroF+QSHrJR5DotOm/r/tg=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY=
github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
github.com/crawlab-team/crawlab-db v0.1.3 h1:RqLoXGZEMUH1B8SQB5OcNmJeyY2xILvwyhv4X9faWl4=
github.com/crawlab-team/crawlab-db v0.1.3/go.mod h1:kPkGZ1P802XdbFFb8byMpZfNG2lWTNoWNRy4beS0/QY=
diff --git a/backend/test/config-worker-03.json b/backend/test/config-worker-03.json
new file mode 100644
index 000000000..70da53f10
--- /dev/null
+++ b/backend/test/config-worker-03.json
@@ -0,0 +1,10 @@
+{
+ "key": "worker-03",
+ "is_master": false,
+ "name": "Worker Node 03",
+ "ip": "",
+ "mac": "",
+ "hostname": "",
+ "description": "",
+ "auth_key": "Crawlab2021!"
+}
\ No newline at end of file
diff --git a/changelog/v0.6.0-beta.20211224-zh.md b/changelog/v0.6.0-beta.20211224-zh.md
new file mode 100644
index 000000000..2de9e2f8d
--- /dev/null
+++ b/changelog/v0.6.0-beta.20211224-zh.md
@@ -0,0 +1,32 @@
+# 更新日志 (v0.6.0-beta.20211224)
+
+## 概览
+
+这是下个重大版本 v0.6.0 的第三次 beta 发布。伴随着更多功能和优化，正式版本 v0.6.0 将会很快发布了。
+
+## 升级优化
+
+- [x] **国际化**. 支持中文.
+- [x] **命令行工具上传爬虫**. [#1020](https://github.com/crawlab-team/crawlab/issues/1020)
+- [x] **官方插件**. 允许用户在 Crawlab 用户界面上安装 [官方插件](https://docs-next.crawlab.cn/zh/guide/use-crawlab/public-plugins/).
+- [x] **更多文档**. 加入了关于插件和命令行工具的文档.
+
+## Bug 修复
+
+- [x] **上传文件问题** [#990](https://github.com/crawlab-team/crawlab/issues/990) [#1014](https://github.com/crawlab-team/crawlab/issues/1014) [#1022](https://github.com/crawlab-team/crawlab/issues/1022)
+- [x] **无法重命名文件** [#1008](https://github.com/crawlab-team/crawlab/issues/1008)
+
+## TODOs
+- [ ] **关联任务**. 如果任务执行模式为 “所有节点” 或 “指定节点”，那么将会有主任务和子任务之分。
+- [ ] **Crontab 编辑器**. 可视化 Crontab 编辑的前端组件。
+- [ ] **结果去重**.
+- [ ] **环境变量**.
+- [ ] **前端易用性优化**. 更多高级功能，例如表格形式保存。
+- [ ] **日志自动清理**.
+- [ ] **跟多文档**.
+- [ ] **前端输出文件大小优化**.
+
+## 未来计划
+
+下一个版本有可能是 v0.6.0 的正式版本，但是没有确定。本次发布版本将会进行更多的测试，以保证其健壮性以及生产可用。
+
diff --git a/changelog/v0.6.0-beta.20211224.md b/changelog/v0.6.0-beta.20211224.md
new file mode 100644
index 000000000..7c7e6daf0
--- /dev/null
+++ b/changelog/v0.6.0-beta.20211224.md
@@ -0,0 +1,32 @@
+# Change Log (v0.6.0-beta.20211224)
+
+## Overview
+
+This is the third beta release for the next major version v0.6.0. With more features and optimization coming in, the release of official version v0.6.0 is approaching soon.
+
+## Enhancement
+
+- [x] **Internationalization**. Support Chinese.
+- [x] **CLI Upload Spider**. [#1020](https://github.com/crawlab-team/crawlab/issues/1020)
+- [x] **Official Plugins**. Allow users to install [official plugins](https://docs-next.crawlab.cn/en/guide/use-crawlab/public-plugins/) on Crawlab web UI.
+- [x] **More Documentation**. Added documentation for plugins and CLI.
+
+## Bug Fixes
+
+- [x] **Upload File Issue** [#990](https://github.com/crawlab-team/crawlab/issues/990) [#1014](https://github.com/crawlab-team/crawlab/issues/1014) [#1022](https://github.com/crawlab-team/crawlab/issues/1022)
+- [x] **Unable to Rename File** [#1008](https://github.com/crawlab-team/crawlab/issues/1008)
+
+## TODOs
+- [ ] **Associated Tasks**. There will be main tasks and their sub-tasks if task mode is "all nodes" or "selected nodes".
+- [ ] **Crontab Editor**. Frontend component that visualize the crontab editing.
+- [ ] **Results Deduplication**.
+- [ ] **Environment Variables**.
+- [ ] **Frontend Utility Enhancement**. Advanced features such as saved table customization.
+- [ ] **Log Auto Cleanup**.
+- [ ] **More Documentation**.
+- [ ] **E2E Tests**.
+- [ ] **Frontend Output File Size Optimization**.
+
+## What Next
+
+The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robustness and production-ready deployment.
diff --git a/frontend/package.json b/frontend/package.json
index a41ba197d..00a9ddada 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
{
"name": "@crawlab/app",
- "version": "0.6.0-beta.20211221.1200",
+ "version": "0.6.0-beta.20211224.1718",
"description": "",
"scripts": {
"serve": "vue-cli-service serve",
@@ -15,7 +15,7 @@
},
"license": "BSD-3-Clause",
"dependencies": {
- "crawlab-ui": "0.6.0-beta.20211219.1200",
+ "crawlab-ui": "0.6.0-beta.20211224.1718",
"vue": "3.0.11",
"vue-router": "^4.0.11"
},
diff --git a/frontend/public/index.html b/frontend/public/index.html
index e307c2bc2..113fdc38d 100644
--- a/frontend/public/index.html
+++ b/frontend/public/index.html
@@ -1,12 +1,12 @@
+ <%= htmlWebpackPlugin.options.title %>
+
+
-
-
- <%= htmlWebpackPlugin.options.title %>
diff --git a/nginx/crawlab.conf b/nginx/crawlab.conf
index 8d1cc3472..f7b03328c 100644
--- a/nginx/crawlab.conf
+++ b/nginx/crawlab.conf
@@ -3,7 +3,7 @@ server {
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_comp_level 2;
- gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
+ gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png image/x-icon;
gzip_vary off;
gzip_disable "MSIE [1-6]\.";
client_max_body_size 200m;
diff --git a/workspace/docker-compose.yml b/workspace/docker-compose.yml
index b20f9cabe..52d35ce39 100644
--- a/workspace/docker-compose.yml
+++ b/workspace/docker-compose.yml
@@ -1,10 +1,24 @@
version: '3.3'
services:
+ build:
+ build:
+ context: ./dockerfiles/golang
+ command: /bin/bash -c "rm -f /backend/dist/crawlab && go build -o ./dist/crawlab ./"
+ volumes:
+ - ./dist:/backend/dist
+ - ./.crawlab/go/pkg/mod:/go/pkg/mod
+ - ../backend:/backend
+ - ../backend/go.mod.local:/backend/go.mod
+ - ../..:/libs/crawlab-team
+ environment:
+ GOPROXY: https://goproxy.cn,direct
+
master:
build:
context: ./dockerfiles/golang
- command: /bin/bash -c "/app/bin/docker-start-master.sh && air -c .air.master.conf"
+ command: /bin/bash -c "/app/bin/docker-start-master.sh && /dist/crawlab master"
volumes:
+ - ./dist:/dist
- ./.crawlab/master:/root/.crawlab
- ./.crawlab/go/pkg/mod:/go/pkg/mod
- ../backend:/backend
@@ -17,22 +31,29 @@ services:
CRAWLAB_NODE_MASTER: "Y"
CRAWLAB_NODE_NAME: "Master Node"
CRAWLAB_MONGO_HOST: "mongo"
+ CRAWLAB_LOG_LEVEL: debug
+ GOPROXY: https://goproxy.cn,direct
ports:
- "9080:8080"
- "9000:8000"
+ - "9866:9666"
+ - "9888:8888"
depends_on:
- mongo
worker01:
build:
context: ./dockerfiles/golang
- command: "air -c .air.worker.conf"
+ command: /bin/bash -c "/dist/crawlab worker"
environment:
CRAWLAB_NODE_MASTER: "N"
CRAWLAB_NODE_NAME: "Worker Node 01"
CRAWLAB_GRPC_ADDRESS: "master"
CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
+ CRAWLAB_LOG_LEVEL: debug
+ GOPROXY: https://goproxy.cn,direct
volumes:
+ - ./dist:/dist
- ./.crawlab/worker01:/root/.crawlab
- ./.crawlab/go/pkg/mod:/go/pkg/mod
- ../backend:/backend
@@ -44,13 +65,16 @@ services:
worker02:
build:
context: ./dockerfiles/golang
- command: "air -c .air.worker.conf"
+ command: /bin/bash -c "/dist/crawlab worker"
environment:
CRAWLAB_NODE_MASTER: "N"
- CRAWLAB_NODE_NAME: "Worker Node 01"
+ CRAWLAB_NODE_NAME: "Worker Node 02"
CRAWLAB_GRPC_ADDRESS: "master"
CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
+ CRAWLAB_LOG_LEVEL: debug
+ GOPROXY: https://goproxy.cn,direct
volumes:
+ - ./dist:/dist
- ./.crawlab/worker02:/root/.crawlab
- ./.crawlab/go/pkg/mod:/go/pkg/mod
- ../backend:/backend
@@ -58,6 +82,7 @@ services:
- ../..:/libs/crawlab-team
depends_on:
- master
+ - worker01
mongo:
image: mongo:4