Merge pull request #1035 from crawlab-team/develop

Develop
crawlab-team · Dec 24, 2021 · e4b1bb5 · e4b1bb5
2 parents aea571d + 6c1f78b
commit e4b1bb5
Show file tree

Hide file tree

Showing 14 changed files with 131 additions and 19 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux
 
 # install backend
 RUN pip install scrapy pymongo bs4 requests
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500
 
 # add files
 COPY ./backend/conf /app/backend/conf

diff --git a/Dockerfile.cn b/Dockerfile.cn
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux
 
 # install backend
 RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500
 
 # add files
 COPY ./backend/conf /app/backend/conf

diff --git a/Dockerfile.local b/Dockerfile.local
@@ -49,7 +49,7 @@ RUN wget https://github.com/crawlab-team/resources/raw/main/seaweedfs/2.79/linux
 
 # install backend
 RUN pip install scrapy pymongo bs4 requests -i https://mirrors.aliyun.com/pypi/simple
-RUN pip install crawlab-sdk==0.6.b20211213-2123
+RUN pip install crawlab-sdk==0.6.b20211224-1500
 
 # add files
 COPY ./backend/conf /app/backend/conf

diff --git a/backend/cmd/root.go b/backend/cmd/root.go
@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/apex/log"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -54,4 +55,17 @@ func initConfig() {
 	if err := viper.ReadInConfig(); err == nil {
 		fmt.Println("Using config file:", viper.ConfigFileUsed())
 	}
+
+	// initialize log level
+	initLogLevel()
+}
+
+func initLogLevel() {
+	// set log level
+	logLevel := viper.GetString("log.level")
+	l, err := log.ParseLevel(logLevel)
+	if err != nil {
+		l = log.InfoLevel
+	}
+	log.SetLevel(l)
 }
diff --git a/backend/go.mod b/backend/go.mod
@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/apex/log v1.9.0
-	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718
 	github.com/crawlab-team/go-trace v0.1.1
 	github.com/gin-gonic/gin v1.7.1
 	github.com/spf13/cobra v1.1.3

diff --git a/backend/go.mod.local b/backend/go.mod.local
@@ -11,11 +11,10 @@ replace (
 
 require (
 	github.com/apex/log v1.9.0
-	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211213.2006
-	github.com/crawlab-team/go-trace v0.1.0
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134
+	github.com/crawlab-team/go-trace v0.1.1
 	github.com/gin-gonic/gin v1.7.1
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/viper v1.7.1
-	go.mongodb.org/mongo-driver v1.6.0 // indirect
 	go.uber.org/dig v1.10.0
 )
diff --git a/backend/go.sum b/backend/go.sum
@@ -63,8 +63,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134 h1:J2NKuIhkXZPlfx9hHT2L4VLS9FMHzEaDGgHP/8ZJKkI=
-github.com/crawlab-team/crawlab-core v0.6.0-beta.20211219.2134/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718 h1:nNmFoR/o+xEZ6ywe9mwx0QroF+QSHrJR5DotOm/r/tg=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20211224.1718/go.mod h1:Y/3Z1WPYvSas5pQtRNypgQoyS3JHvrk2wtdR8n94sHY=
 github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
 github.com/crawlab-team/crawlab-db v0.1.3 h1:RqLoXGZEMUH1B8SQB5OcNmJeyY2xILvwyhv4X9faWl4=
 github.com/crawlab-team/crawlab-db v0.1.3/go.mod h1:kPkGZ1P802XdbFFb8byMpZfNG2lWTNoWNRy4beS0/QY=

diff --git a/backend/test/config-worker-03.json b/backend/test/config-worker-03.json
@@ -0,0 +1,10 @@
+{
+  "key": "worker-03",
+  "is_master": false,
+  "name": "Worker Node 03",
+  "ip": "",
+  "mac": "",
+  "hostname": "",
+  "description": "",
+  "auth_key": "Crawlab2021!"
+}
diff --git a/changelog/v0.6.0-beta.20211224-zh.md b/changelog/v0.6.0-beta.20211224-zh.md
@@ -0,0 +1,32 @@
+# 更新日志 (v0.6.0-beta.20211224)
+
+## 概览
+
+这是下个重大版本 v0.6.0 的第三次 beta 发布。伴随着更多功能和优化，正式版本 v0.6.0 将会很快发布了。
+
+## 升级优化
+
+- [x] **国际化**. 支持中文.
+- [x] **命令行工具上传爬虫**. [#1020](https://github.com/crawlab-team/crawlab/issues/1020)
+- [x] **官方插件**. 允许用户在 Crawlab 用户界面上安装 [官方插件](https://docs-next.crawlab.cn/zh/guide/use-crawlab/public-plugins/).
+- [x] **更多文档**. 加入了关于插件和命令行工具的文档.
+
+## Bug 修复
+
+- [x] **上传文件问题** [#990](https://github.com/crawlab-team/crawlab/issues/990) [#1014](https://github.com/crawlab-team/crawlab/issues/1014) [#1022](https://github.com/crawlab-team/crawlab/issues/1022)
+- [x] **无法重命名文件** [#1008](https://github.com/crawlab-team/crawlab/issues/1008)
+
+## TODOs
+- [ ] **关联任务**. 如果任务执行模式为 “所有节点” 或 “指定节点”，那么将会有主任务和子任务之分。
+- [ ] **Crontab 编辑器**. 可视化 Crontab 编辑的前端组件。
+- [ ] **结果去重**.
+- [ ] **环境变量**.
+- [ ] **前端易用性优化**. 更多高级功能，例如表格形式保存。
+- [ ] **日志自动清理**.
+- [ ] **跟多文档**.
+- [ ] **前端输出文件大小优化**.
+
+## 未来计划
+
+下一个版本有可能是 v0.6.0 的正式版本，但是没有确定。本次发布版本将会进行更多的测试，以保证其健壮性以及生产可用。
+
diff --git a/changelog/v0.6.0-beta.20211224.md b/changelog/v0.6.0-beta.20211224.md
@@ -0,0 +1,32 @@
+# Change Log (v0.6.0-beta.20211224)
+
+## Overview
+
+This is the third beta release for the next major version v0.6.0. With more features and optimization coming in, the release of official version v0.6.0 is approaching soon. 
+
+## Enhancement
+
+- [x] **Internationalization**. Support Chinese.
+- [x] **CLI Upload Spider**. [#1020](https://github.com/crawlab-team/crawlab/issues/1020)
+- [x] **Official Plugins**. Allow users to install [official plugins](https://docs-next.crawlab.cn/en/guide/use-crawlab/public-plugins/) on Crawlab web UI.
+- [x] **More Documentation**. Added documentation for plugins and CLI.
+
+## Bug Fixes
+
+- [x] **Upload File Issue** [#990](https://github.com/crawlab-team/crawlab/issues/990) [#1014](https://github.com/crawlab-team/crawlab/issues/1014) [#1022](https://github.com/crawlab-team/crawlab/issues/1022)
+- [x] **Unable to Rename File** [#1008](https://github.com/crawlab-team/crawlab/issues/1008)
+
+## TODOs
+- [ ] **Associated Tasks**. There will be main tasks and their sub-tasks if task mode is "all nodes" or "selected nodes".
+- [ ] **Crontab Editor**. Frontend component that visualize the crontab editing.
+- [ ] **Results Deduplication**.
+- [ ] **Environment Variables**.
+- [ ] **Frontend Utility Enhancement**. Advanced features such as saved table customization. 
+- [ ] **Log Auto Cleanup**.
+- [ ] **More Documentation**. 
+- [ ] **E2E Tests**. 
+- [ ] **Frontend Output File Size Optimization**.
+
+## What Next
+
+The next version could the official release of v0.6.0, but not determined yet. There will be more tests running against the current beta version to ensure robustness and production-ready deployment.
diff --git a/frontend/package.json b/frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@crawlab/app",
-  "version": "0.6.0-beta.20211221.1200",
+  "version": "0.6.0-beta.20211224.1718",
   "description": "",
   "scripts": {
     "serve": "vue-cli-service serve",
@@ -15,7 +15,7 @@
   },
   "license": "BSD-3-Clause",
   "dependencies": {
-    "crawlab-ui": "0.6.0-beta.20211219.1200",
+    "crawlab-ui": "0.6.0-beta.20211224.1718",
     "vue": "3.0.11",
     "vue-router": "^4.0.11"
   },

diff --git a/frontend/public/index.html b/frontend/public/index.html
@@ -1,12 +1,12 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
+  <title><%= htmlWebpackPlugin.options.title %></title>
+  <link rel="icon" type="image/x-icon" href="<%= BASE_URL %>favicon.ico">
+  <link rel="stylesheet" href="font-awesome.min.css">
   <meta charset="utf-8">
   <meta content="IE=edge" http-equiv="X-UA-Compatible">
   <meta content="width=device-width,initial-scale=1.0" name="viewport">
-  <link href="<%= BASE_URL %>favicon.ico" rel="icon">
-  <link href="font-awesome.min.css" rel="stylesheet">
-  <title><%= htmlWebpackPlugin.options.title %></title>
   <script>
   window.VUE_APP_API_BASE_URL = '<%= VUE_APP_API_BASE_URL %>'
   </script>

diff --git a/nginx/crawlab.conf b/nginx/crawlab.conf
@@ -3,7 +3,7 @@ server {
 	gzip_min_length 1k;
 	gzip_buffers 4 16k;
 	gzip_comp_level 2;
-	gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;
+	gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png image/x-icon;
 	gzip_vary off;
 	gzip_disable "MSIE [1-6]\.";
 	client_max_body_size 200m;

diff --git a/workspace/docker-compose.yml b/workspace/docker-compose.yml
@@ -1,10 +1,24 @@
 version: '3.3'
 services:
+  build:
+    build:
+      context: ./dockerfiles/golang
+    command: /bin/bash -c "rm -f /backend/dist/crawlab && go build -o ./dist/crawlab ./"
+    volumes:
+      - ./dist:/backend/dist
+      - ./.crawlab/go/pkg/mod:/go/pkg/mod
+      - ../backend:/backend
+      - ../backend/go.mod.local:/backend/go.mod
+      - ../..:/libs/crawlab-team
+    environment:
+      GOPROXY: https://goproxy.cn,direct
+
   master:
     build:
       context: ./dockerfiles/golang
-    command: /bin/bash -c "/app/bin/docker-start-master.sh && air -c .air.master.conf"
+    command: /bin/bash -c "/app/bin/docker-start-master.sh && /dist/crawlab master"
     volumes:
+      - ./dist:/dist
       - ./.crawlab/master:/root/.crawlab
       - ./.crawlab/go/pkg/mod:/go/pkg/mod
       - ../backend:/backend
@@ -17,22 +31,29 @@ services:
       CRAWLAB_NODE_MASTER: "Y"
       CRAWLAB_NODE_NAME: "Master Node"
       CRAWLAB_MONGO_HOST: "mongo"
+      CRAWLAB_LOG_LEVEL: debug
+      GOPROXY: https://goproxy.cn,direct
     ports:
       - "9080:8080"
       - "9000:8000"
+      - "9866:9666"
+      - "9888:8888"
     depends_on:
       - mongo
 
   worker01:
     build:
       context: ./dockerfiles/golang
-    command: "air -c .air.worker.conf"
+    command: /bin/bash -c "/dist/crawlab worker"
     environment:
       CRAWLAB_NODE_MASTER: "N"
       CRAWLAB_NODE_NAME: "Worker Node 01"
       CRAWLAB_GRPC_ADDRESS: "master"
       CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
+      CRAWLAB_LOG_LEVEL: debug
+      GOPROXY: https://goproxy.cn,direct
     volumes:
+      - ./dist:/dist
       - ./.crawlab/worker01:/root/.crawlab
       - ./.crawlab/go/pkg/mod:/go/pkg/mod
       - ../backend:/backend
@@ -44,20 +65,24 @@ services:
   worker02:
     build:
       context: ./dockerfiles/golang
-    command: "air -c .air.worker.conf"
+    command: /bin/bash -c "/dist/crawlab worker"
     environment:
       CRAWLAB_NODE_MASTER: "N"
-      CRAWLAB_NODE_NAME: "Worker Node 01"
+      CRAWLAB_NODE_NAME: "Worker Node 02"
       CRAWLAB_GRPC_ADDRESS: "master"
       CRAWLAB_FS_FILER_URL: "http://master:8080/api/filer"
+      CRAWLAB_LOG_LEVEL: debug
+      GOPROXY: https://goproxy.cn,direct
     volumes:
+      - ./dist:/dist
       - ./.crawlab/worker02:/root/.crawlab
       - ./.crawlab/go/pkg/mod:/go/pkg/mod
       - ../backend:/backend
       - ../backend/go.mod.local:/backend/go.mod
       - ../..:/libs/crawlab-team
     depends_on:
       - master
+      - worker01
 
   mongo:
     image: mongo:4