Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: add basic ci workflow #40

Merged
merged 4 commits into from
Jan 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: CI

on:
schedule:
- cron: '0 0 * * *'
pull_request:
branches: [dev]

jobs:
format:
name: Format
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v3

- name: Setup Go
uses: actions/setup-go@v4
with:
go-version-file: go.mod

- name: Format code
uses: iamnotaturtle/auto-gofmt@v2.1.0
with:
only_changed: True

lint:
name: Lint Check
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v3

- name: Setup Go
uses: actions/setup-go@v4
with:
go-version-file: go.mod

- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: latest
args: --timeout=10m

test:
name: Doris Operator Test
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v3

- name: Setup Go
uses: actions/setup-go@v4
with:
go-version-file: go.mod

- name: Run tests
run: make test

spell-check:
name: Spell Check
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v3

- name: Spelling Check
uses: crate-ci/typos@master
2 changes: 1 addition & 1 deletion docs/scale/manual-scale/index.zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ FE、BE、Broker 通常需要 30 秒 - 1分钟完成扩容或缩容。
垂直扩缩容操作指的是通过增加或减少 Pod 的资源限制,来达到集群扩缩容的目的。垂直扩缩容本质上是 Pod 滚动升级的过程。

要对 FE、BE、CN、Broker 进行垂直扩缩容,通过 kubectl 修改集群所对应的 `DorisCluster`
对象的 `spec.<fe/be/cn/broker>.resourecs` 至期望值。
对象的 `spec.<fe/be/cn/broker>.resources` 至期望值。

可以通过以下命令查看垂直扩缩容进度,当所有 Pod 都重建完毕进入 `Running` 状态后,垂直扩缩容完成。

Expand Down
3 changes: 2 additions & 1 deletion internal/discovery/sqlclient.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package discovery
import (
"database/sql"
"fmt"

_ "github.com/go-sql-driver/mysql"
)

Expand Down Expand Up @@ -57,7 +58,7 @@ func ReadAllRowsAsString(rows *sql.Rows) []RowMap {
for rows.Next() {
columns := make([]any, len(cols))
columnPointers := make([]any, len(cols))
for i, _ := range columns {
for i := range columns {
columnPointers[i] = &columns[i]
}
_ = rows.Scan(columnPointers...)
Expand Down
33 changes: 24 additions & 9 deletions internal/discovery/sqlcommand.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,20 @@ import (
"database/sql"
"errors"
"fmt"

ut "github.com/al-assad/doris-operator/internal/util"
logf "sigs.k8s.io/controller-runtime/pkg/log"

u "github.com/rjNemo/underscore"
)

func ShowFrontendHosts(db *sql.DB) ([]string, error) {
rows, err := db.Query("show frontends")
defer rows.Close()
defer func() {
if err = rows.Close(); err != nil {
logf.Log.Error(err, "unable to close rows queried by 'show frontends' command")
}
}()

if err != nil {
return []string{}, ut.MergeErrors(errors.New("failed to execute sql 'show frontends'"), err)
Expand All @@ -42,7 +49,11 @@ func ShowFrontendHosts(db *sql.DB) ([]string, error) {

func ShowBackendHosts(db *sql.DB) ([]string, error) {
rows, err := db.Query("show backends")
defer rows.Close()
defer func() {
if err = rows.Close(); err != nil {
logf.Log.Error(err, "unable to close rows queried by 'show backends' command")
}
}()

if err != nil {
return []string{}, ut.MergeErrors(errors.New("failed to execute sql 'show backends'"), err)
Expand All @@ -57,7 +68,11 @@ func ShowBackendHosts(db *sql.DB) ([]string, error) {
// ShowBrokerNameHosts returns map structure: key is broker name, value is broker host
func ShowBrokerNameHosts(db *sql.DB) (map[string]string, error) {
rows, err := db.Query("show broker")
defer rows.Close()
defer func() {
if err = rows.Close(); err != nil {
logf.Log.Error(err, "unable to close rows queried by 'show broker' command")
}
}()

if err != nil {
return map[string]string{}, ut.MergeErrors(errors.New("failed to execute sql 'show broker'"), err)
Expand All @@ -74,7 +89,7 @@ func AddFrontend(db *sql.DB, feHostPort string) error {
addSql := fmt.Sprintf(`alter system add follower "%s"`, feHostPort)
_, err := db.Exec(addSql)
if err != nil {
return ut.MergeErrors(errors.New(fmt.Sprintf("failed to execute sql '%s'", addSql)), err)
return ut.MergeErrors(fmt.Errorf("failed to execute sql '%s'", addSql), err)
}
return nil
}
Expand All @@ -83,7 +98,7 @@ func AddBackend(db *sql.DB, beHostPort string) error {
addSql := fmt.Sprintf(`alter system add backend "%s"`, beHostPort)
_, err := db.Exec(addSql)
if err != nil {
return ut.MergeErrors(errors.New(fmt.Sprintf("failed to execute sql '%s'", addSql)), err)
return ut.MergeErrors(fmt.Errorf("failed to execute sql '%s'", addSql), err)
}
return nil
}
Expand All @@ -92,7 +107,7 @@ func AddBroker(db *sql.DB, brokerName string, brokerHost string) error {
addSql := fmt.Sprintf(`alter system add broker %s "%s"`, brokerName, brokerHost)
_, err := db.Exec(addSql)
if err != nil {
return ut.MergeErrors(errors.New(fmt.Sprintf("failed to execute sql '%s'", addSql)), err)
return ut.MergeErrors(fmt.Errorf("failed to execute sql '%s'", addSql), err)
}
return nil
}
Expand All @@ -101,7 +116,7 @@ func DropFrontend(db *sql.DB, feHostPort string) error {
addSql := fmt.Sprintf(`alter system drop follower "%s"`, feHostPort)
_, err := db.Exec(addSql)
if err != nil {
return ut.MergeErrors(errors.New(fmt.Sprintf("failed to execute sql '%s'", addSql)), err)
return ut.MergeErrors(fmt.Errorf("failed to execute sql '%s'", addSql), err)
}
return nil
}
Expand All @@ -110,7 +125,7 @@ func DropBackend(db *sql.DB, beHostPort string) error {
addSql := fmt.Sprintf(`alter system drop backend "%s"`, beHostPort)
_, err := db.Exec(addSql)
if err != nil {
return ut.MergeErrors(errors.New(fmt.Sprintf("failed to execute sql '%s'", addSql)), err)
return ut.MergeErrors(fmt.Errorf("failed to execute sql '%s'", addSql), err)
}
return nil
}
Expand All @@ -119,7 +134,7 @@ func DropBroker(db *sql.DB, brokerName string) error {
addSql := fmt.Sprintf(`alter system all broker %s`, brokerName)
_, err := db.Exec(addSql)
if err != nil {
return ut.MergeErrors(errors.New(fmt.Sprintf("failed to execute sql '%s'", addSql)), err)
return ut.MergeErrors(fmt.Errorf("failed to execute sql '%s'", addSql), err)
}
return nil
}
11 changes: 5 additions & 6 deletions internal/reconciler/autoscaler_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
package reconciler

import (
"errors"
"fmt"

dapi "github.com/al-assad/doris-operator/api/v1beta1"
tran "github.com/al-assad/doris-operator/internal/transformer"
"github.com/al-assad/doris-operator/internal/util"
Expand Down Expand Up @@ -65,18 +65,17 @@ func (r *DorisAutoScalerReconciler) Reconcile() (dapi.AutoscalerRecStatus, error
return err
}
if !exist {
return errors.New(fmt.Sprintf("target DorisCluster[name=%s][namespace=%s] not exist",
clusterRef.Name, clusterRef.Name))
return fmt.Errorf("target DorisCluster[name=%s][namespace=%s] not exist",
clusterRef.Name, clusterRef.Name)
}
// check if target DorisCluster already bound another DorisAutoscaler
bound, bErr := r.FindRefDorisAutoScaler(clusterRef)
if bErr != nil {
return err
}
if bound != nil && bound.Name != r.CR.Name && bound.Namespace != r.CR.Namespace {
return errors.New(
fmt.Sprintf("target DorisCluster already bound another DorisAutoscaler[name=%s][namespace=%s]",
bound.Name, bound.Name))
return fmt.Errorf("target DorisCluster already bound another DorisAutoscaler[name=%s][namespace=%s]",
bound.Name, bound.Name)
}
// apply hpa resources
if cnUpHpa := tran.MakeCnScaleUpHpa(r.CR, r.Schema); cnUpHpa != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/template/monitor/grafana-dashboards.json
Original file line number Diff line number Diff line change
Expand Up @@ -1603,7 +1603,7 @@
"type": "prometheus",
"uid": "PEB833E60655F2EBA"
},
"description": "The image wirte of each FE",
"description": "The image write of each FE",
"fieldConfig": {
"defaults": {
"color": {
Expand Down
2 changes: 1 addition & 1 deletion internal/util/value.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func ArrayFallback[T any](array []T, fallback []T) []T {
}

func MapFallback[K comparable, V any](mapValue map[K]V, fallback map[K]V) map[K]V {
if mapValue == nil || len(mapValue) == 0 {
if len(mapValue) == 0 {
return fallback
} else {
return mapValue
Expand Down
6 changes: 3 additions & 3 deletions website/config/_default/hyas/doks.toml
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ docsRepoSubPath = "website" # "" (none, default) or <sub path>
# Colors

## Dark theme
# textDark = "#dee2e6" # "#dee2e6" (default), "#dee2e6" (orignal), or custom color
# textDark = "#dee2e6" # "#dee2e6" (default), "#dee2e6" (original), or custom color
# accentDark = "#5d2f86" # "#5d2f86" (default), "#5d2f86" (original), or custom color

## Light theme
# textLight = "#1d2d35" # "#1d2d35" (default), "#1d2d35" (orignal), or custom color
# accentLight = "#8ed6fb" # "#8ed6fb" (default), "#8ed6fb" (orignal), or custom color
# textLight = "#1d2d35" # "#1d2d35" (default), "#1d2d35" (original), or custom color
# accentLight = "#8ed6fb" # "#8ed6fb" (default), "#8ed6fb" (original), or custom color