From d98699b5fcc25395b2f911c088166e94660cb6a8 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Thu, 17 Aug 2023 09:29:23 +0800 Subject: [PATCH 1/4] Filter artifact without CVE from top 5 dangerous artifacts (#19187) Fixes #19186 Signed-off-by: stonezdj --- src/pkg/securityhub/dao/security.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/securityhub/dao/security.go b/src/pkg/securityhub/dao/security.go index 20495a04da6..bc08ce527d8 100644 --- a/src/pkg/securityhub/dao/security.go +++ b/src/pkg/securityhub/dao/security.go @@ -44,6 +44,7 @@ from artifact a, scan_report s where a.digest = s.digest and s.registration_uuid = ? + and s.critical_cnt+s.high_cnt+s.medium_cnt+s.low_cnt > 0 order by s.critical_cnt desc, s.high_cnt desc, s.medium_cnt desc, s.low_cnt desc limit 5` From 52e66155d49acceb7ed9c5ceec8b466973be25f7 Mon Sep 17 00:00:00 2001 From: Chlins Zhang Date: Thu, 17 Aug 2023 11:11:04 +0800 Subject: [PATCH 2/4] log: change log level to reduce the noise logs (#19146) 1. Change some logs level to reduce the noise. 2. Wrap the go-redis.Nil error as ErrNotFound to avoid confusing Signed-off-by: chlins --- src/cmd/exporter/main.go | 45 ++++++++++++++++++++++++--------- src/core/session/session.go | 7 +++-- src/lib/cache/redis/redis.go | 4 +++ src/pkg/notifier/notifier.go | 2 +- src/server/registry/manifest.go | 2 +- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/cmd/exporter/main.go b/src/cmd/exporter/main.go index 0c5dfa9407e..400d5417876 100644 --- a/src/cmd/exporter/main.go +++ b/src/cmd/exporter/main.go @@ -36,16 +36,7 @@ func main() { viper.SetEnvPrefix("harbor") viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - connMaxLifetime, err := time.ParseDuration(viper.GetString("database.conn_max_lifetime")) - if err != nil { - log.Errorf("Failed to parse database.conn_max_lifetime: %v", err) - connMaxLifetime = 5 * time.Minute - } - connMaxIdleTime, err := time.ParseDuration(viper.GetString("database.conn_max_idle_time")) - if err != nil { - log.Errorf("Failed to parse database.conn_max_idle_time: %v", err) - connMaxIdleTime = 0 - } + dbCfg := &models.Database{ Type: "postgresql", PostGreSQL: &models.PostGreSQL{ @@ -57,8 +48,8 @@ func main() { SSLMode: viper.GetString("database.sslmode"), MaxIdleConns: viper.GetInt("database.max_idle_conns"), MaxOpenConns: viper.GetInt("database.max_open_conns"), - ConnMaxLifetime: connMaxLifetime, - ConnMaxIdleTime: connMaxIdleTime, + ConnMaxLifetime: getConnMaxLifetime(viper.GetString("database.conn_max_lifetime")), + ConnMaxIdleTime: getConnMaxIdleTime(viper.GetString("database.conn_max_idle_time")), }, } if err := dao.InitDatabase(dbCfg); err != nil { @@ -109,3 +100,33 @@ func main() { os.Exit(1) } } + +func getConnMaxLifetime(duration string) time.Duration { + // set conn max life time, 5m by default + connMaxLifetime := 5 * time.Minute + if duration != "" { + maxLifetime, err := time.ParseDuration(duration) + if err == nil { + connMaxLifetime = maxLifetime + } else { + log.Warningf("Failed to parse database.conn_max_lifetime, use default value: %s, err: %v", connMaxLifetime, err) + } + } + + return connMaxLifetime +} + +func getConnMaxIdleTime(duration string) time.Duration { + // set conn max idle time, 0 by default + connMaxIdleTime := time.Duration(0) + if duration != "" { + maxIdleTime, err := time.ParseDuration(duration) + if err == nil { + connMaxIdleTime = maxIdleTime + } else { + log.Warningf("Failed to parse database.conn_max_idle_time, use default value: %s, err: %v", connMaxIdleTime, err) + } + } + + return connMaxIdleTime +} diff --git a/src/core/session/session.go b/src/core/session/session.go index 80d4b389c64..d76f8c01f78 100644 --- a/src/core/session/session.go +++ b/src/core/session/session.go @@ -16,13 +16,12 @@ package session import ( "context" + "errors" "net/http" - "strings" "sync" "time" "github.com/beego/beego/v2/server/web/session" - goredis "github.com/go-redis/redis/v8" "github.com/goharbor/harbor/src/lib/cache" "github.com/goharbor/harbor/src/lib/cache/redis" @@ -131,7 +130,7 @@ func (rp *Provider) SessionRead(ctx context.Context, sid string) (session.Store, ctx = context.TODO() } err := rp.c.Fetch(ctx, sid, &kv) - if err != nil && !strings.Contains(err.Error(), goredis.Nil.Error()) { + if err != nil && !errors.Is(err, cache.ErrNotFound) { return nil, err } @@ -166,7 +165,7 @@ func (rp *Provider) SessionRegenerate(ctx context.Context, oldsid, sid string) ( } else { kv := make(map[interface{}]interface{}) err := rp.c.Fetch(ctx, sid, &kv) - if err != nil && !strings.Contains(err.Error(), goredis.Nil.Error()) { + if err != nil && !errors.Is(err, cache.ErrNotFound) { return nil, err } diff --git a/src/lib/cache/redis/redis.go b/src/lib/cache/redis/redis.go index a72b05e9c87..80beb3b997c 100644 --- a/src/lib/cache/redis/redis.go +++ b/src/lib/cache/redis/redis.go @@ -58,6 +58,10 @@ func (c *Cache) Fetch(ctx context.Context, key string, value interface{}) error // convert internal or Timeout error to be ErrNotFound // so that the caller can continue working without breaking // return cache.ErrNotFound + if err == redis.Nil { + return cache.ErrNotFound + } + return fmt.Errorf("%w:%v", cache.ErrNotFound, err) } diff --git a/src/pkg/notifier/notifier.go b/src/pkg/notifier/notifier.go index 47293b9390b..b8f2a2c36c9 100644 --- a/src/pkg/notifier/notifier.go +++ b/src/pkg/notifier/notifier.go @@ -217,7 +217,7 @@ func (nw *NotificationWatcher) Notify(ctx context.Context, notification Notifica // Currently, we just log the error log.Errorf("Error occurred when triggering handler %s of topic %s: %s\n", reflect.TypeOf(hd).String(), notification.Topic, err.Error()) } else { - log.Infof("Handle notification with Handler '%s' on topic '%s': %+v\n", hd.Name(), notification.Topic, notification.Value) + log.Debugf("Handle notification with Handler '%s' on topic '%s': %+v\n", hd.Name(), notification.Topic, notification.Value) } }() }(h, handlerChan) diff --git a/src/server/registry/manifest.go b/src/server/registry/manifest.go index 39a26fab9ea..4693c3a7e65 100644 --- a/src/server/registry/manifest.go +++ b/src/server/registry/manifest.go @@ -93,7 +93,7 @@ func getManifest(w http.ResponseWriter, req *http.Request) { _, _ = buffer.Write(manifest) } } else { - log.Warningf("failed to get manifest from cache, error: %v", err) + log.Debugf("failed to get manifest from cache, will fallback to registry, error: %v", err) // only write cache when request is GET because HEAD request resp // body is empty. if req.Method == http.MethodGet { From 83ff2b277ae89b12056c73e2dd3c8cca7c9e8b47 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Thu, 17 Aug 2023 14:12:07 +0800 Subject: [PATCH 3/4] Wrong artifact scanned count (#19198) fixes #19197 Signed-off-by: stonezdj --- src/pkg/securityhub/dao/security.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/pkg/securityhub/dao/security.go b/src/pkg/securityhub/dao/security.go index bc08ce527d8..0c87f417930 100644 --- a/src/pkg/securityhub/dao/security.go +++ b/src/pkg/securityhub/dao/security.go @@ -48,15 +48,20 @@ where a.digest = s.digest order by s.critical_cnt desc, s.high_cnt desc, s.medium_cnt desc, s.low_cnt desc limit 5` - // sql to query the total artifact count, exclude the artifact accessory, and child artifact in image index + // sql to query the total artifact count, + // 1. exclude the artifact accessory, + // 2. exclude child artifact without tag + // 3. include top level artifact in image index + // The totalArtifactCountSQL and scannedArtifactCountSQL should use the same criteria to filter the artifact totalArtifactCountSQL = `SELECT COUNT(1) -FROM artifact A +FROM artifact a WHERE NOT EXISTS (select 1 from artifact_accessory acc WHERE acc.artifact_id = a.id) AND (EXISTS (SELECT 1 FROM tag WHERE tag.artifact_id = a.id) OR NOT EXISTS (SELECT 1 FROM artifact_reference ref WHERE ref.child_id = a.id))` - // sql to query the scanned artifact count, exclude the artifact accessory, and child artifact in image index, - // and include the image index artifact which at least one child artifact is scanned + // sql to query the scanned artifact count, + // exclude the artifact accessory, and child artifact in image index (without tag), + // include the image index artifact which at least one child artifact is scanned scannedArtifactCountSQL = `SELECT COUNT(1) FROM artifact a WHERE EXISTS (SELECT 1 @@ -65,13 +70,10 @@ WHERE EXISTS (SELECT 1 AND s.registration_uuid = ?) -- exclude artifact accessory AND NOT EXISTS (SELECT 1 FROM artifact_accessory acc WHERE acc.artifact_id = a.id) - -- exclude artifact without tag and part of the image index - AND EXISTS (SELECT 1 - FROM tag - WHERE tag.artifact_id = id - OR (NOT EXISTS (SELECT 1 FROM artifact_reference ref WHERE ref.child_id = a.id))) - -- include image index which is scanned - OR EXISTS (SELECT 1 + -- not a child without tag + AND NOT EXISTS (SELECT 1 FROM artifact_reference WHERE child_id = a.id AND NOT EXISTS (SELECT 1 FROM tag WHERE artifact_id = a.id)) + -- include image index which is scanned + OR EXISTS (SELECT 1 FROM scan_report s, artifact_reference ref WHERE s.digest = ref.child_digest From c7e25295fef081b168b607c6f850e03bb50ffd29 Mon Sep 17 00:00:00 2001 From: Chlins Zhang Date: Fri, 18 Aug 2023 11:04:16 +0800 Subject: [PATCH 4/4] fix: support customize cache db for business (#19182) Support to configure the customized redis db for cache layer and other misc business for core, by default the behavior is same with previous(stored in db 0). Signed-off-by: chlins --- make/harbor.yml.tmpl | 26 ++++++++ .../migrations/version_2_7_0/harbor.yml.jinja | 59 +++++++++++++++++++ .../migrations/version_2_8_0/harbor.yml.jinja | 59 +++++++++++++++++++ .../migrations/version_2_9_0/harbor.yml.jinja | 59 +++++++++++++++++++ make/photon/prepare/templates/core/env.jinja | 6 ++ .../prepare/templates/jobservice/env.jinja | 3 + make/photon/prepare/utils/configs.py | 19 ++++-- src/controller/quota/controller.go | 2 +- src/core/main.go | 38 +++++++----- src/jobservice/main.go | 18 ------ src/lib/cache/cache.go | 35 +++++++++++ src/lib/redis/client.go | 30 ++++++---- src/lib/redis/client_test.go | 12 ++-- src/pkg/cached/base_manager.go | 16 ++--- 14 files changed, 318 insertions(+), 64 deletions(-) diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index cc5db0fbed9..7ab922bb81d 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -167,6 +167,28 @@ _version: 2.9.0 # max_idle_conns: 2 # max_open_conns: 0 +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_db_index: 7 + +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + # Uncomment external_redis if using external Redis server # external_redis: # # support redis, redis+sentinel @@ -186,6 +208,10 @@ _version: 2.9.0 # jobservice_db_index: 2 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. # uaa: diff --git a/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja b/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja index 59daf5ae9f1..fe931f8d6e3 100644 --- a/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja +++ b/make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja @@ -371,6 +371,49 @@ external_database: # ssl_mode: disable {% endif %} +{% if redis is defined %} +redis: +# # db_index 0 is for core, it's unchangeable +{% if redis.registry_db_index is defined %} + registry_db_index: {{ redis.registry_db_index }} +{% else %} +# # registry_db_index: 1 +{% endif %} +{% if redis.jobservice_db_index is defined %} + jobservice_db_index: {{ redis.jobservice_db_index }} +{% else %} +# # jobservice_db_index: 2 +{% endif %} +{% if redis.trivy_db_index is defined %} + trivy_db_index: {{ redis.trivy_db_index }} +{% else %} +# # trivy_db_index: 5 +{% endif %} +{% if redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} +{% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +{% endif %} +{% if redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} +{% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} +{% else %} +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} + {% if external_redis is defined %} external_redis: # support redis, redis+sentinel @@ -390,6 +433,18 @@ external_redis: jobservice_db_index: {{ external_redis.jobservice_db_index }} trivy_db_index: 5 idle_timeout_seconds: 30 + {% if external_redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} + {% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 + {% endif %} + {% if external_redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} + {% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + {% endif %} {% else %} # Umcomments external_redis if using external Redis server # external_redis: @@ -406,6 +461,10 @@ external_redis: # jobservice_db_index: 2 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 {% endif %} {% if uaa is defined %} diff --git a/make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja b/make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja index bea786619cb..eb061f92e1f 100644 --- a/make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja +++ b/make/photon/prepare/migrations/version_2_8_0/harbor.yml.jinja @@ -381,6 +381,49 @@ external_database: # ssl_mode: disable {% endif %} +{% if redis is defined %} +redis: +# # db_index 0 is for core, it's unchangeable +{% if redis.registry_db_index is defined %} + registry_db_index: {{ redis.registry_db_index }} +{% else %} +# # registry_db_index: 1 +{% endif %} +{% if redis.jobservice_db_index is defined %} + jobservice_db_index: {{ redis.jobservice_db_index }} +{% else %} +# # jobservice_db_index: 2 +{% endif %} +{% if redis.trivy_db_index is defined %} + trivy_db_index: {{ redis.trivy_db_index }} +{% else %} +# # trivy_db_index: 5 +{% endif %} +{% if redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} +{% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +{% endif %} +{% if redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} +{% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} +{% else %} +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} + {% if external_redis is defined %} external_redis: # support redis, redis+sentinel @@ -406,6 +449,18 @@ external_redis: jobservice_db_index: {{ external_redis.jobservice_db_index }} trivy_db_index: 5 idle_timeout_seconds: 30 + {% if external_redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} + {% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 + {% endif %} + {% if external_redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} + {% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + {% endif %} {% else %} # Umcomments external_redis if using external Redis server # external_redis: @@ -424,6 +479,10 @@ external_redis: # jobservice_db_index: 2 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 {% endif %} {% if uaa is defined %} diff --git a/make/photon/prepare/migrations/version_2_9_0/harbor.yml.jinja b/make/photon/prepare/migrations/version_2_9_0/harbor.yml.jinja index 718353ae6bc..4f9a79cf8ec 100644 --- a/make/photon/prepare/migrations/version_2_9_0/harbor.yml.jinja +++ b/make/photon/prepare/migrations/version_2_9_0/harbor.yml.jinja @@ -378,6 +378,49 @@ external_database: # max_open_conns: 0 {% endif %} +{% if redis is defined %} +redis: +# # db_index 0 is for core, it's unchangeable +{% if redis.registry_db_index is defined %} + registry_db_index: {{ redis.registry_db_index }} +{% else %} +# # registry_db_index: 1 +{% endif %} +{% if redis.jobservice_db_index is defined %} + jobservice_db_index: {{ redis.jobservice_db_index }} +{% else %} +# # jobservice_db_index: 2 +{% endif %} +{% if redis.trivy_db_index is defined %} + trivy_db_index: {{ redis.trivy_db_index }} +{% else %} +# # trivy_db_index: 5 +{% endif %} +{% if redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} +{% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +{% endif %} +{% if redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} +{% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} +{% else %} +# Uncomment redis if need to customize redis db +# redis: +# # db_index 0 is for core, it's unchangeable +# # registry_db_index: 1 +# # jobservice_db_index: 2 +# # trivy_db_index: 5 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 +{% endif %} + {% if external_redis is defined %} external_redis: # support redis, redis+sentinel @@ -399,6 +442,18 @@ external_redis: jobservice_db_index: {{ external_redis.jobservice_db_index }} trivy_db_index: 5 idle_timeout_seconds: 30 + {% if external_redis.harbor_db_index is defined %} + harbor_db_index: {{ redis.harbor_db_index }} + {% else %} +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 + {% endif %} + {% if external_redis.cache_layer_db_index is defined %} + cache_layer_db_index: {{ redis.cache_layer_db_index }} + {% else %} +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 + {% endif %} {% else %} # Umcomments external_redis if using external Redis server # external_redis: @@ -417,6 +472,10 @@ external_redis: # jobservice_db_index: 2 # trivy_db_index: 5 # idle_timeout_seconds: 30 +# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it. +# # harbor_db_index: 6 +# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it. +# # cache_layer_db_index: 7 {% endif %} {% if uaa is defined %} diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index 161e0d16690..acfa34467fd 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -1,6 +1,9 @@ CONFIG_PATH=/etc/core/app.conf UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem _REDIS_URL_CORE={{redis_url_core}} +{% if redis_url_harbor %} +_REDIS_URL_HARBOR={{redis_url_harbor}} +{% endif %} SYNC_QUOTA=true _REDIS_URL_REG={{redis_url_reg}} @@ -84,6 +87,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }} {% endif %} {% if cache.enabled %} +{% if redis_url_cache_layer %} +_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}} +{% endif %} CACHE_ENABLED=true CACHE_EXPIRE_HOURS={{ cache.expire_hours }} {% endif %} diff --git a/make/photon/prepare/templates/jobservice/env.jinja b/make/photon/prepare/templates/jobservice/env.jinja index 563d99e51f2..3cfa9e32dda 100644 --- a/make/photon/prepare/templates/jobservice/env.jinja +++ b/make/photon/prepare/templates/jobservice/env.jinja @@ -51,6 +51,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }} {% if cache.enabled %} _REDIS_URL_CORE={{redis_url_core}} +{% if redis_url_cache_layer %} +_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}} +{% endif %} CACHE_ENABLED=true CACHE_EXPIRE_HOURS={{ cache.expire_hours }} {% endif %} \ No newline at end of file diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 16d22adf1a6..bb302493532 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -275,7 +275,7 @@ def parse_yaml_config(config_file_path, with_trivy): config_dict['external_database'] = False # update redis configs - config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy)) + config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy)) # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) @@ -371,7 +371,7 @@ def get_redis_url_param(redis=None): return "" -def get_redis_configs(external_redis=None, with_trivy=True): +def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True): """Returns configs for redis >>> get_redis_configs()['external_redis'] @@ -404,6 +404,8 @@ def get_redis_configs(external_redis=None, with_trivy=True): >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False) True """ + + internal_redis = internal_redis or {} external_redis = external_redis or {} configs = dict(external_redis=bool(external_redis)) @@ -418,13 +420,22 @@ def get_redis_configs(external_redis=None, with_trivy=True): 'idle_timeout_seconds': 30, } - # overwriting existing keys by external_redis - redis.update({key: value for (key, value) in external_redis.items() if value}) + if len(internal_redis) > 0: + # overwriting existing keys by internal_redis + redis.update({key: value for (key, value) in internal_redis.items() if value}) + else: + # overwriting existing keys by external_redis + redis.update({key: value for (key, value) in external_redis.items() if value}) configs['redis_url_core'] = get_redis_url(0, redis) configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis) configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis) + if redis.get('harbor_db_index'): + configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis) + if redis.get('cache_layer_db_index'): + configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis) + if with_trivy: configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis) diff --git a/src/controller/quota/controller.go b/src/controller/quota/controller.go index 951be3f493d..29748b15f1c 100644 --- a/src/controller/quota/controller.go +++ b/src/controller/quota/controller.go @@ -262,7 +262,7 @@ func (c *controller) updateUsageByRedis(ctx context.Context, reference, referenc return retry.Abort(ctx.Err()) } - client, err := libredis.GetCoreClient() + client, err := libredis.GetHarborClient() if err != nil { return retry.Abort(err) } diff --git a/src/core/main.go b/src/core/main.go index 26d772693ee..78fe8e171ee 100644 --- a/src/core/main.go +++ b/src/core/main.go @@ -129,25 +129,35 @@ func main() { web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName web.BConfig.MaxMemory = 1 << 35 // (32GB) web.BConfig.MaxUploadSize = 1 << 35 // (32GB) - - redisURL := os.Getenv("_REDIS_URL_CORE") - if len(redisURL) > 0 { - u, err := url.Parse(redisURL) + // the core db used for beego session + redisCoreURL := os.Getenv("_REDIS_URL_CORE") + if len(redisCoreURL) > 0 { + _, err := url.Parse(redisCoreURL) if err != nil { - panic("bad _REDIS_URL") + panic("bad _REDIS_URL_CORE") } - + // configure the beego session redis web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName - web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL + web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL + } - log.Info("initializing cache ...") - if err := cache.Initialize(u.Scheme, redisURL); err != nil { - log.Fatalf("failed to initialize cache: %v", err) - } - // when config/db init function is called, the cache is not ready, - // enable config cache explicitly when the cache is ready - dbCfg.EnableConfigCache() + log.Info("initializing cache ...") + // the harbor db used for harbor business, use core db if not specified + redisHarborURL := os.Getenv("_REDIS_URL_HARBOR") + if redisHarborURL == "" { + redisHarborURL = redisCoreURL } + u, err := url.Parse(redisHarborURL) + if err != nil { + panic("bad _REDIS_URL_HARBOR") + } + if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil { + log.Fatalf("failed to initialize cache: %v", err) + } + // when config/db init function is called, the cache is not ready, + // enable config cache explicitly when the cache is ready + dbCfg.EnableConfigCache() + web.AddTemplateExt("htm") log.Info("initializing configurations...") diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 184a205fa64..42d531546bc 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -19,8 +19,6 @@ import ( "errors" "flag" "fmt" - "net/url" - "os" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/jobservice/common/utils" @@ -29,7 +27,6 @@ import ( "github.com/goharbor/harbor/src/jobservice/job/impl" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/runtime" - "github.com/goharbor/harbor/src/lib/cache" cfgLib "github.com/goharbor/harbor/src/lib/config" tracelib "github.com/goharbor/harbor/src/lib/trace" _ "github.com/goharbor/harbor/src/pkg/accessory/model/base" @@ -47,21 +44,6 @@ func main() { panic(fmt.Sprintf("failed to load configuration, error: %v", err)) } - // init cache if cache layer enabled - // gc needs to delete artifact by artifact manager, but the artifact cache store in - // core redis db so here require core redis url and init default cache. - if cfgLib.CacheEnabled() { - cacheURL := os.Getenv("_REDIS_URL_CORE") - u, err := url.Parse(cacheURL) - if err != nil { - panic("bad _REDIS_URL_CORE") - } - - if err = cache.Initialize(u.Scheme, cacheURL); err != nil { - panic(fmt.Sprintf("failed to initialize cache: %v", err)) - } - } - // Get parameters configPath := flag.String("c", "", "Specify the yaml config file path") flag.Parse() diff --git a/src/lib/cache/cache.go b/src/lib/cache/cache.go index 85f1b921e6d..ba49c1aa1a2 100644 --- a/src/lib/cache/cache.go +++ b/src/lib/cache/cache.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net/url" + "os" "sync" "time" @@ -137,3 +138,37 @@ func Initialize(typ, addr string) error { func Default() Cache { return cache } + +var ( + // cacheLayer is the global cache layer cache instance. + cacheLayer Cache + // cacheLayerOnce is the once condition for initializing instance. + cacheLayerOnce sync.Once +) + +// LayerCache is the global cache instance for cache layer. +func LayerCache() Cache { + // parse the redis url for cache layer, use the default cache if not specify + redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER") + if redisCacheURL == "" { + if cache != nil { + return cache + } + // use the core url if cache layer url not found + redisCacheURL = os.Getenv("_REDIS_URL_CORE") + } + + u, err := url.Parse(redisCacheURL) + if err != nil { + log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER") + } + + cacheLayerOnce.Do(func() { + cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:")) + if err != nil { + log.Fatalf("failed to initialize cache for cache layer, err: %v", err) + } + }) + + return cacheLayer +} diff --git a/src/lib/redis/client.go b/src/lib/redis/client.go index d0547185734..82256e398ea 100644 --- a/src/lib/redis/client.go +++ b/src/lib/redis/client.go @@ -31,9 +31,9 @@ var ( registry *redis.Client registryOnce = &sync.Once{} - // core is a global redis client for core db - core *redis.Client - coreOnce = &sync.Once{} + // harbor is a global redis client for harbor db + harbor *redis.Client + harborOnce = &sync.Once{} ) // GetRegistryClient returns the registry redis client. @@ -60,26 +60,30 @@ func GetRegistryClient() (*redis.Client, error) { return registry, nil } -// GetCoreClient returns the core redis client. -func GetCoreClient() (*redis.Client, error) { - coreOnce.Do(func() { - url := os.Getenv("_REDIS_URL_CORE") +// GetHarborClient returns the harbor redis client. +func GetHarborClient() (*redis.Client, error) { + harborOnce.Do(func() { + // parse redis url for harbor business, use core url by default + url := os.Getenv("_REDIS_URL_HARBOR") + if url == "" { + url = os.Getenv("_REDIS_URL_CORE") + } c, err := libredis.New(cache.Options{Address: url}) if err != nil { - log.Errorf("failed to initialize redis client for core, error: %v", err) + log.Errorf("failed to initialize redis client for harbor, error: %v", err) // reset the once to support retry if error occurred - coreOnce = &sync.Once{} + harborOnce = &sync.Once{} return } if c != nil { - core = c.(*libredis.Cache).Client + harbor = c.(*libredis.Cache).Client } }) - if core == nil { - return nil, errors.New("no core redis client initialized") + if harbor == nil { + return nil, errors.New("no harbor redis client initialized") } - return core, nil + return harbor, nil } diff --git a/src/lib/redis/client_test.go b/src/lib/redis/client_test.go index 1e288ba44ba..189d0f8eecd 100644 --- a/src/lib/redis/client_test.go +++ b/src/lib/redis/client_test.go @@ -41,22 +41,22 @@ func TestGetRegistryClient(t *testing.T) { } } -func TestGetCoreClient(t *testing.T) { +func TestGetHarborClient(t *testing.T) { // failure case with invalid address - t.Setenv("_REDIS_URL_CORE", "invalid-address") - client, err := GetCoreClient() + t.Setenv("_REDIS_URL_HARBOR", "invalid-address") + client, err := GetHarborClient() assert.Error(t, err) assert.Nil(t, client) // normal case with valid address - t.Setenv("_REDIS_URL_CORE", "redis://localhost:6379/0") - client, err = GetCoreClient() + t.Setenv("_REDIS_URL_HARBOR", "redis://localhost:6379/0") + client, err = GetHarborClient() assert.NoError(t, err) assert.NotNil(t, client) // multiple calls should return the same client for i := 0; i < 10; i++ { - newClient, err := GetCoreClient() + newClient, err := GetHarborClient() assert.NoError(t, err) assert.Equal(t, client, newClient) } diff --git a/src/pkg/cached/base_manager.go b/src/pkg/cached/base_manager.go index bc810372849..d7bf1caddbc 100644 --- a/src/pkg/cached/base_manager.go +++ b/src/pkg/cached/base_manager.go @@ -24,27 +24,27 @@ import ( ) // innerCache is the default cache client, -// actually it is a wrapper for cache.Default(). +// actually it is a wrapper for cache.LayerCache(). var innerCache cache.Cache = &cacheClient{} -// cacheClient is a interceptor for cache.Default, in order to implement specific +// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific // case for cache layer. type cacheClient struct{} func (*cacheClient) Contains(ctx context.Context, key string) bool { - return cache.Default().Contains(ctx, key) + return cache.LayerCache().Contains(ctx, key) } func (*cacheClient) Delete(ctx context.Context, key string) error { - return cache.Default().Delete(ctx, key) + return cache.LayerCache().Delete(ctx, key) } func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error { - return cache.Default().Fetch(ctx, key, value) + return cache.LayerCache().Fetch(ctx, key, value) } func (*cacheClient) Ping(ctx context.Context) error { - return cache.Default().Ping(ctx) + return cache.LayerCache().Ping(ctx) } func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error { @@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp return nil } - return cache.Default().Save(ctx, key, value, expiration...) + return cache.LayerCache().Save(ctx, key, value, expiration...) } func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) { - return cache.Default().Scan(ctx, match) + return cache.LayerCache().Scan(ctx, match) } var _ Manager = &BaseManager{}