Skip to content

Commit

Permalink
Move to secret dumper, cloud run v2 and improve logging init (#302)
Browse files Browse the repository at this point in the history
* Improve response pattern, runs behind the scenes.

* will this work then ?

* increase attempt deadline to make sure gh clients are created

* Moved to using harpocrates and added loglevel to app config

* provider fix

* fix resource to v2

* more fix

* One step closer to a correct config

* Maybe this fixes it :)

* string fix

* fixes to v2

* explicit tell about google-beta providers

* needs more cow bell

* docs

* a better log check
  • Loading branch information
brondum authored Jul 24, 2023
1 parent 8f8b4e9 commit b55b032
Show file tree
Hide file tree
Showing 7 changed files with 143 additions and 113 deletions.
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type Config struct {
HTTP HTTPConfig `yaml:"http"`
Server baseapp.HTTPConfig `yaml:"server"`
BestsellerSpecific BestsellerSpecificConfig `yaml:"bestseller_specific"`
LogLevel *int `yaml:"log_level,omitempty"`
}

type BestsellerSpecificConfig struct {
Expand Down
6 changes: 0 additions & 6 deletions config/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,5 @@ func LoadEnvConfig() error {
return err
}

// If no log level is set, default to info
if EnvVars.LogLevel == nil {
logLevel := 1
EnvVars.LogLevel = &logLevel
}

return nil
}
13 changes: 11 additions & 2 deletions logger/logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,21 @@ import (
func Init() {
log.Debug().Msgf("Setting log level to %d (%s)", *config.EnvVars.LogLevel, zerolog.Level(*config.EnvVars.LogLevel).String())

if config.EnvVars.LogLevel == nil {
// sets default to info if nothing is set
if config.AppConfig.LogLevel == nil && config.EnvVars.LogLevel == nil {
zerolog.SetGlobalLevel(zerolog.InfoLevel)
return
}

if *config.EnvVars.LogLevel <= int(zerolog.Disabled) {
// env is prioritized over config, as this has less dependencies
if config.EnvVars.LogLevel != nil && *config.EnvVars.LogLevel <= int(zerolog.Disabled) {
zerolog.SetGlobalLevel(zerolog.Level(*config.EnvVars.LogLevel))
return
}

// if not set in env, set from config
if *config.AppConfig.LogLevel <= int(zerolog.Disabled) {
zerolog.SetGlobalLevel(zerolog.Level(*config.AppConfig.LogLevel))
return
}
}
64 changes: 11 additions & 53 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/BESTSELLER/dependabot-circleci/api"
"github.com/BESTSELLER/dependabot-circleci/config"
"github.com/BESTSELLER/dependabot-circleci/logger"
"github.com/BESTSELLER/go-vault/gcpss"

"flag"
)
Expand All @@ -21,63 +20,22 @@ func init() {
logger.Init()
log.Debug().Msgf("Logging level: %d", *config.EnvVars.LogLevel)

if err != nil {
log.Fatal().Err(err).Msg("failed to read env config")
}

var appsecret []byte
var dbsecret []byte

if config.EnvVars.Config == "" {
log.Debug().Msg("No config file specified, fetching secrets from vault")
vaultAddr := os.Getenv("VAULT_ADDR")
if vaultAddr == "" {
log.Fatal().Msg("VAULT_ADDR must be set")
}
vaultRole := os.Getenv("VAULT_ROLE")
if vaultRole == "" {
log.Fatal().Msg("VAULT_ROLE must be set")
}

appSecret := os.Getenv("APP_SECRET")
if appSecret == "" {
log.Fatal().Msg("APP_SECRET must be set")
}

dbSecret := os.Getenv("DB_SECRET")
if dbSecret == "" {
log.Fatal().Msg("DB_SECRET must be set")
}

// fetch app secrets
secretData, err := gcpss.FetchVaultSecret(vaultAddr, appSecret, vaultRole)
if err != nil {
log.Fatal().Err(err).Msgf("Unable to fetch secrets from vault. error %v", err)
}
appsecret = []byte(secretData)

// fetch db secrets
secretData, err = gcpss.FetchVaultSecret(vaultAddr, dbSecret, vaultRole)
if err != nil {
log.Fatal().Err(err).Msgf("Unable to fetch secrets from vault. error %v", err)
}
dbsecret = []byte(secretData)

} else {
log.Debug().Msgf("Using config file: %s", config.EnvVars.Config)
bytes, err := os.ReadFile(config.EnvVars.Config)
if err != nil {
log.Fatal().Err(err).Msgf("Unable to read file %s", config.EnvVars.Config)
}
appsecret = bytes
log.Debug().Msgf("Using config file: %s", config.EnvVars.Config)
bytes, err := os.ReadFile(config.EnvVars.Config)
if err != nil {
log.Fatal().Err(err).Msgf("Unable to read file %s", config.EnvVars.Config)
}
appsecret = bytes

log.Debug().Msgf("Using db config file: %s", config.EnvVars.DBConfig)
bytes, err = os.ReadFile(config.EnvVars.DBConfig)
if err != nil {
log.Fatal().Err(err).Msgf("Unable to read file %s", config.EnvVars.DBConfig)
}
dbsecret = bytes
log.Debug().Msgf("Using db config file: %s", config.EnvVars.DBConfig)
bytes, err = os.ReadFile(config.EnvVars.DBConfig)
if err != nil {
log.Fatal().Err(err).Msgf("Unable to read file %s", config.EnvVars.DBConfig)
}
dbsecret = bytes

err = config.ReadAppConfig([]byte(appsecret))
if err != nil {
Expand Down
163 changes: 112 additions & 51 deletions terraform/modules/cloud_run/main.tf
Original file line number Diff line number Diff line change
@@ -1,76 +1,137 @@
resource "google_cloud_run_service" "main" {
resource "google_cloud_run_v2_service" "main" {
provider = google-beta
name = var.name
location = var.location
project = var.project_id
metadata {
labels = {
env = var.env
service = var.service
team = var.team
version = replace(var.tag, ".", "_")
}
template {
service_account = "${var.service}-v3@${var.project_id}.iam.gserviceaccount.com"
timeout = "1800s"
max_instance_request_concurrency = var.container_concurrency
labels = {
env = var.env
service = var.service
team = var.team
version = replace(var.tag, ".", "_")
}
}
template {
metadata {
labels = {
env = var.env
service = var.service
team = var.team
version = replace(var.tag, ".", "_")
}
annotations = {
"autoscaling.knative.dev/maxScale" = var.scaling["max"]
"autoscaling.knative.dev/minScale" = var.scaling["min"]
"run.googleapis.com/cloudsql-instances" = var.db_instance
annotations = {
"autoscaling.knative.dev/maxScale" = var.scaling["max"]
"autoscaling.knative.dev/minScale" = var.scaling["min"]
"run.googleapis.com/cloudsql-instances" = var.db_instance

}
containers {
name = var.name
image = "europe-docker.pkg.dev/artifacts-pub-prod-b57f/public-docker/${var.service}:${var.tag}"
args = var.args
depends_on = ["secret-dumper"]
env {
name = "DEPENDABOT_WORKERURL"
value = var.worker_url
}
env {
name = "DEPENDABOT_CONFIG"
value = "/secrets/app-secrets"
}
env {
name = "DEPENDABOT_DBCONFIG"
value = "/secrets/db-secrets"
}
ports {
name = "http1"
container_port = 3000
}
volume_mounts {
name = "secrets"
mount_path = "/secrets"
}
}
spec {
containers {
image = "europe-docker.pkg.dev/artifacts-pub-prod-b57f/public-docker/${var.service}:${var.tag}"
args = var.args
env {
name = "DEPENDABOT_WORKERURL"
value = var.worker_url
}
env {
name = "VAULT_ADDR"
value = "https://vault.bestsellerit.com"
}
env {
name = "VAULT_ROLE"
value = "dependabot-circleci-v3"
}
env {
name = "APP_SECRET"
value = "ES/data/${var.service}/prod"
}
env {
name = "DB_SECRET"
value = "ES/data/${var.service}/db"
}
ports {
name = "http1"
container_port = 3000
containers {
name = "secret-dumper"
image = "europe-docker.pkg.dev/artifacts-pub-prod-b57f/public-docker/harpocrates:2.4.0"
args = [
jsonencode({
"format" : "json",
"output" : "/secrets",
"secrets" : [
{
"ES/data/${var.service}/prod" : {
"filename" : "app-secrets"
}
},
{
"ES/data/${var.service}/db" : {
"filename" : "db-secrets"
}
}
]
})
]
env {
name = "VAULT_ADDR"
value = "https://vault.bestsellerit.com"
}
env {
name = "AUTH_NAME"
value = "dependabot-circleci-v3"
}
env {
name = "ROLE_NAME"
value = "dependabot-circleci-v3"
}
env {
name = "GCP_WORKLOAD_ID"
value = "true"
}
env {
name = "CONTINUOUS"
value = "true"
}
env {
name = "INTERVAL"
value = "60s"
}
env {
name = "LOG_LEVEL"
value = "warn"
}
ports {
name = "http1"
container_port = 8080
}
volume_mounts {
name = "secrets"
mount_path = "/secrets"
}
startup_probe {
http_get {
path = "/status"
port = 8000
}
initial_delay_seconds = 2
}
service_account_name = "${var.service}-v3@${var.project_id}.iam.gserviceaccount.com"
timeout_seconds = 1800
container_concurrency = var.container_concurrency
}
volumes {
name = "secrets"
empty_dir {}
}
}

traffic {
percent = 100
latest_revision = true
percent = 100
}
}

resource "google_cloud_run_service_iam_member" "allow_unauthenticated" {
resource "google_cloud_run_v2_service_iam_member" "allow_unauthenticated" {
count = var.allow_unauthenticated ? 1 : 0
location = google_cloud_run_service.main.location
project = google_cloud_run_service.main.project
service = google_cloud_run_service.main.name
location = google_cloud_run_v2_service.main.location
project = google_cloud_run_v2_service.main.project
name = google_cloud_run_v2_service.main.name
role = "roles/run.invoker"
member = "allUsers"
}
2 changes: 1 addition & 1 deletion terraform/modules/cloud_run/outputs.tf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
output "url" {
value = google_cloud_run_service.main.status.0.url
value = google_cloud_run_v2_service.main.uri
}

7 changes: 7 additions & 0 deletions terraform/terraform.tf
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ terraform {
source = "hashicorp/google"
version = "4.71.0"
}
google-beta = {
source = "hashicorp/google-beta"
version = "4.71.0"
}
}

backend "remote" {
Expand All @@ -21,3 +25,6 @@ terraform {
provider "google" {
credentials = file("/tmp/cloudrun-admin.json")
}
provider "google-beta" {
credentials = file("/tmp/cloudrun-admin.json")
}

0 comments on commit b55b032

Please sign in to comment.