diff --git a/.circleci/config.yml b/.circleci/config.yml index 75e97180..0f95365a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,8 @@ version: 2.1 executors: + gcp_image: + docker: + - image: google/cloud-sdk:alpine go_image: docker: - image: cimg/go:1.15 @@ -72,84 +75,74 @@ jobs: -Dsonar.links.scm=$CIRCLE_REPOSITORY_URL \ -Dsonar.go.coverage.reportPaths=coverage.out - docker-build-n-push: - docker: - - image: docker:stable-git - auth: - username: $DOCKERHUB_USER - password: $DOCKERHUB_PASS + build-push-serverless: + executor: gcp_image steps: - - attach_workspace: - at: /tmp - checkout - setup_remote_docker + - attach_workspace: + at: /tmp - run: - name: Push and build image + name: install jq command: | - source /tmp/secrets - docker login -u $DOCKER_username -p $DOCKER_password $K8S_CLUSTER_docker_registry_url - docker build \ - -t $K8S_CLUSTER_docker_registry_url/$SHORT/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG \ - -t $K8S_CLUSTER_docker_registry_url/$SHORT/$CIRCLE_PROJECT_REPONAME:latest \ - --build-arg VERSION=$CIRCLE_TAG . - docker push $K8S_CLUSTER_docker_registry_url/$SHORT/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG - docker push $K8S_CLUSTER_docker_registry_url/$SHORT/$CIRCLE_PROJECT_REPONAME:latest + apk add --no-cache jq + - run: + name: Build and push image + command: | + # source /tmp/secrets - deploy: - docker: - - image: google/cloud-sdk:alpine - auth: - username: $DOCKERHUB_USER - password: $DOCKERHUB_PASS + gcloud auth activate-service-account --key-file=/tmp/cluster_secret.json + gcloud auth configure-docker europe-docker.pkg.dev --quiet + + export docker_registry_url=europe-docker.pkg.dev/es-standalone-cb21 + docker build -t $docker_registry_url/es-docker/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG . + docker build -t $docker_registry_url/es-docker/$CIRCLE_PROJECT_REPONAME:$latest . + + docker push $docker_registry_url/es-docker/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG + docker push $docker_registry_url/es-docker/$CIRCLE_PROJECT_REPONAME:latest + + deploy-serverless: + executor: gcp_image steps: - checkout - attach_workspace: at: /tmp - run: - name: gcloud login - command: | - source /tmp/secrets - gcloud auth activate-service-account --key-file=/tmp/cluster_secret.json - - run: - name: install kubectl + name: install jq command: | - curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl - chmod +x ./kubectl - mv ./kubectl /usr/local/bin/kubectl + apk add --no-cache jq - run: - name: connect to the cluster + name: gcloud login command: | - source /tmp/secrets - gcloud container clusters get-credentials $K8S_CLUSTER_cluster_name --project $K8S_CLUSTER_project_id --zone europe-west4 + gcloud components install beta + cat /tmp/cloudrun_admin | jq -r .private_key_data | base64 -d > cloudrun-admin.json + gcloud auth activate-service-account --key-file=./cloudrun-admin.json - run: - name: Install envsubt + name: creating deployment command: | - apk update - apk add gettext + gcloud beta run deploy $CIRCLE_PROJECT_REPONAME \ + --image=europe-docker.pkg.dev/es-standalone-cb21/es-docker/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG \ + --allow-unauthenticated \ + --min-instances=1 \ + --max-instances=100 \ + --port=3000 \ + --platform=managed \ + --region=europe-west4 \ + --project=es-standalone-cb21 \ + --set-env-vars VAULT_ADDR="${VAULT_ADDR}",VAULT_ROLE=dependabot-circleci,VAULT_SECRET=ES/data/dependabot-circleci/prod,DEPENDABOT_VERSION=${$CIRCLE_TAG},DEPENDABOT_CONFIG=/secrets/secrets \ + --service-account=dependabot-circleci@es-standalone-cb21.iam.gserviceaccount.com - run: - name: replace strings + name: create domain mapping command: | - source /tmp/secrets - if [[ ${CIRCLE_BRANCH} = "master" ]] || [[ ${CIRCLE_TAG} =~ ^[0-9]+(\.[0-9]+)*(-.*)*$ ]]; - then - export DEPLOY_VERSION='' - export SECRET_VERSION=prod - else - export DEPLOY_VERSION=-$CIRCLE_BRANCH - echo 'export DEPLOY_VERSION="-$CIRCLE_BRANCH"' >> $BASH_ENV - source ${BASH_ENV} - export SECRET_VERSION=$CIRCLE_BRANCH + if ! gcloud beta run domain-mappings describe --domain=bestsellerit.com --platform=managed --region=europe-west1 --project=es-standalone-cb21; then + gcloud beta run domain-mappings create \ + --service=$CIRCLE_PROJECT_REPONAME \ + --domain=bestsellerit.com \ + --platform=managed \ + --region=europe-west1 \ + --project=es-standalone-cb21 fi - envsubst < ./secrets.yml > ./secrets_var.yml && mv ./secrets_var.yml ./secrets.yml - envsubst < ./deployment.yml > ./deployment_var.yml && mv ./deployment_var.yml ./deployment.yml - - secret-injector/inject: - app-name: $CIRCLE_PROJECT_REPONAME$DEPLOY_VERSION - deploy-file: ./deployment.yml - secret-file: secrets.yml - - run: - name: create kubernetes service - command: | - kubectl apply -f ./deployment.yml + workflows: test: @@ -177,29 +170,25 @@ workflows: tags: ignore: /^[0-9]+(\.[0-9]+)*(-.*)*$/ - test-build-deploy: + + serverless-test-build-deploy: jobs: - secret-injector/dump-secrets: - context: - - es02-prod - - shared + context: es02-prod filters: tags: only: /^[0-9]+(\.[0-9]+)*(-.*)*$/ branches: ignore: /.*/ - test: - context: - - shared + requires: + - secret-injector/dump-secrets filters: tags: only: /^[0-9]+(\.[0-9]+)*(-.*)*$/ branches: - ignore: /.*/ + ignore: /.*/ - test-sonar: - context: - - es02-prod - - shared requires: - secret-injector/dump-secrets filters: @@ -207,25 +196,35 @@ workflows: only: /^[0-9]+(\.[0-9]+)*(-.*)*$/ branches: ignore: /.*/ - - docker-build-n-push: - context: - - es02-prod - - shared + - secret-injector/dump-secrets: + requires: + - test + - test-sonar + name: secret-injector/cloudrun + vault-path: "gcp_landingzone/key/cloudrun-admin" + output-filename: cloudrun_admin + format: json + context: es02-prod + filters: + tags: + only: /^[0-9]+(\.[0-9]+)*(-.*)*$/ + branches: + ignore: /.*/ + - build-push-serverless: requires: - secret-injector/dump-secrets + - secret-injector/cloudrun filters: tags: only: /^[0-9]+(\.[0-9]+)*(-.*)*$/ branches: ignore: /.*/ - - deploy: - context: - - es02-prod - - shared + - deploy-serverless: requires: - - docker-build-n-push + - build-push-serverless + context: es02-prod filters: tags: only: /^[0-9]+(\.[0-9]+)*(-.*)*$/ branches: - ignore: /.*/ \ No newline at end of file + ignore: /.*/ diff --git a/Dockerfile b/Dockerfile index 3601203a..d85b5c1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,5 @@ RUN GOOS=linux GOARCH=amd64 go build -ldflags="-w -s -X gh.version=${VERSION}" - FROM alpine COPY --from=builder /tmp/dependabot-circleci /dependabot-circleci -CMD ["/dependabot-circleci"] \ No newline at end of file +CMD ["/dependabot-circleci"] +EXPOSE 3000 \ No newline at end of file diff --git a/config/config.go b/config/config.go index 5f125e55..d9701201 100644 --- a/config/config.go +++ b/config/config.go @@ -9,10 +9,15 @@ import ( "gopkg.in/yaml.v2" ) +type DatadogConfig struct { + APIKey string `yaml:"api_key"` +} + // Config contains global config type Config struct { - Github githubapp.Config `yaml:"github"` - Server baseapp.HTTPConfig `yaml:"server"` + Datadog DatadogConfig `yaml:"datadog"` + Github githubapp.Config `yaml:"github"` + Server baseapp.HTTPConfig `yaml:"server"` } // RepoConfig contains specific config for each repos diff --git a/datadog/client.go b/datadog/client.go index 76d4cc52..aa950e07 100644 --- a/datadog/client.go +++ b/datadog/client.go @@ -1,33 +1,36 @@ package datadog import ( + "bytes" + "encoding/json" "fmt" + "io/ioutil" + "net/http" + "time" "github.com/BESTSELLER/dependabot-circleci/config" - "github.com/DataDog/datadog-go/statsd" "github.com/rs/zerolog/log" ) -var client *statsd.Client -var metricPrefix = "dependabot_circleci" +type DataDog struct { + Series []Series `json:"series"` +} -// CreateClient creates a statsd client -func CreateClient() (err error) { - client, err = statsd.New(config.EnvVars.DDAddress) - if err != nil { - return err - } - return nil +type Series struct { + Metric string `json:"metric"` + Points [][]int64 `json:"points"` + Tags []string `json:"tags"` + Type string `json:"type"` + Host string `json:"host"` + Interval int64 `json:"interval"` } +var metricPrefix = "dependabot_circleci" + // IncrementCount incrementes a counter based on the input func IncrementCount(metricName string, org string) { - err := client.Incr( - fmt.Sprintf("%s.%s", metricPrefix, metricName), - []string{ - "organistation:" + org, - }, - 1) + metric := metricPrefix + "." + metricName + err := postDataDogMetric(metric, 1, "count", []string{"organistation:" + org}) if err != nil { log.Debug().Err(err).Msgf("could increment datadog counter %s", metricName) } @@ -35,13 +38,72 @@ func IncrementCount(metricName string, org string) { // Gauge incrementes a counter based on the input func Gauge(metricName string, value float64, tags []string) { - err := client.Gauge( - fmt.Sprintf("%s.%s", metricPrefix, metricName), - value, - tags, - 1, - ) + metric := metricPrefix + "." + metricName + err := postDataDogMetric(metric, int64(value), "gauge", tags) if err != nil { log.Debug().Err(err).Msgf("could send gauge to datadog %s", metricName) } + +} + +func postDataDogMetric(metric string, value int64, metricType string, tags []string) error { + apiKey := config.AppConfig.Datadog.APIKey + + url := "https://api.datadoghq.eu/api/v1/series?api_key=" + apiKey + + series := Series{ + Metric: metric, + Type: metricType, + Tags: tags, + } + + point := [][]int64{ + {time.Now().Unix(), value}, + } + series.Points = point + + payload := DataDog{[]Series{series}} + + _, err := postStructAsJSON(url, payload, nil) + if err != nil { + return err + } + + return nil +} + +func postStructAsJSON(url string, payload interface{}, target interface{}) (string, error) { + var myClient = http.Client{} + b := new(bytes.Buffer) + err := json.NewEncoder(b).Encode(payload) + if err != nil { + fmt.Printf("Unable to encode struct: %s", err) + return "", err + } + req, err := http.NewRequest("POST", url, b) + if err != nil { + return "", err + } + req.Header.Add("Accept", "application/json") + req.Header.Add("Content-Type", "application/json") + r, err := myClient.Do(req) + if err != nil { + return "", err + } + defer r.Body.Close() + + // check status code + bodyBytes, _ := ioutil.ReadAll(r.Body) + bodyString := string(bodyBytes) + + if r.StatusCode < 200 || r.StatusCode > 299 { + return "", fmt.Errorf("Request failed, expected status: 2xx got: %d, error message: %s", r.StatusCode, bodyString) + } + decode := json.NewDecoder(r.Body) + err = decode.Decode(&target) + if err != nil { + return "", err + } + + return bodyString, nil } diff --git a/go.mod b/go.mod index d0c3bc02..732c40f9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/BESTSELLER/dependabot-circleci go 1.15 require ( - github.com/CircleCI-Public/circleci-cli v0.1.11924 + github.com/BESTSELLER/go-vault v0.1.2 + github.com/CircleCI-Public/circleci-cli v0.1.11756 github.com/DataDog/datadog-go v4.2.0+incompatible github.com/go-co-op/gocron v0.5.1 github.com/google/go-containerregistry v0.4.0 diff --git a/go.sum b/go.sum index 6cd8002c..4dec1abb 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,10 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AlecAivazis/survey/v2 v2.1.1/go.mod h1:9FJRdMdDm8rnT+zHVbvQT2RTSTLq0Ttd6q3Vl2fahjk= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/BESTSELLER/go-vault v0.1.1 h1:Bb+pQTnkVsH1meEFGzsjJD8tiM9zz/DZC0DniadgnHA= +github.com/BESTSELLER/go-vault v0.1.1/go.mod h1:1vDNl2t7FFuzP9o+rJQpvWHASQWSXht7iZvXbg8KTQk= +github.com/BESTSELLER/go-vault v0.1.2 h1:CLMWRQq9L58kg3d8VPZ3FhWkO1swKrR0NRh29GeriLk= +github.com/BESTSELLER/go-vault v0.1.2/go.mod h1:1vDNl2t7FFuzP9o+rJQpvWHASQWSXht7iZvXbg8KTQk= github.com/BESTSELLER/go-version v1.2.5 h1:naMdfH4XsfyfAQVMSNDmye6pAQx0sIYClV6rPhVBqHg= github.com/BESTSELLER/go-version v1.2.5/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= diff --git a/main.go b/main.go index 2eb143df..216a6d6c 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,9 @@ package main import ( "context" + "fmt" + "io/ioutil" + "os" "sync" "time" @@ -12,6 +15,7 @@ import ( "github.com/BESTSELLER/dependabot-circleci/datadog" "github.com/BESTSELLER/dependabot-circleci/dependabot" "github.com/BESTSELLER/dependabot-circleci/logger" + "github.com/BESTSELLER/go-vault/gcpss" "github.com/BESTSELLER/dependabot-circleci/config" "github.com/BESTSELLER/dependabot-circleci/gh" @@ -19,6 +23,38 @@ import ( var wg sync.WaitGroup +func init() { + vaultAddr := os.Getenv("VAULT_ADDR") + if vaultAddr == "" { + fmt.Println("VAULT_ADDR must be set.") + } + vaultSecret := os.Getenv("VAULT_SECRET") + if vaultSecret == "" { + fmt.Println("VAULT_SECRET must be set.") + } + vaultRole := os.Getenv("VAULT_ROLE") + if vaultRole == "" { + fmt.Println("VAULT_ROLE must be set.") + } + + secret, err := gcpss.FetchVaultSecret(vaultAddr, vaultSecret, vaultRole) + if err != nil { + fmt.Println(err) + } + + err = os.Mkdir("/secrets", 0644) + if err != nil { + fmt.Println(err) + } + + data := []byte(secret) + err = ioutil.WriteFile("/secrets/secrets", data, 0644) + if err != nil { + fmt.Println(err) + } + +} + func main() { err := config.LoadEnvConfig() logger.Init() @@ -29,13 +65,7 @@ func main() { err = config.ReadConfig(config.EnvVars.Config) if err != nil { - log.Fatal().Err(err).Msg("failed to read github app config") - } - - // create statsd client - err = datadog.CreateClient() - if err != nil { - log.Error().Err(err).Msg("failed to register dogstatsd client") + log.Fatal().Err(err).Msg("failed to read github app config:") } //schedule checks