ES-2647: Introduce a feature to update private orbs for BESTSELLER on…

…ly + Fix deprecated dependency `io/ioutil` + Polish documentation + Ease local development + Fix issue with many idle database connections + General code improvements (#297)

* Fix spelling mistake

* Fixing spelling mistake in comment

* Naming convention on secret files

* Adding connection string to dbconf

* Handling connection string

* Able to pass dbconfig through env vars and handle default log level

* Just a debug log

* Just for safety

* docker-compose for local development

* This works for now. Let's see when there's time for documentation and cleanup

* Documentation for DBClient

* Added specific bestseller config to handle our private orbs + added handling of an error + correcting variable types and more

* Added error handling

* Convert("fmt.Print*", "Log.[Debug,Info]().*")

* Polishing documentation and docker-compose stuff

* Fix deprecated dependency

* Better returns

* Error messages should not be capitalised

* Error messages should not be capitalised

* FIxed deprecated dependency

* Fixed vet issues

* Fixed deprecated dependencies

* Unused

* Some how, these dots got away

.circleci/config.yml

@@ -97,7 +97,7 @@ workflows:
   test:
     jobs:
       - secret-injector/dump-secrets-yaml:
-          secret-file: ci-secrets.yaml
+          secret-file: secrets-ci.yaml
           <<: [*prod_context, *no_deploy_filter]
       - cci-common/go_test_unit:
           <<: [*prod_context, *no_deploy_filter]
@@ -127,7 +127,7 @@ workflows:
   deploy:
     jobs:
       - secret-injector/dump-secrets-yaml:
-          secret-file: ci-secrets.yaml
+          secret-file: secrets-ci.yaml
           <<: [*prod_context, *deploy_filter]
       - cci-common/build_n_push_docker:
           tag: ${CIRCLE_TAG}

.gitignore

@@ -22,3 +22,8 @@ terraform/.terraform
 terraform/.terraform.*
 terraform/terraform.tfstate*
 
+# Just for safety
+cloudrun_admin*
+db-secrets
+real-secrets*
+secrets*

README.md

@@ -8,15 +8,15 @@
 
 <br/>
 
-`dependabot-circleci` is, as its name suggests, a small dependabot for circleci orbs and images.
-We have created this as at the time of creation it was nearly impossible getting changes into the official [dependabot](https://github.com/dependabot/dependabot-core).
+`dependabot-circleci` is, as its name suggests, a small dependabot for CircleCI orbs and container images.
+We have created this as at the time of creation it was nearly impossible to get changes into the official [dependabot](https://github.com/dependabot/dependabot-core).
 
 ---
 <br/>
 
 ## Getting Started
-1. Install the `dependabot-circleci` [github app](https://github.com/apps/dependabot-circleci) in your organization.
-2. You enable `dependabot-circleci` on specific repositories creating dependabot-circleci.yml configuration file in to your repository's .github directory. `dependabot-circleci` then raises pull requests to keep the dependencies you configure up-to-date.
+1. Install the `dependabot-circleci` [GitHub App](https://github.com/apps/dependabot-circleci) in your organization.
+2. You enable `dependabot-circleci` on specific repositories by creating a `dependabot-circleci.yml` configuration file in your repository's `.github` directory. `dependabot-circleci` then raise pull requests to keep the dependencies you configure up-to-date.
 
 <br/>
 
@@ -64,3 +64,46 @@ You must store this file in the .github directory of your repository.
 
 ## Contributing
 We are open for issues, pull requests etc.
+
+## Running locally
+1. Clone the repository
+2. Make sure to have your secrets file in place  
+    2.1 BESTSELLER folks can use Harpocrates to get them from Vault.
+      ```bash
+      harpocrates -f secrets-local.yaml --vault-token $(vault token create -format=json | jq -r '.auth.client_token')
+      ```  
+    2.2 Others will have to fill out this template in any other way.  
+      ```json
+      {
+        "datadog": {
+          "api_key": ""
+        },
+        "github": {
+          "app": {
+            "integration_id": ,
+            "private_key": "",
+            "webhook_secret": ""
+          },
+          "oauth": {
+            "client_id": "",
+            "client_secret": ""
+          },
+          "v3_api_url": "https://api.github.com/"
+        },
+        "http": {
+          "token": ""
+        },
+        "server": {
+          "port": 3000,
+          "public_url": ""
+        },
+        "bestseller_specific": {
+          "token": ""
+        }
+      }
+      ```
+3. Run `dependabot-circleci` by using Docker compose
+    > `--build` will ensure that the latest version of the code is used
+    ```bash
+    docker-compose up --build
+    ```

api/controller.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strings"
 	"sync"
@@ -29,16 +29,21 @@ type WorkerPayload struct {
 var wg sync.WaitGroup
 
 func controllerHandler(w http.ResponseWriter, r *http.Request) {
+	log.Debug().Msg("controllerHandler called")
+
 	orgs, err := pullRepos()
 	if err != nil {
 		log.Error().Err(err).Msgf("pull repos from big query failed: %s", err)
 		http.Error(w, "", http.StatusInternalServerError)
 	}
+	log.Debug().Msgf("Found %d organizations", len(orgs))
 
+	log.Debug().Msg("Sending metric to datadog")
 	// send stats to dd
 	go datadog.Gauge("organizations", float64(len(orgs)), nil)
 
-	// should be in parralel
+	log.Debug().Msg("Triggering workers")
+	// should be in parallel
 	for organization, repositories := range orgs {
 		wg.Add(1)
 		go func(org string, repos []db.RepoData) {
@@ -67,22 +72,16 @@ func controllerHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	wg.Wait()
+	log.Debug().Msg("All workers finished")
 }
 func shouldRun(schedule string) bool {
 	// check if an update should be run
 	t := time.Now()
 	schedule = strings.ToLower(schedule)
 	if schedule == "monthly" {
-		if t.Day() == 1 {
-			return true
-		}
-		return false
+		return (t.Day() == 1)
 	} else if schedule == "weekly" {
-		if t.Weekday() == 1 {
-			return true
-		}
-		return false
-
+		return (t.Weekday() == 1)
 	} else if schedule == "daily" || schedule == "" {
 		return true
 	}
@@ -92,30 +91,33 @@ func shouldRun(schedule string) bool {
 // PostJSON posts the structs as json to the specified url
 func PostJSON(url string, payload []byte) error {
 
+	var myClient *http.Client
 	clientWithAuth, err := idtoken.NewClient(context.Background(), url)
 	if err != nil {
-		return fmt.Errorf("idtoken.NewClient: %v", err)
+		log.Warn().Err(err).Msg("Issues getting token from GCP metadata server, trying to continue without auth.")
+		myClient = &http.Client{}
+		// return fmt.Errorf("idtoken.NewClient: %v", err)
+	} else {
+		myClient = httptrace.WrapClient(clientWithAuth)
 	}
 
-	var myClient = httptrace.WrapClient(clientWithAuth)
-
 	req, err := http.NewRequest("POST", url, bytes.NewBuffer(payload))
 	if err != nil {
-		return fmt.Errorf("Unable to create request: %s", err)
+		return fmt.Errorf("unable to create request: %s", err)
 	}
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", config.AppConfig.HTTP.Token))
 	r, err := myClient.Do(req)
 	if err != nil {
-		return fmt.Errorf("Unable to do request: %s", err)
+		return fmt.Errorf("unable to do request: %s", err)
 	}
 	defer r.Body.Close()
 
 	// check response code
 	if r.StatusCode != http.StatusOK {
-		bodyBytes, _ := ioutil.ReadAll(r.Body)
+		bodyBytes, _ := io.ReadAll(r.Body)
 		bodyString := string(bodyBytes)
-		return fmt.Errorf("Request failed, expected status: %d got: %d, error message: %s", http.StatusOK, r.StatusCode, bodyString)
+		return fmt.Errorf("request failed, expected status: %d got: %d, error message: %s", http.StatusOK, r.StatusCode, bodyString)
 	}
 	return nil
 }

api/dep_check.go

@@ -29,17 +29,17 @@ func dependencyHandler(w http.ResponseWriter, r *http.Request) {
 	cc, err := gh.CreateGHClient(config.AppConfig.Github)
 	if err != nil {
 		http.Error(w, "failed to register organization client", http.StatusInternalServerError)
-		log.Fatal().Err(err).Msg("failed to register organization client")
+		log.Fatal().Err(err).Msg("failed to register organization client (gh.CreateGHClient)")
 	}
 
 	client, err := gh.GetSingleOrganizationClient(cc, workerPayload.Org)
 	if err != nil {
 		http.Error(w, "failed to register organization client", http.StatusInternalServerError)
-		log.Fatal().Err(err).Msg("failed to register organization client")
+		log.Fatal().Err(err).Msg("failed to register organization client (gh.GetSingleOrganizationClient)")
 	}
 
 	// do our magic
-	dependabot.Start(context.Background(), client, workerPayload.Repos)
+	dependabot.Start(context.Background(), client, workerPayload.Org, workerPayload.Repos)
 
 	// send stats to DD
 	defer datadog.TimeTrackAndGauge("dependency_check_duration", []string{fmt.Sprintf("organization:%s", workerPayload.Org)}, start)

circleci/orb.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/BESTSELLER/dependabot-circleci/config"
 	"github.com/CircleCI-Public/circleci-cli/api"
 	"github.com/CircleCI-Public/circleci-cli/api/graphql"
 	"github.com/rs/zerolog/log"
@@ -38,7 +39,13 @@ func findNewestOrbVersion(orb string) string {
 		return orbSplitString[1]
 	}
 
-	client := graphql.NewClient(http.DefaultClient, "https://circleci.com/", "graphql-unstable", "", false)
+	CCIApiToken := ""
+	if config.AppConfig.BestsellerSpecific.Running {
+		log.Debug().Msg("Using Bestseller specific token to handle private orbs")
+		CCIApiToken = config.AppConfig.BestsellerSpecific.Token
+	}
+
+	client := graphql.NewClient(http.DefaultClient, "https://circleci.com/", "graphql-unstable", CCIApiToken, false)
 
 	// if requests fails, return current version
 	orbInfo, err := api.OrbInfo(client, orbSplitString[0])

circleci/update_test.go

@@ -2,7 +2,7 @@ package circleci
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -13,7 +13,7 @@ import (
 
 func getTestCases() map[string]*yaml.Node {
 	path := "../.test_cases"
-	files, err := ioutil.ReadDir(path)
+	files, err := os.ReadDir(path)
 	if err != nil {
 		log.Fatal().Err(err)
 	}
@@ -27,7 +27,7 @@ func getTestCases() map[string]*yaml.Node {
 		filePath := filepath.Join(path, fileName)
 		fmt.Println(f.Name())
 
-		content, _ := ioutil.ReadFile(filePath)
+		content, _ := os.ReadFile(filePath)
 		var cciconfig yaml.Node
 		err = yaml.Unmarshal(content, &cciconfig)
 		if err != nil {

config/config.go

@@ -6,6 +6,7 @@ import (
 	"github.com/palantir/go-baseapp/baseapp"
 	"github.com/palantir/go-githubapp/githubapp"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v2"
 )
 
@@ -19,19 +20,26 @@ type HTTPConfig struct {
 
 // Config contains global config
 type Config struct {
-	Datadog DatadogConfig      `yaml:"datadog"`
-	Github  githubapp.Config   `yaml:"github"`
-	HTTP    HTTPConfig         `yaml:"http"`
-	Server  baseapp.HTTPConfig `yaml:"server"`
+	Datadog            DatadogConfig            `yaml:"datadog"`
+	Github             githubapp.Config         `yaml:"github"`
+	HTTP               HTTPConfig               `yaml:"http"`
+	Server             baseapp.HTTPConfig       `yaml:"server"`
+	BestsellerSpecific BestsellerSpecificConfig `yaml:"bestseller_specific"`
+}
+
+type BestsellerSpecificConfig struct {
+	Token   string `yaml:"token"`
+	Running bool
 }
 
 // DBConfig contains global db config
 type DBConfigSpec struct {
-	ConnectionName string `yaml:"connection_name"`
-	DBName         string `yaml:"db_name"`
-	Instance       string `yaml:"instance"`
-	Password       string `yaml:"password"`
-	Username       string `yaml:"username"`
+	ConnectionName   string `yaml:"connection_name"`
+	ConnectionString string `yaml:"connection_string"`
+	DBName           string `yaml:"db_name"`
+	Instance         string `yaml:"instance"`
+	Password         string `yaml:"password"`
+	Username         string `yaml:"username"`
 }
 
 // RepoConfig contains specific config for each repos
@@ -67,6 +75,8 @@ func ReadDBConfig(secrets []byte) error {
 		return errors.Wrap(err, "failed parsing configuration file")
 	}
 
+	log.Debug().Msg("Both DBConficSpec.ConnectionName and DBConfigSpec.ConnectionString are set. DBConfigSpec.ConnectionString is overwriting DBConficSpec.ConnectionName")
+
 	return nil
 }
 

config/env.go

@@ -7,6 +7,7 @@ import (
 // EnvConfig defines the structure of the global configuration parameters
 type EnvConfig struct {
 	Config    string `required:"false"`
+	DBConfig  string `required:"false"`
 	LogLevel  *int   `required:"false"`
 	DDAddress string `required:"false"`
 	Schedule  string `required:"false"`
@@ -22,5 +23,12 @@ func LoadEnvConfig() error {
 	if err != nil {
 		return err
 	}
+
+	// If no log level is set, default to info
+	if EnvVars.LogLevel == nil {
+		logLevel := 1
+		EnvVars.LogLevel = &logLevel
+	}
+
 	return nil
 }

datadog/client.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"time"
 
@@ -102,11 +102,11 @@ func postStructAsJSON(url string, payload interface{}, target interface{}) (stri
 	defer r.Body.Close()
 
 	// check status code
-	bodyBytes, _ := ioutil.ReadAll(r.Body)
+	bodyBytes, _ := io.ReadAll(r.Body)
 	bodyString := string(bodyBytes)
 
 	if r.StatusCode < 200 || r.StatusCode > 299 {
-		return "", fmt.Errorf("Request failed, expected status: 2xx got: %d, error message: %s", r.StatusCode, bodyString)
+		return "", fmt.Errorf("request failed, expected status: 2xx got: %d, error message: %s", r.StatusCode, bodyString)
 	}
 	decode := json.NewDecoder(r.Body)
 	err = decode.Decode(&target)

db/db.go

@@ -21,8 +21,22 @@ type RepoData struct {
 	LastRun  bun.NullTime `bun:"lastrun"`
 }
 
+// client to be returned instead of creating a new one repeatedly
+var client *bun.DB
+
+// According to documentation, it's rarely necessary to close a DB connection.
+// The returned DB is safe for concurrent use by multiple goroutines
+// and maintains its own pool of idle connections. Thus, the OpenDB function should be called just once.
 func DBClient() *bun.DB {
+	if client != nil {
+		return client
+	}
+
 	dsn := fmt.Sprintf("unix:///cloudsql/%s/.s.PGSQL.5432?sslmode=disable", config.DBConfig.ConnectionName)
+	if config.DBConfig.ConnectionString != "" {
+		dsn = config.DBConfig.ConnectionString
+	}
+
 	sqldb := sql.OpenDB(pgdriver.NewConnector(
 		pgdriver.WithDatabase(config.DBConfig.DBName),
 		pgdriver.WithUser(config.DBConfig.Username),
@@ -31,7 +45,8 @@ func DBClient() *bun.DB {
 		pgdriver.WithDSN(dsn),
 	))
 
-	return bun.NewDB(sqldb, pgdialect.New())
+	client = bun.NewDB(sqldb, pgdialect.New())
+	return client
 
 }