implement database postgresql connection and background cost collector

.github/workflows/ci.yml

@@ -31,5 +31,5 @@ jobs:
         go get gopkg.in/check.v1
     - name: Build
       run: go build -v .
-    # - name: Test
-    #   run: go test -v ./...
+    - name: Test
+      run: go test -v ./...

.gitignore

@@ -7,4 +7,5 @@ docs/kubernetes/oingress.yaml
 docs/kubernetes/ns.yaml
 docs/kubernetes/ns-config.yaml
 docs/kubernetes/kubeconfig.yaml
+docs/kubernetes/test.sh
 .DS_Store

Dockerfile

@@ -9,7 +9,6 @@ FROM alpine:latest
 RUN apk --no-cache add ca-certificates
 WORKDIR /root/
 COPY --from=build /go/src/github.com/natron-io/tenant-api/tenant-api ./
-COPY --from=build /go/src/github.com/natron-io/tenant-api/views ./views
-COPY --from=build /go/src/github.com/natron-io/tenant-api/static ./static
+COPY --from=build /go/src/github.com/natron-io/tenant-api/web ./web
 EXPOSE 8000
 CMD ["./tenant-api"]

README.md

@@ -21,7 +21,7 @@ You can also sync your slack broadcast channel to present some important informa
 
 ## api
 
-#### `GET`
+### `GET`
 > **important:** for authenticated access you need to provide the `Authorization` header with the `Bearer` token.
 
 You can add `<tenant>` in front of the path to get the tenant specific data (of everything). 
@@ -47,15 +47,16 @@ You can add `<tenant>` in front of the path to get the tenant specific data (of
 `/api/v1/<tenant>/costs/cpu` - Get the CPU costs by CPU \
 `/api/v1/<tenant>/costs/memory` - Get the memory costs by Memory \
 `/api/v1/<tenant>/costs/storage` - Get the storage costs by StorageClass \
-`/api/v1/<tenant>/costs/ingress` - Get the ingress costs by tenant
+`/api/v1/<tenant>/costs/ingress` - Get the ingress costs by tenant \
+`/api/v1/<tenant>/costs/currentmonth` - Get a list of the average cost consumption for the current month
 
 ##### tenant resource quotas
 `/api/v1/<tenant>/quotas/cpu` - Get the CPU resource Quota by the label defined via env \
 `/api/v1/<tenant>/quotas/memory` - Get the memory resource Quota by the label defined via env \
 `/api/v1/<tenant>/quotas/storage` - Get the storage resource Quota for each storage class by the label**s** defined via env 
 
 
-#### `POST`
+### `POST`
 
 ##### auth
 You can send the github code with json body `{"github_code": "..."}` to the `/login/github` endpoint.
@@ -65,6 +66,7 @@ You can send the github code with json body `{"github_code": "..."}` to the `/lo
 
 ### general
 `CORS` - CORS middleware for Fiber that that can be used to enable Cross-Origin Resource Sharing with various options. (e.g. "https://example.com, https://example2.com")
+`DEBUG` - Disables API authentication / authorization (tenants doesn't work anymore, bc jwt won't get validated and parsed)
 
 ### GitHub
 > There are two ways for authenticating with GitHub. You can authenticate without a dashboard, so the github callback url is not the same as the dashboard.
@@ -94,6 +96,15 @@ You can send the github code with json body `{"github_code": "..."}` to the `/lo
 `INGRESS_COST_PER_DOMAIN` - Calculates only ingress per domain.tld format *optional* (default: false) \
 `EXCLUDE_INGRESS_VCLUSTER` - Excludes the vcluster ingress resource to expose the vcluster Kubernetes API. Name of the ingress must contain the string "vcluster" *optional* (default: false)
 
+#### cost persistency
+`COST_PERSISTENCY` - if set to true, database persistency configuration must be set *optional* (default: false) \
+`COST_PERSISTENCY_INTERVAL` - interval in seconds to log cost data into database *optional* (default: 3600) \
+`DB_HOST` - postgresql db host *optional* (default: localhost) \
+`DB_PORT` - postgresql db port *optional* (default: 5432) \
+`DB_USER` - postgresql db user *optional* (default: postgres) \
+`DB_PASSWORD` - postgresql db password *optional* (default: postgres) \
+`DB_NAME` - postgresql db name *optional* (default: postgres) \
+`DB_SSLMODE` - postgresql sslmode *optional* (default: postgres)
 
 ### resource quotas
 It will get the resource quotas defined in the tenant namespace with the exact name of the tenant.

controllers/authController.go

@@ -116,6 +116,10 @@ func CheckAuth(c *fiber.Ctx) []string {
 	var token *jwt.Token
 	var tokenString string
 
+	if util.DEBUG {
+		return []string{"debug"}
+	}
+
 	// get bearer token from header
 	bearerToken := c.Get("Authorization")
 

controllers/costController.go

@@ -2,7 +2,10 @@ package controllers
 
 import (
 	"github.com/gofiber/fiber/v2"
+	"github.com/natron-io/tenant-api/database"
+	"github.com/natron-io/tenant-api/models"
 	"github.com/natron-io/tenant-api/util"
+	"gorm.io/gorm"
 )
 
 // GetCPUCostSum returns the cpu cost sum per tenant
@@ -196,3 +199,90 @@ func GetIngressCostSum(c *fiber.Ctx) error {
 		return c.JSON(tenantsIngressCostsPerDomain[tenant])
 	}
 }
+
+func GetMonthlyCostSum(c *fiber.Ctx) error {
+
+	util.InfoLogger.Printf("%s %s %s", c.IP(), c.Method(), c.Path())
+	tenant := c.Params("tenant")
+	tenants := CheckAuth(c)
+	if len(tenants) == 0 {
+		return c.Status(401).JSON(fiber.Map{
+			"message": "Unauthorized",
+		})
+	}
+	if tenant != "" && !util.Contains(tenant, tenants) {
+		return c.Status(403).JSON(fiber.Map{
+			"message": "Forbidden",
+		})
+	}
+
+	db := database.DBConn
+
+	// search tenants in database GitHubTeamSlug
+	dbTenant := models.Tenant{
+		GitHubTeamSlug: tenant,
+	}
+
+	err := db.Where(&dbTenant).First(&dbTenant).Error
+	if err != nil {
+		if err == gorm.ErrRecordNotFound {
+			return c.Status(404).JSON(fiber.Map{
+				"message": "Not Found",
+			})
+		}
+		util.ErrorLogger.Printf("%s", err)
+		return c.Status(500).JSON(fiber.Map{
+			"message": "Internal Server Error",
+		})
+	}
+
+	// get cpuCosts by month and tenant
+	cpuCosts, err := util.GetCPUCostsByMonthAndTenant(dbTenant)
+	if err != nil {
+		return c.Status(500).JSON(fiber.Map{
+			"message": "Internal Server Error",
+		})
+	}
+
+	// get memoryCosts by month and tenant
+	memoryCosts, err := util.GetMemoryCostsByMonthAndTenant(dbTenant)
+	if err != nil {
+		return c.Status(500).JSON(fiber.Map{
+			"message": "Internal Server Error",
+		})
+	}
+
+	// get storageCosts by month and tenant
+	storageCosts, err := util.GetStorageCostsByMonthAndTenant(dbTenant)
+	if err != nil {
+		return c.Status(500).JSON(fiber.Map{
+			"message": "Internal Server Error",
+		})
+	}
+
+	var storageClassCostSum map[string]float64
+	for _, storageClass := range storageCosts {
+		if storageClassCostSum == nil {
+			storageClassCostSum = make(map[string]float64)
+		}
+		storageClassCostSum[storageClass.StorageClass] = storageClassCostSum[storageClass.StorageClass] + storageClass.Value
+	}
+
+	ingressCosts, err := util.GetIngressCostsByMonthAndTenant(dbTenant)
+	if err != nil {
+		return c.Status(500).JSON(fiber.Map{
+			"message": "Internal Server Error",
+		})
+	}
+
+	// create JSON object for each cost
+	costsJSON := make(map[string]float64)
+	costsJSON["cpu"] = cpuCosts.Value
+	costsJSON["memory"] = memoryCosts.Value
+	for storageClass, value := range storageClassCostSum {
+		costsJSON[storageClass] = value
+	}
+	costsJSON["ingress"] = ingressCosts.Value
+
+	return c.JSON(costsJSON)
+}

controllers/doc.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2022 Jan Lauber
+Copyright 2022 Netrics AG
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

database/database.go

@@ -0,0 +1,44 @@
+package database
+
+import (
+	"fmt"
+
+	"github.com/natron-io/tenant-api/models"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+var (
+	DBConn      *gorm.DB
+	err         error
+	DB_HOST     string
+	DB_PORT     string
+	DB_USER     string
+	DB_PASSWORD string
+	DB_NAME     string
+	DB_SSLMODE  string
+)
+
+func InitDB() error {
+	// Connect to the database
+	dbUri := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=%s", DB_HOST, DB_USER, DB_PASSWORD, DB_NAME, DB_PORT, DB_SSLMODE)
+	DBConn, err = gorm.Open(postgres.Open(dbUri), &gorm.Config{})
+	if err != nil {
+		return err
+	}
+
+	// Migrate the schema
+	err = DBConn.AutoMigrate(
+		&models.Tenant{},
+		&models.CPUCost{},
+		&models.MemoryCost{},
+		&models.StorageCost{},
+		&models.IngressCost{},
+		&models.MonthlyCost{},
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

database/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2022 Netrics AG
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package controllers is responsible to set up the controllers for the http handlers
+package database

docs/kubernetes/ingress.yaml

@@ -18,7 +18,7 @@ spec:
             name: tenant-api
             port:
               number: 8000
-  - host: api2.example.com
+  - host: api2.example4.com
     http:
       paths:
       - path: /
@@ -68,4 +68,4 @@ spec:
           service:
             name: tenant-api
             port:
-              number: 8000
+              number: 8000

docs/kubernetes/postgresql-values.yaml

@@ -0,0 +1,5 @@
+# Test Helm Values
+global:
+  postgresql:
+    auth:
+      postgresPassword: "12345678"

go.mod

@@ -2,28 +2,41 @@ module github.com/natron-io/tenant-api
 
 go 1.17
 
-require k8s.io/apimachinery v0.23.1
+require (
+	github.com/slack-go/slack v0.10.1
+	k8s.io/api v0.23.1
+	k8s.io/apimachinery v0.23.1
+)
 
 require (
 	github.com/andybalholm/brotli v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.10.1 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.9.1 // indirect
+	github.com/jackc/pgx/v4 v4.14.1 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.4 // indirect
 	github.com/klauspost/compress v1.13.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/slack-go/slack v0.10.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.31.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
 	golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect
 	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.23.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )
@@ -43,6 +56,8 @@ require (
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gorm.io/driver/postgres v1.3.1
+	gorm.io/gorm v1.23.2
 	k8s.io/client-go v0.23.1
 	k8s.io/klog/v2 v2.30.0 // indirect
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect