feat: add occupied slots to game rooms api (#641) (#651)

* feat: add occupied slots to game rooms api (#641)
* feat: add occupied slots to game rooms api
* fix: rename api parameter
* fix: failing tests
* chore: upgrade and remove some dependencies (#644)
* chore: upgrade and remove some dependecies
* feat: allow maestro to deal with multiple matches per game server (#646)
* feat: allow maestro to deaul with multiple matches per game server
* fix: removing cooling status
* feat: add new metrics (#649)
* feat: add new metrics
* fix: metrics reporter tests
* fix: protobuf linter offenses
* fix: update buf dependencies before running linter
* fix: add max validation to minFreeSlots fields
* feat: removing unused field minFreeSlots
* chore: update dependencies
* fix: project dependencies
* chore: upgrading go version on CI
* chore: upgrade docker image for build step
* fix: backward compatibility with exiting schedulers
* fix: linter offenses
* fix: adding default match allocation configuration when not provided
* fix: missing match allocation configuration
* feat: create simulation service
* fix: missing license headers
* fix(policy): compute desired number of matches before rooms (#654)
If we first divide the number of running matches per maxMatches
we might end up with a number between 0 and 1, which is quirky
when using math.Ceil because it would propagate the error to when
computing number of desired rooms. To simplify and fix this, instead
we now compute how many matches is the desired and then convert
the final number to the amount of rooms/instances/pods needed to
provide this number of matches:

1. How many running matches we have?
* Get the value from the redis: scheduler:<scheduler_name>:occupancy
2. How many matches do we need to have based on readyTarget?
* Divide running matches by 1 - readyTarget
3. How many game rooms are needed to provide this amount of matches?
* Get the desired number of matches and divide by the max number of matches
per room as defined in the scheduler

* fix(allocation): maxMatches limit to 35 (#655)
Some games might choose to run larger pod resources and increase number
of matches running in each room/instance, thus increasing from 30 to 35.
The limit in the max matches is useful so configuration stays in a sane
number to help in autoscaling and roundings.

* fix: setting running matches when not given
* fix: adding transition from unready to active
* fix: adding transition from occupied to active
* fix: multiple matches status calculation
* feat: report metric with scheduler max matches
* fix: sync scheduler before sending metrics

---------

Co-authored-by: Pedro Soares <pedro.soares@wildlifestudios.com>

.github/workflows/lint.yaml

@@ -8,7 +8,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: '1.20'
+          go-version: '1.22'
       - name: Checkout
         uses: actions/checkout@v2
       - name: Restore cache

.github/workflows/test.yaml

@@ -8,7 +8,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: '1.20'
+        go-version: '1.22'
     - name: Checkout
       uses: actions/checkout@v2
     - name: Restore cache
@@ -28,7 +28,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: '1.20'
+        go-version: '1.22'
     - name: Checkout
       uses: actions/checkout@v2
     - name: Restore cache
@@ -55,7 +55,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: '1.20'
+        go-version: '1.22'
     - name: Checkout
       uses: actions/checkout@v2
     - name: Restore cache

.gitignore

@@ -53,4 +53,5 @@ kubeconfig.yaml
 e2e/framework/maestro/kubeconfig.yaml
 
 # MkDocs
-site/
+site/
+buf.lock

Dockerfile

@@ -19,7 +19,7 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-FROM golang:1.20-alpine AS build-env
+FROM golang:1.22-alpine AS build-env
 
 WORKDIR /build
 

Makefile

@@ -2,6 +2,8 @@ SOURCES := $(shell \
 	find . -not \( \( -name .git -o -name .go -o -name vendor -o -name '*.pb.go' -o -name '*.pb.gw.go' -o -name '*_gen.go' -o -name '*mock*' \) -prune \) \
 	-name '*.go')
 
+BUF := github.com/bufbuild/buf/cmd/buf@v1.32.1
+
 .PHONY: help
 help: Makefile ## Show list of commands.
 	@echo "Choose a command to run in "$(APP_NAME)":"
@@ -36,7 +38,7 @@ lint/go: ## Execute golangci-lint.
 
 .PHONY: lint/protobuf
 lint/protobuf: ## Execute buf linter.
-	@go run github.com/bufbuild/buf/cmd/buf lint
+	@go run $(BUF) dep update && go run $(BUF) lint
 
 .PHONY: run/tests
 run/tests: run/unit-tests run/integration-tests ## Execute all unit and integration tests
@@ -103,8 +105,10 @@ run/metrics-reporter: build ## Runs maestro metrics-reporter.
 
 .PHONY: generate
 generate: ## Execute code generation.
+	@go run $(BUF) dep update && go run $(BUF) generate
 	@go generate ./gen
 
+
 #-------------------------------------------------------------------------------
 #  Migration and database make targets
 #-------------------------------------------------------------------------------
@@ -148,13 +152,13 @@ deps/down: ## Delete containers dependencies.
 .PHONY: maestro/start
 maestro/start: build-linux-x86_64 ## Start Maestro with all of its dependencies.
 	@echo "Starting maestro..."
-	@cd ./e2e/framework/maestro; docker-compose up --build -d
+	@cd ./e2e/framework/maestro; docker compose up --build -d
 	@MAESTRO_MIGRATION_PATH="file://internal/service/migrations" go run main.go migrate;
-	@cd ./e2e/framework/maestro; docker-compose up --build -d worker runtime-watcher #Worker and watcher do not work before migration, so we start them after it.
+	@cd ./e2e/framework/maestro; docker compose up --build -d worker runtime-watcher #Worker and watcher do not work before migration, so we start them after it.
 	@echo "Maestro is up and running!"
 
 .PHONY: maestro/down
 maestro/down: ## Delete Maestro and all of its dependencies.
 	@echo "Deleting maestro..."
-	@cd ./e2e/framework/maestro; docker-compose down
+	@cd ./e2e/framework/maestro; docker compose down
 	@echo "Maestro was deleted with success!"

buf.gen.yaml

@@ -0,0 +1,14 @@
+version: v2
+plugins:
+  - remote: buf.build/protocolbuffers/go:v1.35.1
+    out: pkg/
+    opt: paths=source_relative
+  - remote: buf.build/grpc/go:v1.5.1
+    out: pkg/
+    opt: paths=source_relative
+  - remote: buf.build/grpc-ecosystem/gateway:v2.22.0
+    out: pkg/
+    opt: paths=source_relative
+  - remote: buf.build/grpc-ecosystem/openapiv2:v2.22.0
+    out: proto/
+    opt: allow_merge=true

buf.yaml

@@ -0,0 +1,18 @@
+version: v2
+deps:
+  - buf.build/googleapis/googleapis
+  - buf.build/grpc-ecosystem/grpc-gateway
+
+modules:
+  - path: proto
+
+breaking:
+  use:
+    - FILE
+
+lint:
+  use:
+    - DEFAULT
+    - COMMENTS
+  except:
+    - PACKAGE_VERSION_SUFFIX

cmd/metricsreporter/wire.go

@@ -50,8 +50,9 @@ func provideMetricsReporterConfig(c config.Config) *workerconfigs.MetricsReporte
 var WorkerOptionsSet = wire.NewSet(
 	service.NewRoomStorageRedis,
 	service.NewGameRoomInstanceStorageRedis,
+	service.NewSchedulerCacheRedis,
 	provideMetricsReporterConfig,
-	wire.Struct(new(worker.WorkerOptions), "RoomStorage", "InstanceStorage", "MetricsReporterConfig"))
+	wire.Struct(new(worker.WorkerOptions), "RoomStorage", "InstanceStorage", "SchedulerCache", "MetricsReporterConfig"))
 
 func initializeMetricsReporter(c config.Config) (*workers.WorkersManager, error) {
 	wire.Build(

e2e/framework/api.go

@@ -28,34 +28,34 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/golang/protobuf/jsonpb"
-	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
 )
 
 type APIClient struct {
 	baseAddr    string
 	httpClient  *http.Client
-	marshaler   *jsonpb.Marshaler
-	unmarshaler *jsonpb.Unmarshaler
+	marshaler   *protojson.MarshalOptions
+	unmarshaler *protojson.UnmarshalOptions
 }
 
 func NewAPIClient(baseAddr string) *APIClient {
 	return &APIClient{
 		httpClient:  &http.Client{},
-		marshaler:   &jsonpb.Marshaler{},
-		unmarshaler: &jsonpb.Unmarshaler{},
+		marshaler:   &protojson.MarshalOptions{},
+		unmarshaler: &protojson.UnmarshalOptions{},
 		baseAddr:    baseAddr,
 	}
 }
 
 func (c *APIClient) Do(verb, path string, request proto.Message, response proto.Message) error {
-	buf := new(bytes.Buffer)
-	err := c.marshaler.Marshal(buf, request)
+	var buf []byte
+	marshalled, err := c.marshaler.MarshalAppend(buf, request)
 	if err != nil {
 		return fmt.Errorf("failed to encode request: %w", err)
 	}
 
-	req, err := http.NewRequest(verb, fmt.Sprintf("%s%s", c.baseAddr, path), buf)
+	req, err := http.NewRequest(verb, fmt.Sprintf("%s%s", c.baseAddr, path), bytes.NewBuffer(marshalled))
 	if err != nil {
 		return fmt.Errorf("failed to build request: %w", err)
 	}
@@ -67,15 +67,16 @@ func (c *APIClient) Do(verb, path string, request proto.Message, response proto.
 
 	defer resp.Body.Close()
 
-	if resp.StatusCode >= 400 {
-		if body, err := io.ReadAll(resp.Body); err == nil {
-			return fmt.Errorf("failed with status %d, response body: %s", resp.StatusCode, string(body))
-		}
+	var body []byte
+	if body, err = io.ReadAll(resp.Body); err != nil {
+		return fmt.Errorf("failed to read response body: %w", err)
+	}
 
-		return fmt.Errorf("failed with status %d", resp.StatusCode)
+	if resp.StatusCode >= 400 {
+		return fmt.Errorf("failed with status: %d, body: %s", resp.StatusCode, string(body))
 	}
 
-	err = c.unmarshaler.Unmarshal(resp.Body, response)
+	err = c.unmarshaler.Unmarshal(body, response)
 	if err != nil {
 		return fmt.Errorf("failed to decode response: %w", err)
 	}

e2e/framework/maestro/components/management_api.go

@@ -23,36 +23,33 @@
 package components
 
 import (
+	"context"
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
-	tc "github.com/testcontainers/testcontainers-go"
-
+	"github.com/docker/compose/v2/pkg/api"
+	tc "github.com/testcontainers/testcontainers-go/modules/compose"
 	"github.com/topfreegames/maestro/e2e/framework/maestro/helpers"
 )
 
 type ManagementApiServer struct {
 	Address string
-	compose tc.DockerCompose
 }
 
-func ProvideManagementApi(maestroPath string) (*ManagementApiServer, error) {
+func ProvideManagementApi(ctx context.Context, compose tc.ComposeStack) (*ManagementApiServer, error) {
 	address := "http://localhost:8080"
 	client := &http.Client{}
 
-	composeFilePaths := []string{fmt.Sprintf("%s/e2e/framework/maestro/docker-compose.yml", maestroPath)}
-	identifier := strings.ToLower("e2e-test")
-
-	compose := tc.NewLocalDockerCompose(composeFilePaths, identifier)
-	composeErr := compose.WithCommand([]string{"up", "-d", "--build", "management-api"}).Invoke()
+	services := compose.Services()
+	services = append(services, "management-api")
 
-	if composeErr.Error != nil {
-		return nil, fmt.Errorf("failed to start management API: %s", composeErr.Error)
+	err := compose.Up(ctx, tc.RunServices(services...), tc.Recreate(api.RecreateNever), tc.RecreateDependencies(api.RecreateNever), tc.Wait(true))
+	if err != nil {
+		return nil, fmt.Errorf("failed to start management-api: %w", err)
 	}
 
-	err := helpers.TimedRetry(func() error {
+	err = helpers.TimedRetry(func() error {
 		res, err := client.Get("http://localhost:8081/healthz")
 		if err != nil {
 			return err
@@ -69,12 +66,5 @@ func ProvideManagementApi(maestroPath string) (*ManagementApiServer, error) {
 		return nil, fmt.Errorf("unable to reach management API: %s", err)
 	}
 
-	return &ManagementApiServer{
-		Address: address,
-		compose: compose,
-	}, nil
-}
-
-func (ms *ManagementApiServer) Teardown() {
-	ms.compose.WithCommand([]string{"rm", "-s", "-v", "-f", "management-api"}).Invoke()
+	return &ManagementApiServer{Address: address}, nil
 }