Merge pull request #62 from YellowBloomKnapsack/feature/cicd

Feature/cicd

.dockerignore

@@ -0,0 +1 @@
+**/uploads

.github/workflows/cicd.yml

@@ -0,0 +1,60 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches:
+        - main
+        - feature/cicd
+  pull_request:
+    branches:
+        - main
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.22.4
+
+    - name: Run tests
+      run: ./tester.sh
+
+  deploy:
+    needs: test
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    steps:
+    - name: Deploy to server
+      uses: appleboy/ssh-action@master
+      env:
+        GITHUB_SHA: ${{ github.sha }}
+      with:
+        host: ${{ secrets.SERVER_HOST }}
+        username: ${{ secrets.SERVER_USER }}
+        password: ${{ secrets.SERVER_PASSWORD }}
+        # key: ${{ secrets.SERVER_SSH_KEY }}
+        port: ${{ secrets.SERVER_SSH_PORT }}
+        envs: GITHUB_SHA
+        script: |
+          # Navigate to the project directory (create if it doesn't exist)
+          mkdir -p ~/mini-yektanet
+          cd ~/mini-yektanet
+
+          # Clone or pull the latest changes
+          if [ -d .git ]; then
+            git pull
+          else
+            git clone https://github.com/YellowBloomKnapsack/mini-yektanet.git .
+          fi
+
+          # Checkout the specific commit that triggered the workflow
+          git checkout $GITHUB_SHA
+
+          # Build and start the containers
+          docker compose up -d --build
+
+          # Prune old images to free up space (optional)
+          docker image prune -af

.gitignore

@@ -30,3 +30,5 @@
 .idea
 
 **/test*.db
+
+**/*.private.sh

Dockerfile

@@ -0,0 +1,44 @@
+FROM golang:1.22.4-bullseye as builder
+
+ARG FOLDER_NAME
+ARG PORT_NUMBER
+ENV BUILDER_HOME /app/
+
+WORKDIR "$BUILDER_HOME"
+
+COPY go.mod ./
+COPY go.sum ./
+RUN go mod download
+RUN go mod verify
+
+COPY ${FOLDER_NAME} ./${FOLDER_NAME}
+COPY common ./common
+COPY adserver/.env adserver/.env
+COPY eventserver/.env eventserver/.env
+COPY panel/.env panel/.env
+COPY publisherwebsite/.env publisherwebsite/.env
+
+WORKDIR ${FOLDER_NAME}
+RUN go build -o output.out
+
+FROM golang:1.22.4-bullseye
+ARG FOLDER_NAME
+ENV envFOLDER_NAME=$FOLDER_NAME
+RUN mkdir -p /app
+ENV APP_HOME /app/${FOLDER_NAME}
+ENV BUILDER_HOME /app/
+RUN mkdir -p "$APP_HOME"
+WORKDIR "$APP_HOME"
+
+COPY --from=builder "$BUILDER_HOME"/"${FOLDER_NAME}"/output.out $APP_HOME
+COPY --from=builder "$BUILDER_HOME"/"${FOLDER_NAME}"/.env $APP_HOME
+COPY --from=builder "$BUILDER_HOME"/adserver /app/adserver
+COPY --from=builder "$BUILDER_HOME"/eventserver /app/eventserver
+COPY --from=builder "$BUILDER_HOME"/panel /app/panel
+COPY --from=builder "$BUILDER_HOME"/publisherwebsite /app/publisherwebsite
+COPY --from=builder "$BUILDER_HOME"/common /app/common
+
+EXPOSE $PORT_NUMBER
+
+CMD ["./output.out"]
+

Dockerfile.redis

@@ -0,0 +1,7 @@
+FROM redislabs/rebloom:2.8.1 as builder
+
+FROM redis:7.2.5-alpine
+COPY --from=builder /usr/lib/redis/modules/redisbloom.so /usr/lib/redis/modules/redisbloom.so
+
+# CMD ["/run.sh", "--loadmodule", "/usr/lib/redis/modules/redisbloom.so"]
+CMD ["redis-server", "--appendonly", "yes", "--loadmodule", "/usr/lib/redis/modules/redisbloom.so"]

adserver/.env.example

@@ -5,3 +5,4 @@ ADS_FETCH_INTERVAL_SECS=5
 NOTIFY_API_PATH="/brake"
 BRAKE_DURATION_SECS=10
 AD_SERVER_HOSTNAME=localhost
+AD_SERVER_PUBLIC_HOSTNAME=localhost

adserver/handlers/handlers.go

@@ -35,7 +35,7 @@ func (h *AdServerHandler) GetAd(c *gin.Context) {
 	}
 
 	eventServerPort := os.Getenv("EVENT_SERVER_PORT")
-	hostName := os.Getenv("EVENT_SERVER_HOSTNAME")
+	hostName := os.Getenv("EVENT_SERVER_PUBLIC_HOSTNAME")
 	eventServerURL := "http://" + hostName + ":" + eventServerPort
 
 	clickReqPath := os.Getenv("CLICK_REQ_PATH")

adserver/handlers/handlers_test.go

@@ -47,7 +47,7 @@ func (m *MockTokenHandler) VerifyToken(encryptedToken string, key []byte) (*dto.
 
 func setupEnv() {
 	os.Setenv("EVENT_SERVER_PORT", "8080")
-	os.Setenv("EVENT_SERVER_HOSTNAME", "localhost")
+	os.Setenv("EVENT_SERVER_PUBLIC_HOSTNAME", "localhost")
 	os.Setenv("CLICK_REQ_PATH", "click")
 	os.Setenv("IMPRESSION_REQ_PATH", "impression")
 	os.Setenv("PRIVATE_KEY", base64.StdEncoding.EncodeToString([]byte("mysecretkey123456")))

adserver/main.go

@@ -39,5 +39,5 @@ func main() {
 	if port == "" {
 		port = "8081"
 	}
-	r.Run(os.Getenv("AD_SERVER_HOSTNAME") + ":" + port)
+	r.Run(os.Getenv("GIN_HOSTNAME") + ":" + port)
 }

common/.env.example

@@ -4,6 +4,7 @@ DB_USER=postgres
 DB_PASSWORD=
 DB_NAME=mini_yektanet_db
 PRIVATE_KEY=
+GIN_HOSTNAME=0.0.0.0
 KAFKA_TOPIC_CLICK=click_events
 KAFKA_TOPIC_IMPRESSION=impression_events
 KAFKA_CLICK_BUFF_LIMIT=3

copyenvs.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Declare variables
+SERVER_USER=
+SERVER_IP=
+SERVER_PORT=
+REMOTE_DIR="/home/$SERVER_USER/mini-yektanet"
+
+# Function to perform SCP
+perform_scp() {
+    local source_file="$1"
+    local dest_path="$2"
+    scp -P $SERVER_PORT "$source_file" "$SERVER_USER@$SERVER_IP:$dest_path"
+}
+
+# Perform SCP for each .env file
+perform_scp "panel/.env" "$REMOTE_DIR/panel/.env"
+perform_scp "adserver/.env" "$REMOTE_DIR/adserver/.env"
+perform_scp "common/.env" "$REMOTE_DIR/common/.env"
+perform_scp "eventserver/.env" "$REMOTE_DIR/eventserver/.env"
+perform_scp "publisherwebsite/.env" "$REMOTE_DIR/publisherwebsite/.env"

docker-compose.yml

@@ -0,0 +1,161 @@
+services:
+  panel:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - FOLDER_NAME=panel
+        - PORT_NUMBER=8083
+    restart: unless-stopped
+    ports:
+      - "8083:8083"
+      # env_file:
+      # - ./panel/.env
+    volumes:
+      - "panel_static:/app/panel/static"
+    depends_on:
+      db:
+        condition: service_healthy
+  publisherwebsite:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - FOLDER_NAME=publisherwebsite
+        - PORT_NUMBER=8084
+    restart: unless-stopped
+    ports:
+      - "8084:8084"
+    volumes:
+      - "publisherwebsite_static:/app/publisherwebsite/static"
+      # - "publisherwebsite_html:/app/publisherwebsite/html"
+  adserver:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - FOLDER_NAME=adserver
+        - PORT_NUMBER=8081
+    restart: unless-stopped
+    ports:
+      - "8081:8081"
+    depends_on:
+      kafka:
+        condition: service_healthy
+      panel:
+        condition: service_started
+  eventserver:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - FOLDER_NAME=eventserver
+        - PORT_NUMBER=8082
+    restart: unless-stopped
+    ports:
+      - "8082:8082"
+    depends_on:
+      kafka:
+        condition: service_healthy
+  db:
+    image: 'postgres:15.7-alpine'
+    environment:
+      - 'POSTGRES_DB=mini_yektanet_db'
+      - 'POSTGRES_USER=postgres'
+      - 'POSTGRES_PASSWORD=1234'
+    ports:
+      - '5433:5432'
+    restart: unless-stopped
+    volumes:
+      - "postgres_data:/var/lib/postgresql/data"
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready", "-d", "db_prod" ]
+      interval: 30s
+      timeout: 60s
+      retries: 5
+      start_period: 80s
+  redis:
+    # image: redis:7.2.5-alpine
+    build:
+      context: .
+      dockerfile: Dockerfile.redis
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis-data:/data
+    # command: redis-server --appendonly yes --loadmodule /opt/redis-stack/lib/redisbloom.so
+    restart: unless-stopped
+    healthcheck:
+      test: [ "CMD", "redis-cli", "ping" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+  zookeeper:
+    # image: zookeeper:3.8.4
+    image: confluentinc/cp-zookeeper:7.5.0
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    restart: unless-stopped
+    ports:
+      - 2181:2181
+    volumes:
+      # - zookeeper-data:/var/lib/zookeeper/data
+      - zookeeper-data:/data
+      # - zookeeper-logs:/var/lib/zookeeper/log
+      - zookeeper-logs:/log
+    healthcheck:
+      test: nc -z localhost 2181 || exit -1
+      interval: 10s
+      timeout: 5s
+      retries: 3
+  kafka:
+    # image: apache/kafka:3.7.1
+    image: &kafka-image confluentinc/cp-server:7.5.0
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+    ports:
+      - 29092:29092
+      - 9092:9092
+    restart: unless-stopped
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      # KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      # KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      # KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      # KAFKA_CREATE_TOPICS: "click_events:3:1,impression_events:3:1"
+
+    volumes:
+      # - kafka-data:/var/lib/kafka/data
+      - kafka-data:/opt/kafka/data
+      - kafka-docker-sock:/var/run/docker.sock
+    healthcheck:
+      test: kafka-topics --bootstrap-server kafka:9092 --list
+      interval: 30s
+      timeout: 10s
+      retries: 3
+  init-kafka:
+    image: *kafka-image
+    entrypoint: /bin/bash -c "kafka-topics --create --if-not-exists --topic click_events --replication-factor=1 --partitions=3 --bootstrap-server kafka:9092 && kafka-topics --create --if-not-exists --topic impression_events --replication-factor=1 --partitions=3 --bootstrap-server kafka:9092 && exit 0"
+    depends_on:
+      kafka:
+        condition: service_healthy
+
+volumes:
+  panel_static:
+  postgres_data:
+  publisherwebsite_static: # publisherwebsite_html:
+  redis-data:
+  zookeeper-data:
+  zookeeper-logs:
+  kafka-data:
+  kafka-docker-sock:

eventserver/.env.example

@@ -8,3 +8,4 @@ EVENT_SERVER_HOSTNAME=localhost
 KAFKA_BOOTSTRAP_SERVERS=localhost
 KAFKA_TOPIC_CLICK=click_events
 KAFKA_TOPIC_IMPRESSION=impression_events
+EVENT_SERVER_PUBLIC_HOSTNAME=localhost

eventserver/handlers/handlers.go

@@ -2,7 +2,6 @@ package handlers
 
 import (
 	"encoding/base64"
-	"fmt"
 	"net/http"
 	"os"
 	"time"

eventserver/main.go

@@ -42,5 +42,5 @@ func main() {
 	if port == "" {
 		port = "8082"
 	}
-	r.Run(os.Getenv("EVENT_SERVER_HOSTNAME") + ":" + port)
+	r.Run(os.Getenv("GIN_HOSTNAME") + ":" + port)
 }

go.mod

@@ -3,7 +3,7 @@ module YellowBloomKnapsack/mini-yektanet
 go 1.22.4
 
 require (
-	github.com/beevik/guid v1.0.0
+	// github.com/beevik/guid v1.0.0
 	github.com/gin-contrib/cors v1.7.2
 	github.com/gin-gonic/gin v1.10.0
 	github.com/joho/godotenv v1.5.1

panel/.env.example

@@ -7,3 +7,4 @@ SCRIPT_TEMPLATE_LOCATION="asset/scriptTemplate.js"
 TEST_DB_PATH="asset/test.db"
 AD_COST_CHECK_DURATION_SECS=60
 PANEL_HOSTNAME=localhost
+PANEL_PUBLIC_HOSTNAME=localhost

panel/handlers/adhandler.go

@@ -27,7 +27,7 @@ func GetActiveAds(c *gin.Context) {
 		adDTO := dto.AdDTO{
 			ID:                ad.ID,
 			Text:              ad.Text,
-			ImagePath:         "http://" + os.Getenv("PANEL_HOSTNAME") + ":" + os.Getenv("PANEL_PORT") + ad.ImagePath,
+			ImagePath:         "http://" + os.Getenv("PANEL_PUBLIC_HOSTNAME") + ":" + os.Getenv("PANEL_PORT") + ad.ImagePath,
 			Bid:               ad.Bid,
 			Website:           ad.Website,
 			TotalCost:         ad.TotalCost,