Move to public registries (#321)

* Synchronize for mfdlabs/grid-bot-libraries#27 * Drop nuget.config for mfdlabs/grid-bot-libraries#27 * Update README.md Change the installation section to better reflect the current environment and it's build capabilities. * Update actions build.yml: ~ Update images to the public Docker registry format. ~ Make the primary job run on a public GitHub action runner, therefore changing all the root directories to the standard format (good by custom build directories :( ) ~ Split the deploy nomad parts out to a seperate Job that can only run on push to master, make it independant from downloading the source code to the repository. ~ Use the deployer token to update deployment statuses deploy.yml: ~ Update images to the public Docker registry format. ~ Make it not checkout, but instead download the job template url for the specified ref ~ Use the deployer token to update deployment statuses
mfdlabs · Jun 22, 2024 · 0bf8131 · 0bf8131
1 parent 6c97314
commit 0bf8131
Show file tree

Hide file tree

Showing 10 changed files with 110 additions and 121 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -29,11 +29,11 @@ on:
       image:
         description: "Image Name"
         required: true
-        default: "cloud-registry.simulprod.com/grid/grid-bot"
+        default: "mfdlabs/grid-bot"
       registry:
         description: "Docker Registry"
         required: true
-        default: "cloud-registry.simulprod.com"
+        default: "docker.io"
 
       create_release:
         description: "Create Release"
@@ -86,47 +86,39 @@ jobs:
   build:
     if: ${{ !contains(github.event.head_commit.message, '#!skip-build-and-deploy!#') }}
 
-    runs-on: grid-bot-infra
+    runs-on: ubuntu-latest
     env:
       BUILD_CONFIGURATION: ${{ github.event.inputs.build_configuration || (github.event_name == 'push' && github.ref == 'refs/heads/master' && 'Release') || 'Debug' }}
 
-      IMAGE: ${{ github.event.inputs.image || (github.event_name == 'push' && github.ref == 'refs/heads/master' && 'cloud-registry.simulprod.com/grid/grid-bot') || 'registry.simulpong.com/grid/grid-bot' }}
-      DOCKER_REGISTRY: ${{ github.event.inputs.registry || (github.event_name == 'push' && github.ref == 'refs/heads/master' && 'cloud-registry.simulprod.com') || 'registry.simulpong.com' }}
+      IMAGE: ${{ github.event.inputs.image || 'mfdlabs/grid-bot' }}
+      DOCKER_REGISTRY: ${{ github.event.inputs.registry || 'docker.io' }}
 
       CREATE_RELEASE: ${{ github.event.inputs.create_release || !contains(github.event.head_commit.message, '#!skip-release!#') }}
       CREATE_IMAGE: ${{ github.event.inputs.create_image || !contains(github.event.head_commit.message, '#!skip-image!#') }}
       UPLOAD_ARTIFACT: ${{ github.event.inputs.upload_artifact || contains(github.event.head_commit.message, '#!upload-artifact!#') }}
 
-      DEPLOY_TO_NOMAD: ${{ github.event.inputs.deploy_to_nomad || (github.event_name == 'push' && github.ref == 'refs/heads/master' && !contains(github.event.head_commit.message, '#!skip-deploy!#')) }}
-
-      NOMAD_ADDR: ${{ vars.NOMAD_ADDR }}
-      NOMAD_TOKEN: ${{ secrets.NOMAD_TOKEN }}
-      NOMAD_ENVIRONMENT: ${{ github.event.inputs.nomad_enviroment || 'production' }}
-      NOMAD_JOB_NAME: ${{ github.event.inputs.nomad_job_name || 'grid-bot' }}-${{ (github.event.inputs.nomad_enviroment || 'production') == 'production' && 'prod' || 'stage' }}
-      NOMAD_RESOURCES: ${{ github.event.inputs.nomad_resources || '2000:1024' }}
-
       DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }}
       DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      image-step-outcome: ${{ steps.image.outcome }}
+      image: ${{ env.IMAGE }}
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           submodules: recursive
 
-      - name: Move to Checkout Path
-        run: |
-          mkdir -p /_/_work/${{ github.repository }}/${{ github.sha }}
-          (shopt -s dotglob; mv * /_/_work/${{ github.repository }}/${{ github.sha }})
-
       - name: Setup .NET 6
         uses: actions/setup-dotnet@v3
         with:
           dotnet-version: 8.0.x
 
       - name: Create Deployment Directory
         run: |
-          mkdir -p /_/_work/${{ github.repository }}/${{ github.sha }}/deploy
+          mkdir -p ./deploy
 
       # Generate version and allow it as an output
       - name: Generate Version
@@ -146,19 +138,18 @@ jobs:
 
       - name: Build
         run: |
-          cd /_/_work/${{ github.repository }}/${{ github.sha }} && \
-          dotnet publish /_/_work/${{ github.repository }}/${{ github.sha }}/src/Grid.Bot.csproj \
+          dotnet publish ./src/Grid.Bot.csproj \
             -c ${{ env.BUILD_CONFIGURATION }} \
-            -o /_/_work/${{ github.repository }}/${{ github.sha }}/deploy/${{ steps.version.outputs.version }} \
+            -o ./deploy/${{ steps.version.outputs.version }} \
             -p:IMAGE_TAG=${{ steps.version.outputs.version }} \
             -p:CI=true
             
       - name: Create Release Artifact
         if: ${{ env.CREATE_RELEASE == 'true' }}
         id: zip
         run: |
-          cd /_/_work/${{ github.repository }}/${{ github.sha }}/deploy/${{ steps.version.outputs.version }} && \
-          zip -r /_/_work/${{ github.repository }}/${{ github.sha }}/deploy/${{ steps.version.outputs.version }}.zip \
+          cd ./deploy/${{ steps.version.outputs.version }} && \
+          zip -r ../${{ steps.version.outputs.version }}.zip \
           .
 
       - name: Write Release Artifact
@@ -168,7 +159,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.DEPLOYER_TOKEN }}
         with:
           files: |
-            /_/_work/${{ github.repository }}/${{ github.sha }}/deploy/${{ steps.version.outputs.version }}.zip
+            ./deploy/${{ steps.version.outputs.version }}.zip
           tag_name: ${{ steps.version.outputs.version }}
           generate_release_notes: true
           prerelease: ${{ env.BUILD_CONFIGURATION == 'Debug' }}
@@ -180,7 +171,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ${{ steps.version.outputs.version }}.zip
-          path: /_/_work/${{ github.repository }}/${{ github.sha }}/deploy/${{ steps.version.outputs.version }}.zip
+          path: ./deploy/${{ steps.version.outputs.version }}.zip
           retention-days: 1
 
       - name: Create Image
@@ -189,7 +180,7 @@ jobs:
         run: |
           echo "${{ env.DOCKER_PASSWORD }}" | docker login -u "${{ env.DOCKER_USERNAME }}" --password-stdin "${{ env.DOCKER_REGISTRY }}"
 
-          cd /_/_work/${{ github.repository }}/${{ github.sha }} && \
+          cd . && \
             docker build -t ${{ env.IMAGE }}:${{ steps.version.outputs.version }} --build-arg IMAGE_TAG=${{ steps.version.outputs.version }} .
 
           if [ "${{ github.ref }}" == "refs/heads/master" ]; then
@@ -202,25 +193,38 @@ jobs:
             docker push ${{ env.IMAGE }}:latest
           fi
 
+  deploy:
+    runs-on: grid-bot-infra
+    needs: build
+
+    if: ${{ needs.build.outputs.image-step-outcome == 'success' && !contains(github.event.head_commit.message, '#!skip-build-and-deploy!#') && github.event.inputs.deploy_to_nomad || (github.event_name == 'push' && github.ref == 'refs/heads/master' && !contains(github.event.head_commit.message, '#!skip-deploy!#')) && vars.NOMAD_ADDR }}
+
+    env:
+      NOMAD_ADDR: ${{ vars.NOMAD_ADDR }}
+      NOMAD_TOKEN: ${{ secrets.NOMAD_TOKEN }}
+      NOMAD_ENVIRONMENT: ${{ github.event.inputs.nomad_enviroment || 'production' }}
+      NOMAD_JOB_NAME: ${{ github.event.inputs.nomad_job_name || 'grid-bot' }}-${{ (github.event.inputs.nomad_enviroment || 'production') == 'production' && 'prod' || 'stage' }}
+      NOMAD_RESOURCES: ${{ github.event.inputs.nomad_resources || '2000:1024' }}
+
+      RAW_NOMAD_JOB_TEMPLATE_DOWNLOAD_URL: "https://raw.githubusercontent.com/mfdlabs/grid-bot/master/nomad/grid-bot.nomad"
+
+    steps:
       - name: Setup Nomad CLI
-        if: ${{ env.DEPLOY_TO_NOMAD == 'true' && env.NOMAD_ADDR && steps.image.outcome == 'success' }}
         uses: nferch/setup-nomad@v4.0.0
         env:
           NOMAD_TLS_SKIP_VERIFY: 1
 
       - name: Set Initial GitHub Deployment Status
-        if: ${{ env.DEPLOY_TO_NOMAD == 'true' && env.NOMAD_ADDR && steps.image.outcome == 'success' }}
         uses: chrnorm/deployment-action@v2.0.7
         id: deployment
         continue-on-error: true
         with:
-          token: "${{ github.token }}"
+          token: "${{ secrets.DEPLOYER_TOKEN }}"
           environment: ${{ env.NOMAD_ENVIRONMENT }}
           environment-url: "${{ env.NOMAD_ADDR }}/ui/jobs/${{ env.NOMAD_JOB_NAME }}"
-          description: "Version: ${{ steps.version.outputs.version }}"
+          description: "Version: ${{ needs.build.outputs.version }}"
 
       - name: Deploy to Nomad
-        if: ${{ env.DEPLOY_TO_NOMAD == 'true' && env.NOMAD_ADDR && steps.image.outcome == 'success' }}
         env:
           NOMAD_TLS_SKIP_VERIFY: 1
         id: deploy
@@ -230,40 +234,38 @@ jobs:
           # Replace {{{IMAGE_NAME}}} and {{{IMAGE_TAG}}} with the actual values
 
           # Create a temporary file
+          NOMAD_TEMPLATE_FILE=$(mktemp)
           NOMAD_JOB_FILE=$(mktemp)
 
+          # Write the template to the temporary file
+          curl -sL $RAW_NOMAD_JOB_TEMPLATE_DOWNLOAD_URL > $NOMAD_TEMPLATE_FILE
+
           # Replace the template with the actual values
-          sed -e "s,{{{IMAGE_NAME}}},${{ env.IMAGE }}," \
-              -e "s/{{{IMAGE_TAG}}}/${{ steps.version.outputs.version }}/" \
+          sed -e "s,{{{IMAGE_NAME}}},${{ needs.build.outputs.image }}," \
+              -e "s/{{{IMAGE_TAG}}}/${{ needs.build.outputs.version }}/" \
               -e "s/{{{NOMAD_JOB_NAME}}}/${{ env.NOMAD_JOB_NAME }}/" \
               -e "s/{{{NOMAD_ENVIRONMENT}}}/${{ env.NOMAD_ENVIRONMENT }}/" \
               -e "s/{{{NOMAD_CPU}}}/$(echo ${{ env.NOMAD_RESOURCES }} | cut -d: -f1)/" \
               -e "s/{{{NOMAD_MEMORY}}}/$(echo ${{ env.NOMAD_RESOURCES }} | cut -d: -f2)/" \
               -e "s/{{{NOMAD_LOG_LEVEL}}}/Information/" \
-              /_/_work/${{ github.repository }}/${{ github.sha }}/nomad/grid-bot.nomad > $NOMAD_JOB_FILE
+              $NOMAD_TEMPLATE_FILE > $NOMAD_JOB_FILE
 
           # Run the job but do not wait for longer than 5 minutes
           nomad job run $NOMAD_JOB_FILE
 
           # Cleanup
           rm $NOMAD_JOB_FILE
+          rm $NOMAD_TEMPLATE_FILE
 
       - name: Set GitHub Deployment Status
-        if: ${{ env.DEPLOY_TO_NOMAD == 'true' && env.NOMAD_ADDR && steps.image.outcome == 'success' }}
         uses: chrnorm/deployment-status@v2
         continue-on-error: true
         with:
-          token: "${{ github.token }}"
+          token: "${{ secrets.DEPLOYER_TOKEN }}"
           deployment-id: ${{ steps.deployment.outputs.deployment_id }}
           environment-url: "${{ steps.deployment.outputs.environment_url }}"
-          description: "Version: ${{ steps.version.outputs.version }}"
+          description: "Version: ${{ needs.build.outputs.version }}"
 
           # If the "deploy" step fails, the deployment status will be set to "failure"
           # If the "deploy" step succeeds, the deployment status will be set to "success"
           state: "${{ steps.deploy.outcome }}"
-
-      - name: Cleanup
-        if: always()
-        continue-on-error: true
-        run: |
-          rm -rf /_/_work/${{ github.repository }}/${{ github.sha }}
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -6,15 +6,11 @@ on:
       image:
         description: "Image Name"
         required: true
-        default: "cloud-registry.simulprod.com/grid/grid-bot"
+        default: "mfdlabs/grid-bot"
       image_tag:
         description: "Image Tag"
         required: true
         default: "latest"
-      registry:
-        description: "Docker Registry"
-        required: true
-        default: "cloud-registry.simulprod.com"
 
       nomad_job_name:
         description: "Nomad Job Name"
@@ -55,9 +51,8 @@ jobs:
   deploy:
     runs-on: grid-bot-infra
     env:
-      IMAGE: ${{ github.event.inputs.image || 'cloud-registry.simulprod.com/grid/grid-bot'}}
+      IMAGE: ${{ github.event.inputs.image || 'mfdlabs/grid-bot'}}
       IMAGE_TAG: ${{ github.event.inputs.image_tag || 'latest' }}
-      DOCKER_REGISTRY: ${{ github.event.inputs.registry || 'cloud-registry.simulprod.com' }}
 
       NOMAD_ADDR: ${{ vars.NOMAD_ADDR }}
       NOMAD_TOKEN: ${{ secrets.NOMAD_TOKEN }}
@@ -66,10 +61,9 @@ jobs:
       NOMAD_RESOURCES: ${{ github.event.inputs.nomad_resources || '2000:1024' }}
       NOMAD_LOG_LEVEL: ${{ github.event.inputs.log_level || 'Information' }}
 
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      RAW_NOMAD_JOB_TEMPLATE_DOWNLOAD_URL: "https://raw.githubusercontent.com/mfdlabs/grid-bot/${{ github.sha }}/nomad/grid-bot.nomad"
 
+    steps:
       - name: Setup Nomad CLI
         uses: nferch/setup-nomad@v4.0.0
         env:
@@ -80,7 +74,7 @@ jobs:
         id: deployment
         continue-on-error: true
         with:
-          token: "${{ github.token }}"
+          token: "${{ secrets.DEPLOYER_TOKEN }}"
           environment: ${{ env.NOMAD_ENVIRONMENT }}
           environment-url: "${{ env.NOMAD_ADDR }}/ui/jobs/${{ env.NOMAD_JOB_NAME }}"
           description: "Version: ${{ env.IMAGE_TAG }}"
@@ -95,8 +89,12 @@ jobs:
           # Replace {{{IMAGE_NAME}}} and {{{IMAGE_TAG}}} with the actual values
 
           # Create a temporary file
+          NOMAD_TEMPLATE_FILE=$(mktemp)
           NOMAD_JOB_FILE=$(mktemp)
 
+          # Write the template to the temporary file
+          curl -sL $RAW_NOMAD_JOB_TEMPLATE_DOWNLOAD_URL > $NOMAD_TEMPLATE_FILE
+
           # Replace the template with the actual values
           sed -e "s,{{{IMAGE_NAME}}},${{ env.IMAGE }}," \
               -e "s/{{{IMAGE_TAG}}}/${{ env.IMAGE_TAG }}/" \
@@ -105,19 +103,20 @@ jobs:
               -e "s/{{{NOMAD_CPU}}}/$(echo ${{ env.NOMAD_RESOURCES }} | cut -d: -f1)/" \
               -e "s/{{{NOMAD_MEMORY}}}/$(echo ${{ env.NOMAD_RESOURCES }} | cut -d: -f2)/" \
               -e "s/{{{NOMAD_LOG_LEVEL}}}/${{ env.NOMAD_LOG_LEVEL }}/" \
-              ./nomad/grid-bot.nomad > $NOMAD_JOB_FILE
+              $NOMAD_TEMPLATE_FILE > $NOMAD_JOB_FILE
 
           # Run the job but do not wait for longer than 5 minutes
           nomad job run $NOMAD_JOB_FILE
 
           # Cleanup
           rm $NOMAD_JOB_FILE
+          rm $NOMAD_TEMPLATE_FILE
 
       - name: Set GitHub Deployment Status
         uses: chrnorm/deployment-status@v2
         continue-on-error: true
         with:
-          token: "${{ github.token }}"
+          token: "${{ secrets.DEPLOYER_TOKEN }}"
           deployment-id: ${{ steps.deployment.outputs.deployment_id }}
           environment-url: "${{ steps.deployment.outputs.environment_url }}"
           description: "Version: ${{ env.IMAGE_TAG }}"

diff --git a/README.md b/README.md
@@ -41,18 +41,18 @@ This notice only serves the purpose of giving a clear understanding of project b
 
 The installation of this project is simple, and can go as follows:
 
-- Either through the basic method of cloning the repository and having it fetch dependencies from the private NuGet repository.
-- or by removing [nuget.config](nuget.config), cloning the repository with `--recurse-submodules` and then running a target within [Makefile](Makefile) that has `local` in the name (e.g. `make build-local-debug`), or going into each library manually [Grid.Bot](./src/Grid.Bot.csproj), [Shared.Commands](./shared/commands/Shared.Commands.csproj), [Shared.Events](./shared/events/Shared.Events.csproj), [Shared.Settings](./shared/settings/Shared.Settings.csproj) and [Shared.Utility](./shared/utility/Shared.Utility.csproj), and then defining the property `LocalBuild` in the configuration:
+- Either through the basic method of cloning the repository and having it fetch dependencies from NuGet by using the solution [grid-bot-bare.sln](./grid-bot-bare.sln).
+- or by cloning the repository with `--recurse-submodules` and then setting the property `LocalBuild` in [Directory.Build.props](./Directory.Build.props) and [shared/Directory.Build.props](./shared/Directory.Build.props), and using the solution [grid-bot.sln](./grid-bot.sln)
 
     ```xml
     <PropertyGroup>
         <LocalBuild>true</LocalBuild>
     </PropertyGroup>
     ```
 
-This repository also provides a [Dockerfile](Dockerfile) to build the project in a container, but the current setup only works in CI/CD environments, as the container requires that the code be 'published' to the ./deploy directory beforehand (see [build.yml](.github/workflows/build.yml)).
+This repository provides Docker builds at [Docker](https://hub.docker.com/r/mfdlabs/grid-bot).
 
-This repository also supplies [releases](https://github.com/mfdlabs/grid-bot/releases), which can be ran with any distribution of [.NET 8.0.1](https://dotnet.microsoft.com/download/dotnet/8.0) or higher:
+This repository also supplies [releases](https://github.com/mfdlabs/grid-bot/releases), which can be ran with any distribution of [.NET 8.0.1](https://dotnet.microsoft.com/download/dotnet/8.0) or higher
 
 ```bash
 dotnet Grid.Bot.dll

diff --git a/nuget.config b/nuget.config
diff --git a/services/recovery/Grid.Bot.Recovery.csproj b/services/recovery/Grid.Bot.Recovery.csproj
@@ -14,13 +14,13 @@
     <Description>Recovery executable for the MFDLABS grid-bot.</Description>
   </PropertyGroup> 
 
-  <ItemGroup Label="PrivatePackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="Configuration" Version="1.0.7" />
-    <PackageReference Include="Logging" Version="1.0.4" />
-    <PackageReference Include="Networking" Version="1.0.5" />
+  <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
+    <PackageReference Include="mfdlabs.Configuration" Version="1.0.7" />
+    <PackageReference Include="mfdlabs.Logging" Version="1.0.4" />
+    <PackageReference Include="mfdlabs.Networking" Version="1.0.5" />
   </ItemGroup>
 
-  <ItemGroup Label="PrivatePackages" Condition=" '$(LocalBuild)' == 'true' ">
+  <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">
     <ProjectReference Include="../../lib/src/configuration/configuration/Configuration.csproj" />
     <ProjectReference Include="..\..\lib\src\logging\logging\Logging.csproj" />
     <ProjectReference Include="../../lib/src/networking/networking/Networking.csproj" />

diff --git a/shared/commands/Shared.Commands.csproj b/shared/commands/Shared.Commands.csproj
@@ -13,16 +13,16 @@
     <ProjectReference Include="../utility/Shared.Utility.csproj" />
   </ItemGroup>
 
-  <ItemGroup Label="PrivatePackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="ClientSettings.Client" Version="1.0.4" />
-    <PackageReference Include="FileSystem" Version="1.0.4" />
-    <PackageReference Include="FloodCheckers.Core" Version="1.0.4" />
-    <PackageReference Include="FloodCheckers.Redis" Version="1.0.4" />
-    <PackageReference Include="Text" Version="1.0.4" />
-    <PackageReference Include="Networking" Version="1.0.5" />
+  <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
+    <PackageReference Include="mfdlabs.ClientSettings.Client" Version="1.0.4" />
+    <PackageReference Include="mfdlabs.FileSystem" Version="1.0.4" />
+    <PackageReference Include="mfdlabs.FloodCheckers.Core" Version="1.0.4" />
+    <PackageReference Include="mfdlabs.FloodCheckers.Redis" Version="1.0.4" />
+    <PackageReference Include="mfdlabs.Text" Version="1.0.4" />
+    <PackageReference Include="mfdlabs.Networking" Version="1.0.5" />
   </ItemGroup>
 
-  <ItemGroup Label="PrivatePackages" Condition=" '$(LocalBuild)' == 'true' ">
+  <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">
     <ProjectReference Include="../../lib/src/clients/client-settings-client/ClientSettings.Client.csproj" />
     <ProjectReference Include="../../lib/src/file-system/file-system/FileSystem.csproj" />
     <ProjectReference Include="../../lib/src/floodcheckers/core/FloodCheckers.Core.csproj" />

diff --git a/shared/events/Shared.Events.csproj b/shared/events/Shared.Events.csproj
@@ -17,11 +17,11 @@
     <PackageReference Include="prometheus-net" Version="8.0.1" />
   </ItemGroup>
 
-  <ItemGroup Label="PrivatePackages" Condition=" '$(LocalBuild)' != 'true' ">
-    <PackageReference Include="Logging" Version="1.0.4" />
+  <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' != 'true' ">
+    <PackageReference Include="mfdlabs.Logging" Version="1.0.4" />
   </ItemGroup>
 
-  <ItemGroup Label="PrivatePackages" Condition=" '$(LocalBuild)' == 'true' ">
+  <ItemGroup Label="CustomPackages" Condition=" '$(LocalBuild)' == 'true' ">
     <ProjectReference Include="../../lib/src/logging/logging/Logging.csproj" />
   </ItemGroup>