Merge pull request #468 from boozallen/467-fix-ci

[#467] document self-hosted runner setup

.github/workflows/build.yml

@@ -49,6 +49,7 @@ jobs:
       - name: Load docker build cache
         id: cached-docker-build
         uses: runs-on/cache/restore@v4
+        if: ${{ ! github.event.schedule }}
         with:
           path: ~/.docker/cache
           key: docker-cache-${{ hashFiles('**/Dockerfile') }}
@@ -57,6 +58,7 @@ jobs:
       - name: Load m2 repository cache # Manually caching .m2 repo as the setup-java caching isn't falling back to older caches
         id: cached-m2-repo
         uses: runs-on/cache/restore@v4
+        if: ${{ ! github.event.schedule }}
         with:
           path: ~/.m2/repository
           key: maven-${{ hashFiles('**/pom.xml') }}
@@ -65,6 +67,7 @@ jobs:
       - name: Load m2 build cache
         id: cached-m2-build
         uses: runs-on/cache/restore@v4
+        if: ${{ ! github.event.schedule }}
         with:
           path: ~/.m2/build-cache
           key: maven-build-cache-${{ hashFiles('**/pom.xml') }}
@@ -74,6 +77,7 @@ jobs:
       - name: Poetry cache
         id: cached-poetry
         uses: runs-on/cache@v4
+        if: ${{ ! github.event.schedule }}
         with:
           path: ~/.cache/pypoetry
           key: poetry-cache-${{ hashFiles('**/pyproject.toml') }}

devops/ARC_README.md

@@ -0,0 +1,76 @@
+# Actions Runner Controller
+
+The [Actions Runner Controller](https://github.com/actions/actions-runner-controller) (ARC) is a project to ease the
+creation of self-hosted GitHub action runners.  The following content is largely in line with the information provided
+in the GitHub documentation but is preserved here as an audit trail in case the online information changes, since ARC is
+still pre-1.0.
+
+## Standing up ARC from scratch
+
+### Creating the Controller
+
+The controller only needs to be created once for the whole cluster.  The controller MUST be created before the runner
+sets and the versions must match between the two.  The controller is responsible for detecting new ARC CRD manifests and
+provisioning the appropriate resources for the runner-set, including the listener which recieves action run requests
+from GitHub.
+
+```sh
+helm install arc-controller oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \
+             --namespace gh-actions-controller \
+             --create-namespace \
+             -f controller-values.yaml
+```
+
+### Creating the Runner Scale Set
+
+Previously, we needed special permissions on the runner pod to execute `helm install --dry-run` as an integration test
+for our baseline charts. However, this requires pretty expansive permissions (cluster level + secrets retrieval). The
+dry-run IT wasn't really adding much over the simple strict linting approach so it's been dropped. The custom service
+account setup has been left in place simply to demonstrate how this _would_ be done if we need permissions for some
+other use case in the future.
+
+>[!NOTE]
+>The namespace is currently hard-coded in the YAML file, so if you intend to deploy to a different namespace when
+>installing the runner-set, you must update the YAML file as well.  This is required because a RoleBinding object in
+>Kubernetes must specify the ServiceAccount namespace explicitly.
+
+```sh
+kubectl apply -f arc-runner-service-account.yaml
+```
+
+Finally, the runner-set can be created via Helm.  The Github token is any Personal Access Token (Classic) that has Repo
+permissions.  In the future, this could be changed to use a Github App so that it isn't tied to a specific user.
+Additionally, we could consider using SealedSecrets or a pre-defined secret for the App settings.
+
+>[!NOTE]
+>The installation name (`arc-runner-set-aissemble`) will be the label used to select the runner set in a workflow file.
+
+```sh
+helm install arc-runner-set-aissemble oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \
+             --namespace gh-actions-aissemble \
+             --create-namespace \
+             --set githubConfigSecret.github_token="{TOKEN}" \
+             -f runnerset-values.yaml
+```
+
+## Upgrading ARC or Updating configuration
+
+### Controller
+
+The controller cannot be upgraded in place according to the Github documentation, so all runnersets in the cluster must
+be uninstalled, then the controller uninstalled, and then the new controller version installed.  A `helm upgrade` to
+simply update the values in the values file has not been tested, so it is unclear if a full uninstall is needed for that
+case.
+
+### Runner Set
+
+If the values file has been updated and simply needs to be re-applied, the `--reuse-values` flag can be used to preserve
+the existing GH PAT.  It is unclear whether `--reuse-values` would allow removal of values from the values file, and so
+the token may be required to achieve this.
+
+```sh
+helm upgrade arc-runner-set-aissemble oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \
+             --namespace gh-actions-aissemble \
+             --reuse-values \
+             -f runnerset-values.yaml
+```

devops/arc-runner-service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: gh-actions-aissemble
+  name: arc-runner-set-aissemble-gha-rs-custom-permissions

devops/clusterrole-configmap-manager-default.yml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"annotations":{},"name":"configmap-manager-default"},"rules":[{"apiGroups":[""],"resources":["configmaps"],"verbs":["get","list","watch","create","update","patch","delete"]}]}
+  creationTimestamp: "2024-07-31T14:58:32Z"
+  name: configmap-manager-default
+  resourceVersion: "65116720"
+  uid: e5b50b8c-dbbd-43ed-bb6e-93d7a2e13709
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete

devops/clusterrole-configmap-manager.yml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"annotations":{},"name":"configmap-manager"},"rules":[{"apiGroups":[""],"resources":["configmaps"],"verbs":["get","list","watch","create","update","patch","delete"]}]}
+  creationTimestamp: "2024-07-31T00:42:13Z"
+  name: configmap-manager
+  resourceVersion: "64977423"
+  uid: ed40e11b-9499-46e9-881f-9316600e0183
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete

devops/controller-values.yaml

@@ -0,0 +1,2 @@
+flags:
+  logLevel: "info"

devops/helm-it-user.yml

@@ -0,0 +1,107 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"annotations":{},"name":"helm-it-user"},"rules":[{"apiGroups":[""],"resources":["namespaces","pods","services","configmaps","secrets","serviceaccounts","persistentvolumeclaims"],"verbs":["create","delete","get","list","patch","update","watch"]},{"apiGroups":["apps"],"resources":["deployments","statefulsets","daemonsets","replicasets"],"verbs":["create","delete","get","list","patch","update","watch"]},{"apiGroups":["batch"],"resources":["jobs","cronjobs"],"verbs":["create","delete","get","list","patch","update","watch"]},{"apiGroups":["networking.k8s.io"],"resources":["ingresses","networkpolicies"],"verbs":["create","delete","get","list","patch","update","watch"]},{"apiGroups":["rbac.authorization.k8s.io"],"resources":["roles","rolebindings","clusterroles","clusterrolebindings"],"verbs":["create","delete","get","list","patch","update","watch"]},{"apiGroups":["apiextensions.k8s.io"],"resources":["customresourcedefinitions"],"verbs":["get","list"]},{"apiGroups":["admissionregistration.k8s.io"],"resources":["validatingwebhookconfigurations","mutatingwebhookconfigurations"],"verbs":["get","list"]},{"apiGroups":["policy"],"resources":["poddisruptionbudgets"],"verbs":["get","list"]}]}
+  creationTimestamp: "2024-08-01T15:18:57Z"
+  name: helm-it-user
+  resourceVersion: "65406556"
+  uid: 0e3b9688-f1a5-46ed-8b09-62769f2df128
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - services
+  - configmaps
+  - secrets
+  - serviceaccounts
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  - statefulsets
+  - daemonsets
+  - replicasets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  - cronjobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  - networkpolicies
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  - rolebindings
+  - clusterroles
+  - clusterrolebindings
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  - mutatingwebhookconfigurations
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - get
+  - list

devops/runnerset-values.yaml

@@ -0,0 +1,7 @@
+containerMode:
+  type: "dind"
+githubConfigUrl: "https://github.com/boozallen/aissemble"
+maxRunners: 3
+template:
+  spec:
+    serviceAccountName: "arc-runner-set-aissemble-gha-rs-custom-permissions"

extensions/extensions-helm/pom.xml

@@ -67,28 +67,6 @@
                 <overwrite.dependencies>true</overwrite.dependencies>
             </properties>
         </profile>
-        <profile>
-            <id>integration-test</id>
-            <build>
-                <pluginManagement>
-                    <plugins>
-                        <plugin>
-                            <groupId>${group.helm.plugin}</groupId>
-                            <artifactId>helm-maven-plugin</artifactId>
-                            <executions>
-                                <execution>
-                                    <id>simulates-an-install</id>
-                                    <phase>integration-test</phase>
-                                    <goals>
-                                        <goal>dry-run</goal>
-                                    </goals>
-                                </execution>
-                            </executions>
-                        </plugin>
-                    </plugins>
-                </pluginManagement>
-            </build>
-        </profile>
     </profiles>
 
     <build>