feat: route proxy to server internal network through internal DNS (#357)

* feat(shulker-crds): add lifecycle strategy to MinecraftServer CRD

* feat(shulker-operator): inject SHULKER_SERVER_LIFECYCLE_STRATEGY env to server

* feat(shulker-server-agent): implement lifecycle strategies

* docs: add lifecycle strategies recipe

* chore: fix some warnings

* feat(shulker-operator): create headless service to have all cluster pods inside it

* feat(shulker-proxy-agent): use internal DNS through headless service rather than GameServer status

packages/shulker-operator/src/reconcilers/minecraft_cluster/headless_service.rs

@@ -0,0 +1,105 @@
+use std::collections::BTreeMap;
+
+use k8s_openapi::api::core::v1::Service;
+use k8s_openapi::api::core::v1::ServiceSpec;
+use kube::core::ObjectMeta;
+use kube::Api;
+use kube::Client;
+use kube::ResourceExt;
+
+use shulker_crds::v1alpha1::minecraft_cluster::MinecraftCluster;
+use shulker_kube_utils::reconcilers::builder::ResourceBuilder;
+
+use super::MinecraftClusterReconciler;
+
+pub struct HeadlessServiceBuilder {
+    client: Client,
+}
+
+#[async_trait::async_trait]
+impl<'a> ResourceBuilder<'a> for HeadlessServiceBuilder {
+    type OwnerType = MinecraftCluster;
+    type ResourceType = Service;
+    type Context = ();
+
+    fn name(cluster: &Self::OwnerType) -> String {
+        format!("{}-cluster", cluster.name_any())
+    }
+
+    fn api(&self, cluster: &Self::OwnerType) -> kube::Api<Self::ResourceType> {
+        Api::namespaced(self.client.clone(), cluster.namespace().as_ref().unwrap())
+    }
+
+    async fn build(
+        &self,
+        cluster: &Self::OwnerType,
+        name: &str,
+        _existing_service: Option<&Self::ResourceType>,
+        _context: Option<Self::Context>,
+    ) -> Result<Self::ResourceType, anyhow::Error> {
+        let service = Service {
+            metadata: ObjectMeta {
+                name: Some(name.to_string()),
+                namespace: Some(cluster.namespace().unwrap().clone()),
+                labels: Some(MinecraftClusterReconciler::get_labels(
+                    cluster,
+                    "service".to_string(),
+                    "minecraft-server-headless".to_string(),
+                )),
+                ..ObjectMeta::default()
+            },
+            spec: Some(ServiceSpec {
+                selector: Some(BTreeMap::from([(
+                    "minecraftcluster.shulkermc.io/name".to_string(),
+                    cluster.name_any(),
+                )])),
+                type_: None,
+                cluster_ip: Some("None".to_string()),
+                ports: Some(vec![]),
+                ..ServiceSpec::default()
+            }),
+            ..Service::default()
+        };
+
+        Ok(service)
+    }
+}
+
+impl HeadlessServiceBuilder {
+    pub fn new(client: Client) -> Self {
+        HeadlessServiceBuilder { client }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use shulker_kube_utils::reconcilers::builder::ResourceBuilder;
+
+    use crate::reconcilers::minecraft_cluster::fixtures::{create_client_mock, TEST_CLUSTER};
+
+    #[test]
+    fn name_contains_cluster_name() {
+        // W
+        let name = super::HeadlessServiceBuilder::name(&TEST_CLUSTER);
+
+        // T
+        assert_eq!(name, "my-cluster-cluster");
+    }
+
+    #[tokio::test]
+    async fn build_snapshot() {
+        // G
+        let client = create_client_mock();
+        let builder = super::HeadlessServiceBuilder::new(client);
+        let name = super::HeadlessServiceBuilder::name(&TEST_CLUSTER);
+
+        // W
+        let service = builder
+            .build(&TEST_CLUSTER, &name, None, None)
+            .await
+            .unwrap();
+
+        // T
+        insta::assert_yaml_snapshot!(service);
+    }
+}

packages/shulker-operator/src/reconcilers/minecraft_cluster/mod.rs

@@ -24,7 +24,8 @@ use shulker_crds::v1alpha1::minecraft_cluster::MinecraftCluster;
 use crate::reconcilers::ReconcilerError;
 
 use self::{
-    forwarding_secret::ForwardingSecretBuilder, minecraft_server_role::MinecraftServerRoleBuilder,
+    forwarding_secret::ForwardingSecretBuilder, headless_service::HeadlessServiceBuilder,
+    minecraft_server_role::MinecraftServerRoleBuilder,
     minecraft_server_role_binding::MinecraftServerRoleBindingBuilder,
     minecraft_server_service_account::MinecraftServerServiceAccountBuilder,
     proxy_role::ProxyRoleBuilder, proxy_role_binding::ProxyRoleBindingBuilder,
@@ -35,6 +36,7 @@ use self::{
 use super::Result;
 
 mod forwarding_secret;
+mod headless_service;
 mod minecraft_server_role;
 mod minecraft_server_role_binding;
 mod minecraft_server_service_account;
@@ -54,6 +56,7 @@ struct MinecraftClusterReconciler {
 
     // Builders
     forwarding_secret_builder: ForwardingSecretBuilder,
+    headless_service_builder: HeadlessServiceBuilder,
     proxy_service_account_builder: ProxyServiceAccountBuilder,
     proxy_role_builder: ProxyRoleBuilder,
     proxy_role_binding_builder: ProxyRoleBindingBuilder,
@@ -73,6 +76,9 @@ impl MinecraftClusterReconciler {
         reconcile_builder(&self.forwarding_secret_builder, cluster.as_ref(), None)
             .await
             .map_err(ReconcilerError::BuilderError)?;
+        reconcile_builder(&self.headless_service_builder, cluster.as_ref(), None)
+            .await
+            .map_err(ReconcilerError::BuilderError)?;
         reconcile_builder(&self.proxy_service_account_builder, cluster.as_ref(), None)
             .await
             .map_err(ReconcilerError::BuilderError)?;
@@ -191,6 +197,7 @@ pub async fn run(client: Client) {
     let context = MinecraftClusterReconciler {
         client: client.clone(),
         forwarding_secret_builder: ForwardingSecretBuilder::new(client.clone()),
+        headless_service_builder: HeadlessServiceBuilder::new(client.clone()),
         proxy_service_account_builder: ProxyServiceAccountBuilder::new(client.clone()),
         proxy_role_builder: ProxyRoleBuilder::new(client.clone()),
         proxy_role_binding_builder: ProxyRoleBindingBuilder::new(client.clone()),

packages/shulker-operator/src/reconcilers/minecraft_cluster/redis_service.rs

@@ -94,7 +94,7 @@ mod tests {
     use crate::reconcilers::minecraft_cluster::fixtures::{create_client_mock, TEST_CLUSTER};
 
     #[test]
-    fn name_contains_fleet_name() {
+    fn name_contains_cluster_name() {
         // W
         let name = super::RedisServiceBuilder::name(&TEST_CLUSTER);
 

packages/shulker-operator/src/reconcilers/minecraft_cluster/snapshots/shulker_operator__reconcilers__minecraft_cluster__headless_service__tests__build_snapshot.snap

@@ -0,0 +1,22 @@
+---
+source: packages/shulker-operator/src/reconcilers/minecraft_cluster/headless_service.rs
+expression: service
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: minecraft-server-headless
+    app.kubernetes.io/instance: service-my-cluster
+    app.kubernetes.io/managed-by: shulker-operator
+    app.kubernetes.io/name: service
+    app.kubernetes.io/part-of: cluster-my-cluster
+    minecraftcluster.shulkermc.io/name: my-cluster
+  name: my-cluster-cluster
+  namespace: default
+spec:
+  clusterIP: None
+  ports: []
+  selector:
+    minecraftcluster.shulkermc.io/name: my-cluster
+

packages/shulker-operator/src/reconcilers/minecraft_cluster/snapshots/shulker_operator__reconcilers__minecraft_cluster__minecraft_server_headless_service__tests__build_snapshot.snap

@@ -0,0 +1,27 @@
+---
+source: packages/shulker-operator/src/reconcilers/minecraft_cluster/minecraft_server_headless_service.rs
+expression: service
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: minecraft-server-headless
+    app.kubernetes.io/instance: service-my-cluster
+    app.kubernetes.io/managed-by: shulker-operator
+    app.kubernetes.io/name: service
+    app.kubernetes.io/part-of: cluster-my-cluster
+    minecraftcluster.shulkermc.io/name: my-cluster
+  name: my-cluster-servers
+  namespace: default
+spec:
+  clusterIP: None
+  ports:
+    - name: minecraft
+      port: 25565
+      protocol: TCP
+      targetPort: minecraft
+  selector:
+    app.kubernetes.io/component: minecraft-server
+    minecraftcluster.shulkermc.io/name: my-cluster
+

packages/shulker-operator/src/reconcilers/minecraft_server/gameserver.rs

@@ -3,6 +3,7 @@ use std::collections::BTreeMap;
 use k8s_openapi::api::core::v1::Capabilities;
 use k8s_openapi::api::core::v1::ConfigMapVolumeSource;
 use k8s_openapi::api::core::v1::Container;
+use k8s_openapi::api::core::v1::ContainerPort;
 use k8s_openapi::api::core::v1::EmptyDirVolumeSource;
 use k8s_openapi::api::core::v1::EnvVar;
 use k8s_openapi::api::core::v1::EnvVarSource;
@@ -29,7 +30,6 @@ use crate::resources::resourceref_resolver::ResourceRefResolver;
 use google_agones_crds::v1::game_server::GameServer;
 use google_agones_crds::v1::game_server::GameServerEvictionSpec;
 use google_agones_crds::v1::game_server::GameServerHealthSpec;
-use google_agones_crds::v1::game_server::GameServerPortSpec;
 use google_agones_crds::v1::game_server::GameServerSpec;
 use shulker_crds::v1alpha1::minecraft_server::MinecraftServer;
 use shulker_kube_utils::reconcilers::builder::ResourceBuilder;
@@ -136,11 +136,7 @@ impl<'a> GameServerBuilder {
             Self::get_pod_template_spec(resourceref_resolver, context, minecraft_server).await?;
 
         let game_server_spec = GameServerSpec {
-            ports: Some(vec![GameServerPortSpec {
-                name: "minecraft".to_string(),
-                container_port: 25565,
-                protocol: "TCP".to_string(),
-            }]),
+            ports: Some(vec![]),
             eviction: Some(GameServerEvictionSpec {
                 safe: "OnUpgrade".to_string(),
             }),
@@ -191,6 +187,11 @@ impl<'a> GameServerBuilder {
             containers: vec![Container {
                 image: Some(MINECRAFT_SERVER_IMAGE.to_string()),
                 name: "minecraft-server".to_string(),
+                ports: Some(vec![ContainerPort {
+                    name: Some("minecraft".to_string()),
+                    container_port: 25565,
+                    ..ContainerPort::default()
+                }]),
                 env: Some(Self::get_env(resourceref_resolver, context, minecraft_server).await?),
                 image_pull_policy: Some("IfNotPresent".to_string()),
                 security_context: Some(PROXY_SECURITY_CONTEXT.clone()),
@@ -213,6 +214,10 @@ impl<'a> GameServerBuilder {
                 ]),
                 ..Container::default()
             }],
+            subdomain: Some(format!(
+                "{}-cluster",
+                &minecraft_server.spec.cluster_ref.name
+            )),
             service_account_name: Some(format!(
                 "shulker-{}-server",
                 &minecraft_server.spec.cluster_ref.name

packages/shulker-operator/src/reconcilers/minecraft_server/snapshots/shulker_operator__reconcilers__minecraft_server__gameserver__tests__build_snapshot.snap

@@ -17,10 +17,7 @@ metadata:
   name: my-server
   namespace: default
 spec:
-  ports:
-    - name: minecraft
-      containerPort: 25565
-      protocol: TCP
+  ports: []
   health:
     disabled: false
     periodSeconds: 15
@@ -82,6 +79,9 @@ spec:
           image: "itzg/minecraft-server:2023.10.1-java17"
           imagePullPolicy: IfNotPresent
           name: minecraft-server
+          ports:
+            - containerPort: 25565
+              name: minecraft
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -136,6 +136,7 @@ spec:
         beta.kubernetes.io/os: linux
       restartPolicy: Never
       serviceAccountName: shulker-my-cluster-server
+      subdomain: my-cluster-cluster
       volumes:
         - configMap:
             name: my-server-config

packages/shulker-operator/src/reconcilers/minecraft_server_fleet/snapshots/shulker_operator__reconcilers__minecraft_server_fleet__fleet__tests__build_snapshot.snap

@@ -35,10 +35,7 @@ spec:
         minecraftserverfleet.shulkermc.io/name: my-server
         test-label/shulkermc.io: my-value
     spec:
-      ports:
-        - name: minecraft
-          containerPort: 25565
-          protocol: TCP
+      ports: []
       health:
         disabled: false
         periodSeconds: 15
@@ -104,6 +101,9 @@ spec:
               image: "itzg/minecraft-server:2023.10.1-java17"
               imagePullPolicy: IfNotPresent
               name: minecraft-server
+              ports:
+                - containerPort: 25565
+                  name: minecraft
               securityContext:
                 allowPrivilegeEscalation: false
                 capabilities:
@@ -158,6 +158,7 @@ spec:
             beta.kubernetes.io/os: linux
           restartPolicy: Never
           serviceAccountName: shulker-my-cluster-server
+          subdomain: my-cluster-cluster
           volumes:
             - configMap:
                 name: my-server-config

packages/shulker-operator/src/reconcilers/proxy_fleet/fleet.rs

@@ -415,6 +415,11 @@ impl<'a> FleetBuilder {
         let redis_ref = RedisRef::from_cluster(context.cluster)?;
 
         let mut env: Vec<EnvVar> = vec![
+            EnvVar {
+                name: "SHULKER_CLUSTER_NAME".to_string(),
+                value: Some(context.cluster.name_any()),
+                ..EnvVar::default()
+            },
             EnvVar {
                 name: "SHULKER_PROXY_NAME".to_string(),
                 value_from: Some(EnvVarSource {

packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__fleet__tests__build_snapshot.snap

@@ -61,6 +61,8 @@ spec:
         spec:
           containers:
             - env:
+                - name: SHULKER_CLUSTER_NAME
+                  value: my-cluster
                 - name: SHULKER_PROXY_NAME
                   valueFrom:
                     fieldRef:

packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/Configuration.kt

@@ -4,6 +4,8 @@ import java.util.Optional
 import java.util.UUID
 
 object Configuration {
+    val CLUSTER_NAME = getStringEnv("SHULKER_CLUSTER_NAME")
+
     val PROXY_NAMESPACE = getStringEnv("SHULKER_PROXY_NAMESPACE")
     val PROXY_NAME = getStringEnv("SHULKER_PROXY_NAME")
     val PROXY_TTL_SECONDS = getLongEnv("SHULKER_PROXY_TTL_SECONDS")

packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/kubernetes/models/AgonesV1GameServer.kt

@@ -39,39 +39,15 @@ class AgonesV1GameServer : CustomResource<AgonesV1GameServer.Spec, AgonesV1GameS
     @JsonInclude(JsonInclude.Include.NON_NULL)
     @JsonIgnoreProperties(ignoreUnknown = true)
     @JsonPropertyOrder(
-        "address",
-        "ports",
         "state"
     )
     class Status : KubernetesResource {
-        @set:JsonProperty("address")
-        @get:JsonProperty("address")
-        @JsonProperty("address")
-        var address: String? = null
-
-        @set:JsonProperty("ports")
-        @get:JsonProperty("ports")
-        @JsonProperty("ports")
-        var ports: kotlin.collections.List<Port>? = null
-
         @set:JsonProperty("state")
         @get:JsonProperty("state")
         @JsonProperty("state")
         var state: String? = null
 
         fun isReady(): Boolean = this.state == "Ready" || this.state == "Reserved" || this.state == "Allocated"
-
-        class Port {
-            @get:JsonProperty("name")
-            @set:JsonProperty("name")
-            @JsonProperty("name")
-            var name: String? = null
-
-            @get:JsonProperty("port")
-            @set:JsonProperty("port")
-            @JsonProperty("port")
-            var port: Int? = null
-        }
     }
 
     @JsonDeserialize(using = JsonDeserializer.None::class)