From 3bb23daddd914a2540afd8539dafa0727f50feaa Mon Sep 17 00:00:00 2001 From: kirinnee Date: Mon, 2 Sep 2024 13:48:46 +0800 Subject: [PATCH] feat: runbook to create/destroy phy vultr clusters --- .../vultr.ts | 230 ++++++++++++++++++ .../digital-ocean.ts | 2 +- src/books/physical-cluster-creation/vultr.ts | 196 +++++++++++++++ src/init/runbooks.ts | 21 ++ src/tasks/tasks.ts | 4 + 5 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 src/books/graceful-physical-cluster-destruction/vultr.ts create mode 100644 src/books/physical-cluster-creation/vultr.ts diff --git a/src/books/graceful-physical-cluster-destruction/vultr.ts b/src/books/graceful-physical-cluster-destruction/vultr.ts new file mode 100644 index 0000000..80e4071 --- /dev/null +++ b/src/books/graceful-physical-cluster-destruction/vultr.ts @@ -0,0 +1,230 @@ +import type { + LandscapeCluster, + ServiceTreeLandscapePrincipal, + ServiceTreeService, +} from '../../lib/service-tree-def.ts'; +import path from 'node:path'; +import { $ } from 'bun'; +import { KubectlUtil, type ResourceSearch } from '../../lib/utility/kubectl-util.ts'; +import type { TaskRunner } from '../../tasks/tasks.ts'; +import type { YamlManipulator } from '../../lib/utility/yaml-manipulator.ts'; +import type { UtilPrompter } from '../../lib/prompts/util-prompter.ts'; +import type { GracefulClusterCloudDestructor } from './cloud.ts'; + +class VultrGracefulPhysicalClusterDestructor implements GracefulClusterCloudDestructor { + slug: string; + + constructor( + private task: TaskRunner, + private y: YamlManipulator, + private k: KubectlUtil, + private up: UtilPrompter, + private sulfoxideTofu: ServiceTreeService, + private sulfoxideHelium: ServiceTreeService, + private virtualLandscapes: ServiceTreeLandscapePrincipal[], + slug: string, + ) { + this.slug = slug; + } + + async Run( + [phyLandscape, phyCluster]: LandscapeCluster, + [adminLandscape, adminCluster]: LandscapeCluster, + ): Promise { + const phy = { landscape: phyLandscape, cluster: phyCluster }; + const admin = { landscape: adminLandscape, cluster: adminCluster }; + + // common variables + const argo = this.sulfoxideHelium; + const tofu = this.sulfoxideTofu; + const phyContextSlug = `${phy.landscape.slug}-${phy.cluster.principal.slug}`; + const adminContextSlug = `${admin.landscape.slug}-${admin.cluster.principal.slug}`; + const adminNamespaceSlug = `${argo.platform.slug}-${argo.principal.slug}`; + const tofuDir = `./platforms/${tofu.platform.slug}/${tofu.principal.slug}`; + const heliumDir = `./platforms/${argo.platform.slug}/${argo.principal.slug}`; + const yamlPath = path.join(heliumDir, 'chart', `values.${admin.landscape.slug}.${admin.cluster.set.slug}.yaml`); + + // Update ArgoCD configurations + await this.task.Run([ + 'Update Helium Configuration', + async () => { + console.log(`🗑️ Removing ArgoCD configurations. Path: ${yamlPath}`); + await this.y.Mutate(yamlPath, [ + [['connector', 'clusters', phy.landscape.slug, phy.cluster.principal.slug, 'enable'], false], + [['connector', 'clusters', phy.landscape.slug, phy.cluster.principal.slug, 'deployAppSet'], false], + [['connector', 'clusters', phy.landscape.slug, phy.cluster.principal.slug, 'aoa', 'enable'], false], + [['connector', 'clusters', phy.landscape.slug, phy.cluster.principal.slug, 'destination'], ''], + ]); + }, + ]); + + // Apply ArgoCD configurations + const adminPls = `${admin.landscape.slug}:${admin.cluster.set.slug}`; + await this.task.Run([ + 'Apply Helium Configuration', + async () => { + await $`pls ${{ raw: adminPls }}:install -- --kube-context ${adminContextSlug} --namespace ${adminNamespaceSlug}`.cwd( + heliumDir, + ); + }, + ]); + + // delete applications from ArgoCD + const appsToRemove: ResourceSearch = { + kind: 'app', + context: adminContextSlug, + namespace: adminNamespaceSlug, + selector: [['atomi.cloud/cluster', phy.cluster.principal.slug]], + }; + + const deleteApps = async () => { + console.log(`🗑️ Delete Root Application: ${phy.landscape.slug}-${phy.cluster.principal.slug}-carbon`); + await this.k.Delete({ + kind: 'app', + context: adminContextSlug, + namespace: adminNamespaceSlug, + name: `${phy.landscape.slug}-${phy.cluster.principal.slug}-carbon`, + }); + console.log('✅ Root Application deleted'); + + console.log('🗑️ Deleting applications...'); + await this.k.DeleteRange(appsToRemove); + console.log('✅ Applications deleted'); + }; + + await this.task.Run(['Delete Applications', deleteApps]); + + await this.task.Run([ + 'Wait for Applications to be deleted', + async () => { + return await this.k.Wait(0, 3, appsToRemove, { + count: 3, + action: async () => { + const deleteApp = await this.up.YesNo('Do you want to manually delete the applications?'); + if (deleteApp) await deleteApps(); + return false; + }, + }); + }, + ]); + + // Delete all validating webhooks + await this.task.Run([ + 'Delete Validating Webhooks', + async () => { + await $`kubectl --context ${phyContextSlug} delete validatingwebhookconfigurations --all`.nothrow(); + }, + ]); + + // Delete all namespaces + await this.task.Run([ + 'Delete Namespaces', + async () => { + for (const namespace of ['pichu', 'pikachu', 'raichu', 'sulfoxide']) { + console.log(` 🗑️ Removing namespace: ${namespace}`); + await this.k.DeleteNamespace({ + context: phyContextSlug, + namespace, + }); + console.log(` ✅ Namespace removed: ${namespace}`); + } + }, + ]); + + // setup tofu repository correctly + await this.task.Run([ + 'Setup Tofu', + async () => { + await $`pls setup`.cwd(tofuDir); + }, + ]); + + // destroy generic infrastructure + const L1G = `${phy.landscape.slug}:l1:${phy.cluster.set.slug}`; + await this.task.Run([ + `Destroy Generic Infrastructure ${L1G}`, + async () => { + await $`pls ${{ raw: L1G }}:destroy -- -auto-approve`.cwd(tofuDir); + }, + ]); + + // destroy L1 infrastructure + const L1 = `${phy.landscape.slug}:l1:${phy.cluster.principal.slug}`; + await this.task.Run([ + `Destroy L1 Infrastructure ${L1}`, + async () => { + await $`pls ${{ raw: L1 }}:state:rm -- 'kubernetes_namespace.sulfoxide'`.cwd(tofuDir).nothrow(); + await $`pls ${{ raw: L1 }}:destroy -- -auto-approve`.cwd(tofuDir); + }, + ]); + + // destroy L0 infrastructure + const L0 = `${phy.landscape.slug}:l0:${phy.cluster.principal.slug}`; + await this.task.Run([ + `Destroy L0 Infrastructure ${L0}`, + async () => { + await $`pls ${{ raw: L0 }}:state:rm -- 'module.cluster.module.proxy_secret.kubernetes_namespace.kubernetes-access'` + .cwd(tofuDir) + .nothrow(); + await $`pls ${{ raw: L0 }}:destroy -- -auto-approve`.cwd(tofuDir); + }, + ]); + + // update kubectl configurations + await this.task.Run([ + 'Retrieve Kubectl Configurations', + async () => { + await $`pls kubectl`; + }, + ]); + + await this.task.Run([ + 'Delete ExternalSecret in admin', + async () => { + for (const ns of this.virtualLandscapes.map(x => x.slug)) { + await this.k.Delete({ + kind: 'externalsecret', + context: adminContextSlug, + namespace: adminNamespaceSlug, + name: `phase-5-${ns}-${phy.cluster.principal.slug}-cluster-secret`, + }); + } + await this.k.Delete({ + kind: 'externalsecret', + context: adminContextSlug, + namespace: adminNamespaceSlug, + name: `${phy.landscape.slug}-${phy.cluster.principal.slug}-external-secret`, + }); + await this.k.Delete({ + kind: 'externalsecret', + context: adminContextSlug, + namespace: adminNamespaceSlug, + name: `${phy.landscape.slug}-${phy.cluster.principal.slug}-external-secret-bearer-token`, + }); + await this.k.Delete({ + kind: 'externalsecret', + context: adminContextSlug, + namespace: adminNamespaceSlug, + name: `${phy.landscape.slug}-${phy.cluster.principal.slug}-external-secret-ca-crt`, + }); + }, + ]); + + // delete pointers to old cluster in admin + await this.task.Run([ + 'Delete SecretStore in admin', + async () => { + for (const ns of this.virtualLandscapes.map(x => x.slug)) { + await this.k.Delete({ + kind: 'secretstore', + context: adminContextSlug, + namespace: adminNamespaceSlug, + name: `phase-5-${ns}-${phy.cluster.principal.slug}`, + }); + } + }, + ]); + } +} + +export { VultrGracefulPhysicalClusterDestructor }; diff --git a/src/books/physical-cluster-creation/digital-ocean.ts b/src/books/physical-cluster-creation/digital-ocean.ts index d50d2ec..863ce12 100644 --- a/src/books/physical-cluster-creation/digital-ocean.ts +++ b/src/books/physical-cluster-creation/digital-ocean.ts @@ -129,7 +129,7 @@ class DigitalOceanPhysicalClusterCreator implements PhysicalClusterCloudCreator context: adminContextSlug, namespace: adminNamespaceSlug, selector: [ - ['atomi.cloud/sync-wave', 'wave-5'], + ['atomi.cloud/sync-wave', 'wave-4'], ['atomi.cloud/landscape', phyLandscape.slug], ['atomi.cloud/cluster', phyCluster.principal.slug], ], diff --git a/src/books/physical-cluster-creation/vultr.ts b/src/books/physical-cluster-creation/vultr.ts new file mode 100644 index 0000000..898f90b --- /dev/null +++ b/src/books/physical-cluster-creation/vultr.ts @@ -0,0 +1,196 @@ +import type { PhysicalClusterCloudCreator } from "./cloud.ts"; +import { $ } from "bun"; +import * as path from "node:path"; +import type { UtilPrompter } from "../../lib/prompts/util-prompter.ts"; +import { input } from "@inquirer/prompts"; +import type { YamlManipulator } from "../../lib/utility/yaml-manipulator.ts"; +import type { KubectlUtil } from "../../lib/utility/kubectl-util.ts"; +import type { LandscapeCluster, ServiceTreeService } from "../../lib/service-tree-def.ts"; +import type { TaskRunner } from "../../tasks/tasks.ts"; + +class VultrPhysicalClusterCreator implements PhysicalClusterCloudCreator { + slug: string; + + constructor( + private task: TaskRunner, + private y: YamlManipulator, + private up: UtilPrompter, + private k: KubectlUtil, + private sulfoxideTofu: ServiceTreeService, + private sulfoxideHelium: ServiceTreeService, + slug: string, + ) { + this.slug = slug; + } + + async Run( + [phyLandscape, phyCluster]: LandscapeCluster, + [adminLandscape, adminCluster]: LandscapeCluster, + ): Promise { + // constants + const tofu = this.sulfoxideTofu; + const He = this.sulfoxideHelium; + + const tofuDir = `./platforms/${tofu.platform.slug}/${tofu.principal.slug}`; + const He_Dir = `./platforms/${He.platform.slug}/${He.principal.slug}`; + + const He_YamlPath = path.join(He_Dir, 'chart', `values.${adminLandscape.slug}.${adminCluster.set.slug}.yaml`); + const aCtx = `${adminLandscape.slug}-${adminCluster.principal.slug}`; + const aNS = `${He.platform.slug}-${He.principal.slug}`; + + // Check if we want to inject the DO secrets + const vultrSecret = await this.up.YesNo('Do you want to inject Vultr API Token?'); + if (vultrSecret) { + const access = await input({ message: 'Enter your Vultr API Token' }); + await $`infisical secrets set --projectId=${tofu.principal.projectId} --env=${phyLandscape.slug} ${phyCluster.principal.slug.toUpperCase()}_VULTR_TOKEN=${access}`; + console.log('✅ Vultr API Token injected'); + } + + const L0 = `${phyLandscape.slug}:l0:${phyCluster.principal.slug}`; + await this.task.Run([ + `Build L0 Infrastructure ${L0}`, + async () => { + await $`pls setup`.cwd(tofuDir); + await $`pls ${{ raw: L0 }}:init`.cwd(tofuDir); + await $`pls ${{ raw: L0 }}:apply -- -auto-approve`.cwd(tofuDir); + }, + ]); + + await this.task.Run([ + 'Retrieve Kubectl Configurations', + async () => { + await $`pls kubectl`; + }, + ]); + + // extract endpoint to use + console.log('📤 Extract endpoint to use...'); + const output = await $`pls ${{ raw: L0 }}:output -- -json`.cwd(tofuDir).json(); + const endpoint = output.cluster_endpoint.value; + console.log(`✅ Extracted endpoint: ${endpoint}`); + + // build L1 generic infrastructure + const L1G = `${phyLandscape.slug}:l1:${phyCluster.set.slug}`; + await this.task.Run([ + `Build L1 Generic Infrastructure ${L1G}`, + async () => { + await $`pls ${{ raw: L1G }}:init`.cwd(tofuDir); + await $`pls ${{ raw: L1G }}:apply -- -auto-approve`.cwd(tofuDir); + }, + ]); + + // build L1 infrastructure + const L1 = `${phyLandscape.slug}:l1:${phyCluster.principal.slug}`; + await this.task.Run([ + `Build L1 Infrastructure ${L1}`, + async () => { + await $`pls ${{ raw: L1 }}:init`.cwd(tofuDir); + await $`pls ${{ raw: L1 }}:apply -- -auto-approve`.cwd(tofuDir); + }, + ]); + + // retrieve yaml in helium folder and replace + await this.task.Run([ + 'Update Helium Configuration', + async () => { + await this.y.Mutate(He_YamlPath, [ + [['connector', 'clusters', phyLandscape.slug, phyCluster.principal.slug, 'enable'], true], + [['connector', 'clusters', phyLandscape.slug, phyCluster.principal.slug, 'deployAppSet'], true], + [['connector', 'clusters', phyLandscape.slug, phyCluster.principal.slug, 'aoa', 'enable'], true], + [['connector', 'clusters', phyLandscape.slug, phyCluster.principal.slug, 'destination'], endpoint], + ]); + }, + ]); + + // apply ArgoCD configurations + const HePls = `${adminLandscape.slug}:${adminCluster.set.slug}`; + await this.task.Run([ + 'Apply Helium Configuration', + async () => { + await $`pls ${{ raw: HePls }}:install -- --kube-context ${aCtx} -n ${aNS}`.cwd(He_Dir); + }, + ]); + + // retrieve kubectl configurations again + await this.task.Run([ + 'Retrieve Kubectl Configurations', + async () => { + await $`pls kubectl`; + }, + ]); + + // wait for iodine to be ready + console.log('🕙 Waiting for iodine to be ready...'); + + await this.task.Exec([ + 'Wait for iodine applications to be ready', + async () => { + await this.k.WaitForApplications(3, { + kind: 'app', + context: aCtx, + namespace: aNS, + selector: [ + ['atomi.cloud/sync-wave', 'wave-4'], + ['atomi.cloud/landscape', phyLandscape.slug], + ['atomi.cloud/cluster', phyCluster.principal.slug], + ], + }); + }, + ]); + + await this.task.Exec([ + 'Wait for statefulset (etcd) to be ready', + async () => { + for (const ns of ['pichu', 'pikachu', 'raichu']) { + await this.k.WaitForReplica({ + kind: 'statefulset', + context: `${phyLandscape.slug}-${phyCluster.principal.slug}`, + namespace: ns, + name: `${phyLandscape.slug}-${ns}-iodine-etcd`, + }); + } + }, + ]); + + await this.task.Exec([ + 'Wait for deployment (iodine) to be ready', + async () => { + for (const ns of ['pichu', 'pikachu', 'raichu']) { + await this.k.WaitForReplica({ + kind: 'deployment', + context: `${phyLandscape.slug}-${phyCluster.principal.slug}`, + namespace: ns, + name: `${phyLandscape.slug}-${ns}-iodine`, + }); + } + }, + ]); + + // retrieve kubectl configurations again + await this.task.Run([ + 'Retrieve Kubectl Configurations', + async () => { + await $`pls kubectl`; + }, + ]); + + // last applications to be ready + await this.task.Exec([ + "Wait for vcluster carbon's last sync wave to be ready", + async () => { + await this.k.WaitForApplications(3, { + kind: 'app', + context: aCtx, + namespace: aNS, + selector: [ + ['atomi.cloud/sync-wave', 'wave-5'], + ['atomi.cloud/element', 'silicon'], + ['atomi.cloud/cluster', phyCluster.principal.slug], + ], + }); + }, + ]); + } +} + +export { VultrPhysicalClusterCreator }; diff --git a/src/init/runbooks.ts b/src/init/runbooks.ts index 152c834..d5436f7 100644 --- a/src/init/runbooks.ts +++ b/src/init/runbooks.ts @@ -19,12 +19,23 @@ import { AdminClusterMigrator } from '../books/admin-cluster-migration'; import { AdminClusterTransitioner } from '../books/admin-cluster-migration/transition.ts'; import { AwsPhysicalClusterCreator } from '../books/physical-cluster-creation/aws.ts'; import { AwsGracefulPhysicalClusterDestructor } from '../books/graceful-physical-cluster-destruction/aws.ts'; +import { VultrPhysicalClusterCreator } from "../books/physical-cluster-creation/vultr.ts"; +import { VultrGracefulPhysicalClusterDestructor } from "../books/graceful-physical-cluster-destruction/vultr.ts"; function initRunBooks(d: Dependencies, t: TaskGenerator): RunBook[] { const sulfoxide = SERVICE_TREE.sulfoxide; // physical cluster creation const phyClusterCreators: PhysicalClusterCloudCreator[] = [ + new VultrPhysicalClusterCreator( + d.taskRunner, + d.yamlManipulator, + d.utilPrompter, + d.kubectl, + sulfoxide.services.tofu, + sulfoxide.services.argocd, + CLOUDS.Vultr.slug, + ), new DigitalOceanPhysicalClusterCreator( d.taskRunner, d.yamlManipulator, @@ -58,6 +69,16 @@ function initRunBooks(d: Dependencies, t: TaskGenerator): RunBook[] { // graceful physical cluster destruction const phyGracefulDestructors = [ + new VultrGracefulPhysicalClusterDestructor( + d.taskRunner, + d.yamlManipulator, + d.kubectl, + d.utilPrompter, + sulfoxide.services.tofu, + sulfoxide.services.argocd, + LANDSCAPE_TREE.v, + CLOUDS.Vultr.slug, + ), new DigitalOceanGracefulPhysicalClusterDestructor( d.taskRunner, d.yamlManipulator, diff --git a/src/tasks/tasks.ts b/src/tasks/tasks.ts index 64aef75..465c119 100644 --- a/src/tasks/tasks.ts +++ b/src/tasks/tasks.ts @@ -1,5 +1,6 @@ import pc from 'picocolors'; import type { UtilPrompter } from '../lib/prompts/util-prompter.ts'; +import { $ } from 'bun'; // name, action interface ComplexTask { @@ -43,10 +44,12 @@ class TaskRunner { const shouldExit = await task(); if (shouldExit) process.exit(0); console.log(pc.green(`✅ Task ${quoted} completed`)); + await $`say "completed"`.nothrow(); break; } catch (e) { Bun.inspect(e); console.log(pc.red(`❌ Task ${quoted} failed`)); + $`say "failed, do you want to retry?"`.nothrow().then(); const loop = await this.up.YesNo(`Do you want to retry?`); if (!loop) { const exit = await this.up.YesNo('Do you want to exit? (no will skip to next step'); @@ -67,6 +70,7 @@ class TaskRunner { * @constructor */ async Run(task: Task): Promise { + $`say "Awaiting confirmation"`.nothrow().then(); const r = await this.up.YesNoExit(`Do you want to run task '${this.extractName(task)}'?`); if (r === 'exit') process.exit(0); if (r) await this.Exec(task);