From adf32569b5a14f7f0ecded66de3fa12257c60deb Mon Sep 17 00:00:00 2001 From: Md Anam Raihan Date: Fri, 18 Oct 2024 16:50:49 +0530 Subject: [PATCH] feat: fully remove support for Log Analysis agent. Log Analysis is a deprecated service and has been replaced by IBM Cloud Logs (#411) BREAKING CHANGE: No longer possible to provision the Log Analysis agent --- .secrets.baseline | 4 +- README.md | 40 ----------- examples/obs-agent-iks/main.tf | 8 +-- examples/obs-agent-iks/provider.tf | 16 ----- examples/obs-agent-iks/version.tf | 5 -- examples/obs-agent-ocp/main.tf | 8 +-- examples/obs-agent-ocp/provider.tf | 16 ----- examples/obs-agent-ocp/version.tf | 4 -- main.tf | 102 +---------------------------- moved.tf | 10 --- variables.tf | 94 -------------------------- 11 files changed, 7 insertions(+), 300 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 1496b7e9..1755472d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2024-10-02T13:57:09Z", + "generated_at": "2024-10-16T12:26:02Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -82,7 +82,7 @@ "hashed_secret": "3f0155e75563ab3adc0505000a86da5baa207d1f", "is_secret": false, "is_verified": false, - "line_number": 64, + "line_number": 59, "type": "Secret Keyword", "verified_result": null } diff --git a/README.md b/README.md index c1d2de0f..a80e0daa 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,6 @@ This module deploys the following observability agents to an IBM Cloud Red Hat O - [Logs agent](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-about) - [Monitoring agent](https://cloud.ibm.com/docs/monitoring?topic=monitoring-about-collect-metrics) -- [DEPRECATED: Log Analysis agent](https://cloud.ibm.com/docs/log-analysis?topic=log-analysis-log_analysis_agent) - -> [!IMPORTANT] -> The IBM Log Analysis service is deprecated. [IBM Cloud Logs](https://www.ibm.com/products/cloud-logs) is the replacement service and is now the default agent created with this module. - @@ -84,28 +79,6 @@ module "observability_agents" { } ``` -### (DEPRECATED) Log Analysis agent configuration for Kubernetes metadata filtering - -You can configure the logging agent to filter log lines according to the Kubernetes resources metadata by setting the exclusion and inclusion parameters. - -For example, to set the agent to return all log lines coming from the `default` Kubernetes namespace and exclude anything with a label `app.kubernetes.io/name` with value `sample-app` or an annotation `annotation.user` with value `sample-user`, include these parameters: - -```text -custom_log_analysis_at_agent_line_exclusion = "label.app.kubernetes.io/name:sample-app\\, annotation.user:sample-user" -custom_log_analysis_at_agent_line_inclusion = "namespace:default" -``` - -The following is the corresponding DaemonSet configuration: - -```text -- name: LOGDNA_K8S_METADATA_LINE_INCLUSION - value: "label.app.kubernetes.io/name:sample-app, annotation.user:sample-user" -- name: LOGDNA_K8S_METADATA_LINE_EXCLUSION - value: "namespace:default" -``` - -For more information, see [Configuration for Kubernetes Metadata Filtering](https://github.com/logdna/logdna-agent-v2/blob/3.8/docs/KUBERNETES.md#configuration-for-kubernetes-metadata-filtering). - ### Required IAM access policies You need the following permissions to run this module. @@ -136,7 +109,6 @@ You need the following permissions to run this module. | Name | Type | |------|------| | [helm_release.cloud_monitoring_agent](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.log_analysis_agent](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [ibm_container_cluster.cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster) | data source | | [ibm_container_cluster_config.cluster_config](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster_config) | data source | | [ibm_container_vpc_cluster.cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_vpc_cluster) | data source | @@ -162,18 +134,6 @@ You need the following permissions to run this module. | [cluster\_id](#input\_cluster\_id) | The ID of the cluster you wish to deploy the agents in | `string` | n/a | yes | | [cluster\_resource\_group\_id](#input\_cluster\_resource\_group\_id) | The Resource Group ID of the cluster | `string` | n/a | yes | | [is\_vpc\_cluster](#input\_is\_vpc\_cluster) | Specify true if the target cluster for the observability agents is a VPC cluster, false if it is a classic cluster. | `bool` | `true` | no | -| [log\_analysis\_add\_cluster\_name](#input\_log\_analysis\_add\_cluster\_name) | DEPRECATED: If true, configure the Log Analysis agent to attach a tag containing the cluster name to all log messages. | `bool` | `true` | no | -| [log\_analysis\_agent\_custom\_line\_exclusion](#input\_log\_analysis\_agent\_custom\_line\_exclusion) | DEPRECATED: Log Analysis agent custom configuration for line exclusion setting LOGDNA\_K8S\_METADATA\_LINE\_EXCLUSION. See https://github.com/logdna/logdna-agent-v2/blob/master/docs/KUBERNETES.md#configuration-for-kubernetes-metadata-filtering for more info. | `string` | `null` | no | -| [log\_analysis\_agent\_custom\_line\_inclusion](#input\_log\_analysis\_agent\_custom\_line\_inclusion) | DEPRECATED: Log Analysis agent custom configuration for line inclusion setting LOGDNA\_K8S\_METADATA\_LINE\_INCLUSION. See https://github.com/logdna/logdna-agent-v2/blob/master/docs/KUBERNETES.md#configuration-for-kubernetes-metadata-filtering for more info. | `string` | `null` | no | -| [log\_analysis\_agent\_name](#input\_log\_analysis\_agent\_name) | DEPRECATED: Log Analysis agent name. Used for naming all kubernetes and helm resources on the cluster. | `string` | `"logdna-agent"` | no | -| [log\_analysis\_agent\_namespace](#input\_log\_analysis\_agent\_namespace) | DEPRECATED: Namespace where to deploy the Log Analysis agent. Default value is 'ibm-observe' | `string` | `"ibm-observe"` | no | -| [log\_analysis\_agent\_tags](#input\_log\_analysis\_agent\_tags) | DEPRECATED: List of tags to associate to all log records that the agent collects so that you can identify the agent's data quicker in the logging UI. NOTE: Use the 'log\_analysis\_add\_cluster\_name' variable to add the cluster name as a tag. | `list(string)` | `[]` | no | -| [log\_analysis\_agent\_tolerations](#input\_log\_analysis\_agent\_tolerations) | DEPRECATED: List of tolerations to apply to Log Analysis agent. | 
list(object({
    key               = optional(string)
    operator          = optional(string)
    value             = optional(string)
    effect            = optional(string)
    tolerationSeconds = optional(number)
  }))
| 
[
  {
    "operator": "Exists"
  }
]
| no | -| [log\_analysis\_enabled](#input\_log\_analysis\_enabled) | DEPRECATED: Deploy IBM Cloud Log Analysis agent | `bool` | `false` | no | -| [log\_analysis\_endpoint\_type](#input\_log\_analysis\_endpoint\_type) | DEPRECATED: Specify the IBM Log Analysis instance endpoint type (public or private) to use. Used to construct the ingestion endpoint. | `string` | `"private"` | no | -| [log\_analysis\_ingestion\_key](#input\_log\_analysis\_ingestion\_key) | DEPRECATED: Ingestion key for the Log Analysis agent to communicate with the instance | `string` | `null` | no | -| [log\_analysis\_instance\_region](#input\_log\_analysis\_instance\_region) | DEPRECATED: The IBM Log Analysis instance region. Used to construct the ingestion endpoint. | `string` | `null` | no | -| [log\_analysis\_secret\_name](#input\_log\_analysis\_secret\_name) | DEPRECATED: The name of the secret which will store the Log Analysis ingestion key. | `string` | `"logdna-agent"` | no | | [logs\_agent\_additional\_log\_source\_paths](#input\_logs\_agent\_additional\_log\_source\_paths) | The list of additional log sources. By default, the Logs agent collects logs from a single source at `/var/log/containers/*.log`. | `list(string)` | `[]` | no | | [logs\_agent\_additional\_metadata](#input\_logs\_agent\_additional\_metadata) | The list of additional metadata fields to add to the routed logs. | 
list(object({
    key   = optional(string)
    value = optional(string)
  }))
| `[]` | no | | [logs\_agent\_enable\_scc](#input\_logs\_agent\_enable\_scc) | Whether to enable creation of Security Context Constraints in Openshift. When installing on an OpenShift cluster, this setting is mandatory to configure permissions for pods within your cluster. | `bool` | `true` | no | diff --git a/examples/obs-agent-iks/main.tf b/examples/obs-agent-iks/main.tf index df63ec39..c73cbe08 100644 --- a/examples/obs-agent-iks/main.tf +++ b/examples/obs-agent-iks/main.tf @@ -128,12 +128,8 @@ resource "time_sleep" "wait_operators" { ############################################################################## module "observability_instances" { - source = "terraform-ibm-modules/observability-instances/ibm" - version = "2.19.1" - providers = { - logdna.at = logdna.at - logdna.ld = logdna.ld - } + source = "terraform-ibm-modules/observability-instances/ibm" + version = "3.0.2" resource_group_id = module.resource_group.resource_group_id region = var.region cloud_logs_plan = "standard" diff --git a/examples/obs-agent-iks/provider.tf b/examples/obs-agent-iks/provider.tf index 5c11f46f..76346099 100644 --- a/examples/obs-agent-iks/provider.tf +++ b/examples/obs-agent-iks/provider.tf @@ -22,19 +22,3 @@ provider "kubernetes" { token = data.ibm_container_cluster_config.cluster_config.token cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate } - -locals { - at_endpoint = "https://api.${var.region}.logging.cloud.ibm.com" -} - -provider "logdna" { - alias = "at" - servicekey = module.observability_instances.activity_tracker_resource_key != null ? module.observability_instances.activity_tracker_resource_key : "" - url = local.at_endpoint -} - -provider "logdna" { - alias = "ld" - servicekey = module.observability_instances.log_analysis_resource_key != null ? module.observability_instances.log_analysis_resource_key : "" - url = local.at_endpoint -} diff --git a/examples/obs-agent-iks/version.tf b/examples/obs-agent-iks/version.tf index c32ea61b..4cd6e426 100644 --- a/examples/obs-agent-iks/version.tf +++ b/examples/obs-agent-iks/version.tf @@ -22,10 +22,5 @@ terraform { source = "hashicorp/time" version = ">= 0.9.1" } - # The logdna provider is not actually required by the module itself, just this example, so OK to use ">=" here instead of locking into a version - logdna = { - source = "logdna/logdna" - version = ">= 1.14.2" - } } } diff --git a/examples/obs-agent-ocp/main.tf b/examples/obs-agent-ocp/main.tf index 9069c34e..04719351 100644 --- a/examples/obs-agent-ocp/main.tf +++ b/examples/obs-agent-ocp/main.tf @@ -128,12 +128,8 @@ data "ibm_container_cluster_config" "cluster_config" { ############################################################################## module "observability_instances" { - source = "terraform-ibm-modules/observability-instances/ibm" - version = "2.19.1" - providers = { - logdna.at = logdna.at - logdna.ld = logdna.ld - } + source = "terraform-ibm-modules/observability-instances/ibm" + version = "3.0.2" resource_group_id = module.resource_group.resource_group_id region = var.region cloud_logs_plan = "standard" diff --git a/examples/obs-agent-ocp/provider.tf b/examples/obs-agent-ocp/provider.tf index 5c11f46f..76346099 100644 --- a/examples/obs-agent-ocp/provider.tf +++ b/examples/obs-agent-ocp/provider.tf @@ -22,19 +22,3 @@ provider "kubernetes" { token = data.ibm_container_cluster_config.cluster_config.token cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate } - -locals { - at_endpoint = "https://api.${var.region}.logging.cloud.ibm.com" -} - -provider "logdna" { - alias = "at" - servicekey = module.observability_instances.activity_tracker_resource_key != null ? module.observability_instances.activity_tracker_resource_key : "" - url = local.at_endpoint -} - -provider "logdna" { - alias = "ld" - servicekey = module.observability_instances.log_analysis_resource_key != null ? module.observability_instances.log_analysis_resource_key : "" - url = local.at_endpoint -} diff --git a/examples/obs-agent-ocp/version.tf b/examples/obs-agent-ocp/version.tf index 3b3c31b0..e131816d 100644 --- a/examples/obs-agent-ocp/version.tf +++ b/examples/obs-agent-ocp/version.tf @@ -16,9 +16,5 @@ terraform { source = "hashicorp/kubernetes" version = ">= 2.16.1" } - logdna = { - source = "logdna/logdna" - version = ">= 1.14.2" - } } } diff --git a/main.tf b/main.tf index 9c756319..df0f7eea 100644 --- a/main.tf +++ b/main.tf @@ -25,15 +25,7 @@ data "ibm_container_cluster_config" "cluster_config" { locals { # LOCALS - cluster_name = var.is_vpc_cluster ? data.ibm_container_vpc_cluster.cluster[0].resource_name : data.ibm_container_cluster.cluster[0].resource_name # Not publically documented in provider. See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4485 - log_analysis_chart_location = "${path.module}/chart/logdna-agent" - log_analysis_image_tag_digest = "3.10.1-20240827.12afa351b661bc07@sha256:3a7ebc7fb58de67db2af15f35ba827c96a92c06e933abb4c67431854a24bd156" # datasource: icr.io/ext/logdna-agent versioning=regex:^(?\d+)\.(?\d+)\.(?\d+)-(?\d+) - log_analysis_agent_registry = "icr.io/ext/logdna-agent" - log_analysis_agent_tags = var.log_analysis_add_cluster_name ? concat([local.cluster_name], var.log_analysis_agent_tags) : var.log_analysis_agent_tags - log_analysis_host = var.log_analysis_enabled ? var.log_analysis_endpoint_type == "private" ? "logs.private.${var.log_analysis_instance_region}.logging.cloud.ibm.com" : "logs.${var.log_analysis_instance_region}.logging.cloud.ibm.com" : null - # The directory in which the logdna agent will store its state database. - # Note that the agent must have write access to the directory (handlded by the initContainer) and be a persistent volume. - log_analysis_agent_db_path = "/var/lib/logdna" + cluster_name = var.is_vpc_cluster ? data.ibm_container_vpc_cluster.cluster[0].resource_name : data.ibm_container_cluster.cluster[0].resource_name # Not publically documented in provider. See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4485 cloud_monitoring_chart_location = "${path.module}/chart/sysdig-agent" cloud_monitoring_image_tag_digest = "13.4.1@sha256:469f3eee8d00ce563041770e875555dbabf02daa57cc489d9e66010707cdc621" # datasource: icr.io/ext/sysdig/agent cloud_monitoring_agent_registry = "icr.io/ext/sysdig/agent" @@ -42,11 +34,6 @@ locals { # TODO: Move this into variable.tf since module requires 1.9 now # VARIABLE VALIDATION - log_analysis_key_validate_condition = var.log_analysis_enabled == true && var.log_analysis_instance_region == null && var.log_analysis_ingestion_key == null - log_analysis_key_validate_msg = "Values for 'log_analysis_ingestion_key' and 'log_analysis_instance_region' variables must be passed when 'log_analysis_enabled = true'" - # tflint-ignore: terraform_unused_declarations - log_analysis_key_validate_check = regex("^${local.log_analysis_key_validate_msg}$", (!local.log_analysis_key_validate_condition ? local.log_analysis_key_validate_msg : "")) - cloud_monitoring_key_validate_condition = var.cloud_monitoring_enabled == true && var.cloud_monitoring_instance_region == null && var.cloud_monitoring_access_key == null cloud_monitoring_key_validate_msg = "Values for 'cloud_monitoring_access_key' and 'log_analysis_instance_region' variables must be passed when 'cloud_monitoring_enabled = true'" # tflint-ignore: terraform_unused_declarations @@ -60,93 +47,6 @@ locals { validate_icl_ingress_endpoint = var.logs_agent_enabled == true && (var.cloud_logs_ingress_endpoint == null || var.cloud_logs_ingress_endpoint == "") ? tobool("When 'logs_agent_enabled' is enabled, you cannot set 'cloud_logs_ingress_endpoint' as null or empty string.") : true } -/** Log Analysis Configuration Start **/ -resource "helm_release" "log_analysis_agent" { - count = var.log_analysis_enabled ? 1 : 0 - name = var.log_analysis_agent_name - chart = local.log_analysis_chart_location - namespace = var.log_analysis_agent_namespace - create_namespace = true - timeout = 1200 - wait = true - recreate_pods = true - force_update = true - - set { - name = "metadata.name" - type = "string" - value = var.log_analysis_agent_name - } - set { - name = "image.version" - type = "string" - value = local.log_analysis_image_tag_digest - } - set { - name = "image.registry" - type = "string" - value = local.log_analysis_agent_registry - } - set { - name = "env.host" - type = "string" - value = local.log_analysis_host - } - set { - name = "secret.name" - type = "string" - value = var.log_analysis_secret_name - } - set_sensitive { - name = "secret.key" - type = "string" - value = var.log_analysis_ingestion_key - } - set { - name = "agent.tags" - type = "string" - value = join("\\,", local.log_analysis_agent_tags) - } - set { - name = "agent.dbPath" - type = "string" - value = local.log_analysis_agent_db_path - } - - values = [ - yamlencode({ - tolerations = var.log_analysis_agent_tolerations - }) - ] - - dynamic "set" { - for_each = var.log_analysis_agent_custom_line_inclusion != null ? [var.log_analysis_agent_custom_line_inclusion] : [] - content { - name = "agentMetadataLineInclusion" - type = "string" - value = set.value - } - } - - dynamic "set" { - for_each = var.log_analysis_agent_custom_line_exclusion != null ? [var.log_analysis_agent_custom_line_exclusion] : [] - content { - name = "agentMetadataLineExclusion" - type = "string" - value = set.value - } - } - - provisioner "local-exec" { - command = "${path.module}/scripts/confirm-rollout-status.sh ${var.log_analysis_agent_name} ${var.log_analysis_agent_namespace}" - interpreter = ["/bin/bash", "-c"] - environment = { - KUBECONFIG = data.ibm_container_cluster_config.cluster_config.config_file_path - } - } -} -/** Log Analysis Configuration End **/ - /** Cloud Monitoring Configuration Start **/ resource "helm_release" "cloud_monitoring_agent" { count = var.cloud_monitoring_enabled ? 1 : 0 diff --git a/moved.tf b/moved.tf index 8d615763..296389f1 100644 --- a/moved.tf +++ b/moved.tf @@ -1,20 +1,10 @@ # The following moved blocks allow consumers to upgrade without instances being destroyed -moved { - from = helm_release.logdna_agent[0] - to = helm_release.log_analysis_agent[0] -} - moved { from = helm_release.sysdig_agent[0] to = helm_release.cloud_monitoring_agent[0] } -moved { - from = data.ibm_resource_instance.logdna_instance[0] - to = data.ibm_resource_instance.log_analysis_instance[0] -} - moved { from = data.ibm_resource_instance.sysdig_instance[0] to = data.ibm_resource_instance.cloud_monitoring_instance[0] diff --git a/variables.tf b/variables.tf index f266f14b..005f2536 100644 --- a/variables.tf +++ b/variables.tf @@ -29,100 +29,6 @@ variable "is_vpc_cluster" { default = true } -############################################################################## -# Log Analysis variables -############################################################################## - -variable "log_analysis_enabled" { - type = bool - description = "DEPRECATED: Deploy IBM Cloud Log Analysis agent" - default = false -} - - -variable "log_analysis_agent_tags" { - type = list(string) - description = "DEPRECATED: List of tags to associate to all log records that the agent collects so that you can identify the agent's data quicker in the logging UI. NOTE: Use the 'log_analysis_add_cluster_name' variable to add the cluster name as a tag." - default = [] - nullable = false -} - -variable "log_analysis_add_cluster_name" { - type = bool - description = "DEPRECATED: If true, configure the Log Analysis agent to attach a tag containing the cluster name to all log messages." - default = true -} - -variable "log_analysis_ingestion_key" { - type = string - description = "DEPRECATED: Ingestion key for the Log Analysis agent to communicate with the instance" - sensitive = true - default = null -} - -variable "log_analysis_secret_name" { - type = string - description = "DEPRECATED: The name of the secret which will store the Log Analysis ingestion key." - default = "logdna-agent" - nullable = false -} - -variable "log_analysis_instance_region" { - type = string - description = "DEPRECATED: The IBM Log Analysis instance region. Used to construct the ingestion endpoint." - default = null -} - -variable "log_analysis_endpoint_type" { - type = string - description = "DEPRECATED: Specify the IBM Log Analysis instance endpoint type (public or private) to use. Used to construct the ingestion endpoint." - default = "private" - validation { - error_message = "The specified endpoint_type can be private or public only." - condition = contains(["private", "public"], var.log_analysis_endpoint_type) - } -} - -variable "log_analysis_agent_custom_line_inclusion" { - description = "DEPRECATED: Log Analysis agent custom configuration for line inclusion setting LOGDNA_K8S_METADATA_LINE_INCLUSION. See https://github.com/logdna/logdna-agent-v2/blob/master/docs/KUBERNETES.md#configuration-for-kubernetes-metadata-filtering for more info." - type = string - default = null -} - -variable "log_analysis_agent_custom_line_exclusion" { - description = "DEPRECATED: Log Analysis agent custom configuration for line exclusion setting LOGDNA_K8S_METADATA_LINE_EXCLUSION. See https://github.com/logdna/logdna-agent-v2/blob/master/docs/KUBERNETES.md#configuration-for-kubernetes-metadata-filtering for more info." - type = string - default = null -} - -variable "log_analysis_agent_name" { - description = "DEPRECATED: Log Analysis agent name. Used for naming all kubernetes and helm resources on the cluster." - type = string - default = "logdna-agent" - nullable = false -} - -variable "log_analysis_agent_namespace" { - type = string - description = "DEPRECATED: Namespace where to deploy the Log Analysis agent. Default value is 'ibm-observe'" - default = "ibm-observe" - nullable = false -} - -variable "log_analysis_agent_tolerations" { - description = "DEPRECATED: List of tolerations to apply to Log Analysis agent." - type = list(object({ - key = optional(string) - operator = optional(string) - value = optional(string) - effect = optional(string) - tolerationSeconds = optional(number) - })) - default = [{ - operator = "Exists" - }] -} - ############################################################################## # Cloud Monitoring variables ##############################################################################