From 4f98f5940059d43e468a91effd3d510dbcc82a83 Mon Sep 17 00:00:00 2001
From: Vladislav Tropnikov <vladislav.tropnikov@genestack.com>
Date: Tue, 27 Feb 2024 21:51:28 +0100
Subject: [PATCH 1/4] Added cloudwatch decompression and data message
 extraction

---
 locals.tf    | 24 ++++++++++++++++++++++--
 variables.tf | 22 ++++++++++++++++++++++
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/locals.tf b/locals.tf
index 573722d..338e148 100644
--- a/locals.tf
+++ b/locals.tf
@@ -29,7 +29,7 @@ locals {
   is_search_destination = contains(["elasticsearch", "opensearch", "opensearchserverless"], local.destination) ? true : false
 
   # Data Transformation
-  enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning
+  enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning || var.enable_cloudwatch_logs_decompression
   lambda_processor = var.enable_lambda_transform ? {
     type = "Lambda"
     parameters = [
@@ -97,11 +97,31 @@ locals {
   record_deaggregation_processor = (var.enable_dynamic_partitioning && var.dynamic_partition_enable_record_deaggregation ?
     (var.dynamic_partition_record_deaggregation_type == "JSON" ? local.record_deaggregation_processor_json : local.record_deaggregation_processor_delimiter)
   : null)
+  cloudwatch_logs_decompression_processor = var.enable_cloudwatch_logs_decompression ? {
+    type = "Decompression"
+    parameters = [
+      {
+        name  = "CompressionFormat"
+        value = var.cloudwatch_logs_compression_type
+      }
+    ]
+  } : null
+  cloudwatch_logs_data_message_extraction_processor = var.enable_cloudwatch_logs_decompression && var.enable_cloudwatch_logs_data_message_extraction ? {
+    type = "CloudWatchLogProcessing"
+    parameters = [
+      {
+        name  = "DataMessageExtraction"
+        value = tostring(var.enable_cloudwatch_logs_data_message_extraction)
+      },
+    ]
+  } : null
   processors = [for each in [
     local.lambda_processor,
     local.metadata_extractor_processor,
     local.append_delimiter_processor,
-    local.record_deaggregation_processor
+    local.record_deaggregation_processor,
+    local.cloudwatch_logs_decompression_processor,
+    local.cloudwatch_logs_data_message_extraction_processor
   ] : each if local.enable_processing && each != null]
 
   # Data Format conversion
diff --git a/variables.tf b/variables.tf
index b30e403..a95b22e 100644
--- a/variables.tf
+++ b/variables.tf
@@ -433,6 +433,28 @@ variable "msk_source_connectivity_type" {
 ######
 # S3 Destination Configurations
 ######
+variable "enable_cloudwatch_logs_decompression" {
+  description = "Enables or disables Cloudwatch Logs decompression"
+  type        = bool
+  default     = false
+}
+
+variable "cloudwatch_logs_compression_type" {
+  description = "Cloudwatch Logs decompression types"
+  type        = string
+  default     = "GZIP"
+  validation {
+    error_message = "Valid value is GZIP."
+    condition     = contains(["GZIP"], var.cloudwatch_logs_compression_type)
+  }
+}
+
+variable "enable_cloudwatch_logs_data_message_extraction" {
+  description = "Cloudwatch Logs data message extraction"
+  type        = bool
+  default     = false
+}
+
 variable "enable_dynamic_partitioning" {
   description = "Enables or disables dynamic partitioning"
   type        = bool

From 193775895008795ea1faed97d8ace6ac242d4152 Mon Sep 17 00:00:00 2001
From: Vladislav Tropnikov <vladislav.tropnikov@genestack.com>
Date: Tue, 27 Feb 2024 21:53:49 +0100
Subject: [PATCH 2/4] run pre-commit

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index ccd1228..823d04d 100644
--- a/README.md
+++ b/README.md
@@ -923,6 +923,7 @@ No modules.
 | <a name="input_associate_role_to_redshift_cluster"></a> [associate\_role\_to\_redshift\_cluster](#input\_associate\_role\_to\_redshift\_cluster) | Set it to false if don't want the module associate the role to redshift cluster | `bool` | `true` | no |
 | <a name="input_buffering_interval"></a> [buffering\_interval](#input\_buffering\_interval) | Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination | `number` | `300` | no |
 | <a name="input_buffering_size"></a> [buffering\_size](#input\_buffering\_size) | Buffer incoming data to the specified size, in MBs, before delivering it to the destination. | `number` | `5` | no |
+| <a name="input_cloudwatch_logs_compression_type"></a> [cloudwatch\_logs\_compression\_type](#input\_cloudwatch\_logs\_compression\_type) | Cloudwatch Logs decompression types | `string` | `"GZIP"` | no |
 | <a name="input_configure_existing_application_role"></a> [configure\_existing\_application\_role](#input\_configure\_existing\_application\_role) | Set it to True if want use existing application role to add the firehose Policy | `bool` | `false` | no |
 | <a name="input_coralogix_endpoint_location"></a> [coralogix\_endpoint\_location](#input\_coralogix\_endpoint\_location) | Endpoint Location to coralogix destination | `string` | `"ireland"` | no |
 | <a name="input_coralogix_parameter_application_name"></a> [coralogix\_parameter\_application\_name](#input\_coralogix\_parameter\_application\_name) | By default, your delivery stream arn will be used as applicationName | `string` | `null` | no |
@@ -981,6 +982,8 @@ No modules.
 | <a name="input_elasticsearch_index_rotation_period"></a> [elasticsearch\_index\_rotation\_period](#input\_elasticsearch\_index\_rotation\_period) | The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data | `string` | `"OneDay"` | no |
 | <a name="input_elasticsearch_retry_duration"></a> [elasticsearch\_retry\_duration](#input\_elasticsearch\_retry\_duration) | The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt | `string` | `300` | no |
 | <a name="input_elasticsearch_type_name"></a> [elasticsearch\_type\_name](#input\_elasticsearch\_type\_name) | The Elasticsearch type name with maximum length of 100 characters | `string` | `null` | no |
+| <a name="input_enable_cloudwatch_logs_data_message_extraction"></a> [enable\_cloudwatch\_logs\_data\_message\_extraction](#input\_enable\_cloudwatch\_logs\_data\_message\_extraction) | Cloudwatch Logs data message extraction | `bool` | `false` | no |
+| <a name="input_enable_cloudwatch_logs_decompression"></a> [enable\_cloudwatch\_logs\_decompression](#input\_enable\_cloudwatch\_logs\_decompression) | Enables or disables Cloudwatch Logs decompression | `bool` | `false` | no |
 | <a name="input_enable_data_format_conversion"></a> [enable\_data\_format\_conversion](#input\_enable\_data\_format\_conversion) | Set it to true if you want to disable format conversion. | `bool` | `false` | no |
 | <a name="input_enable_destination_log"></a> [enable\_destination\_log](#input\_enable\_destination\_log) | The CloudWatch Logging Options for the delivery stream | `bool` | `true` | no |
 | <a name="input_enable_dynamic_partitioning"></a> [enable\_dynamic\_partitioning](#input\_enable\_dynamic\_partitioning) | Enables or disables dynamic partitioning | `bool` | `false` | no |

From 95951cd97921fcdcd35d1bd2b6512a92747cb8be Mon Sep 17 00:00:00 2001
From: Vladislav Tropnikov <vladislav.tropnikov@genestack.com>
Date: Wed, 28 Feb 2024 10:47:23 +0100
Subject: [PATCH 3/4] delete cloudwatch_logs_compression_type

---
 locals.tf    |  2 +-
 variables.tf | 10 ----------
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/locals.tf b/locals.tf
index 338e148..b051d2a 100644
--- a/locals.tf
+++ b/locals.tf
@@ -102,7 +102,7 @@ locals {
     parameters = [
       {
         name  = "CompressionFormat"
-        value = var.cloudwatch_logs_compression_type
+        value = "GZIP"
       }
     ]
   } : null
diff --git a/variables.tf b/variables.tf
index a95b22e..36988f8 100644
--- a/variables.tf
+++ b/variables.tf
@@ -439,16 +439,6 @@ variable "enable_cloudwatch_logs_decompression" {
   default     = false
 }
 
-variable "cloudwatch_logs_compression_type" {
-  description = "Cloudwatch Logs decompression types"
-  type        = string
-  default     = "GZIP"
-  validation {
-    error_message = "Valid value is GZIP."
-    condition     = contains(["GZIP"], var.cloudwatch_logs_compression_type)
-  }
-}
-
 variable "enable_cloudwatch_logs_data_message_extraction" {
   description = "Cloudwatch Logs data message extraction"
   type        = bool

From c9e39e54f261412b780436bbad5d2344a72ab01d Mon Sep 17 00:00:00 2001
From: Fabio Santos <fabiodanielmonteirosantos@gmail.com>
Date: Wed, 28 Feb 2024 22:48:55 +0000
Subject: [PATCH 4/4] Update README.md

---
 README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 823d04d..3be1419 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,6 @@ Supports all destinations and all Kinesis Firehose Features.
   * [Version 3.1.0](#version-310)
 * [License](#license)
 
-
 ## Module versioning rule
 
 | Module version | AWS Provider version |
@@ -95,6 +94,7 @@ Supports all destinations and all Kinesis Firehose Features.
 - Original Data Backup in S3
 - Logging and Encryption
 - Application Role to Direct Put Sources
+- Turn on/off cloudwatch logs decompressing and data message extraction
 - Permissions
   - IAM Roles
   - Opensearch / Opensearch Serverless Service Role
@@ -923,7 +923,6 @@ No modules.
 | <a name="input_associate_role_to_redshift_cluster"></a> [associate\_role\_to\_redshift\_cluster](#input\_associate\_role\_to\_redshift\_cluster) | Set it to false if don't want the module associate the role to redshift cluster | `bool` | `true` | no |
 | <a name="input_buffering_interval"></a> [buffering\_interval](#input\_buffering\_interval) | Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination | `number` | `300` | no |
 | <a name="input_buffering_size"></a> [buffering\_size](#input\_buffering\_size) | Buffer incoming data to the specified size, in MBs, before delivering it to the destination. | `number` | `5` | no |
-| <a name="input_cloudwatch_logs_compression_type"></a> [cloudwatch\_logs\_compression\_type](#input\_cloudwatch\_logs\_compression\_type) | Cloudwatch Logs decompression types | `string` | `"GZIP"` | no |
 | <a name="input_configure_existing_application_role"></a> [configure\_existing\_application\_role](#input\_configure\_existing\_application\_role) | Set it to True if want use existing application role to add the firehose Policy | `bool` | `false` | no |
 | <a name="input_coralogix_endpoint_location"></a> [coralogix\_endpoint\_location](#input\_coralogix\_endpoint\_location) | Endpoint Location to coralogix destination | `string` | `"ireland"` | no |
 | <a name="input_coralogix_parameter_application_name"></a> [coralogix\_parameter\_application\_name](#input\_coralogix\_parameter\_application\_name) | By default, your delivery stream arn will be used as applicationName | `string` | `null` | no |