From 4f98f5940059d43e468a91effd3d510dbcc82a83 Mon Sep 17 00:00:00 2001 From: Vladislav Tropnikov <vladislav.tropnikov@genestack.com> Date: Tue, 27 Feb 2024 21:51:28 +0100 Subject: [PATCH 1/4] Added cloudwatch decompression and data message extraction --- locals.tf | 24 ++++++++++++++++++++++-- variables.tf | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/locals.tf b/locals.tf index 573722d..338e148 100644 --- a/locals.tf +++ b/locals.tf @@ -29,7 +29,7 @@ locals { is_search_destination = contains(["elasticsearch", "opensearch", "opensearchserverless"], local.destination) ? true : false # Data Transformation - enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning + enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning || var.enable_cloudwatch_logs_decompression lambda_processor = var.enable_lambda_transform ? { type = "Lambda" parameters = [ @@ -97,11 +97,31 @@ locals { record_deaggregation_processor = (var.enable_dynamic_partitioning && var.dynamic_partition_enable_record_deaggregation ? (var.dynamic_partition_record_deaggregation_type == "JSON" ? local.record_deaggregation_processor_json : local.record_deaggregation_processor_delimiter) : null) + cloudwatch_logs_decompression_processor = var.enable_cloudwatch_logs_decompression ? { + type = "Decompression" + parameters = [ + { + name = "CompressionFormat" + value = var.cloudwatch_logs_compression_type + } + ] + } : null + cloudwatch_logs_data_message_extraction_processor = var.enable_cloudwatch_logs_decompression && var.enable_cloudwatch_logs_data_message_extraction ? { + type = "CloudWatchLogProcessing" + parameters = [ + { + name = "DataMessageExtraction" + value = tostring(var.enable_cloudwatch_logs_data_message_extraction) + }, + ] + } : null processors = [for each in [ local.lambda_processor, local.metadata_extractor_processor, local.append_delimiter_processor, - local.record_deaggregation_processor + local.record_deaggregation_processor, + local.cloudwatch_logs_decompression_processor, + local.cloudwatch_logs_data_message_extraction_processor ] : each if local.enable_processing && each != null] # Data Format conversion diff --git a/variables.tf b/variables.tf index b30e403..a95b22e 100644 --- a/variables.tf +++ b/variables.tf @@ -433,6 +433,28 @@ variable "msk_source_connectivity_type" { ###### # S3 Destination Configurations ###### +variable "enable_cloudwatch_logs_decompression" { + description = "Enables or disables Cloudwatch Logs decompression" + type = bool + default = false +} + +variable "cloudwatch_logs_compression_type" { + description = "Cloudwatch Logs decompression types" + type = string + default = "GZIP" + validation { + error_message = "Valid value is GZIP." + condition = contains(["GZIP"], var.cloudwatch_logs_compression_type) + } +} + +variable "enable_cloudwatch_logs_data_message_extraction" { + description = "Cloudwatch Logs data message extraction" + type = bool + default = false +} + variable "enable_dynamic_partitioning" { description = "Enables or disables dynamic partitioning" type = bool From 193775895008795ea1faed97d8ace6ac242d4152 Mon Sep 17 00:00:00 2001 From: Vladislav Tropnikov <vladislav.tropnikov@genestack.com> Date: Tue, 27 Feb 2024 21:53:49 +0100 Subject: [PATCH 2/4] run pre-commit --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ccd1228..823d04d 100644 --- a/README.md +++ b/README.md @@ -923,6 +923,7 @@ No modules. | <a name="input_associate_role_to_redshift_cluster"></a> [associate\_role\_to\_redshift\_cluster](#input\_associate\_role\_to\_redshift\_cluster) | Set it to false if don't want the module associate the role to redshift cluster | `bool` | `true` | no | | <a name="input_buffering_interval"></a> [buffering\_interval](#input\_buffering\_interval) | Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination | `number` | `300` | no | | <a name="input_buffering_size"></a> [buffering\_size](#input\_buffering\_size) | Buffer incoming data to the specified size, in MBs, before delivering it to the destination. | `number` | `5` | no | +| <a name="input_cloudwatch_logs_compression_type"></a> [cloudwatch\_logs\_compression\_type](#input\_cloudwatch\_logs\_compression\_type) | Cloudwatch Logs decompression types | `string` | `"GZIP"` | no | | <a name="input_configure_existing_application_role"></a> [configure\_existing\_application\_role](#input\_configure\_existing\_application\_role) | Set it to True if want use existing application role to add the firehose Policy | `bool` | `false` | no | | <a name="input_coralogix_endpoint_location"></a> [coralogix\_endpoint\_location](#input\_coralogix\_endpoint\_location) | Endpoint Location to coralogix destination | `string` | `"ireland"` | no | | <a name="input_coralogix_parameter_application_name"></a> [coralogix\_parameter\_application\_name](#input\_coralogix\_parameter\_application\_name) | By default, your delivery stream arn will be used as applicationName | `string` | `null` | no | @@ -981,6 +982,8 @@ No modules. | <a name="input_elasticsearch_index_rotation_period"></a> [elasticsearch\_index\_rotation\_period](#input\_elasticsearch\_index\_rotation\_period) | The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data | `string` | `"OneDay"` | no | | <a name="input_elasticsearch_retry_duration"></a> [elasticsearch\_retry\_duration](#input\_elasticsearch\_retry\_duration) | The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt | `string` | `300` | no | | <a name="input_elasticsearch_type_name"></a> [elasticsearch\_type\_name](#input\_elasticsearch\_type\_name) | The Elasticsearch type name with maximum length of 100 characters | `string` | `null` | no | +| <a name="input_enable_cloudwatch_logs_data_message_extraction"></a> [enable\_cloudwatch\_logs\_data\_message\_extraction](#input\_enable\_cloudwatch\_logs\_data\_message\_extraction) | Cloudwatch Logs data message extraction | `bool` | `false` | no | +| <a name="input_enable_cloudwatch_logs_decompression"></a> [enable\_cloudwatch\_logs\_decompression](#input\_enable\_cloudwatch\_logs\_decompression) | Enables or disables Cloudwatch Logs decompression | `bool` | `false` | no | | <a name="input_enable_data_format_conversion"></a> [enable\_data\_format\_conversion](#input\_enable\_data\_format\_conversion) | Set it to true if you want to disable format conversion. | `bool` | `false` | no | | <a name="input_enable_destination_log"></a> [enable\_destination\_log](#input\_enable\_destination\_log) | The CloudWatch Logging Options for the delivery stream | `bool` | `true` | no | | <a name="input_enable_dynamic_partitioning"></a> [enable\_dynamic\_partitioning](#input\_enable\_dynamic\_partitioning) | Enables or disables dynamic partitioning | `bool` | `false` | no | From 95951cd97921fcdcd35d1bd2b6512a92747cb8be Mon Sep 17 00:00:00 2001 From: Vladislav Tropnikov <vladislav.tropnikov@genestack.com> Date: Wed, 28 Feb 2024 10:47:23 +0100 Subject: [PATCH 3/4] delete cloudwatch_logs_compression_type --- locals.tf | 2 +- variables.tf | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/locals.tf b/locals.tf index 338e148..b051d2a 100644 --- a/locals.tf +++ b/locals.tf @@ -102,7 +102,7 @@ locals { parameters = [ { name = "CompressionFormat" - value = var.cloudwatch_logs_compression_type + value = "GZIP" } ] } : null diff --git a/variables.tf b/variables.tf index a95b22e..36988f8 100644 --- a/variables.tf +++ b/variables.tf @@ -439,16 +439,6 @@ variable "enable_cloudwatch_logs_decompression" { default = false } -variable "cloudwatch_logs_compression_type" { - description = "Cloudwatch Logs decompression types" - type = string - default = "GZIP" - validation { - error_message = "Valid value is GZIP." - condition = contains(["GZIP"], var.cloudwatch_logs_compression_type) - } -} - variable "enable_cloudwatch_logs_data_message_extraction" { description = "Cloudwatch Logs data message extraction" type = bool From c9e39e54f261412b780436bbad5d2344a72ab01d Mon Sep 17 00:00:00 2001 From: Fabio Santos <fabiodanielmonteirosantos@gmail.com> Date: Wed, 28 Feb 2024 22:48:55 +0000 Subject: [PATCH 4/4] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 823d04d..3be1419 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,6 @@ Supports all destinations and all Kinesis Firehose Features. * [Version 3.1.0](#version-310) * [License](#license) - ## Module versioning rule | Module version | AWS Provider version | @@ -95,6 +94,7 @@ Supports all destinations and all Kinesis Firehose Features. - Original Data Backup in S3 - Logging and Encryption - Application Role to Direct Put Sources +- Turn on/off cloudwatch logs decompressing and data message extraction - Permissions - IAM Roles - Opensearch / Opensearch Serverless Service Role @@ -923,7 +923,6 @@ No modules. | <a name="input_associate_role_to_redshift_cluster"></a> [associate\_role\_to\_redshift\_cluster](#input\_associate\_role\_to\_redshift\_cluster) | Set it to false if don't want the module associate the role to redshift cluster | `bool` | `true` | no | | <a name="input_buffering_interval"></a> [buffering\_interval](#input\_buffering\_interval) | Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination | `number` | `300` | no | | <a name="input_buffering_size"></a> [buffering\_size](#input\_buffering\_size) | Buffer incoming data to the specified size, in MBs, before delivering it to the destination. | `number` | `5` | no | -| <a name="input_cloudwatch_logs_compression_type"></a> [cloudwatch\_logs\_compression\_type](#input\_cloudwatch\_logs\_compression\_type) | Cloudwatch Logs decompression types | `string` | `"GZIP"` | no | | <a name="input_configure_existing_application_role"></a> [configure\_existing\_application\_role](#input\_configure\_existing\_application\_role) | Set it to True if want use existing application role to add the firehose Policy | `bool` | `false` | no | | <a name="input_coralogix_endpoint_location"></a> [coralogix\_endpoint\_location](#input\_coralogix\_endpoint\_location) | Endpoint Location to coralogix destination | `string` | `"ireland"` | no | | <a name="input_coralogix_parameter_application_name"></a> [coralogix\_parameter\_application\_name](#input\_coralogix\_parameter\_application\_name) | By default, your delivery stream arn will be used as applicationName | `string` | `null` | no |