diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 9717ee2..4f02853 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.2 +current_version = 2.0.0 commit = True tag = True message = Bump version: {current_version} → {new_version} @@ -12,14 +12,11 @@ replace = v{new_version} search = v{current_version} replace = v{new_version} -[bumpversion:file:cloudformation.json] +[bumpversion:file:deploy-using-profile.sh] search = v{current_version} replace = v{new_version} -[bumpversion:file:cloudformation-no-trail.json] +[bumpversion:file:cloudformation.json] search = v{current_version} replace = v{new_version} -[bumpversion:file:cloudformation-test.json] -search = v{current_version} -replace = v{new_version} diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e007b..c0857e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,38 @@ # Changelog +## 2.0.0 (2023-01-06) +Updated naming where possible to LogScale. +Added functionality to only support hosting the integration files in one bucket in one region. +Added functionality to choose which S3 bucket and S3 key should be used for retrieving the code files for the AWS Lambda functions. +Removed the option to choose older versions of the integration. +Removed unnecessary permissions for the CloudFormation resources. + +### Added +- Introduced parameter groups in the CF template. +- Added parameter for setting the S3 bucket containing the lambda code files. +- Added parameter for setting the S3 key in the S3 bucket containing the lambda code files. +- Created resources for copying the lambda code files from the specified S3 bucket, so that it is no longer required to create a new bucket in each region supported. The default S3 bucket hosted by LogScale is named _logscale-public-us-east-1_. +- Added a parameter for determining whether the CloudTrail and S3 bucket for autosubscription should be created, which also made it possible to delete the additional CF template. +- Made it possible for users to set which S3 bucket and S3 key they want to retrieve the lambda code files from, which also made it possible to delete the CF template for testing. + +### Changed +- Made metric ingester and metric statistics ingester creation optional. +- Updated the README of the project to contain a section for how the integration works, how versioning works, and how to support new regions. +- Updated the deployment scripts. +- Replaced 'Humio' with 'LogScale' where possible. +- Merged permissions regarding the subscriber into one. + +### Removed +- Removed the CF template with no trail. +- Removed the CF template for testing. +- Removed the parameter related to versions, so that there is only the newest version of the code uploaded. +- Removed unnecessary permissions for some of the AWS Lambdas. + ## 1.2.2 (2022-08-30) -Added additional login at ingest error +Added additional login at ingest error. ### Added -- Logging for when ingest into Humio fails +- Logging for when ingest into Humio fails. ## 1.2.1 (2021-01-12) Performance regression fix. diff --git a/CopyZipLambda.py b/CopyZipLambda.py new file mode 100644 index 0000000..a7102d1 --- /dev/null +++ b/CopyZipLambda.py @@ -0,0 +1,92 @@ +# This file contains the inline code for the lambda named "LogScaleCloudWatchCopyZipLambda". +# This code is copied directly into the CF resource, and thus this file is only to have an overview. +# This code is based on the inline code from the following GitHub library https://github.com/aws-quickstart/quickstart-examples/blob/main/patterns/LambdaZips/example.yaml. + +import json +import logging +import os +import threading +import boto3 +import cfnresponse + +level = os.getenv("log_level", "INFO") +logging.basicConfig(level=level) +logger = logging.getLogger() +logger.setLevel(level) + + +def copy_objects(source_bucket, dest_bucket, key): + """ + Copy key from source bucket to destination bucket. + + :param source_bucket: S3 bucket containing zip file with code. + :param dest_bucket: S3 bucket where zip file with code should be copied to. + :param key: File name to be copied. + :return: None + """ + s3 = boto3.client('s3') + copy_source = { + 'Bucket': source_bucket, + 'Key': key + } + logger.debug(('copy_source: %s' % copy_source)) + logger.debug(('dest_bucket = %s' % dest_bucket)) + logger.debug(('key = %s' % key)) + s3.copy_object(CopySource=copy_source, Bucket=dest_bucket, Key=key) + + +def delete_objects(bucket, key): + """ + Delete a bucket and its objects. + + :param bucket: S3 bucket to be deleted. + :param key: S3 key to object that should be deleted. + :return: + """ + s3 = boto3.client('s3') + objects = {'Objects': [{'Key': key}]} + s3.delete_objects(Bucket=bucket, Delete=objects) + + +def timeout(event, context): + """ + Send a response to the custom resource if it times out. + + :param event: Event data from the lambda. + :param context: Lambda context object. + :return: None + """ + logging.error('Execution is about to time out, sending failure response to CloudFormation') + cfnresponse.send(event, context, cfnresponse.FAILED, {}, None) + + +def handler(event, context): + """ + Lambda handler that will copy the zip file from the source bucket to the + destination bucket. + + It will send a failure to CloudFormation if the function is going to timeout. + + :param event: Event data from caller. + :param context: Lambda context object. + :return: None + """ + timer = threading.Timer((context.get_remaining_time_in_millis() + / 1000.00) - 0.5, timeout, args=[event, context]) + timer.start() + logger.debug(('Received event: %s' % json.dumps(event))) + status = cfnresponse.SUCCESS + try: + source_bucket = event['ResourceProperties']['SourceBucket'] + dest_bucket = event['ResourceProperties']['DestBucket'] + key = event['ResourceProperties']['Key'] + if event['RequestType'] == 'Delete': + delete_objects(dest_bucket, key) + else: + copy_objects(source_bucket, dest_bucket, key) + except Exception as e: + logging.error('Exception: %s' % e, exc_info=True) + status = cfnresponse.FAILED + finally: + timer.cancel() + cfnresponse.send(event, context, status, {}, None) \ No newline at end of file diff --git a/Makefile b/Makefile index e3d2c18..582c702 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -build: clean target copy dependencies target/cloudwatch2humio.zip +build: clean target copy dependencies target/cloudwatch2logscale.zip clean: rm -rf target/* @@ -12,8 +12,8 @@ copy: dependencies: pip3 install -r requirements.txt -t target -target/cloudwatch2humio.zip: - (cd target/ && zip -r ../target/v1.2.2_cloudwatch2humio.zip * ) +target/cloudwatch2logscale.zip: + (cd target/ && zip -r ../target/v2.0.0_cloudwatch2logscale.zip * ) clean: rm -rf target diff --git a/README.md b/README.md index deb611a..821e03d 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,18 @@ -# CloudWatch2Humio -This repository contains a set of lambdas for shipping Cloudwatch Logs and Metrics to Humio. +# CloudWatch2LogScale +This repository contains a CloudFormation template as well as Python code files for setting up an integration +which is used for shipping Cloudwatch Logs and Metrics to LogScale. -The full documentation regarding installing and using this integration can be found in the official [humio library](https://library.humio.com/reference/log-formats/amazon-cloudwatch/). +The full documentation regarding installing and using this integration can be found in the official +[humio library](https://library.humio.com/reference/log-formats/amazon-cloudwatch/). ## Vision -The vision for the CloudWatch to Humio integration, is to create a bridge between Humio and AWS, which enables users to ingest logs and metrics from the AWS CloudWatch service, so that Humio can be used to manage this data. +The vision for the CloudWatch to LogScale integration, is to create a bridge between LogScale and AWS, +which enables users to ingest logs and metrics from the AWS CloudWatch service, +so that LogScale can be used to manage this data. ## Governance -This project is maintained by employees at Humio ApS. -As a general rule, only employees at Humio can become maintainers and have commit privileges to this repository. +This project is maintained by employees at CrowdStrike. +As a general rule, only employees at CrowdStrike can become maintainers and have commit privileges to this repository. Therefore, if you want to contribute to the project, which we very much encourage, you must first fork the repository. Maintainers will have the final say on accepting or rejecting pull requests. As a rule of thumb, pull requests will be accepted if: @@ -18,25 +22,50 @@ As a rule of thumb, pull requests will be accepted if: * The contribution is of a quality comparable to the rest of the project The maintainers will attempt to react to issues and pull requests quickly, but their ability to do so can vary. -If you haven't heard back from a maintainer within 7 days of creating an issue or making a pull request, please feel free to ping them on the relevant post. +If you haven't heard back from a maintainer within 7 days of creating an issue or making a pull request , +please feel free to ping them on the relevant post. -Maintainers will also be in charge of both versioning and publishing future releases of the project. This includes adding versioning tags and adding to the changelog file. +Maintainers will also be in charge of both versioning and publishing future releases of the project. +This includes adding versioning tags and adding to the changelog file. The active maintainers involved with this project include: * [Suzanna Volkov](https://github.com/Suzanna-Volkov) -## CloudFormation Files -There are two versions of the CloudFormation file; one which creates a CloudTrail with a S3 bucket for log storage, -and one which does not create these resources. -The reason for this is that every AWS account only allows for one free CloudTrail, -and adding another will therefore be an additional cost for setting up this integration if one is already present. -By introducing two versions of the CloudFormation file, -it is possible for the user to choose whether they want to create a CloudTrail specifically to be used with this integration, -or to use a one which already exists. +## How this integration works +The integration consists of a CloudFormation template and some Python code files used for lambda functions. +The lambda code files are zipped and uploaded to a public S3 bucket hosted by LogScale. +When creating a CloudStack using this CloudFormation template, +some additional helper resources are created to help copy the lambda code files from the S3 bucket hosting the +files to a newly created S3 bucket in the user's AWS environment. +This is so that we do not have to create buckets in each supported region as the CloudFormation has a +restriction that it can only retrieve lambda code files from a bucket located in the same region as the stack. -***It is thus required when creating a stack using the "no-trail" CloudFormation file that an existing CloudTrail with "management events" enabled already to be present for the account!*** +When all the resources are created, the lambda functions will start sending logs to the LogScale host, +which was determined during setup. +Chosen log groups are subscribed to, and whenever new logs arrive, these will be forwarded to LogScale. +This means that existing logs will not be forwarded to LogScale using this integration. +To get existing logs into LogScale another approach is required, which will probably include manually downloading +the existing logs and then sending them to LogScale using some sort of shipper. -The CloudTrail is used by the integration for discovering new log groups in CloudWatch, -which makes it possible for the integration to automatically subscribe to these when they appear. -This is only done if the parameter `EnableCloudWatchLogsAutoSubscription` is set to **true**. +## Versioning +Versioning is used for the ZIP package containing the deployment files. +Versioning of the deployment package is required to make it possible to update an already installed +integration's code files. +A CloudStack will only update its files if the name changes, so you cannot use a file with the +same name as is already used to update with as it won't be recognized as a change. +So to force CloudStack to run an update, the deployment package name needs to be changed. +This means that only one file version is necessary to maintain in the public S3 bucket, +but the name needs to be updated every time there are new changes to the code files. + +A flow can be setup where older versions are deleted when a new one is uploaded, this does not yet exist. + +It is thus not necessary to make a new release of the integration unless the Python files are changed, +however, changes to the CloudFormation file will still be included in the CHANGELOG. Therefore, +version updates has happened whenever there are changes pushed to the GitHub repository to keep these +two numbers aligned. + +## Adding new regions +The most common request from users is that a new region be supported. Since the move to one bucket in one region, +it is no longer necessary to add new buckets, and thus this code does not need any updates. To support a new region +there simply needs to be added a link in the documentation. diff --git a/cloudformation-no-trail.json b/cloudformation-no-trail.json deleted file mode 100644 index 064b4cf..0000000 --- a/cloudformation-no-trail.json +++ /dev/null @@ -1,517 +0,0 @@ -{ - "AWSTemplateFormatVersion" : "2010-09-09", - "Description" : "CloudWatch to Humio Integration for sending CloudWatch Logs and Metrics to Humio.", - "Parameters" : { - "HumioProtocol" : { - "Type" : "String", - "Description" : "The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended.", - "Default" : "https" - }, - "HumioHost" : { - "Type" : "String", - "Description" : "The host to ship Humio log/metric events to.", - "Default" : "cloud.humio.com" - }, - "HumioIngestToken" : { - "Type" : "String", - "Description" : "The value of the ingest token for the repository from your Humio account to ship log/metric events to.", - "Default" : "", - "NoEcho" : true - }, - "HumioLambdaLogRetention" : { - "Type" : "Number", - "Description" : "Number of days to retain CloudWatch logs from the Humio Lambda functions.", - "Default": 1 - }, - "EnableCloudWatchLogsAutoSubscription" : { - "Type" : "String", - "AllowedValues" : [ - "true", - "false" - ], - "Description" : "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable.", - "Default" : "true" - }, - "HumioCloudWatchLogsSubscriptionPrefix" : { - "Type" : "String", - "Description" : "Humio will only subscribe to log groups with the prefix specified.", - "Default" : "" - }, - "EnableCloudWatchLogsBackfillerAutoRun" : { - "Type" : "String", - "AllowedValues" : [ - "true", - "false" - ], - "Description" : "Make the backfiller run automatically when created. Set to 'true' to enable.", - "Default" : "true" - }, - "EnableVPCForIngesterLambdas" : { - "Type" : "String", - "AllowedValues" : [ - "true", - "false" - ], - "Description" : "Use a VPC for the lambda ingester functions. Set to 'true' to enable.", - "Default" : "false" - }, - "SecurityGroupIds" : { - "Type" : "CommaDelimitedList", - "Description" : "A comma separated list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled." - }, - "SubnetIds" : { - "Type" : "CommaDelimitedList", - "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled." - }, - "HumioLambdaLogLevel" : { - "Type" : "String", - "AllowedValues" : ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - "Default" : "INFO" - }, - "Version" : { - "Type" : "String", - "Description" : "The version of the integration you want installed.", - "Default" : "v1.2.2" - } - }, - "Conditions" : { - "CreateAutoSubscriptionResources" : { - "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsAutoSubscription" }, "true" ] - }, - "CreateHumioBackfillerAutoRunner" : { - "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsBackfillerAutoRun" }, "true" ] - }, - "ConfigureVPCForIngesterLambdas" : { - "Fn::Equals" : [ { "Ref" : "EnableVPCForIngesterLambdas" }, "true" ] - } - }, - "Resources" : { - "HumioCloudWatchRole" : { - "Type" : "AWS::IAM::Role", - "Properties" : { - "AssumeRolePolicyDocument" : { - "Version" : "2012-10-17", - "Statement" : [ - { - "Action" : "sts:AssumeRole", - "Principal" : { - "Service" : [ - "lambda.amazonaws.com", - "apigateway.amazonaws.com", - "logs.amazonaws.com" - ] - }, - "Effect" : "Allow", - "Sid" : "" - } - ] - }, - "Policies" : [ - { - "PolicyName" : "humio_cloudwatch_role", - "PolicyDocument" : { - "Fn::If" : [ "ConfigureVPCForIngesterLambdas", - { - "Version" : "2012-10-17", - "Statement" : [ - { - "Effect" : "Allow", - "Action" : [ - "lambda:GetFunction", - "lambda:InvokeFunction", - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:DescribeSubscriptionFilters", - "logs:PutSubscriptionFilter", - "logs:DeleteSubscriptionFilter", - "logs:PutLogEvents", - "logs:GetLogEvents", - "logs:FilterLogEvents", - "cloudwatch:GetMetricData", - "cloudwatch:GetMetricStatistics", - "ec2:CreateNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "ec2:CreateNetworkInterfacePermission", - "ec2:DeleteNetworkInterface" - ], - "Resource" : "*" - } - ] - }, - { - "Version" : "2012-10-17", - "Statement" : [ - { - "Effect" : "Allow", - "Action" : [ - "lambda:GetFunction", - "lambda:InvokeFunction", - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:DescribeSubscriptionFilters", - "logs:PutSubscriptionFilter", - "logs:DeleteSubscriptionFilter", - "logs:PutLogEvents", - "logs:GetLogEvents", - "logs:FilterLogEvents", - "cloudwatch:GetMetricData", - "cloudwatch:GetMetricStatistics" - ], - "Resource" : "*" - } - ] - } - ] - } - } - ] - } - }, - "HumioCloudWatchLogsIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "VpcConfig" : { - "Fn::If" : [ "ConfigureVPCForIngesterLambdas", - { - "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, - "SubnetIds" : { "Ref" : "SubnetIds" } - }, - { - "Ref" : "AWS::NoValue" - } - ] - }, - "Description" : "CloudWatch Logs to Humio ingester.", - "Handler" : "logs_ingester.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchLogsIngesterPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchLogsIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsIngester" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - }, - "HumioCloudWatchLogsSubscriber" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_log_ingester_arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] - }, - "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "Description" : "CloudWatch Logs to Humio log group subscriber.", - "Handler" : "logs_subscriber.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchLogsSubscriberPermission" : { - "Condition" : "CreateAutoSubscriptionResources", - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] - }, - "Principal" : "events.amazonaws.com", - "SourceAccount" : { "Ref" : "AWS::AccountId" } - } - }, - "HumioCloudWatchLogsSubscriberPermission2" : { - "Condition" : "CreateAutoSubscriptionResources", - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] - }, - "Principal" : "events.amazonaws.com", - "SourceArn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberEventRule", "Arn" ] - } - } - }, - "HumioCloudWatchLogsSubscriberLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsSubscriber" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - }, - "HumioCloudWatchLogsBackfiller" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_log_ingester_arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] - }, - "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "Description" : "CloudWatch Logs to Humio logs backfiller.", - "Handler" : "logs_backfiller.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchLogsBackfillerPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchLogsBackfillerLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsBackfiller" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - }, - "HumioBackfillerAutoRunner" : { - "Condition" : "CreateHumioBackfillerAutoRunner", - "DependsOn" : [ "HumioCloudWatchLogsBackfiller" ], - "Type" : "Custom::BackfillerAutoRunner", - "Properties" : { - "ServiceToken" : { "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] }, - "StackName" : { "Ref" : "AWS::StackName" } - } - }, - "HumioCloudWatchLogsSubscriberEventRule" : { - "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriber" ], - "Type" : "AWS::Events::Rule", - "Properties" : { - "Description" : "Humio log group auto subscription event rule.", - "EventPattern" : { - "source" : [ "aws.logs" ], - "detail-type" : [ "AWS API Call via CloudTrail" ], - "detail" : { - "eventSource" : [ "logs.amazonaws.com" ], - "eventName" : [ "CreateLogGroup" ] - } - }, - "Name" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] - }, - "Targets" : [ - { - "Id" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] - }, - "Arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] - } - } - ] - } - }, - "HumioCloudWatchMetricIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "VpcConfig" : { - "Fn::If" : [ "ConfigureVPCForIngesterLambdas", - { - "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, - "SubnetIds" : { "Ref" : "SubnetIds" } - }, - { - "Ref" : "AWS::NoValue" - } - ] - }, - "Description" : "CloudWatch Metrics to Humio ingester.", - "Handler" : "metric_ingester.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchMetricIngesterPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchMetricIngester", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchMetricIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricIngester" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - }, - "HumioCloudWatchMetricStatisticsIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "VpcConfig" : { - "Fn::If" : [ "ConfigureVPCForIngesterLambdas", - { - "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, - "SubnetIds" : { "Ref" : "SubnetIds" } - }, - { - "Ref" : "AWS::NoValue" - } - ] - }, - "Description" : "CloudWatch Metrics Statistics to Humio ingester.", - "Handler" : "metric_statistics_ingester.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchMetricStatisticsIngesterPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchMetricStatisticsIngester", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchMetricStatisticsIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricStatisticsIngester" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - } - } -} diff --git a/cloudformation-test.json b/cloudformation-test.json deleted file mode 100644 index e52547a..0000000 --- a/cloudformation-test.json +++ /dev/null @@ -1,572 +0,0 @@ -{ - "AWSTemplateFormatVersion" : "2010-09-09", - "Description" : "CloudWatch to Humio Integration for sending CloudWatch Logs and Metrics to Humio.", - "Parameters" : { - "HumioProtocol" : { - "Type" : "String", - "Description" : "The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended.", - "Default" : "https" - }, - "HumioHost" : { - "Type" : "String", - "Description" : "The host to ship Humio log/metric events to.", - "Default" : "cloud.humio.com" - }, - "HumioIngestToken" : { - "Type" : "String", - "Description" : "The value of the ingest token for the repository from your Humio account to ship log/metric events to.", - "Default" : "", - "NoEcho" : true - }, - "HumioLambdaLogRetention" : { - "Type" : "Number", - "Description" : "Number of days to retain CloudWatch logs from the Humio Lambda functions.", - "Default": 1 - }, - "EnableCloudWatchLogsAutoSubscription" : { - "Type" : "String", - "AllowedValues" : [ - "true", - "false" - ], - "Description" : "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable.", - "Default" : "true" - }, - "HumioCloudWatchLogsSubscriptionPrefix" : { - "Type" : "String", - "Description" : "Humio will only subscribe to log groups with the prefix specified.", - "Default" : "" - }, - "EnableCloudWatchLogsBackfillerAutoRun" : { - "Type" : "String", - "AllowedValues" : [ - "true", - "false" - ], - "Description" : "Make the backfiller run automatically when created. Set to 'true' to enable.", - "Default" : "true" - }, - "EnableVPCForIngesterLambdas" : { - "Type" : "String", - "AllowedValues" : [ - "true", - "false" - ], - "Description" : "Use a VPC for the lambda ingester functions. Set to 'true' to enable.", - "Default" : "false" - }, - "SecurityGroupIds" : { - "Type" : "CommaDelimitedList", - "Description" : "A comma separated list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled." - }, - "SubnetIds" : { - "Type" : "CommaDelimitedList", - "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lamda functions will be deployed into. Only required if VPC is enabled." - }, - "HumioLambdaLogLevel" : { - "Type" : "String", - "AllowedValues" : ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - "Default" : "INFO" - }, - "Version" : { - "Type" : "String", - "Description" : "The version of the integration you want installed.", - "Default" : "v1.2.2" - } - }, - "Conditions" : { - "CreateAutoSubscriptionResources" : { - "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsAutoSubscription" }, "true" ] - }, - "CreateHumioBackfillerAutoRunner" : { - "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsBackfillerAutoRun" }, "true" ] - }, - "ConfigureVPCForIngesterLambdas" : { - "Fn::Equals" : [ { "Ref" : "EnableVPCForIngesterLambdas" }, "true" ] - } - }, - "Resources" : { - "HumioCloudWatchRole" : { - "Type" : "AWS::IAM::Role", - "Properties" : { - "AssumeRolePolicyDocument" : { - "Version" : "2012-10-17", - "Statement" : [ - { - "Action" : "sts:AssumeRole", - "Principal" : { - "Service" : [ - "lambda.amazonaws.com", - "apigateway.amazonaws.com", - "logs.amazonaws.com" - ] - }, - "Effect" : "Allow", - "Sid" : "" - } - ] - }, - "Policies" : [ - { - "PolicyName" : "humio_cloudwatch_role", - "PolicyDocument" : { - "Fn::If" : [ "ConfigureVPCForIngesterLambdas", - { - "Version" : "2012-10-17", - "Statement" : [ - { - "Effect" : "Allow", - "Action" : [ - "lambda:GetFunction", - "lambda:InvokeFunction", - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:DescribeSubscriptionFilters", - "logs:PutSubscriptionFilter", - "logs:DeleteSubscriptionFilter", - "logs:PutLogEvents", - "logs:GetLogEvents", - "logs:FilterLogEvents", - "cloudwatch:GetMetricData", - "cloudwatch:GetMetricStatistics", - "ec2:CreateNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "ec2:CreateNetworkInterfacePermission", - "ec2:DeleteNetworkInterface" - ], - "Resource" : "*" - } - ] - }, - { - "Version" : "2012-10-17", - "Statement" : [ - { - "Effect" : "Allow", - "Action" : [ - "lambda:GetFunction", - "lambda:InvokeFunction", - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:DescribeSubscriptionFilters", - "logs:PutSubscriptionFilter", - "logs:DeleteSubscriptionFilter", - "logs:PutLogEvents", - "logs:GetLogEvents", - "logs:FilterLogEvents", - "cloudwatch:GetMetricData", - "cloudwatch:GetMetricStatistics" - ], - "Resource" : "*" - } - ] - } - ] - } - } - ] - } - }, - "HumioCloudWatchLogsIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : "cloudwatch2humio", - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "VpcConfig" : { - "Fn::If" : [ "ConfigureVPCForIngesterLambdas", - { - "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, - "SubnetIds" : { "Ref" : "SubnetIds" } - }, - { - "Ref" : "AWS::NoValue" - } - ] - }, - "Description" : "CloudWatch Logs to Humio ingester.", - "Handler" : "logs_ingester.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchLogsIngesterPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchLogsIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsIngester" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - }, - "HumioCloudWatchLogsSubscriber" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : "cloudwatch2humio", - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_log_ingester_arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] - }, - "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "Description" : "CloudWatch Logs to Humio log group subscriber.", - "Handler" : "logs_subscriber.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchLogsSubscriberPermission" : { - "Condition" : "CreateAutoSubscriptionResources", - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] - }, - "Principal" : "events.amazonaws.com", - "SourceAccount" : { "Ref" : "AWS::AccountId" } - } - }, - "HumioCloudWatchLogsSubscriberPermission2" : { - "Condition" : "CreateAutoSubscriptionResources", - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] - }, - "Principal" : "events.amazonaws.com", - "SourceArn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberEventRule", "Arn" ] - } - } - }, - "HumioCloudWatchLogsSubscriberLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsSubscriber" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - }, - "HumioCloudWatchLogsBackfiller" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : "cloudwatch2humio", - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_log_ingester_arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] - }, - "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "Description" : "CloudWatch Logs to Humio logs backfiller.", - "Handler" : "logs_backfiller.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchLogsBackfillerPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchLogsBackfillerLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsBackfiller" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - }, - "HumioBackfillerAutoRunner" : { - "Condition" : "CreateHumioBackfillerAutoRunner", - "DependsOn" : [ "HumioCloudWatchLogsBackfiller" ], - "Type" : "Custom::BackfillerAutoRunner", - "Properties" : { - "ServiceToken" : { "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] }, - "StackName" : { "Ref" : "AWS::StackName" } - } - }, - "HumioCloudWatchLogsSubscriberS3Bucket" : { - "Condition" : "CreateAutoSubscriptionResources", - "Type" : "AWS::S3::Bucket", - "Properties" : { - "AccessControl" : "BucketOwnerFullControl", - "BucketName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] - } - } - }, - "HumioCloudWatchLogsSubscriberS3BucketPolicy" : { - "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3Bucket" ], - "Type" : "AWS::S3::BucketPolicy", - "Properties" : { - "Bucket" : { "Ref" : "HumioCloudWatchLogsSubscriberS3Bucket" }, - "PolicyDocument" : { - "Version" : "2012-10-17", - "Statement" : [ - { - "Sid" : "AWSCloudTrailAclCheck20150319", - "Effect" : "Allow", - "Principal" : { - "Service" : "cloudtrail.amazonaws.com" - }, - "Action" : "s3:GetBucketAcl", - "Resource" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] - } - }, - { - "Sid" : "AWSCloudTrailWrite20150319", - "Effect" : "Allow", - "Principal" : { - "Service" : "cloudtrail.amazonaws.com" - }, - "Action" : "s3:PutObject", - "Resource" : { - "Fn::Join" : [ "", [ { "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] }, "/AWSLogs/", { "Ref" : "AWS::AccountId" }, "/*" ] ] - }, - "Condition" : { - "StringEquals" : { "s3:x-amz-acl" : "bucket-owner-full-control" } - } - } - ] - } - } - }, - "HumioCloudWatchLogsSubscriberCloudTrail" : { - "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3BucketPolicy" ], - "Type" : "AWS::CloudTrail::Trail", - "Properties" : { - "EnableLogFileValidation" : false, - "IncludeGlobalServiceEvents" : true, - "IsMultiRegionTrail" : true, - "IsLogging" : true, - "S3BucketName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] - }, - "TrailName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", { "Ref" : "AWS::AccountId" } ] ] - } - } - }, - "HumioCloudWatchLogsSubscriberEventRule" : { - "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriber" ], - "Type" : "AWS::Events::Rule", - "Properties" : { - "Description" : "Humio log group auto subscription event rule.", - "EventPattern" : { - "source" : [ "aws.logs" ], - "detail-type" : [ "AWS API Call via CloudTrail" ], - "detail" : { - "eventSource" : [ "logs.amazonaws.com" ], - "eventName" : [ "CreateLogGroup" ] - } - }, - "Name" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] - }, - "Targets" : [ - { - "Id" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] - }, - "Arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] - } - } - ] - } - }, - "HumioCloudWatchMetricIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : "cloudwatch2humio", - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "VpcConfig" : { - "Fn::If" : [ "ConfigureVPCForIngesterLambdas", - { - "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, - "SubnetIds" : { "Ref" : "SubnetIds" } - }, - { - "Ref" : "AWS::NoValue" - } - ] - }, - "Description" : "CloudWatch Metrics to Humio ingester.", - "Handler" : "metric_ingester.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchMetricIngesterPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchMetricIngester", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchMetricIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricIngester" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - }, - "HumioCloudWatchMetricStatisticsIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Lambda::Function", - "Properties" : { - "Code" : { - "S3Bucket" : "cloudwatch2humio", - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } - }, - "Environment" : { - "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } - } - }, - "VpcConfig" : { - "Fn::If" : [ "ConfigureVPCForIngesterLambdas", - { - "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, - "SubnetIds" : { "Ref" : "SubnetIds" } - }, - { - "Ref" : "AWS::NoValue" - } - ] - }, - "Description" : "CloudWatch Metrics Statistics to Humio ingester.", - "Handler" : "metric_statistics_ingester.lambda_handler", - "MemorySize" : "128", - "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] - }, - "Runtime" : "python3.8", - "Timeout" : "300" - } - }, - "HumioCloudWatchMetricStatisticsIngesterPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchMetricStatisticsIngester", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchMetricStatisticsIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], - "Type" : "AWS::Logs::LogGroup", - "Properties" : { - "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricStatisticsIngester" } ] ] - }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } - } - } - } -} diff --git a/cloudformation.json b/cloudformation.json index a7d0d75..5ea7dac 100644 --- a/cloudformation.json +++ b/cloudformation.json @@ -1,26 +1,26 @@ { "AWSTemplateFormatVersion" : "2010-09-09", - "Description" : "CloudWatch to Humio Integration for sending CloudWatch Logs and Metrics to Humio.", + "Description" : "CloudWatch to LogScale Integration for sending CloudWatch Logs and Metrics to LogScale.", "Parameters" : { - "HumioProtocol" : { + "LogScaleProtocol" : { "Type" : "String", - "Description" : "The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended.", + "Description" : "The transport protocol used for delivering log/metric events to LogScale. HTTPS is default and recommended.", "Default" : "https" }, - "HumioHost" : { + "LogScaleHost" : { "Type" : "String", - "Description" : "The host to ship Humio log/metric events to.", + "Description" : "The host to ship LogScale log/metric events to.", "Default" : "cloud.humio.com" }, - "HumioIngestToken" : { + "LogScaleIngestToken" : { "Type" : "String", - "Description" : "The value of the ingest token for the repository from your Humio account to ship log/metric events to.", + "Description" : "The value of the ingest token for the repository from your LogScale account to ship log/metric events to.", "Default" : "", "NoEcho" : true }, - "HumioLambdaLogRetention" : { + "LogScaleLambdaLogRetention" : { "Type" : "Number", - "Description" : "Number of days to retain CloudWatch logs from the Humio Lambda functions.", + "Description" : "The number of days to retain CloudWatch logs for the created lambda functions.", "Default": 1 }, "EnableCloudWatchLogsAutoSubscription" : { @@ -32,9 +32,18 @@ "Description" : "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable.", "Default" : "true" }, - "HumioCloudWatchLogsSubscriptionPrefix" : { + "CreateCloudTrailForAutoSubscription" :{ "Type" : "String", - "Description" : "Humio will only subscribe to log groups with the prefix specified.", + "AllowedValues" : [ + "true", + "false" + ], + "Description" : "If autosubscription to new logs is desired, but instead of the needed resources (a CloudTrail and a S3 bucket) being created now existing resources can be used instead. To use existing resources set this to 'false'. Note that this parameter is only considered if autosubscription is enabled.", + "Default" : "true" + }, + "LogScaleCloudWatchLogsSubscriptionPrefix" : { + "Type" : "String", + "Description" : "LogScale will only subscribe using either the backfiller or the autosubscriber to log groups with the prefix specified.", "Default" : "" }, "EnableCloudWatchLogsBackfillerAutoRun" : { @@ -46,6 +55,15 @@ "Description" : "Make the backfiller run automatically when created. Set to 'true' to enable.", "Default" : "true" }, + "CreateCloudWatchMetricIngesterAndMetricStatisticsIngesterLambdas" : { + "Type" : "String", + "AllowedValues" : [ + "true", + "false" + ], + "Description" : "Choose whether the metric ingester and the metric statistics ingester lambdas should be created. These are not necessary for getting CloudWatch logs into LogScale. Set to 'true' to create.", + "Default" : "false" + }, "EnableVPCForIngesterLambdas" : { "Type" : "String", "AllowedValues" : [ @@ -61,32 +79,82 @@ }, "SubnetIds" : { "Type" : "CommaDelimitedList", - "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lamda functions will be deployed into. Only required if VPC is enabled." + "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled." }, - "HumioLambdaLogLevel" : { + "LogScaleLambdaLogLevel" : { "Type" : "String", "AllowedValues" : ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - "Default" : "INFO" + "Default" : "INFO", + "Description" : "The log level printed by the lambda functions when they run. Note that a higher log level will result in more CloudWatch logs." + }, + "S3BucketContainingLambdaCodeFiles" : { + "Type" : "String", + "Description" : "The name of the S3 bucket containing the lambda code files used for the integration. Change this if you want to retrieve the code files from another S3 bucket than the default 'logscale-public-us-east-1'.", + "Default" : "logscale-public-us-east-1" }, - "Version" : { + "S3KeyOfTheDeploymentPackageContainingLambdaCodeFiles" : { "Type" : "String", - "Description" : "The version of the integration you want installed.", - "Default" : "v1.2.2" + "Description" : "The name of the S3 key in the S3 bucket containing the lambda code files used for the integration. Change this if you have named the deployment package containing the code files something other than the default 'v2.0.0_cloudwatch2logscale.zip'.", + "Default" : "v2.0.0_cloudwatch2logscale.zip" } }, "Conditions" : { "CreateAutoSubscriptionResources" : { "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsAutoSubscription" }, "true" ] }, - "CreateHumioBackfillerAutoRunner" : { + "CreateCloudTrailAndS3BucketForAutoSubscription" : { + "Fn::Equals" : [ { "Ref" : "CreateCloudTrailForAutoSubscription" }, "true" ] + }, + "CreateCloudTrailAndS3BucketAndEnableAutoSubscription" : { + "Fn::And" : [ + { + "Condition": "CreateAutoSubscriptionResources" + }, + { + "Condition": "CreateCloudTrailAndS3BucketForAutoSubscription" + } + ] + }, + "CreateLogScaleBackfillerAutoRunner" : { "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsBackfillerAutoRun" }, "true" ] }, "ConfigureVPCForIngesterLambdas" : { "Fn::Equals" : [ { "Ref" : "EnableVPCForIngesterLambdas" }, "true" ] + }, + "CreateMetricAndMetricStatisticsLambdas" : { + "Fn::Equals" : [ { "Ref" : "CreateCloudWatchMetricIngesterAndMetricStatisticsIngesterLambdas" }, "true" ] + } + }, + "Metadata": { + "AWS::CloudFormation::Interface" : { + "ParameterGroups" : [ + { + "Label" : { "default" : "LogScale Settings" }, + "Parameters" : [ "LogScaleProtocol", "LogScaleHost", "LogScaleIngestToken" ] + }, + { + "Label" : { "default" : "VPC Settings"}, + "Parameters" : [ "EnableVPCForIngesterLambdas", "SecurityGroupIds", "SubnetIds" ] + }, + { + "Label" : { "default" : "Lambda Settings" }, + "Parameters" : [ + "EnableCloudWatchLogsAutoSubscription", + "CreateCloudTrailForAutoSubscription", + "EnableCloudWatchLogsBackfillerAutoRun", + "LogScaleCloudWatchLogsSubscriptionPrefix", + "CreateCloudWatchMetricIngesterAndMetricStatisticsIngesterLambdas", + "S3BucketContainingLambdaCodeFiles", + "S3KeyOfTheDeploymentPackageContainingLambdaCodeFiles", + "LogScaleLambdaLogLevel", + "LogScaleLambdaLogRetention" + ] + } + ] } }, "Resources" : { - "HumioCloudWatchRole" : { + "LogScaleCloudWatchRole" : { "Type" : "AWS::IAM::Role", "Properties" : { "AssumeRolePolicyDocument" : { @@ -108,7 +176,7 @@ }, "Policies" : [ { - "PolicyName" : "humio_cloudwatch_role", + "PolicyName" : "logscale_cloudwatch_role", "PolicyDocument" : { "Fn::If" : [ "ConfigureVPCForIngesterLambdas", { @@ -134,7 +202,10 @@ "ec2:CreateNetworkInterface", "ec2:DescribeNetworkInterfaces", "ec2:CreateNetworkInterfacePermission", - "ec2:DeleteNetworkInterface" + "ec2:DeleteNetworkInterface", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" ], "Resource" : "*" } @@ -159,7 +230,10 @@ "logs:GetLogEvents", "logs:FilterLogEvents", "cloudwatch:GetMetricData", - "cloudwatch:GetMetricStatistics" + "cloudwatch:GetMetricStatistics", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" ], "Resource" : "*" } @@ -171,24 +245,56 @@ ] } }, - "HumioCloudWatchLogsIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchLambdaZipBucket" : { + "Type" : "AWS::S3::Bucket" + }, + "LogScaleCloudWatchCopyZipCustom" : { + "Type" : "Custom::CopyZip", + "Properties": { + "ServiceToken": { "Fn::GetAtt" : [ "LogScaleCloudWatchCopyZipLambda", "Arn" ] }, + "DestBucket" : { "Ref" : "LogScaleCloudWatchLambdaZipBucket" }, + "SourceBucket" : { "Ref" : "S3BucketContainingLambdaCodeFiles" }, + "Key" : { "Ref" : "S3KeyOfTheDeploymentPackageContainingLambdaCodeFiles" } + } + }, + "LogScaleCloudWatchCopyZipLambda" : { + "DependsOn" : [ "LogScaleCloudWatchRole" ], + "Type": "AWS::Lambda::Function", + "Properties": { + "Description" : "Copies objects from a S3 bucket to a destination.", + "Handler" : "index.handler", + "Runtime" : "python3.8", + "Timeout" : "300", + "Role" : { "Fn::GetAtt": [ "LogScaleCloudWatchRole", "Arn" ] }, + "Code" : { + "ZipFile" : "import json\nimport logging\nimport os\nimport threading\nimport boto3\nimport cfnresponse\n\nlevel = os.getenv(\"log_level\", \"INFO\")\nlogging.basicConfig(level=level)\nlogger = logging.getLogger()\nlogger.setLevel(level)\n\n\ndef copy_objects(source_bucket, dest_bucket, key):\n \"\"\"\n Copy key from source bucket to destination bucket.\n\n :param source_bucket: S3 bucket containing ZIP file with code.\n :param dest_bucket: S3 bucket where ZIP file with code should be copied to.\n :param key: File name to be copied.\n :return: None\n \"\"\"\n s3 = boto3.client('s3')\n copy_source = {\n 'Bucket': source_bucket,\n 'Key': key\n }\n logger.debug(('copy_source: %s' % copy_source))\n logger.debug(('dest_bucket = %s' % dest_bucket))\n logger.debug(('key = %s' % key))\n s3.copy_object(CopySource=copy_source, Bucket=dest_bucket, Key=key)\n\n\ndef delete_objects(bucket, key):\n \"\"\"\n Delete a bucket specified by the key.\n\n :param bucket: S3 bucket to be deleted.\n :param key: S3 key to object that should be deleted.\n :return:\n \"\"\"\n s3 = boto3.client('s3')\n objects = {'Objects': [{'Key': key}]}\n s3.delete_objects(Bucket=bucket, Delete=objects)\n\n\ndef timeout(event, context):\n \"\"\"\n Send a response to the custom resource if it times out.\n\n :param event: Event data from the Lambda.\n :param context: Lambda context object.\n :return: None\n \"\"\"\n logging.error('Execution is about to time out, sending failure response to CloudFormation')\n cfnresponse.send(event, context, cfnresponse.FAILED, {}, None)\n\n\ndef handler(event, context):\n \"\"\"\n Lambda handler that will copy the ZIP file from the source bucket to the\n destination bucket.\n\n It will send a failure to CloudFormation if the function is going to timeout.\n\n :param event: Event data from caller.\n :param context: Lambda context object.\n :return: None\n \"\"\"\n timer = threading.Timer((context.get_remaining_time_in_millis()\n / 1000.00) - 0.5, timeout, args=[event, context])\n timer.start()\n logger.debug(('Received event: %s' % json.dumps(event)))\n status = cfnresponse.SUCCESS\n try:\n source_bucket = event['ResourceProperties']['SourceBucket']\n dest_bucket = event['ResourceProperties']['DestBucket']\n key = event['ResourceProperties']['Key']\n if event['RequestType'] == 'Delete':\n delete_objects(dest_bucket, key)\n else:\n copy_objects(source_bucket, dest_bucket, key)\n except Exception as e:\n logging.error('Exception: %s' % e, exc_info=True)\n status = cfnresponse.FAILED\n finally:\n timer.cancel()\n cfnresponse.send(event, context, status, {}, None)" + } + } + }, + "LogScaleCloudWatchCopyZipLambdaLogGroup" : { + "DependsOn" : [ "LogScaleCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "LogScaleCloudWatchCopyZipLambda" } ] ] + }, + "RetentionInDays" : { "Ref" : "LogScaleLambdaLogRetention" } + } + }, + "LogScaleCloudWatchLogsIngester" : { + "DependsOn" : [ "LogScaleCloudWatchRole", "LogScaleCloudWatchCopyZipCustom" ], "Type" : "AWS::Lambda::Function", "Properties" : { "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } + "S3Bucket" : { "Ref" : "LogScaleCloudWatchLambdaZipBucket" }, + "S3Key" : { "Ref" : "S3KeyOfTheDeploymentPackageContainingLambdaCodeFiles" } }, "Environment" : { "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } + "logscale_protocol" : { "Ref" : "LogScaleProtocol" }, + "logscale_host" : { "Ref" : "LogScaleHost" }, + "logscale_ingest_token" : { "Ref" : "LogScaleIngestToken" }, + "log_level" : { "Ref" : "LogScaleLambdaLogLevel" } } }, "VpcConfig" : { @@ -202,182 +308,149 @@ } ] }, - "Description" : "CloudWatch Logs to Humio ingester.", + "Description" : "CloudWatch Logs to LogScale ingester.", "Handler" : "logs_ingester.lambda_handler", "MemorySize" : "128", "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + "Fn::GetAtt" : [ "LogScaleCloudWatchRole", "Arn" ] }, "Runtime" : "python3.8", "Timeout" : "300" } }, - "HumioCloudWatchLogsIngesterPermission" : { + "LogScaleCloudWatchLogsIngesterPermission" : { "Type" : "AWS::Lambda::Permission", "Properties" : { "Action" : "lambda:InvokeFunction", "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + "Fn::GetAtt" : [ "LogScaleCloudWatchLogsIngester", "Arn" ] }, "Principal" : "logs.amazonaws.com" } }, - "HumioCloudWatchLogsIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchLogsIngesterLogGroup" : { + "DependsOn" : [ "LogScaleCloudWatchRole" ], "Type" : "AWS::Logs::LogGroup", "Properties" : { "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsIngester" } ] ] + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "LogScaleCloudWatchLogsIngester" } ] ] }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + "RetentionInDays" : { "Ref" : "LogScaleLambdaLogRetention" } } }, - "HumioCloudWatchLogsSubscriber" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchLogsSubscriber" : { + "DependsOn" : [ "LogScaleCloudWatchRole", "LogScaleCloudWatchCopyZipCustom" ], "Type" : "AWS::Lambda::Function", "Properties" : { "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } + "S3Bucket" : { "Ref" : "LogScaleCloudWatchLambdaZipBucket" }, + "S3Key" : { "Ref" : "S3KeyOfTheDeploymentPackageContainingLambdaCodeFiles" } }, "Environment" : { "Variables" : { - "humio_log_ingester_arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + "logscale_log_ingester_arn" : { + "Fn::GetAtt" : [ "LogScaleCloudWatchLogsIngester", "Arn" ] }, - "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } + "logscale_subscription_prefix" : { "Ref" : "LogScaleCloudWatchLogsSubscriptionPrefix" }, + "log_level" : { "Ref" : "LogScaleLambdaLogLevel" } } }, - "Description" : "CloudWatch Logs to Humio log group subscriber.", + "Description" : "CloudWatch Logs to LogScale log group subscriber.", "Handler" : "logs_subscriber.lambda_handler", "MemorySize" : "128", "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + "Fn::GetAtt" : [ "LogScaleCloudWatchRole", "Arn" ] }, "Runtime" : "python3.8", "Timeout" : "300" } }, - "HumioCloudWatchLogsSubscriberPermission" : { + "LogScaleCloudWatchLogsSubscriberPermission" : { "Condition" : "CreateAutoSubscriptionResources", "Type" : "AWS::Lambda::Permission", "Properties" : { "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] - }, + "FunctionName" : { "Fn::GetAtt" : [ "LogScaleCloudWatchLogsSubscriber", "Arn" ] }, "Principal" : "events.amazonaws.com", - "SourceAccount" : { "Ref" : "AWS::AccountId" } + "SourceAccount" : { "Ref" : "AWS::AccountId" }, + "SourceArn" : { "Fn::GetAtt" : [ "LogScaleCloudWatchLogsSubscriberEventRule", "Arn" ] } } }, - "HumioCloudWatchLogsSubscriberPermission2" : { - "Condition" : "CreateAutoSubscriptionResources", - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] - }, - "Principal" : "events.amazonaws.com", - "SourceArn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberEventRule", "Arn" ] - } - } - }, - "HumioCloudWatchLogsSubscriberLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchLogsSubscriberLogGroup" : { + "DependsOn" : [ "LogScaleCloudWatchRole" ], "Type" : "AWS::Logs::LogGroup", "Properties" : { "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsSubscriber" } ] ] + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "LogScaleCloudWatchLogsSubscriber" } ] ] }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + "RetentionInDays" : { "Ref" : "LogScaleLambdaLogRetention" } } }, - "HumioCloudWatchLogsBackfiller" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchLogsBackfiller" : { + "DependsOn" : [ "LogScaleCloudWatchRole", "LogScaleCloudWatchCopyZipCustom" ], "Type" : "AWS::Lambda::Function", "Properties" : { "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } + "S3Bucket" : { "Ref" : "LogScaleCloudWatchLambdaZipBucket" }, + "S3Key" : { "Ref" : "S3KeyOfTheDeploymentPackageContainingLambdaCodeFiles" } }, "Environment" : { "Variables" : { - "humio_log_ingester_arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + "logscale_log_ingester_arn" : { + "Fn::GetAtt" : [ "LogScaleCloudWatchLogsIngester", "Arn" ] }, - "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } + "logscale_subscription_prefix" : { "Ref" : "LogScaleCloudWatchLogsSubscriptionPrefix" }, + "logscale_protocol" : { "Ref" : "LogScaleProtocol" }, + "logscale_host" : { "Ref" : "LogScaleHost" }, + "logscale_ingest_token" : { "Ref" : "LogScaleIngestToken" }, + "log_level" : { "Ref" : "LogScaleLambdaLogLevel" } } }, - "Description" : "CloudWatch Logs to Humio logs backfiller.", + "Description" : "CloudWatch Logs to LogScale logs backfiller.", "Handler" : "logs_backfiller.lambda_handler", "MemorySize" : "128", "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + "Fn::GetAtt" : [ "LogScaleCloudWatchRole", "Arn" ] }, "Runtime" : "python3.8", "Timeout" : "300" } }, - "HumioCloudWatchLogsBackfillerPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchLogsBackfillerLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchLogsBackfillerLogGroup" : { + "DependsOn" : [ "LogScaleCloudWatchRole" ], "Type" : "AWS::Logs::LogGroup", "Properties" : { "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsBackfiller" } ] ] + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "LogScaleCloudWatchLogsBackfiller" } ] ] }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + "RetentionInDays" : { "Ref" : "LogScaleLambdaLogRetention" } } }, - "HumioBackfillerAutoRunner" : { - "Condition" : "CreateHumioBackfillerAutoRunner", - "DependsOn" : [ "HumioCloudWatchLogsBackfiller" ], + "LogScaleBackfillerAutoRunner" : { + "Condition" : "CreateLogScaleBackfillerAutoRunner", + "DependsOn" : [ "LogScaleCloudWatchLogsBackfiller" ], "Type" : "Custom::BackfillerAutoRunner", "Properties" : { - "ServiceToken" : { "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] }, + "ServiceToken" : { "Fn::GetAtt" : [ "LogScaleCloudWatchLogsBackfiller", "Arn" ] }, "StackName" : { "Ref" : "AWS::StackName" } } }, - "HumioCloudWatchLogsSubscriberS3Bucket" : { - "Condition" : "CreateAutoSubscriptionResources", + "LogScaleCloudWatchLogsSubscriberS3Bucket" : { + "Condition" : "CreateCloudTrailAndS3BucketAndEnableAutoSubscription", "Type" : "AWS::S3::Bucket", "Properties" : { "AccessControl" : "BucketOwnerFullControl", "BucketName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "logscale", "cloudtrail" ] ] } } }, - "HumioCloudWatchLogsSubscriberS3BucketPolicy" : { - "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3Bucket" ], + "LogScaleCloudWatchLogsSubscriberS3BucketPolicy" : { + "Condition" : "CreateCloudTrailAndS3BucketAndEnableAutoSubscription", + "DependsOn" : [ "LogScaleCloudWatchLogsSubscriberS3Bucket" ], "Type" : "AWS::S3::BucketPolicy", "Properties" : { - "Bucket" : { "Ref" : "HumioCloudWatchLogsSubscriberS3Bucket" }, + "Bucket" : { "Ref" : "LogScaleCloudWatchLogsSubscriberS3Bucket" }, "PolicyDocument" : { "Version" : "2012-10-17", "Statement" : [ @@ -389,7 +462,7 @@ }, "Action" : "s3:GetBucketAcl", "Resource" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] + "Fn::GetAtt" : [ "LogScaleCloudWatchLogsSubscriberS3Bucket", "Arn" ] } }, { @@ -400,7 +473,7 @@ }, "Action" : "s3:PutObject", "Resource" : { - "Fn::Join" : [ "", [ { "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] }, "/AWSLogs/", { "Ref" : "AWS::AccountId" }, "/*" ] ] + "Fn::Join" : [ "", [ { "Fn::GetAtt" : [ "LogScaleCloudWatchLogsSubscriberS3Bucket", "Arn" ] }, "/AWSLogs/", { "Ref" : "AWS::AccountId" }, "/*" ] ] }, "Condition" : { "StringEquals" : { "s3:x-amz-acl" : "bucket-owner-full-control" } @@ -410,9 +483,9 @@ } } }, - "HumioCloudWatchLogsSubscriberCloudTrail" : { - "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3BucketPolicy" ], + "LogScaleCloudWatchLogsSubscriberCloudTrail" : { + "Condition" : "CreateCloudTrailAndS3BucketAndEnableAutoSubscription", + "DependsOn" : [ "LogScaleCloudWatchLogsSubscriberS3BucketPolicy" ], "Type" : "AWS::CloudTrail::Trail", "Properties" : { "EnableLogFileValidation" : false, @@ -420,19 +493,19 @@ "IsMultiRegionTrail" : true, "IsLogging" : true, "S3BucketName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "logscale", "cloudtrail" ] ] }, "TrailName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", { "Ref" : "AWS::AccountId" } ] ] + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "logscale", { "Ref" : "AWS::AccountId" } ] ] } } }, - "HumioCloudWatchLogsSubscriberEventRule" : { + "LogScaleCloudWatchLogsSubscriberEventRule" : { "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriber" ], + "DependsOn" : [ "LogScaleCloudWatchLogsSubscriber" ], "Type" : "AWS::Events::Rule", "Properties" : { - "Description" : "Humio log group auto subscription event rule.", + "Description" : "LogScale log group auto subscription event rule.", "EventPattern" : { "source" : [ "aws.logs" ], "detail-type" : [ "AWS API Call via CloudTrail" ], @@ -442,38 +515,35 @@ } }, "Name" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "logscale-auto-subscription-rule" ] ] }, "Targets" : [ { "Id" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "logscale-auto-subscription-rule" ] ] }, "Arn" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] + "Fn::GetAtt" : [ "LogScaleCloudWatchLogsSubscriber", "Arn" ] } } ] } }, - "HumioCloudWatchMetricIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchMetricIngester" : { + "Condition" : "CreateMetricAndMetricStatisticsLambdas", + "DependsOn" : [ "LogScaleCloudWatchRole", "LogScaleCloudWatchCopyZipCustom" ], "Type" : "AWS::Lambda::Function", "Properties" : { "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } + "S3Bucket" : { "Ref" : "LogScaleCloudWatchLambdaZipBucket" }, + "S3Key" : { "Ref" : "S3KeyOfTheDeploymentPackageContainingLambdaCodeFiles" } }, "Environment" : { "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } + "logscale_protocol" : { "Ref" : "LogScaleProtocol" }, + "logscale_host" : { "Ref" : "LogScaleHost" }, + "logscale_ingest_token" : { "Ref" : "LogScaleIngestToken" }, + "log_level" : { "Ref" : "LogScaleLambdaLogLevel" } } }, "VpcConfig" : { @@ -487,54 +557,42 @@ } ] }, - "Description" : "CloudWatch Metrics to Humio ingester.", + "Description" : "CloudWatch Metrics to LogScale ingester.", "Handler" : "metric_ingester.lambda_handler", "MemorySize" : "128", "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + "Fn::GetAtt" : [ "LogScaleCloudWatchRole", "Arn" ] }, "Runtime" : "python3.8", "Timeout" : "300" } }, - "HumioCloudWatchMetricIngesterPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchMetricIngester", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchMetricIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchMetricIngesterLogGroup" : { + "Condition" : "CreateMetricAndMetricStatisticsLambdas", + "DependsOn" : [ "LogScaleCloudWatchRole" ], "Type" : "AWS::Logs::LogGroup", "Properties" : { "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricIngester" } ] ] + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "LogScaleCloudWatchMetricIngester" } ] ] }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + "RetentionInDays" : { "Ref" : "LogScaleLambdaLogRetention" } } }, - "HumioCloudWatchMetricStatisticsIngester" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchMetricStatisticsIngester" : { + "Condition" : "CreateMetricAndMetricStatisticsLambdas", + "DependsOn" : [ "LogScaleCloudWatchRole", "LogScaleCloudWatchCopyZipCustom" ], "Type" : "AWS::Lambda::Function", "Properties" : { "Code" : { - "S3Bucket" : { - "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] - }, - "S3Key" : { - "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] - } + "S3Bucket" : { "Ref" : "LogScaleCloudWatchLambdaZipBucket" }, + "S3Key" : { "Ref" : "S3KeyOfTheDeploymentPackageContainingLambdaCodeFiles" } }, "Environment" : { "Variables" : { - "humio_protocol" : { "Ref" : "HumioProtocol" }, - "humio_host" : { "Ref" : "HumioHost" }, - "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, - "log_level" : { "Ref" : "HumioLambdaLogLevel" } + "logscale_protocol" : { "Ref" : "LogScaleProtocol" }, + "logscale_host" : { "Ref" : "LogScaleHost" }, + "logscale_ingest_token" : { "Ref" : "LogScaleIngestToken" }, + "log_level" : { "Ref" : "LogScaleLambdaLogLevel" } } }, "VpcConfig" : { @@ -548,34 +606,25 @@ } ] }, - "Description" : "CloudWatch Metrics Statistics to Humio ingester.", + "Description" : "CloudWatch Metrics Statistics to LogScale ingester.", "Handler" : "metric_statistics_ingester.lambda_handler", "MemorySize" : "128", "Role" : { - "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + "Fn::GetAtt" : [ "LogScaleCloudWatchRole", "Arn" ] }, "Runtime" : "python3.8", "Timeout" : "300" } }, - "HumioCloudWatchMetricStatisticsIngesterPermission" : { - "Type" : "AWS::Lambda::Permission", - "Properties" : { - "Action" : "lambda:InvokeFunction", - "FunctionName" : { - "Fn::GetAtt" : [ "HumioCloudWatchMetricStatisticsIngester", "Arn" ] - }, - "Principal" : "logs.amazonaws.com" - } - }, - "HumioCloudWatchMetricStatisticsIngesterLogGroup" : { - "DependsOn" : [ "HumioCloudWatchRole" ], + "LogScaleCloudWatchMetricStatisticsIngesterLogGroup" : { + "Condition" : "CreateMetricAndMetricStatisticsLambdas", + "DependsOn" : [ "LogScaleCloudWatchRole" ], "Type" : "AWS::Logs::LogGroup", "Properties" : { "LogGroupName" : { - "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricStatisticsIngester" } ] ] + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "LogScaleCloudWatchMetricStatisticsIngester" } ] ] }, - "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + "RetentionInDays" : { "Ref" : "LogScaleLambdaLogRetention" } } } } diff --git a/deploy-using-profile.sh b/deploy-using-profile.sh index c0c0c69..b3f9cf3 100755 --- a/deploy-using-profile.sh +++ b/deploy-using-profile.sh @@ -3,18 +3,5 @@ set -e cat cloudformation.json | jq make build -aws s3 cp --acl public-read cloudformation.json s3://humio-public-us-east-1/ --region us-east-1 --profile cloudwatch -aws s3 cp --acl public-read cloudformation-no-trail.json s3://humio-public-us-east-1/ --region us-east-1 --profile cloudwatch - -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-central-1/ --region eu-central-1 --profile cloudwatch -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-west-1/ --region eu-west-1 --profile cloudwatch -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-west-2/ --region eu-west-2 --profile cloudwatch -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-west-3/ --region eu-west-3 --profile cloudwatch -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-north-1/ --region eu-north-1 --profile cloudwatch - -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-us-east-1/ --region us-east-1 --profile cloudwatch -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-us-east-2/ --region us-east-2 --profile cloudwatch -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-us-west-2/ --region us-west-2 --profile cloudwatch - -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-ap-southeast-1/ --region ap-southeast-1 --profile cloudwatch -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-ap-southeast-2/ --region ap-southeast-2 --profile cloudwatch \ No newline at end of file +aws s3 cp --acl public-read cloudformation.json s3://logscale-public-us-east-1/ --region us-east-1 --profile cloudwatch +aws s3 cp --acl public-read target/v2.0.0_cloudwatch2logscale.zip s3://logscale-public-us-east-1/ --region us-east-1 --profile cloudwatch diff --git a/deploy.sh b/deploy.sh index 2ba5af1..9608474 100755 --- a/deploy.sh +++ b/deploy.sh @@ -3,18 +3,5 @@ set -e cat cloudformation.json | jq make build -aws s3 cp --acl public-read cloudformation.json s3://humio-public-us-east-1/ --region us-east-1 -aws s3 cp --acl public-read cloudformation-no-trail.json s3://humio-public-us-east-1/ --region us-east-1 - -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-central-1/ --region eu-central-1 -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-west-1/ --region eu-west-1 -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-west-2/ --region eu-west-2 -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-west-3/ --region eu-west-3 -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-eu-north-1/ --region eu-north-1 - -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-us-east-1/ --region us-east-1 -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-us-east-2/ --region us-east-2 -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-us-west-2/ --region us-west-2 - -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-ap-southeast-1/ --region ap-southeast-1 -aws s3 cp --acl public-read target/v1.2.2_cloudwatch2humio.zip s3://humio-public-ap-southeast-2/ --region ap-southeast-2 \ No newline at end of file +aws s3 cp --acl public-read cloudformation.json s3://logscale-public-us-east-1/ --region us-east-1 +aws s3 cp --acl public-read target/v2.0.0_cloudwatch2logscale.zip s3://logscale-public-us-east-1/ --region us-east-1 \ No newline at end of file diff --git a/src/helpers.py b/src/helpers.py index 1a32916..572bbfa 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -21,29 +21,29 @@ def setup(): This can be called every invocation but we will only run it once per Lambda instance. """ - global humio_host - global humio_protocol - global humio_ingest_token + global logscale_host + global logscale_protocol + global logscale_ingest_token global http_session global _is_setup if _is_setup: return - humio_host = os.environ["humio_host"] - humio_protocol = os.environ["humio_protocol"] - humio_ingest_token = os.environ["humio_ingest_token"] + logscale_host = os.environ["logscale_host"] + logscale_protocol = os.environ["logscale_protocol"] + logscale_ingest_token = os.environ["logscale_ingest_token"] http_session = requests.Session() _is_setup = True -def ingest_events(humio_events, host_type): +def ingest_events(logscale_events, host_type): """ - Wrap and send CloudWatch Logs/Metrics to Humio repository. + Wrap and send CloudWatch Logs/Metrics to LogScale repository. - :param humio_events: Structured events to be ingested into Humio. - :type humio_events: list + :param logscale_events: Structured events to be ingested into LogScale. + :type logscale_events: list :param host_type: Type of host from which the events are being sent. :type host_type: str @@ -51,29 +51,29 @@ def ingest_events(humio_events, host_type): :return: Response object from request. :rtype: obj """ - humio_url = "%s://%s/api/v1/ingest/humio-structured" % (humio_protocol, humio_host) - humio_headers = { + logscale_url = "%s://%s/api/v1/ingest/humio-structured" % (logscale_protocol, logscale_host) + logscale_headers = { "Content-Type": "application/json", - "Authorization": "Bearer %s" % humio_ingest_token + "Authorization": "Bearer %s" % logscale_ingest_token } - # Prepare events to be sent to Humio. - wrapped_data = [{"tags": {"host": host_type}, "events": humio_events}] + # Prepare events to be sent to LogScale. + wrapped_data = [{"tags": {"host": host_type}, "events": logscale_events}] - logger.debug("Data being sent to Humio: %s" % wrapped_data) + logger.debug("Data being sent to LogScale: %s" % wrapped_data) # Make request. response = http_session.post( - humio_url, + logscale_url, data=json.dumps(wrapped_data), - headers=humio_headers + headers=logscale_headers ) try: response.raise_for_status() except requests.exceptions.RequestException as e: - logger.error("Got error %s from Humio." % response.text, exc_info= e) + logger.error("Got error %s from LogScale." % response.text, exc_info= e) else: - logger.debug("Got response %s from Humio." % response.text) + logger.debug("Got response %s from LogScale." % response.text) return response @@ -94,7 +94,7 @@ def decode_event(event): return decoded_event -def create_subscription(log_client, log_group_name, humio_log_ingester_arn, context): +def create_subscription(log_client, log_group_name, logscale_log_ingester_arn, context): """ Create subscription to CloudWatch Logs specified log group. @@ -104,8 +104,8 @@ def create_subscription(log_client, log_group_name, humio_log_ingester_arn, cont :param log_group_name: Name of the log group. :type log_group_name: str - :param humio_log_ingester_arn: Name of the logs ingester resource. - :type humio_log_ingester_arn: str + :param logscale_log_ingester_arn: Name of the logs ingester resource. + :type logscale_log_ingester_arn: str :param context: Lambda context object. :type context: obj @@ -115,17 +115,20 @@ def create_subscription(log_client, log_group_name, humio_log_ingester_arn, cont # We cannot subscribe to the log group that our stdout/err goes to. if context.log_group_name == log_group_name: logger.debug("Skipping our own log group name...") - # And we do not want to subscribe to other Humio log ingesters - if there are any. + # And we do not want to subscribe to other LogScale log ingesters - if there are any. if "HumioCloudWatchLogsIngester" in log_group_name: - logger.debug("Skipping cloudwatch2humio ingesters...") + logger.debug("Skipping cloudwatch2logscale ingesters...") + # And we do not want to subscribe to other LogScale log ingesters - if there are any. + if "LogScaleCloudWatchLogsIngester" in log_group_name: + logger.debug("Skipping cloudwatch2logscale ingesters...") else: logger.info("Creating subscription for %s" % log_group_name) try: log_client.put_subscription_filter( logGroupName=log_group_name, - filterName="%s-humio_ingester" % log_group_name, + filterName="%s-logscale_ingester" % log_group_name, filterPattern="", # Matching everything. - destinationArn=humio_log_ingester_arn, + destinationArn=logscale_log_ingester_arn, distribution="ByLogStream" ) logger.debug("Successfully subscribed to %s!" % log_group_name) diff --git a/src/logs_backfiller.py b/src/logs_backfiller.py index 3ec4514..90fa8eb 100644 --- a/src/logs_backfiller.py +++ b/src/logs_backfiller.py @@ -28,8 +28,8 @@ def lambda_handler(event, context): send_custom_resource_response(event, context) # Set environment variables. - humio_log_ingester_arn = os.environ["humio_log_ingester_arn"] - humio_subscription_prefix = os.environ.get("humio_subscription_prefix") + logscale_log_ingester_arn = os.environ["logscale_log_ingester_arn"] + logscale_subscription_prefix = os.environ.get("logscale_subscription_prefix") # Set up CloudWatch Logs client. log_client = boto3.client("logs") @@ -37,9 +37,9 @@ def lambda_handler(event, context): # Grab all log groups with a token and/or prefix if we have them. if "nextToken" in event.keys(): next_token = event["nextToken"] - if humio_subscription_prefix: + if logscale_subscription_prefix: log_groups = log_client.describe_log_groups( - logGroupNamePrefix=humio_subscription_prefix, + logGroupNamePrefix=logscale_subscription_prefix, nextToken=next_token ) else: @@ -47,9 +47,9 @@ def lambda_handler(event, context): nextToken=next_token ) else: - if humio_subscription_prefix: + if logscale_subscription_prefix: log_groups = log_client.describe_log_groups( - logGroupNamePrefix=humio_subscription_prefix, + logGroupNamePrefix=logscale_subscription_prefix, ) else: log_groups = log_client.describe_log_groups() @@ -74,7 +74,8 @@ def lambda_handler(event, context): # First we check to see if there are any filters at all. if all_subscription_filters["subscriptionFilters"]: # If our function is not subscribed, delete subscription and create ours. - if all_subscription_filters["subscriptionFilters"][0]["destinationArn"] != humio_log_ingester_arn: + if all_subscription_filters["subscriptionFilters"][0]["destinationArn"] != logscale_log_ingester_arn: + # TODO: Do not delete subscriptions! It is better that it fails in this case instead of potentially ruining infrastructure. helpers.delete_subscription( log_client, log_group["logGroupName"], @@ -83,7 +84,7 @@ def lambda_handler(event, context): helpers.create_subscription( log_client, log_group["logGroupName"], - humio_log_ingester_arn, + logscale_log_ingester_arn, context ) # We are now subscribed. @@ -94,19 +95,19 @@ def lambda_handler(event, context): helpers.create_subscription( log_client, log_group["logGroupName"], - humio_log_ingester_arn, context + logscale_log_ingester_arn, context ) def send_custom_resource_response(event, context): if "LogicalResourceId" in event.keys(): - if event["LogicalResourceId"] == "HumioBackfillerAutoRunner": + if event["LogicalResourceId"] == "LogScaleBackfillerAutoRunner": response_content = { "Status" : "SUCCESS", "RequestId" : event["RequestId"], "LogicalResourceId" : event["LogicalResourceId"], "StackId" : event["StackId"], - "PhysicalResourceId" : event["ResourceProperties"]["StackName"] + "-HumioBackfillerAutoRunner" + "PhysicalResourceId" : event["ResourceProperties"]["StackName"] + "-LogScaleBackfillerAutoRunner" } response = requests.put( event["ResponseURL"], diff --git a/src/logs_ingester.py b/src/logs_ingester.py index f1399ab..d973959 100644 --- a/src/logs_ingester.py +++ b/src/logs_ingester.py @@ -13,7 +13,7 @@ def lambda_handler(event, context): """ Extract log data from CloudWatch Logs events and - pass the data onto the Humio ingester. + pass the data onto the LogScale ingester. :param event: Event data from CloudWatch Logs. :type event: dict @@ -52,7 +52,7 @@ def lambda_handler(event, context): ) # Flatten the events from CloudWatch Logs. - humio_events = [] + logscale_events = [] for log_event in decoded_event["logEvents"]: message = log_event["message"] @@ -62,7 +62,7 @@ def lambda_handler(event, context): attributes.update(helpers.parse_message(message)) # Append the flattened event - humio_events.append({ + logscale_events.append({ "timestamp": log_event["timestamp"], "rawstring": message, "kvparse": True, @@ -70,4 +70,4 @@ def lambda_handler(event, context): }) # Make request to Humio. - helpers.ingest_events(humio_events, 'cloudwatch_logs') + helpers.ingest_events(logscale_events, 'cloudwatch_logs') diff --git a/src/logs_subscriber.py b/src/logs_subscriber.py index 2a722f1..70423d6 100644 --- a/src/logs_subscriber.py +++ b/src/logs_subscriber.py @@ -3,8 +3,8 @@ import helpers # Set environment variables. -humio_log_ingester_arn = os.environ["humio_log_ingester_arn"] -humio_subscription_prefix = os.environ.get("humio_subscription_prefix") +logscale_log_ingester_arn = os.environ["logscale_log_ingester_arn"] +logscale_subscription_prefix = os.environ.get("logscale_subscription_prefix") # Set up CloudWatch Logs client. log_client = boto3.client("logs") @@ -26,14 +26,14 @@ def lambda_handler(event, context): log_group_name = event["detail"]["requestParameters"]["logGroupName"] # Check whether the prefix is set - the prefix is used to determine which logs we want. - if not humio_subscription_prefix: + if not logscale_subscription_prefix: helpers.create_subscription( - log_client, log_group_name, humio_log_ingester_arn, context + log_client, log_group_name, logscale_log_ingester_arn, context ) else: # Check whether the log group's name starts with the set prefix. - if log_group_name.startswith(humio_subscription_prefix): + if log_group_name.startswith(logscale_subscription_prefix): helpers.create_subscription( - log_client, log_group_name, humio_log_ingester_arn, context + log_client, log_group_name, logscale_log_ingester_arn, context ) diff --git a/src/metric_ingester.py b/src/metric_ingester.py index 1fec0e8..837f049 100644 --- a/src/metric_ingester.py +++ b/src/metric_ingester.py @@ -13,7 +13,7 @@ def lambda_handler(event, context): """ - Ingest CloudWatch Metrics data to Humio repository. + Ingest CloudWatch Metrics data to LogScale repository. :param event: Event data. :type event: dict @@ -65,11 +65,11 @@ def lambda_handler(event, context): Payload=json.dumps(event) ) - # Format metric data to Humio event data. - humio_events = create_humio_events(metric_data, configurations) + # Format metric data to LogsScale event data. + logscale_events = create_logscale_events(metric_data, configurations) - # Send Humio event data to Humio. - helpers.ingest_events(humio_events, "cloudwatch_metrics") + # Send LogScale event data to LogScale. + helpers.ingest_events(logscale_events, "cloudwatch_metrics") def get_metric_data(configurations): @@ -92,9 +92,9 @@ def get_metric_data(configurations): return metric_data -def create_humio_events(metrics, configurations): +def create_logscale_events(metrics, configurations): """ - Create list of Humio events based on metrics. + Create list of LogScale events based on metrics. :param metrics: Metrics received from GetMetricData. :type metrics: dict @@ -102,12 +102,12 @@ def create_humio_events(metrics, configurations): :param configurations: User defined API request parameters for the boto client. :type configurations: dict - :return: Events to be sent to Humio. + :return: Events to be sent to LogScale. :rtype: list """ - humio_events = [] + logscale_events = [] - # Create Humio event based on each extracted timestamp. + # Create LogScale event based on each extracted timestamp. for result in metrics["MetricDataResults"]: count = 0 for timestamp in result["Timestamps"]: @@ -126,7 +126,7 @@ def create_humio_events(metrics, configurations): "requestParameters": configurations } } - humio_events.append(event) + logscale_events.append(event) count += 1 - return humio_events + return logscale_events diff --git a/src/metric_statistics_ingester.py b/src/metric_statistics_ingester.py index be4e73e..194ad82 100644 --- a/src/metric_statistics_ingester.py +++ b/src/metric_statistics_ingester.py @@ -13,7 +13,7 @@ def lambda_handler(event, context): """ - Ingest CloudWatch Metric statistics to Humio repository. + Ingest CloudWatch Metric statistics to LogScale repository. :param event: Event data. :type event: dict @@ -34,11 +34,11 @@ def lambda_handler(event, context): # Used for debugging. logger.debug("Statistics from CloudWatch Metrics: %s" % metric_statistics) - # Format metric data to Humio event data. - humio_events = create_humio_events(metric_statistics, api_parameters) + # Format metric data to LogScale event data. + logscale_events = create_logscale_events(metric_statistics, api_parameters) - # Send Humio event data to Humio. - helpers.ingest_events(humio_events, "cloudwatch_metrics") + # Send LogScale event data to LogScale. + helpers.ingest_events(logscale_events, "cloudwatch_metrics") def get_metric_statistics(configurations): @@ -77,9 +77,9 @@ def get_metric_statistics(configurations): return metric_statistics, api_parameters -def create_humio_events(metrics, api_parameters): +def create_logscale_events(metrics, api_parameters): """ - Create list of Humio events based on metrics. + Create list of LogScale events based on metrics. :param metrics: Metrics received from GetMetricStatistics. :type metrics: dict @@ -88,15 +88,15 @@ def create_humio_events(metrics, api_parameters): to retrieve the metric statistics. :type lambda_event: dict - :return: List of events to be sent to Humio. + :return: List of events to be sent to LogScale. :rtype: list """ - humio_events = [] + logscale_events = [] # Used for debuggin. logger.debug("Datapoints: %s" % metrics["Datapoints"]) - # Create one Humio event per datapoint/timestamp. + # Create one LogScale event per datapoint/timestamp. for datapoint in metrics["Datapoints"]: # Create event data. event = { @@ -117,6 +117,6 @@ def create_humio_events(metrics, api_parameters): "requestParameters ": api_parameters } } - humio_events.append(event) + logscale_events.append(event) - return humio_events + return logscale_events