improve data collection database management (#904)

cfn-templates/cid-cfn.yml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: Deployment of Cloud Intelligence Dashboards v0.3.9
+Description: Deployment of Cloud Intelligence Dashboards v0.3.10
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -1611,7 +1611,7 @@ Resources:
       Description: An AWS managed layer with a cid-cmd package installed
       Content:
         S3Bucket: !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}'
-        S3Key: 'cid-resource-lambda-layer/cid-0.3.9.zip' #replace version here if needed
+        S3Key: 'cid-resource-lambda-layer/cid-0.3.10.zip' #replace version here if needed
       CompatibleRuntimes:
         - python3.10
         - python3.11

cid/_version.py

@@ -1 +1 @@
-__version__ = '0.3.9'
+__version__ = '0.3.10'

cid/common.py

@@ -348,16 +348,29 @@ def get_template_parameters(self, parameters: dict, param_prefix: str='', others
                     choices=self.cur.tag_and_cost_category_fields + ["'none'"],
                 )
             elif isinstance(value, dict) and value.get('type') == 'athena':
-                if 'query' not in value:
-                    raise CidCritical(f'Failed fetching parameter {prefix}{key}: paramter with type ahena must have query value.')
-                query = value['query']
-                try:
-                    res = self.athena.query(query)[0]
-                except (self.athena.client.exceptions.ClientError, CidError, CidCritical) as exc:
-                    raise CidCritical(f'Failed fetching parameter {prefix}{key}: {exc}') from exc
-                if not res:
-                    raise CidCritical(f'Failed fetching parameter {prefix}{key}, {value}. Athena returns empty result')
-                params[key] = res[0]
+                if get_parameters().get(prefix + key): # priority to user input
+                    params[key] = get_parameters().get(prefix + key)
+                else:
+                    if 'query' not in value:
+                        raise CidCritical(f'Failed fetching parameter {prefix}{key}: parameter with type Athena must have query value.')
+                    query = value['query']
+                    try:
+                        res_list = self.athena.query(query)
+                    except (self.athena.client.exceptions.ClientError, CidError, CidCritical) as exc:
+                        raise CidCritical(f'Failed fetching parameter {prefix}{key}: {exc}') from exc
+                    if not res_list:
+                        raise CidCritical(f'Failed fetching parameter {prefix}{key}, {value}. Athena returns empty results')
+                    elif len(res_list) == 1:
+                        params[key] = '-'.join(res_list[0])
+                    else:
+                        options = ['-'.join(res) for res in res_list]
+                        default = value.get('default')
+                        params[key] = get_parameter(
+                            param_name=prefix + key,
+                            message=f"Required parameter: {key} ({value.get('description')})",
+                            choices=options,
+                            default=default if default in options else None,
+                        )
             elif isinstance(value, dict):
                 params[key] = value.get('value')
                 while params[key] is None:

dashboards/aws-feeds/aws-feeds.yaml

@@ -2939,6 +2939,7 @@ datasets:
       - aws_feeds
     schedules:
     - default
+
   aws_feeds_blog_post:
     data:
       DataSetId: 173bf994-4763-4044-a96d-1cb6cce591f4
@@ -2998,9 +2999,12 @@ datasets:
     - default
     parameters:
       data_collection_database_name:
+        type: athena
+        query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
         default: optimization_data
         description: "Enter the name of the data collection database"
         global: True
+
   aws_feeds_security_bulletin:
     data:
       DataSetId: cd5b5e17-fdde-4563-a944-1c5fc3b27684
@@ -3054,9 +3058,12 @@ datasets:
     - default
     parameters:
       data_collection_database_name:
+        type: athena
+        query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
         default: optimization_data
         description: "Enter the name of the data collection database"
         global: True
+
   aws_feeds_whats_new:
     data:
       DataSetId: 00e4f86c-96b8-49b7-b68f-8b540e4fa6b7
@@ -3116,9 +3123,12 @@ datasets:
     - default
     parameters:
       data_collection_database_name:
+        type: athena
+        query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
         default: optimization_data
         description: "Enter the name of the data collection database"
         global: True
+
   aws_feeds_youtube:
     data:
       DataSetId: 5258b9ee-48d0-48f3-9ec4-baa4acd1d006
@@ -3236,6 +3246,8 @@ views:
         "${data_collection_database_name}"."aws_feeds_youtube"
     parameters:
       data_collection_database_name:
+        type: athena
+        query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
         default: optimization_data
         description: "Enter the name of the data collection database"
         global: True

dashboards/aws-marketplace/aws-marketplace-spg.yaml

@@ -165,6 +165,6 @@ views:
       , "sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" ELSE 0 END)) "usage_quantity"
       , "sum"("line_item_unblended_cost") "unblended_cost"
       FROM
-        ${cur_table_name}
+        "${cur_table_name}"
       WHERE ((("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '36' MONTH)) AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '36' MONTH))) AND ("bill_billing_entity" = 'AWS Marketplace'))
       GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21