Feature crawler

cid/builtin/core/data/resources.yaml

@@ -267,47 +267,186 @@ views:
     riFile: cid/summary_view_ri.sql
     File: cid/summary_view.sql
     dependsOn:
-      cur: true
+      cur:
+        - bill_billing_entity
+        - bill_billing_period_start_date
+        - bill_invoice_id
+        - bill_payer_account_id
+        - line_item_availability_zone
+        - line_item_legal_entity
+        - line_item_line_item_description
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_resource_id
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - pricing_public_on_demand_cost
+        - pricing_unit
+        - product_current_generation
+        - product_database_engine
+        - product_from_location
+        - product_group
+        - product_operating_system
+        - product_physical_processor
+        - product_processor_features
+        - product_product_family
+        - product_product_name
+        - product_region
+        - product_servicecode
+        - product_tenancy
+        - product_to_location
+        - reservation_effective_cost
+        - reservation_reservation_a_r_n
+        - reservation_unused_amortized_upfront_fee_for_billing_period
+        - reservation_unused_recurring_fee
+        - savings_plan_amortized_upfront_commitment_for_billing_period
+        - savings_plan_savings_plan_a_r_n
+        - savings_plan_savings_plan_effective_cost
+        - savings_plan_total_commitment_to_date
+        - savings_plan_used_commitment
 
   ec2_running_cost:
     spriFile: cid/ec2_running_cost_sp_ri.sql
     spFile: cid/ec2_running_cost_sp.sql
     riFile: cid/ec2_running_cost_ri.sql
     File: cid/ec2_running_cost.sql
     dependsOn:
-      cur: true
+      cur:
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_type
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - reservation_effective_cost
+        - reservation_reservation_a_r_n
+        - savings_plan_savings_plan_a_r_n
+        - savings_plan_savings_plan_effective_cost
 
   compute_savings_plan_eligible_spend:
     File: cid/compute_savings_plan_eligible_spend.sql
     dependsOn:
       cur: true
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - product_servicecode
 
   s3_view:
     File: cid/s3.sql
     dependsOn:
-      cur: true
+      cur:
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_resource_id
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - pricing_public_on_demand_cost
+        - pricing_unit
+        - product_region
 
   ri_sp_mapping:
     spriFile: cid/ri_sp_mapping_sp_ri.sql
     spFile: cid/ri_sp_mapping_sp.sql
     riFile: cid/ri_sp_mapping_ri.sql
     File: cid/ri_sp_mapping.sql
     dependsOn:
-      cur: true
+      cur:
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_type
+        - pricing_lease_contract_length
+        - pricing_offering_class
+        - pricing_purchase_option
+        - reservation_reservation_a_r_n
+        - savings_plan_end_time
+        - savings_plan_savings_plan_a_r_n
+
   hourly_view:
     spriFile: cudos/hourly_view_sp_ri.sql
     spFile: cudos/hourly_view_sp.sql
     riFile: cudos/hourly_view_ri.sql
     File: cudos/hourly_view.sql
     dependsOn:
-      cur: true 
+      cur:
+        - bill_billing_period_start_date
+        - bill_payer_account_id
+        - line_item_line_item_description
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - pricing_term
+        - pricing_unit
+        - product_region
+        - product_servicecode
+        - reservation_effective_cost
+        - reservation_reservation_a_r_n
+        - savings_plan_savings_plan_a_r_n
+        - savings_plan_savings_plan_effective_cost
+
   resource_view:
     spriFile: cudos/resource_view_sp_ri.sql
     spFile: cudos/resource_view_sp.sql
     riFile: cudos/resource_view_ri.sql
     File: cudos/resource_view.sql
     dependsOn:
-      cur: true 
+      cur:
+        - bill_billing_entity
+        - bill_payer_account_id
+        - line_item_legal_entity
+        - line_item_line_item_description
+        - line_item_line_item_type
+        - line_item_operation
+        - line_item_product_code
+        - line_item_resource_id
+        - line_item_unblended_cost
+        - line_item_usage_account_id
+        - line_item_usage_amount
+        - line_item_usage_start_date
+        - line_item_usage_type
+        - pricing_term
+        - pricing_unit
+        - product_database_engine
+        - product_deployment_option
+        - product_from_location
+        - product_group
+        - product_instance_type
+        - product_instance_type_family
+        - product_operating_system
+        - product_product_family
+        - product_product_name
+        - product_region
+        - product_servicecode
+        - product_storage
+        - product_to_location
+        - product_volume_api_name
+        - reservation_effective_cost
+        - reservation_reservation_a_r_n
+        - savings_plan_savings_plan_a_r_n
+        - savings_plan_savings_plan_effective_cost
+
   # Trends
   daily_anomaly_detection:
     File: trends/daily_anomaly_detection.sql
@@ -484,6 +623,19 @@ views:
         - aws_accounts
   ta_descriptions:
     File: shared/ta_descriptions.sql
+  cur:
+    type: Glue_Table
+    File: shared/CUR.yaml
+    crawler: cur
+    parameters:
+      partitions:
+        type: cur.partitions
+        description: 'comma separated list of CUR partitions ex: ["source_account_id","cur_name_1","cur_name_2","year","month"] ex2: ["year","month"]'
+        default: '[{"Name":"source_account_id","Type":"string"},{"Name":"cur_name_1","Type":"string"},{"Name":"cur_name_2","Type":"string"},{"Name":"year","Type":"string"},{"Name":"month","Type":"string"}]'
+      location:
+        type: cur.location
+        description: 's3 path'
+        default: 's3://cid-{account_id}-shared/cur/'
 
 # Refresh Schedules for QuickSight DataSets
 schedules:
@@ -493,3 +645,45 @@ schedules:
       Interval: DAILY
       TimeOfTheDay: '02:00-05:00'
     RefreshType: FULL_REFRESH
+
+crawlers:
+  cur:
+    data:
+      Name:  'CidCurCrawler'
+      Description: A recurring crawler that keeps your CUR table in Athena up-to-date.
+      Role:  ${crawler_role_arn}
+      DatabaseName: "${athena_database_name}"
+      Targets:
+        S3Targets:
+          - Path: ${location}
+            Exclusions:
+              - '**.json'
+              - '**.yml'
+              - '**.sql'
+              - '**.csv'
+              - '**.csv.metadata'
+              - '**.gz'
+              - '**.zip'
+              - '**/cost_and_usage_data_status/*'
+              - 'aws-programmatic-access-test-object'
+      SchemaChangePolicy:
+        DeleteBehavior: LOG
+      RecrawlPolicy:
+        RecrawlBehavior: CRAWL_EVERYTHING
+      Schedule: cron(0 2 * * ? *)
+      Configuration: |
+        {
+          "Version":1.0,
+          "Grouping": {
+            "TableGroupingPolicy": "CombineCompatibleSchemas"
+          },
+          "CrawlerOutput":{
+            "Tables":{
+              "AddOrUpdateBehavior":"MergeNewColumns"
+            }
+          }
+        }
+    parameters:
+      s3path:
+        default: 's3://cid-{account_id}-cur/cur/{account_id}/cid/cid'
+        description: Compute Optimiser EBS report S3 path

cid/cli.py

@@ -245,5 +245,19 @@ def initqs(ctx, **kwargs):
 
     ctx.obj.initqs(**kwargs)
 
+@click.option('-v', '--verbose', count=True)
+@cid_command
+def init_cur(ctx, **kwargs):
+    """Initialize CUR
+
+    \b
+
+     --enable-quicksight-enterprise (yes|no) Confirm the activation of QuickSight
+     --account-name NAME                     Unique QuickSight account name (Unique across all AWS users) 
+     --notification-email EMAIL              User's email for QuickSight notifications
+    """
+
+    ctx.obj.init_cur(**kwargs)
+
 if __name__ == '__main__':
     main()