Fixes for AWS key checker (#646)

aws-exposed-key-checker-infra/lambda_source/exposed_key_checker/exposed_keys.py

@@ -1,22 +1,37 @@
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timedelta
 import re
 from exposed_key_checker.ticket_manager import TicketData
 
+MAX_PROCESS_AGE_DAYS = 7
+
+
+def get_recent_open_tickets(tickets):
+    return [
+        ticket
+        for ticket in tickets
+        if (datetime.now() - ticket.created_dt) <= timedelta(days=MAX_PROCESS_AGE_DAYS)
+        and ticket.status not in ["solved", "closed"]
+    ]
+
 
 def parse_tickets(
     tickets: "list[TicketData]",
 ) -> "tuple[list[ExposedKeyData], list[int]]":
     exposed_data: list[ExposedKeyData] = []
+    ignorable_tickets: list[int] = []
     parse_error_ids: list[int] = []
     for ticket in tickets:
         data = ExposedKeyData.from_ticket(ticket)
-        if data is None and not _should_ignore_ticket_parse_failure(ticket):
+        ignorable = _should_ignore_ticket_parse_failure(ticket)
+        if ignorable:
+            ignorable_tickets.append(ticket.id)
+        elif data is None:
             parse_error_ids.append(ticket.id)
-        elif data is not None:
+        else:
             exposed_data.append(data)
 
-    return exposed_data, parse_error_ids
+    return exposed_data, ignorable_tickets, parse_error_ids
 
 
 def _should_ignore_ticket_parse_failure(ticket: TicketData) -> bool:
@@ -41,8 +56,7 @@ def _should_ignore_ticket_parse_failure(ticket: TicketData) -> bool:
     for match_str in ignore_strs:
         if match_str in text:
             return True
-    else:
-        return False
+    return False
 
 
 @dataclass
@@ -61,16 +75,6 @@ def tokens_server(self) -> str:
     def token(self) -> str:
         return self.iam_user.split("@@")[1]
 
-    def to_db_item(self) -> dict:
-        return {
-            "IamUser": {"S": self.iam_user.lower()},
-            "AccessKey": {"S": self.access_key.lower()},
-            "PublicLocation": {"S": self.public_location},
-            "ProcessedAt": {"N": f"{datetime.now().timestamp():.0f}"},
-            "ZendeskTicketId": {"N": str(self.ticket.id)},
-            "TicketCreatedAt": {"N": f"{self.ticket.created_dt.timestamp():.0f}"},
-        }
-
     @classmethod
     def from_ticket(cls, ticket: TicketData) -> "ExposedKeyData | None":
         iam_user = ""

aws-exposed-key-checker-infra/lambda_source/exposed_key_checker/lambda_handler.py

@@ -5,13 +5,15 @@
 import boto3
 from botocore.config import Config
 
-from exposed_key_checker.database import Database
 from exposed_key_checker.ticket_manager import ZendeskTicketManager
-from exposed_key_checker.exposed_keys import ExposedKeyData, parse_tickets
+from exposed_key_checker.exposed_keys import (
+    ExposedKeyData,
+    parse_tickets,
+    get_recent_open_tickets,
+    MAX_PROCESS_AGE_DAYS,
+)
 from exposed_key_checker import support_ticketer
 
-DB_TABLE_NAME = "ExposedKeyCheckerProcessed"
-MAX_PROCESS_AGE_DAYS = 7
 ZENDESK_EXPOSED_TICKET_TAG = os.environ["ZENDESK_EXPOSED_TICKET_TAG"]
 ZENDESK_AUTH_SECRET_ID = os.environ["ZENDESK_AUTH_SECRET_ID"]
 TOKENS_SERVERS_ALLOW_LIST = [
@@ -23,11 +25,9 @@
 
 
 def lambda_handler(_event, _context):
-    db = Database()
-
     try:
         ticket_manager = ZendeskTicketManager(*get_zendesk_auth())
-        key_data, failed_ids = gather_data(ticket_manager)
+        key_data, ignorable_ids, failed_ids = gather_data(ticket_manager)
     except Exception as e:
         text = f"The key checker could not query the Zendesk API for tickets.\nThe exception was {e}."
         support_ticketer.create_ticket(
@@ -37,7 +37,13 @@ def lambda_handler(_event, _context):
         )
         return
 
-    process_data(db, key_data)
+    process_data(key_data, ticket_manager)
+
+    for ticket_id in ignorable_ids:
+        print(f"Ignoring follow-up ticket: {ticket_id}")
+        ticket_manager.set_ticket_as_solved(
+            ticket_id, comment="Ignored by AWS key checker."
+        )
 
     if failed_ids:
         text = f"The key checker could not parse the following Zendesk ticket IDs: {failed_ids}"
@@ -48,13 +54,9 @@ def lambda_handler(_event, _context):
         )
 
 
-def process_data(db: Database, data: "list[ExposedKeyData]"):
-    unprocessed_items = [d for d in data if not db.has_key_been_processed(d)]
-    processed_count = len(data) - len(unprocessed_items)
-    print(f"Skipping {processed_count} items that were already processed.")
-
+def process_data(data: "list[ExposedKeyData]", ticket_manager: "ZendeskTicketManager"):
     items_to_process = []
-    for item in unprocessed_items:
+    for item in data:
         if item.tokens_server not in TOKENS_SERVERS_ALLOW_LIST:
             print(
                 f"Ignoring the following item because its server is not in the allow list: {item}"
@@ -76,30 +78,36 @@ def process_data(db: Database, data: "list[ExposedKeyData]"):
                 "exposed-aws-key-checker-post-error",
             )
         else:
-            db.mark_key_as_processed(item)
+            ticket_manager.set_ticket_as_solved(
+                item.ticket.id, comment="Resolved by AWS key checker."
+            )
 
 
 def gather_data(
     ticket_manager: "ZendeskTicketManager",
 ) -> "tuple[list[ExposedKeyData], list[int]]":
     data: list[ExposedKeyData] = []
+    ignorable_ids: list[int] = []
     error_ids: list[int] = []
 
     num_tickets = 0
     for tickets in ticket_manager.read_all_tickets_in_batches():
-        num_tickets += len(tickets)
-        key_data, eids = parse_tickets(tickets)
+        last_ticket = tickets[-1]
+        tickets_in_window = get_recent_open_tickets(tickets)
+
+        num_tickets += len(tickets_in_window)
+        key_data, ignorable_tickets, eids = parse_tickets(tickets_in_window)
         data.extend(key_data)
+        ignorable_ids.extend(ignorable_tickets)
         error_ids.extend(eids)
 
-        age = datetime.now() - key_data[-1].ticket.created_dt
+        age = datetime.now() - last_ticket.created_dt
         if age > timedelta(days=MAX_PROCESS_AGE_DAYS):
             # Only check the last week's data
             break
-
     print(f"Got {len(data)} exposed keys from {num_tickets} tickets.")
 
-    return data, error_ids
+    return data, ignorable_ids, error_ids
 
 
 def send_to_tokens_server(data: "ExposedKeyData"):
@@ -125,4 +133,4 @@ def get_zendesk_auth():
     client = boto3.client("secretsmanager", config=BOTO_CONFIG)
     res = client.get_secret_value(SecretId=ZENDESK_AUTH_SECRET_ID)
     data = json.loads(res.get("SecretString"))
-    return data["api_token"], data["user"], data["search_endpoint"]
+    return data["api_token"], data["user"], data["api_base_url"]

aws-exposed-key-checker-infra/lambda_source/exposed_key_checker/ticket_manager.py

@@ -5,19 +5,21 @@
 import requests
 
 ZENDESK_EXPOSED_TICKET_TAG = os.environ["ZENDESK_EXPOSED_TICKET_TAG"]
+ZENDESK_CLOSED_TICKET_TAG = os.environ["ZENDESK_CLOSED_TICKET_TAG"]
+ZENDESK_ASSIGNEE = os.environ["ZENDESK_ASSIGNEE"]
 
 
 class ZendeskTicketManager:
     ZENDESK_SEARCH_LAST_PAGE = (
         10  # the search endpoint has a maximum number of pages that it allows
     )
 
-    def __init__(self, api_token: str, user: str, search_endpoint: str):
+    def __init__(self, api_token: str, user: str, api_base_url: str):
         self._auth = (user, api_token)
-        self._search_endpoint = search_endpoint
+        self._api_base_url = api_base_url
 
     def read_all_tickets_in_batches(self) -> "Generator[list[TicketData], None, None]":
-        next_url = self._search_endpoint
+        next_url = f"{self._api_base_url}/api/v2/search.json"
         for _ in range(self.ZENDESK_SEARCH_LAST_PAGE):
             if next_url is None:
                 break
@@ -27,10 +29,10 @@ def read_all_tickets_in_batches(self) -> "Generator[list[TicketData], None, None
 
     def get_tickets_from_api(
         self,
-        url: str | None = None,
+        url: str,
     ) -> "tuple[list[TicketData], str | None]":
         r = requests.get(
-            url or self._search_endpoint,
+            url or f"{self._api_base_url}/api/v2/search.json",
             auth=self._auth,
             data={
                 "query": f'tags:"{ZENDESK_EXPOSED_TICKET_TAG}"',
@@ -45,13 +47,40 @@ def get_tickets_from_api(
         tickets = [TicketData.from_dict(t) for t in res["results"]]
         return tickets, res["next_page"]
 
+    def set_ticket_as_solved(self, ticket_id: "TicketData", comment: str):
+        print(f"Ticket to set to solved: {ticket_id}")
+
+        url = f"{self._api_base_url}/api/v2/tickets/update_many.json?ids={ticket_id}"
+
+        headers = {"Content-Type": "application/json"}
+
+        payload = {
+            "ticket": {
+                "status": "solved",
+                "assignee_id": ZENDESK_ASSIGNEE,
+                "comment": {
+                    "body": comment,
+                    "public": True,
+                },
+                "additional_tags": ["closed-by-aws-exposed-key-checker"],
+            }
+        }
+
+        response = requests.put(url, headers=headers, json=payload, auth=self._auth)
+        if response.status_code == 200:
+            print("Ticket successfully updated to solved and tag added.")
+        else:
+            print(f"Failed to update ticket. Status code: {response.status_code}")
+            print(f"Response: {response.text}")
+
 
 @dataclass
 class TicketData:
     url: str
     id: int
     created_at: str
     updated_at: str
+    status: str
     subject: str
     description: str
 
@@ -62,6 +91,7 @@ def from_dict(cls, data: dict) -> "TicketData":
             data["id"],
             data["created_at"],
             data["updated_at"],
+            data["status"],
             data["subject"],
             "".join(data["description"].splitlines()),
         )