Feat/improve alerts

README.md

@@ -67,6 +67,9 @@ This allows our on-chain verifier to authenticate the email sender and authorize
 
 For detailed setup instructions, see [here](./packages/circuits/README.md).
 
+### Infrastructure Package
+For detailed instructions on how to manage infrastructure metrics and alerts in Google Cloud Platform (GCP), please refer to the [Infrastructure Management Scripts](./infrastructure/README.md) documentation.
+
 ### `contracts` Package
 It has Solidity contracts that help smart contracts based on our SDK verify the email-auth message. Among them, there are three significant contracts: verifier, DKIM registry, and email-auth contracts.
 

infrastructure/README.md

@@ -0,0 +1,42 @@
+
+# Infrastructure Management Scripts
+
+This document provides instructions on how to use the provided scripts to apply metrics and alerts to your Google Cloud Platform (GCP) environment.
+
+## Prerequisites
+
+- Ensure you have the [Google Cloud SDK](https://cloud.google.com/sdk) installed and authenticated.
+- Make sure `jq` is installed for JSON processing.
+
+## Applying Metrics
+
+To apply a specific metric from a JSON file, use the `apply_metric.sh` script. This script reads the metric configuration from the specified JSON file and applies it to your GCP project.
+
+### Usage
+
+```bash
+cd /path/to/your/project/infrastructure/metrics
+./apply_metric.sh metric_file.json
+```
+
+- Replace `/path/to/your/project` with the root directory of your project.
+- Replace `metric_file.json` with the path to the metric JSON file in the metrics directory.
+
+## Applying Alerts
+
+To apply a specific alert policy from a JSON file, use the `apply_alert.sh` script. This script reads the alert policy configuration from the specified JSON file and applies it to your GCP project.
+
+### Usage
+
+```bash
+cd /path/to/your/project/infrastructure/alerts
+./apply_alert.sh alert_file.json
+```
+
+- Replace `/path/to/your/project` with the root directory of your project.
+- Replace `alert_file.json` with the path to your alert policy JSON file in the alerts directory.
+
+## Additional Notes
+
+- Ensure that you have the necessary permissions to apply metrics and alerts in your GCP project.
+- If you encounter any errors, refer to the script logs for more details.

infrastructure/alerts/Uptime_Check_for_AR_Relayer_(Base_Sepolia).json

@@ -0,0 +1,46 @@
+{
+  "alertStrategy": {
+    "autoClose": "604800s",
+    "notificationPrompts": [
+      "OPENED"
+    ]
+  },
+  "combiner": "OR",
+  "conditions": [
+    {
+      "conditionThreshold": {
+        "aggregations": [
+          {
+            "alignmentPeriod": "300s",
+            "perSeriesAligner": "ALIGN_FRACTION_TRUE"
+          }
+        ],
+        "comparison": "COMPARISON_LT",
+        "duration": "0s",
+        "filter": "resource.type = \"uptime_url\" AND resource.labels.host = \"auth-base-sepolia-staging.prove.email\" AND metric.type = \"monitoring.googleapis.com/uptime_check/check_passed\"",
+        "thresholdValue": 0.9,
+        "trigger": {
+          "count": 1
+        }
+      },
+      "displayName": "Uptime Check URL - Check passed",
+      "name": "projects/zkairdrop/alertPolicies/14540598362673761778/conditions/9717783705450711541"
+    }
+  ],
+  "displayName": "Uptime Check for AR Relayer (Base Sepolia)",
+  "documentation": {
+    "content": "Uptime check failed for account recovery relayer (Base Sepolia).",
+    "mimeType": "text/markdown",
+    "subject": "Uptime Check Failed for AR Base Sepolia Relayer"
+  },
+  "enabled": true,
+  "name": "projects/zkairdrop/alertPolicies/14540598362673761778",
+  "notificationChannels": [
+    "projects/zkairdrop/notificationChannels/8151570453739639273",
+    "projects/zkairdrop/notificationChannels/7073288447352854381",
+    "projects/zkairdrop/notificationChannels/2385336445405127098",
+    "projects/zkairdrop/notificationChannels/17871058532858569683",
+    "projects/zkairdrop/notificationChannels/13717630568524701111"
+  ],
+  "severity": "WARNING"
+}

infrastructure/alerts/Uptime_Check_for_AR_Relayer_(ZKsync_Sepolia).json

@@ -0,0 +1,46 @@
+{
+  "alertStrategy": {
+    "autoClose": "604800s",
+    "notificationPrompts": [
+      "OPENED"
+    ]
+  },
+  "combiner": "OR",
+  "conditions": [
+    {
+      "conditionThreshold": {
+        "aggregations": [
+          {
+            "alignmentPeriod": "300s",
+            "perSeriesAligner": "ALIGN_FRACTION_TRUE"
+          }
+        ],
+        "comparison": "COMPARISON_LT",
+        "duration": "0s",
+        "filter": "resource.type = \"uptime_url\" AND resource.labels.host = \"auth-zksync-sepolia-staging.prove.email\" AND metric.type = \"monitoring.googleapis.com/uptime_check/check_passed\"",
+        "thresholdValue": 0.9,
+        "trigger": {
+          "count": 1
+        }
+      },
+      "displayName": "Uptime Check URL - Check passed",
+      "name": "projects/zkairdrop/alertPolicies/1785328578980889069/conditions/12793827239849115908"
+    }
+  ],
+  "displayName": "Uptime Check for AR Relayer (ZKsync Sepolia)",
+  "documentation": {
+    "content": "Uptime check failed for account recovery relayer (ZKsync Sepolia).",
+    "mimeType": "text/markdown",
+    "subject": "Uptime Check Failed for AR ZKsync Sepolia Relayer"
+  },
+  "enabled": true,
+  "name": "projects/zkairdrop/alertPolicies/1785328578980889069",
+  "notificationChannels": [
+    "projects/zkairdrop/notificationChannels/13717630568524701111",
+    "projects/zkairdrop/notificationChannels/17871058532858569683",
+    "projects/zkairdrop/notificationChannels/2385336445405127098",
+    "projects/zkairdrop/notificationChannels/7073288447352854381",
+    "projects/zkairdrop/notificationChannels/8151570453739639273"
+  ],
+  "severity": "WARNING"
+}

infrastructure/alerts/Uptime_Check_for_Email_Wallet_Relayer.json

@@ -0,0 +1,46 @@
+{
+  "alertStrategy": {
+    "autoClose": "604800s",
+    "notificationPrompts": [
+      "OPENED"
+    ]
+  },
+  "combiner": "OR",
+  "conditions": [
+    {
+      "conditionThreshold": {
+        "aggregations": [
+          {
+            "alignmentPeriod": "300s",
+            "perSeriesAligner": "ALIGN_FRACTION_TRUE"
+          }
+        ],
+        "comparison": "COMPARISON_LT",
+        "duration": "0s",
+        "filter": "resource.type = \"uptime_url\" AND resource.labels.host = \"relayerapi.emailwallet.org\" AND metric.type = \"monitoring.googleapis.com/uptime_check/check_passed\"",
+        "thresholdValue": 0.9,
+        "trigger": {
+          "count": 1
+        }
+      },
+      "displayName": "Uptime Check URL - Check passed",
+      "name": "projects/zkairdrop/alertPolicies/8542470102587481494/conditions/1395222843874964873"
+    }
+  ],
+  "displayName": "Uptime Check for Email Wallet Relayer",
+  "documentation": {
+    "content": "Uptime check failed for email wallet relayer.",
+    "mimeType": "text/markdown",
+    "subject": "Uptime Check Failed for Email Wallet"
+  },
+  "enabled": true,
+  "name": "projects/zkairdrop/alertPolicies/8542470102587481494",
+  "notificationChannels": [
+    "projects/zkairdrop/notificationChannels/13717630568524701111",
+    "projects/zkairdrop/notificationChannels/2385336445405127098",
+    "projects/zkairdrop/notificationChannels/7073288447352854381",
+    "projects/zkairdrop/notificationChannels/8151570453739639273",
+    "projects/zkairdrop/notificationChannels/17871058532858569683"
+  ],
+  "severity": "WARNING"
+}

infrastructure/alerts/apply_alert.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Check if a file name is provided
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <file_name>"
+  exit 1
+fi
+
+FILE_NAME=$1
+
+# Check if the file exists
+if [ ! -f "$FILE_NAME" ]; then
+  echo "Error: File '$FILE_NAME' does not exist."
+  exit 1
+fi
+
+# Extract the display name from the JSON file
+DISPLAY_NAME=$(jq -r '.displayName' "$FILE_NAME")
+
+# Create the alert policy using gcloud
+gcloud alpha monitoring policies create --policy-from-file="$FILE_NAME"
+
+# Check if gcloud was successful
+if [ $? -eq 0 ]; then
+  echo "Alert policy '$DISPLAY_NAME' applied successfully."
+else
+  echo "Error applying alert policy '$DISPLAY_NAME'."
+  exit 1
+fi

infrastructure/alerts/logging_user_error-from-prover_[SUM].json

@@ -0,0 +1,40 @@
+{
+  "alertStrategy": {
+    "autoClose": "1800s",
+    "notificationPrompts": [
+      "OPENED",
+      "CLOSED"
+    ]
+  },
+  "combiner": "OR",
+  "conditions": [
+    {
+      "conditionThreshold": {
+        "aggregations": [
+          {
+            "alignmentPeriod": "60s",
+            "crossSeriesReducer": "REDUCE_SUM",
+            "perSeriesAligner": "ALIGN_COUNT"
+          }
+        ],
+        "comparison": "COMPARISON_GT",
+        "duration": "0s",
+        "filter": "metric.type=\"logging.googleapis.com/user/error-from-prover\" AND resource.type=\"k8s_container\"",
+        "trigger": {
+          "count": 1
+        }
+      },
+      "displayName": "logging/user/error-from-prover [SUM]"
+    }
+  ],
+  "displayName": "logging/user/error-from-prover [SUM]",
+  "documentation": {
+    "content": "This error happens when the prover get some error",
+    "mimeType": "text/markdown"
+  },
+  "enabled": true,
+  "notificationChannels": [
+    "projects/zkairdrop/notificationChannels/16929435408944174030"
+  ],
+  "severity": "ERROR"
+}

infrastructure/alerts/logging_user_error-handling-email-event_[SUM].json

@@ -0,0 +1,41 @@
+{
+  "alertStrategy": {
+    "autoClose": "1800s",
+    "notificationPrompts": [
+      "OPENED",
+      "CLOSED"
+    ]
+  },
+  "combiner": "OR",
+  "conditions": [
+    {
+      "conditionThreshold": {
+        "aggregations": [
+          {
+            "alignmentPeriod": "60s",
+            "crossSeriesReducer": "REDUCE_SUM",
+            "perSeriesAligner": "ALIGN_COUNT"
+          }
+        ],
+        "comparison": "COMPARISON_GT",
+        "duration": "0s",
+        "filter": "resource.type = \"k8s_container\" AND metric.type = \"logging.googleapis.com/user/error-handling-email-event\" AND resource.type=\"k8s_container\"",
+        "trigger": {
+          "count": 1
+        }
+      },
+      "displayName": "logging/user/error-handling-email-event [SUM]"
+    }
+  ],
+  "displayName": "logging/user/error-handling-email-event [SUM]",
+  "documentation": {
+    "content": "This error happens when the relayer get in trouble about sending mail",
+    "mimeType": "text/markdown",
+    "subject": "error-handling-email-event"
+  },
+  "enabled": true,
+  "notificationChannels": [
+    "projects/zkairdrop/notificationChannels/16929435408944174030"
+  ],
+  "severity": "WARNING"
+}

infrastructure/alerts/logging_user_error-handling-email_[SUM].json

@@ -0,0 +1,41 @@
+{
+  "alertStrategy": {
+    "autoClose": "1800s",
+    "notificationPrompts": [
+      "OPENED",
+      "CLOSED"
+    ]
+  },
+  "combiner": "OR",
+  "conditions": [
+    {
+      "conditionThreshold": {
+        "aggregations": [
+          {
+            "alignmentPeriod": "60s",
+            "crossSeriesReducer": "REDUCE_SUM",
+            "perSeriesAligner": "ALIGN_COUNT"
+          }
+        ],
+        "comparison": "COMPARISON_GT",
+        "duration": "0s",
+        "filter": "resource.type = \"k8s_container\" AND metric.type = \"logging.googleapis.com/user/error-handling-email\" AND resource.type=\"k8s_container\"",
+        "trigger": {
+          "count": 1
+        }
+      },
+      "displayName": "logging/user/error-handling-email [SUM]"
+    }
+  ],
+  "displayName": "logging/user/error-handling-email [SUM]",
+  "documentation": {
+    "content": "This error happens when the relayer got the contract call error or the zkregx errors.",
+    "mimeType": "text/markdown",
+    "subject": "error-handling-email"
+  },
+  "enabled": true,
+  "notificationChannels": [
+    "projects/zkairdrop/notificationChannels/16929435408944174030"
+  ],
+  "severity": "WARNING"
+}

infrastructure/metrics/apply_metric.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Check if a file name is provided
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <file_name>"
+  exit 1
+fi
+
+FILE_NAME=$1
+
+# Check if the file exists
+if [ ! -f "$FILE_NAME" ]; then
+  echo "Error: File '$FILE_NAME' does not exist."
+  exit 1
+fi
+
+# Extract the metric name using jq
+METRIC_NAME=$(jq -r '.name' "$FILE_NAME")
+
+# Check if jq was successful
+if [ -z "$METRIC_NAME" ]; then
+  echo "Error: Could not extract metric name from '$FILE_NAME'."
+  exit 1
+fi
+
+# Apply the metric using gcloud
+gcloud logging metrics create "$METRIC_NAME" --config-from-file="$FILE_NAME"
+
+# Check if gcloud was successful
+if [ $? -eq 0 ]; then
+  echo "Metric '$METRIC_NAME' applied successfully."
+else
+  echo "Error applying metric '$METRIC_NAME'."
+  exit 1
+fi