feat: adding ci for simulator testing

.github/check_latest_simulators.py

@@ -0,0 +1,40 @@
+import os
+import requests
+import re
+
+def check_path_exists(version, simulator):
+    path = f"/software/{simulator}{version}"
+    if os.path.exists(path):
+        print(f"Latest {simulator} version {version} already present on the machine.\n")
+    else:
+        s = """
+                       _          
+ _    _____ ________  (_)__  ___ _
+| |/|/ / _ `/ __/ _ \/ / _ \/ _ `/
+|__,__/\_,_/_/ /_//_/_/_//_/\_, / 
+                           /___/  
+        """
+        print(s)
+        print(f"A new {simulator} version ({version}) needs to be installed and tested in CI.\n")
+
+print("Checking for CARLA...")
+carla_url = 'https://github.com/carla-simulator/carla/releases/latest'
+carla_response = requests.get(carla_url)
+carla_match = re.search(r'carla-simulator/carla/releases/tag/([^"]+)', carla_response.text)
+
+try:
+    version = carla_match.group(1)
+    check_path_exists(version, "CARLA_")
+except AttributeError:
+    print("Error: Unable to find the latest CARLA version using regex.")
+
+print("Checking for Webots...")
+webots_url = 'https://github.com/cyberbotics/webots/releases/latest'
+webots_response = requests.get(webots_url)
+webots_match = re.search(r'cyberbotics/webots/releases/tag/([^"]+)', webots_response.text)
+
+try:
+    version = webots_match.group(1)
+    check_path_exists(version, 'webots')
+except AttributeError:
+    print("Error: Unable to find the latest Webots version using regex.")

.github/workflows/run-simulators.yml

@@ -0,0 +1,113 @@
+name: run_simulators
+on:
+  # IMPORTANT: this workflow should only be triggered manually via the Actions
+  # portal of the repo!!! Do not modify this workflow's trigger!
+  workflow_dispatch:
+
+jobs:
+    build:
+        name: run_simulators
+        runs-on: ubuntu-latest
+        concurrency:
+          group: sim
+
+        steps:
+            - name: Start EC2 Instance
+              env:
+                INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }}
+                AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+              run: |
+                # Get the instance state
+                instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
+
+                # If the machine is stopping wait for it to fully stop
+                while [ "$instance_state" == "stopping" ]; do
+                    echo "Instance is stopping, waiting for it to fully stop..."
+                    sleep 10
+                    instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
+                done
+
+                # Check if instance state is "stopped"
+                if [[ "$instance_state" == "stopped" ]]; then
+                    echo "Instance is stopped, starting it..."
+                    aws ec2 start-instances --instance-ids $INSTANCE_ID
+                elif [[ "$instance_state" == "pending" ]]; then
+                    echo "Instance startup is pending, continuing..."
+                elif [[ "$instance_state" == "running" ]]; then
+                    echo "Instance is already running..."
+                    exit 0
+                else
+                    echo "Unknown instance state: $instance_state"
+                    exit 1
+                fi
+
+                # Wait for 2 minutes
+                echo "Waiting 2 minutes for full startup..."
+                sleep 120
+
+            - name: Check for Simulator Version Updates
+              env:
+                PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+                HOSTNAME: ${{secrets.SSH_HOST}}
+                USER_NAME: ${{secrets.SSH_USERNAME}}
+              run: |
+                echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
+                ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
+                  cd /home/ubuntu/actions/ &&
+                  rm -rf Scenic &&
+                  git clone --branch $(basename "${{ github.ref }}") --single-branch https://$GH_ACCESS_TOKEN@github.com/BerkeleyLearnVerify/Scenic.git &&
+                  cd Scenic &&
+                  python3 .github/check_latest_simulators.py
+                '
+
+            - name: SSH into EC2 and Run Simulator Tests
+              env:
+                PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+                HOSTNAME: ${{secrets.SSH_HOST}}
+                USER_NAME: ${{secrets.SSH_USERNAME}}
+              run: |
+                echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
+                ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
+                  cd /home/ubuntu/actions/Scenic &&
+                  python3 -m venv venv &&
+                  source venv/bin/activate &&
+                  python3 -m pip install -e . &&
+                  python3 -m pip install -e .[test-full] &&
+                  python3 -m pip install -e .[guideways] &&
+                  pip install carla &&
+                  export DISPLAY=:1 &&
+                  pytest tests/simulators/* &&
+                '
+
+            - name: Stop EC2 Instance
+              env:
+                INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }}
+                AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+              run: |
+                # Get the instance state
+                instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
+
+                # If the machine is pending wait for it to fully start
+                while [ "$instance_state" == "pending" ]; do
+                    echo "Instance is pending startup, waiting for it to fully start..."
+                    sleep 10
+                    instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
+                done
+
+                # Check if instance state is "stopped"
+                if [[ "$instance_state" == "running" ]]; then
+                    echo "Instance is running, stopping it..."
+                    aws ec2 stop-instances --instance-ids $INSTANCE_ID
+                elif [[ "$instance_state" == "stopping" ]]; then
+                    echo "Instance is stopping..."
+                elif [[ "$instance_state" == "stopped" ]]; then
+                    echo "Instance is already stopped..."
+                    exit 0
+                else
+                    echo "Unknown instance state: $instance_state"
+                    exit 1
+                fi