feat: adding dynamic scenario and object creation tests for CARLA (#227)

* feat: [wip] adding dynamic scenario tests for CARLA

* fix: attempting to add some stateful behaviors

* feat: adding throttle and brake dynamic scenario tests

* feat: apply formatting for dynamic scenarios

* feat: adding object creation tests

* fix: reformatting CarlaSimulator import

* fix: fixing BusStop error and adding all objects test

* fix: removing typo

* fix: adding compileScenic inline for testing files

* fix: adding old blueprint names

* fix: adding syntax error fix

* fix: adding reworked examples with blueprint

* fix: adding test cases using pytest parameterize

* fix: attempting to rework some logic

* fix: removing blueprint tests

* fix: applying black and isort

* fix: updating tests with specific position

* fix: editing pyproject.toml to separate carla related tests

* fix: add env variable for CARLA path and blueprint tests with manual skip

* fix: correcting environment references

* fix: reformatting file with black

* fix: adding generic CARLA version to install

* fix: CARLA not supported with python 3.11

* feat: adding flaky

* fix: moving flaky to .[test]

* fix: adding linux and windows check for CARLA pip installation

* fix: adding suggested fixes, testing fixes soon

* fix: adding fixes

* fix: attempting to ignore all tests dependent on carla package in test_carla.py

* fix: adding launchCarlaServer to getCarlaSimulator

* fix: adding reformatting

* fix: dividing dynamic tests into test_actions.py file

* fix: reverting test/conftest.py to original settings

* fix: blueprints getCarlaSimulator reference

* fix: reworking CARLA simulator code

* fix: removing unnecessary pytest skips in blueprints file

* fix: further paramterizing blueprint tests and adding package scope for getCarlaSimulator

* test: making changes to throttle and brake tests, will check if working

* fix: addressing remaining revisions

* fix: addressing more round of comments

* fix: addressing more comments

pyproject.toml

@@ -64,7 +64,7 @@ test-full = [  # like 'test' but adds dependencies for optional features
 	"scenic[test]",       # all dependencies from 'test' extra above
 	"scenic[guideways]",  # for running guideways modules
 	"astor >= 0.8.1",
-	'carla >= 0.9.12; python_version <= "3.8" and (platform_system == "Linux" or platform_system == "Windows")',
+	'carla >= 0.9.12; python_version <= "3.10" and (platform_system == "Linux" or platform_system == "Windows")',
 	"dill",
 	"exceptiongroup",
 	"inflect ~= 5.5",

src/scenic/simulators/carla/model.scenic

@@ -259,7 +259,7 @@ class Chair(Prop):
 
 
 class BusStop(Prop):
-    blueprint: Uniform(*blueprints.busStopsModels)
+    blueprint: Uniform(*blueprints.busStopModels)
 
 
 class Advertisement(Prop):

tests/conftest.py

@@ -2,7 +2,6 @@
 import os.path
 from pathlib import Path
 import re
-import subprocess
 import sys
 
 import pytest
@@ -41,7 +40,7 @@ def manager():
     return manager
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def getAssetPath():
     base = Path(__file__).parent.parent / "assets"
 

tests/simulators/carla/test_actions.py

@@ -0,0 +1,120 @@
+import os
+from pathlib import Path
+import signal
+import socket
+import subprocess
+import time
+
+import pytest
+
+try:
+    import carla
+
+    from scenic.simulators.carla import CarlaSimulator
+except ModuleNotFoundError:
+    pytest.skip("carla package not installed", allow_module_level=True)
+
+from tests.utils import compileScenic, sampleScene
+
+
+def checkCarlaPath():
+    CARLA_ROOT = os.environ.get("CARLA_ROOT")
+    if not CARLA_ROOT:
+        pytest.skip("CARLA_ROOT env variable not set.")
+    return CARLA_ROOT
+
+
+def isCarlaServerRunning(host="localhost", port=2000):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.settimeout(1)
+        try:
+            sock.connect((host, port))
+            return True
+        except (socket.timeout, ConnectionRefusedError):
+            return False
+
+
+@pytest.fixture(scope="package")
+def getCarlaSimulator(getAssetPath):
+    carla_process = None
+    if not isCarlaServerRunning():
+        CARLA_ROOT = checkCarlaPath()
+        carla_process = subprocess.Popen(
+            f"bash {CARLA_ROOT}/CarlaUE4.sh -RenderOffScreen", shell=True
+        )
+
+        for _ in range(30):
+            if isCarlaServerRunning():
+                break
+            time.sleep(1)
+
+        # Extra 5 seconds to ensure server startup
+        time.sleep(5)
+
+    base = getAssetPath("maps/CARLA")
+
+    def _getCarlaSimulator(town):
+        path = os.path.join(base, f"{town}.xodr")
+        simulator = CarlaSimulator(map_path=path, carla_map=town)
+        return simulator, town, path
+
+    yield _getCarlaSimulator
+
+    if carla_process:
+        subprocess.run("killall -9 CarlaUE4-Linux-Shipping", shell=True)
+
+
+def test_throttle(getCarlaSimulator):
+    simulator, town, mapPath = getCarlaSimulator("Town01")
+    code = f"""
+        param map = r'{mapPath}'
+        param carla_map = '{town}'
+        param time_step = 1.0/10
+
+        model scenic.simulators.carla.model
+
+        behavior DriveWithThrottle():
+            while True:
+                take SetThrottleAction(1)
+        
+        ego = new Car at (369, -326), with behavior DriveWithThrottle
+        record ego.speed as CarSpeed
+        terminate after 5 steps
+    """
+    scenario = compileScenic(code, mode2D=True)
+    scene = sampleScene(scenario)
+    simulation = simulator.simulate(scene)
+    records = simulation.result.records["CarSpeed"]
+    assert records[len(records) // 2][1] < records[-1][1]
+
+
+def test_brake(getCarlaSimulator):
+    simulator, town, mapPath = getCarlaSimulator("Town01")
+    code = f"""
+        param map = r'{mapPath}'
+        param carla_map = '{town}'
+        param time_step = 1.0/10
+
+        model scenic.simulators.carla.model
+
+        behavior DriveWithThrottle():
+            while True:
+                take SetThrottleAction(1)
+
+        behavior Brake():
+            while True:
+                take SetThrottleAction(0), SetBrakeAction(1)
+
+        behavior DriveThenBrake():
+            do DriveWithThrottle() for 2 steps
+            do Brake() for 4 steps
+
+        ego = new Car at (369, -326), with behavior DriveThenBrake
+        record final ego.speed as CarSpeed
+        terminate after 6 steps
+    """
+    scenario = compileScenic(code, mode2D=True)
+    scene = sampleScene(scenario)
+    simulation = simulator.simulate(scene)
+    finalSpeed = simulation.result.records["CarSpeed"]
+    assert finalSpeed == pytest.approx(0.0, abs=1e-1)

tests/simulators/carla/test_blueprints.py

@@ -0,0 +1,98 @@
+import pytest
+
+try:
+    import carla
+except ModuleNotFoundError:
+    pytest.skip("carla package not installed", allow_module_level=True)
+
+from test_actions import getCarlaSimulator
+
+from scenic.simulators.carla.blueprints import (
+    advertisementModels,
+    atmModels,
+    barrelModels,
+    barrierModels,
+    benchModels,
+    bicycleModels,
+    boxModels,
+    busStopModels,
+    carModels,
+    caseModels,
+    chairModels,
+    coneModels,
+    containerModels,
+    creasedboxModels,
+    debrisModels,
+    garbageModels,
+    gnomeModels,
+    ironplateModels,
+    kioskModels,
+    mailboxModels,
+    motorcycleModels,
+    plantpotModels,
+    tableModels,
+    trafficwarningModels,
+    trashModels,
+    truckModels,
+    vendingMachineModels,
+    walkerModels,
+)
+from tests.utils import compileScenic, sampleScene
+
+
+def model_blueprint(simulator, mapPath, town, modelType, modelName):
+    code = f"""
+            param map = r'{mapPath}'
+            param carla_map = '{town}'
+            param time_step = 1.0/10
+
+            model scenic.simulators.carla.model
+            ego = new {modelType} with blueprint '{modelName}'
+            terminate after 1 steps
+        """
+    scenario = compileScenic(code, mode2D=True)
+    scene = sampleScene(scenario)
+    simulation = simulator.simulate(scene)
+    obj = simulation.objects[0]
+    assert obj.blueprint == modelName
+
+
+model_data = {
+    "Car": carModels,
+    "Bicycle": bicycleModels,
+    "Motorcycle": motorcycleModels,
+    "Truck": truckModels,
+    "Trash": trashModels,
+    "Cone": coneModels,
+    "Debris": debrisModels,
+    "VendingMachine": vendingMachineModels,
+    "Chair": chairModels,
+    "BusStop": busStopModels,
+    "Advertisement": advertisementModels,
+    "Garbage": garbageModels,
+    "Container": containerModels,
+    "Table": tableModels,
+    "Barrier": barrierModels,
+    "PlantPot": plantpotModels,
+    "Mailbox": mailboxModels,
+    "Gnome": gnomeModels,
+    "CreasedBox": creasedboxModels,
+    "Case": caseModels,
+    "Box": boxModels,
+    "Bench": benchModels,
+    "Barrel": barrelModels,
+    "ATM": atmModels,
+    "Kiosk": kioskModels,
+    "IronPlate": ironplateModels,
+    "TrafficWarning": trafficwarningModels,
+    "Pedestrian": walkerModels,
+}
+
+
+@pytest.mark.parametrize(
+    "modelType, modelName",
+    [(type, name) for type, names in model_data.items() for name in names],
+)
+def test_model_blueprints(getCarlaSimulator, modelType, modelName):
+    simulator, town, mapPath = getCarlaSimulator("Town01")
+    model_blueprint(simulator, mapPath, town, modelType, modelName)