From 6c3bc2b0a870e8def5be23b323e6919483dd4048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armando=20Ba=C3=B1uelos?= <32311654+abanuelo@users.noreply.github.com> Date: Fri, 12 Jul 2024 10:13:16 -0700 Subject: [PATCH] feat: adding dynamic scenario and object creation tests for CARLA (#227) * feat: [wip] adding dynamic scenario tests for CARLA * fix: attempting to add some stateful behaviors * feat: adding throttle and brake dynamic scenario tests * feat: apply formatting for dynamic scenarios * feat: adding object creation tests * fix: reformatting CarlaSimulator import * fix: fixing BusStop error and adding all objects test * fix: removing typo * fix: adding compileScenic inline for testing files * fix: adding old blueprint names * fix: adding syntax error fix * fix: adding reworked examples with blueprint * fix: adding test cases using pytest parameterize * fix: attempting to rework some logic * fix: removing blueprint tests * fix: applying black and isort * fix: updating tests with specific position * fix: editing pyproject.toml to separate carla related tests * fix: add env variable for CARLA path and blueprint tests with manual skip * fix: correcting environment references * fix: reformatting file with black * fix: adding generic CARLA version to install * fix: CARLA not supported with python 3.11 * feat: adding flaky * fix: moving flaky to .[test] * fix: adding linux and windows check for CARLA pip installation * fix: adding suggested fixes, testing fixes soon * fix: adding fixes * fix: attempting to ignore all tests dependent on carla package in test_carla.py * fix: adding launchCarlaServer to getCarlaSimulator * fix: adding reformatting * fix: dividing dynamic tests into test_actions.py file * fix: reverting test/conftest.py to original settings * fix: blueprints getCarlaSimulator reference * fix: reworking CARLA simulator code * fix: removing unnecessary pytest skips in blueprints file * fix: further paramterizing blueprint tests and adding package scope for getCarlaSimulator * test: making changes to throttle and brake tests, will check if working * fix: addressing remaining revisions * fix: addressing more round of comments * fix: addressing more comments --- pyproject.toml | 2 +- src/scenic/simulators/carla/model.scenic | 2 +- tests/conftest.py | 3 +- tests/simulators/carla/test_actions.py | 120 ++++++++++++++++++ .../carla/{test_carla.py => test_basic.py} | 0 tests/simulators/carla/test_blueprints.py | 98 ++++++++++++++ 6 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 tests/simulators/carla/test_actions.py rename tests/simulators/carla/{test_carla.py => test_basic.py} (100%) create mode 100644 tests/simulators/carla/test_blueprints.py diff --git a/pyproject.toml b/pyproject.toml index 8ef20b1c1..960bc4724 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ test-full = [ # like 'test' but adds dependencies for optional features "scenic[test]", # all dependencies from 'test' extra above "scenic[guideways]", # for running guideways modules "astor >= 0.8.1", - 'carla >= 0.9.12; python_version <= "3.8" and (platform_system == "Linux" or platform_system == "Windows")', + 'carla >= 0.9.12; python_version <= "3.10" and (platform_system == "Linux" or platform_system == "Windows")', "dill", "exceptiongroup", "inflect ~= 5.5", diff --git a/src/scenic/simulators/carla/model.scenic b/src/scenic/simulators/carla/model.scenic index c9cac9dfb..66433de53 100644 --- a/src/scenic/simulators/carla/model.scenic +++ b/src/scenic/simulators/carla/model.scenic @@ -259,7 +259,7 @@ class Chair(Prop): class BusStop(Prop): - blueprint: Uniform(*blueprints.busStopsModels) + blueprint: Uniform(*blueprints.busStopModels) class Advertisement(Prop): diff --git a/tests/conftest.py b/tests/conftest.py index e5cfadcde..3fe2f4d92 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,6 @@ import os.path from pathlib import Path import re -import subprocess import sys import pytest @@ -41,7 +40,7 @@ def manager(): return manager -@pytest.fixture +@pytest.fixture(scope="session") def getAssetPath(): base = Path(__file__).parent.parent / "assets" diff --git a/tests/simulators/carla/test_actions.py b/tests/simulators/carla/test_actions.py new file mode 100644 index 000000000..e3dd51980 --- /dev/null +++ b/tests/simulators/carla/test_actions.py @@ -0,0 +1,120 @@ +import os +from pathlib import Path +import signal +import socket +import subprocess +import time + +import pytest + +try: + import carla + + from scenic.simulators.carla import CarlaSimulator +except ModuleNotFoundError: + pytest.skip("carla package not installed", allow_module_level=True) + +from tests.utils import compileScenic, sampleScene + + +def checkCarlaPath(): + CARLA_ROOT = os.environ.get("CARLA_ROOT") + if not CARLA_ROOT: + pytest.skip("CARLA_ROOT env variable not set.") + return CARLA_ROOT + + +def isCarlaServerRunning(host="localhost", port=2000): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(1) + try: + sock.connect((host, port)) + return True + except (socket.timeout, ConnectionRefusedError): + return False + + +@pytest.fixture(scope="package") +def getCarlaSimulator(getAssetPath): + carla_process = None + if not isCarlaServerRunning(): + CARLA_ROOT = checkCarlaPath() + carla_process = subprocess.Popen( + f"bash {CARLA_ROOT}/CarlaUE4.sh -RenderOffScreen", shell=True + ) + + for _ in range(30): + if isCarlaServerRunning(): + break + time.sleep(1) + + # Extra 5 seconds to ensure server startup + time.sleep(5) + + base = getAssetPath("maps/CARLA") + + def _getCarlaSimulator(town): + path = os.path.join(base, f"{town}.xodr") + simulator = CarlaSimulator(map_path=path, carla_map=town) + return simulator, town, path + + yield _getCarlaSimulator + + if carla_process: + subprocess.run("killall -9 CarlaUE4-Linux-Shipping", shell=True) + + +def test_throttle(getCarlaSimulator): + simulator, town, mapPath = getCarlaSimulator("Town01") + code = f""" + param map = r'{mapPath}' + param carla_map = '{town}' + param time_step = 1.0/10 + + model scenic.simulators.carla.model + + behavior DriveWithThrottle(): + while True: + take SetThrottleAction(1) + + ego = new Car at (369, -326), with behavior DriveWithThrottle + record ego.speed as CarSpeed + terminate after 5 steps + """ + scenario = compileScenic(code, mode2D=True) + scene = sampleScene(scenario) + simulation = simulator.simulate(scene) + records = simulation.result.records["CarSpeed"] + assert records[len(records) // 2][1] < records[-1][1] + + +def test_brake(getCarlaSimulator): + simulator, town, mapPath = getCarlaSimulator("Town01") + code = f""" + param map = r'{mapPath}' + param carla_map = '{town}' + param time_step = 1.0/10 + + model scenic.simulators.carla.model + + behavior DriveWithThrottle(): + while True: + take SetThrottleAction(1) + + behavior Brake(): + while True: + take SetThrottleAction(0), SetBrakeAction(1) + + behavior DriveThenBrake(): + do DriveWithThrottle() for 2 steps + do Brake() for 4 steps + + ego = new Car at (369, -326), with behavior DriveThenBrake + record final ego.speed as CarSpeed + terminate after 6 steps + """ + scenario = compileScenic(code, mode2D=True) + scene = sampleScene(scenario) + simulation = simulator.simulate(scene) + finalSpeed = simulation.result.records["CarSpeed"] + assert finalSpeed == pytest.approx(0.0, abs=1e-1) diff --git a/tests/simulators/carla/test_carla.py b/tests/simulators/carla/test_basic.py similarity index 100% rename from tests/simulators/carla/test_carla.py rename to tests/simulators/carla/test_basic.py diff --git a/tests/simulators/carla/test_blueprints.py b/tests/simulators/carla/test_blueprints.py new file mode 100644 index 000000000..4b7dcba25 --- /dev/null +++ b/tests/simulators/carla/test_blueprints.py @@ -0,0 +1,98 @@ +import pytest + +try: + import carla +except ModuleNotFoundError: + pytest.skip("carla package not installed", allow_module_level=True) + +from test_actions import getCarlaSimulator + +from scenic.simulators.carla.blueprints import ( + advertisementModels, + atmModels, + barrelModels, + barrierModels, + benchModels, + bicycleModels, + boxModels, + busStopModels, + carModels, + caseModels, + chairModels, + coneModels, + containerModels, + creasedboxModels, + debrisModels, + garbageModels, + gnomeModels, + ironplateModels, + kioskModels, + mailboxModels, + motorcycleModels, + plantpotModels, + tableModels, + trafficwarningModels, + trashModels, + truckModels, + vendingMachineModels, + walkerModels, +) +from tests.utils import compileScenic, sampleScene + + +def model_blueprint(simulator, mapPath, town, modelType, modelName): + code = f""" + param map = r'{mapPath}' + param carla_map = '{town}' + param time_step = 1.0/10 + + model scenic.simulators.carla.model + ego = new {modelType} with blueprint '{modelName}' + terminate after 1 steps + """ + scenario = compileScenic(code, mode2D=True) + scene = sampleScene(scenario) + simulation = simulator.simulate(scene) + obj = simulation.objects[0] + assert obj.blueprint == modelName + + +model_data = { + "Car": carModels, + "Bicycle": bicycleModels, + "Motorcycle": motorcycleModels, + "Truck": truckModels, + "Trash": trashModels, + "Cone": coneModels, + "Debris": debrisModels, + "VendingMachine": vendingMachineModels, + "Chair": chairModels, + "BusStop": busStopModels, + "Advertisement": advertisementModels, + "Garbage": garbageModels, + "Container": containerModels, + "Table": tableModels, + "Barrier": barrierModels, + "PlantPot": plantpotModels, + "Mailbox": mailboxModels, + "Gnome": gnomeModels, + "CreasedBox": creasedboxModels, + "Case": caseModels, + "Box": boxModels, + "Bench": benchModels, + "Barrel": barrelModels, + "ATM": atmModels, + "Kiosk": kioskModels, + "IronPlate": ironplateModels, + "TrafficWarning": trafficwarningModels, + "Pedestrian": walkerModels, +} + + +@pytest.mark.parametrize( + "modelType, modelName", + [(type, name) for type, names in model_data.items() for name in names], +) +def test_model_blueprints(getCarlaSimulator, modelType, modelName): + simulator, town, mapPath = getCarlaSimulator("Town01") + model_blueprint(simulator, mapPath, town, modelType, modelName)