Fix for petab 0.2.x (#1073)

* Adapt to new petab test suite structure
* Adapt  to changes in AMICI's experimental PySB-PEtab support
* Separate petab tests into separate pytest cases
* Run additional tests (newly added sbml cases + pysb cases)
* Fix parameter mapping failure for petab-sbml test case 17&18

Co-authored-by: Daniel Weindl <dweindl@users.noreply.github.com>

.github/workflows/ci.yml

@@ -73,7 +73,7 @@ jobs:
       run: .github/workflows/install_deps.sh amici pysb
 
     - name: Run tests
-      timeout-minutes: 25
+      timeout-minutes: 35
       run: tox -e petab
 
     - name: Coverage

.github/workflows/install_deps.sh

@@ -37,7 +37,7 @@ for par in "$@"; do
     pysb)
       # bionetgen
       wget -q -O bionetgen.tar \
-        https://github.com/RuleWorld/bionetgen/releases/download/BioNetGen-2.6.0/BioNetGen-2.6.0-linux.tgz
+        https://github.com/RuleWorld/bionetgen/releases/download/BioNetGen-2.8.5/BioNetGen-2.8.5-linux.tar.gz
       tar -xf bionetgen.tar
     ;;
 

pypesto/objective/amici/amici_util.py

@@ -67,8 +67,13 @@ def map_par_opt_to_par_sim(
     # iterate over simulation parameter indices
     for ix, val in enumerate(par_sim_vals):
         if not isinstance(val, numbers.Number):
-            # value is optimization parameter id
-            par_sim_vals[ix] = x_dct[val]
+            try:
+                # value is optimization parameter id
+                par_sim_vals[ix] = x_dct[val]
+            except KeyError:
+                # this may happen in case of states with NaN in the conditions
+                #  table
+                par_sim_vals[ix] = np.nan
 
     # return the created simulation parameter vector
     return np.array(par_sim_vals)

pypesto/petab/importer.py

@@ -335,14 +335,15 @@ def compile_model(self, **kwargs):
 
         Parameters
         ----------
-        kwargs: Extra arguments passed to `amici.SbmlImporter.sbml2amici`.
-
+        kwargs:
+            Extra arguments passed to :meth:`amici.SbmlImporter.sbml2amici`
+            or :meth:`amici.pysb_import.pysb2amici`.
         """
         # delete output directory
         if os.path.exists(self.output_folder):
             shutil.rmtree(self.output_folder)
 
-        amici.petab_import.import_model(
+        amici.petab_import.import_petab_problem(
             petab_problem=self.petab_problem,
             model_name=self.model_name,
             model_output_dir=self.output_folder,
@@ -857,10 +858,10 @@ def _find_output_folder_name(
     """
     Find a name for storing the compiled amici model in.
 
-    If available, use the sbml model name from the `petab_problem` or the
-    provided `model_name` (latter is given priority), otherwise create a
+    If available, use the model name from the ``petab_problem`` or the
+    provided ``model_name`` (latter is given priority), otherwise create a
     unique name. The folder will be located in the
-    `PetabImporter.MODEL_BASE_DIR` subdirectory of the current directory.
+    ``PetabImporter.MODEL_BASE_DIR`` subdirectory of the current directory.
     """
     # check whether location for amici model is a file
     if os.path.exists(PetabImporter.MODEL_BASE_DIR) and not os.path.isdir(
@@ -875,14 +876,14 @@ def _find_output_folder_name(
     if not os.path.exists(PetabImporter.MODEL_BASE_DIR):
         os.makedirs(PetabImporter.MODEL_BASE_DIR)
 
-    # try sbml model id
-    sbml_model_id = petab_problem.sbml_model.getId()
+    # try model id
+    model_id = petab_problem.model.model_id
     if model_name is not None:
-        sbml_model_id = model_name
+        model_id = model_name
 
-    if sbml_model_id:
+    if model_id:
         output_folder = os.path.abspath(
-            os.path.join(PetabImporter.MODEL_BASE_DIR, sbml_model_id)
+            os.path.join(PetabImporter.MODEL_BASE_DIR, model_id)
         )
     else:
         # create random folder name

pypesto/petab/pysb_importer.py

@@ -32,7 +32,7 @@ def __init__(
             Passed to `PetabImporter.__init__`.
         """
         if "model_name" not in kwargs:
-            kwargs["model_name"] = petab_problem.pysb_model.name
+            kwargs["model_name"] = petab_problem.model.model_id
         super().__init__(
             petab_problem,
             validate_petab=validate_petab,

setup.cfg

@@ -83,7 +83,6 @@ all =
     %(example)s
     %(select)s
     %(test)s
-    %(test_petab)s
 all_optimizers =
     %(ipopt)s
     %(dlib)s
@@ -93,9 +92,9 @@ all_optimizers =
     %(pyswarms)s
     %(fides)s
 amici =
-    amici >= 0.14.0
+    amici >= 0.18.0
 petab =
-    petab >= 0.1.14
+    petab >= 0.2.0
 ipopt =
     ipopt
 dlib =
@@ -158,8 +157,6 @@ test =
     gitpython >= 3.1.7
     pytest-rerunfailures >= 9.1.1
     autograd >= 1.3
-test_petab =
-    petabtests >= 0.0.0a6
 
 [bdist_wheel]
 # Requires Python 3

test/petab/test_petab_import_pysb.py

@@ -6,18 +6,15 @@
 import os
 
 import numpy as np
+import petab
 import petabtests
-import yaml
-
-# must import after previous, otherwise circular import issues :(
-from amici.petab_import_pysb import PysbPetabProblem
 
 import pypesto.optimize as optimize
 from pypesto.petab import PetabImporterPysb
 
-# In CI, bionetgen is install here
+# In CI, bionetgen is installed here
 BNGPATH = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), '..', '..', 'BioNetGen-2.6.0')
+    os.path.join(os.path.dirname(__file__), '..', '..', 'BioNetGen-2.8.5')
 )
 if 'BNGPATH' not in os.environ:
     logging.warning(f"Env var BNGPATH was not set. Setting to {BNGPATH}")
@@ -26,16 +23,16 @@
 
 def test_petab_pysb_optimization():
     test_case = '0001'
-    test_case_dir = os.path.join(petabtests.PYSB_DIR, test_case)
-    petab_yaml = os.path.join(test_case_dir, f'_{test_case}.yaml')
-    solution_yaml = os.path.join(test_case_dir, f'_{test_case}_solution.yaml')
-
+    test_case_dir = petabtests.get_case_dir(
+        test_case, version='v2.0.0', format_='pysb'
+    )
+    petab_yaml = test_case_dir / petabtests.problem_yaml_name(test_case)
     # expected results
-    with open(solution_yaml) as f:
-        solution = yaml.full_load(f)
-
-    petab_problem = PysbPetabProblem.from_yaml(petab_yaml)
+    solution = petabtests.load_solution(
+        test_case, format='pysb', version='v2.0.0'
+    )
 
+    petab_problem = petab.Problem.from_yaml(petab_yaml)
     importer = PetabImporterPysb(petab_problem)
     problem = importer.create_problem()
 

test/petab/test_petab_suite.py

@@ -1,14 +1,11 @@
 """Execute petab test suite."""
 
 import logging
-import os
-import sys
 
 import amici.petab_objective
 import petab
 import petabtests
 import pytest
-from _pytest.outcomes import Skipped
 
 import pypesto
 import pypesto.petab
@@ -17,32 +14,18 @@
 logger = logging.getLogger(__name__)
 
 
-def test_petab_suite():
-    """Execute all cases from the petab test suite, report performance."""
-    n_success = n_skipped = 0
-    for case in petabtests.CASES_LIST:
-        try:
-            execute_case(case)
-            n_success += 1
-        except Skipped:
-            n_skipped += 1
-        except Exception as e:
-            # run all despite failures
-            logger.error(f"Case {case} failed.")
-            logger.error(e)
-
-    logger.info(
-        f"{n_success} / {len(petabtests.CASES_LIST)} successful, "
-        f"{n_skipped} skipped"
-    )
-    if n_success + n_skipped != len(petabtests.CASES_LIST):
-        sys.exit(1)
-
-
-def execute_case(case):
+@pytest.mark.parametrize(
+    "case, model_type, version",
+    [
+        (case, model_type, version)
+        for (model_type, version) in (("sbml", "v1.0.0"), ("pysb", "v2.0.0"))
+        for case in petabtests.get_cases(format_=model_type, version=version)
+    ],
+)
+def test_petab_case(case, model_type, version):
     """Wrapper for _execute_case for handling test outcomes"""
     try:
-        _execute_case(case)
+        _execute_case(case, model_type, version)
     except Exception as e:
         if isinstance(
             e, NotImplementedError
@@ -56,16 +39,18 @@ def execute_case(case):
             raise e
 
 
-def _execute_case(case):
+def _execute_case(case, model_type, version):
     """Run a single PEtab test suite case"""
     case = petabtests.test_id_str(case)
     logger.info(f"Case {case}")
 
     # case folder
-    case_dir = os.path.join(petabtests.CASES_DIR, case)
+    case_dir = petabtests.get_case_dir(case, model_type, version)
 
     # load solution
-    solution = petabtests.load_solution(case, 'sbml')
+    solution = petabtests.load_solution(
+        case, format=model_type, version=version
+    )
     gt_chi2 = solution[petabtests.CHI2]
     gt_llh = solution[petabtests.LLH]
     gt_simulation_dfs = solution[petabtests.SIMULATION_DFS]
@@ -74,17 +59,22 @@ def _execute_case(case):
     tol_simulations = solution[petabtests.TOL_SIMULATIONS]
 
     # import petab problem
-    yaml_file = os.path.join(case_dir, petabtests.problem_yaml_name(case))
+    yaml_file = case_dir / petabtests.problem_yaml_name(case)
 
     # unique folder for compiled amici model
-    output_folder = f'amici_models/model_{case}'
+    model_name = (
+        f'petab_test_case_{case}_{model_type}_{version.replace(".", "_" )}'
+    )
+    output_folder = f'amici_models/{model_name}'
 
     # import and create objective function
     if case.startswith('0006'):
         petab_problem = petab.Problem.from_yaml(yaml_file)
         petab.flatten_timepoint_specific_output_overrides(petab_problem)
         importer = pypesto.petab.PetabImporter(
-            petab_problem=petab_problem, output_folder=output_folder
+            petab_problem=petab_problem,
+            output_folder=output_folder,
+            model_name=model_name,
         )
         petab_problem = petab.Problem.from_yaml(yaml_file)
     else:
@@ -143,9 +133,9 @@ def _execute_case(case):
     )
 
     if not all([llhs_match, chi2s_match, simulations_match]):
-        logger.error(f"Case {case} failed.")
+        logger.error(f"Case {version}/{model_type}/{case} failed.")
         raise AssertionError(
             f"Case {case}: Test results do not match " "expectations"
         )
 
-    logger.info(f"Case {case} passed.")
+    logger.info(f"Case {version}/{model_type}/{case} passed.")

tox.ini

@@ -62,13 +62,14 @@ description =
     Test basic functionality
 
 [testenv:petab]
-extras = test,test_petab,amici,petab,pyswarm
+extras = test,amici,petab,pyswarm
 deps =
     git+https://github.com/Benchmarking-Initiative/Benchmark-Models-PEtab.git@master\#subdirectory=src/python
+    git+https://github.com/PEtab-dev/petab_test_suite@main
+    git+https://github.com/FFroehlich/pysb@fix_pattern_matching
+    git+https://github.com/AMICI-dev/amici.git@develop\#egg=amici&subdirectory=python/sdist
+
 commands =
-    # install pysb
-    pip install \
-      git+https://github.com/pysb/pysb.git@5cbb147527814770ffc42e8e7d96e91dcb2b9857#egg=pysb
     pytest --cov=pypesto --cov-report=xml --cov-append \
         test/petab -s
 description =