diff --git a/examples/setup_config.toml b/examples/setup_config.toml new file mode 100644 index 00000000..a5c42993 --- /dev/null +++ b/examples/setup_config.toml @@ -0,0 +1,34 @@ +####################### +# Setup Model Configs # +####################### + +# this is where user want to create the MTC model +ROOT_DIR = "V:/projects/MTC/US0024934.9168/setup_model" + +# user-defined model folder name (naming convention: ModelYear_ModelVersion_Project_Scenario) +MODEL_FOLDER_NAME = "2023_TM22_Demo" + +# location of the networks +# this includes file such as tolls, transit fares etc. +INPUT_NETWORK_DIR = "V:/projects/MTC/US0024934.9168/setup_model/model_inputs/2015-tm22-dev-sprint-02" + +# this is location/folder for Emme Netorks (built from Lasso) +INPUT_EMME_NETWORK_DIR = "V:/projects/MTC/US0024934.9168/setup_model/emme_network" + +# location of the populationsim and land use inputs +INPUT_POPLU_DIR = "V:/projects/MTC/US0024934.9168/setup_model/model_inputs/2015-tm22-dev-sprint-02" + +# location of nonres inputs +INPUT_NONRES_DIR = "V:/projects/MTC/US0024934.9168/setup_model/model_inputs/2015-tm22-dev-sprint-02" + +# this is location of demand matrices for warm start +WARMSTART_DEMAND_DIR = "V:/projects/MTC/transit_path_builder_calib/TM2.2.1.2/warmstart" + +# this is location/folder for Emme Netorks +EMME_TEMPLATE_PROJECT_DIR = "V:/projects/MTC/US0024934.9168/setup_model/emme_23_project_template" + +# github raw link for toml config files +CONFIGS_GITHUB_PATH = "https://raw.githubusercontent.com/BayAreaMetro/travel-model-two-config/sprint_02/2015-tm22-dev-sprint-02" + +# travel model two release tag to fetch +TRAVEL_MODEL_TWO_RELEASE_TAG = "TM2.2.2" \ No newline at end of file diff --git a/tm2py/__init__.py b/tm2py/__init__.py index e2c0d299..5dafbbca 100644 --- a/tm2py/__init__.py +++ b/tm2py/__init__.py @@ -11,6 +11,7 @@ from .controller import RunController from .examples import get_example from .logger import Logger, LogStartEnd +from .setup_model.setup import SetupModel __all__ = [ # component @@ -24,6 +25,8 @@ "TimePeriodConfig", # controller "RunController", + # setupmodel + "SetupModel", # logger "Logger", "LogStartEnd", diff --git a/tm2py/acceptance/simulated.py b/tm2py/acceptance/simulated.py index 9a08d660..1f2b430c 100644 --- a/tm2py/acceptance/simulated.py +++ b/tm2py/acceptance/simulated.py @@ -115,9 +115,11 @@ def __init__( scenario_file: str = None, model_file: str = None, on_board_assign_summary: bool = False, + iteration: int = 3, ) -> None: self.c = canonical self.scenario_file = scenario_file + self.iter = iteration self._load_configs(scenario=True, model=False) if not on_board_assign_summary: @@ -605,7 +607,8 @@ def _reduce_simulated_station_to_station(self): df["boarding"] = df["boarding"].astype(int) df["alighting"] = df["alighting"].astype(int) - df["simulated"] = df["simulated"].astype(float) + # there are occasional odd simulated values with characters, such as '309u4181' + df["simulated"] = pd.to_numeric(df['simulated'], errors='coerce').fillna(0) file.close() @@ -1022,7 +1025,7 @@ def _read_transit_demand(self): out_df = pd.DataFrame() for time_period in self.model_time_periods: - filename = os.path.join(dem_dir, "trn_demand_{}.omx".format(time_period)) + filename = os.path.join(dem_dir, "trn_demand_{}_{}.omx".format(time_period, self.iter)) omx_handle = omx.open_file(filename) running_df = None diff --git a/tm2py/components/demand/household.py b/tm2py/components/demand/household.py index 3375acfa..e395c58a 100644 --- a/tm2py/components/demand/household.py +++ b/tm2py/components/demand/household.py @@ -34,6 +34,7 @@ def run(self): self._stop_java() # consume ctramp person trip list and create trip tables for assignment self._prepare_demand_for_assignment() + self._copy_auto_maz_demand() def _prepare_demand_for_assignment(self): prep_demand = PrepareHighwayDemand(self.controller) @@ -77,6 +78,31 @@ def _run_resident_model(self): def _stop_java(): run_process(['taskkill /im "java.exe" /F']) + def _copy_auto_maz_demand(self): + time_period_names = self.time_period_names + + for period in time_period_names: + for maz_group in [1, 2, 3]: + output_path = ( + self.controller.get_abs_path( + self.controller.config.highway.maz_to_maz.demand_file + ) + .__str__() + .format( + period=period, number=maz_group, iter=self.controller.iteration + ) + ) + + input_path = ( + self.controller.get_abs_path( + self.config.highway_maz_ctramp_output_file + ) + .__str__() + .format(period=period, number=maz_group) + ) + + _shutil.copyfile(input_path, output_path) + def _consolidate_demand_for_assign(self): """ CTRAMP writes out demands in separate omx files, e.g. @@ -123,27 +149,7 @@ def _consolidate_demand_for_assign(self): output_omx.close() # auto MAZ - for period in time_period_names: - for maz_group in [1, 2, 3]: - output_path = ( - self.controller.get_abs_path( - self.controller.config.highway.maz_to_maz.demand_file - ) - .__str__() - .format( - period=period, number=maz_group, iter=self.controller.iteration - ) - ) - - input_path = ( - self.controller.get_abs_path( - self.config.highway_maz_ctramp_output_file - ) - .__str__() - .format(period=period, number=maz_group) - ) - - _shutil.copyfile(input_path, output_path) + self._copy_auto_maz_demand() # transit TAP # for period in time_period_names: diff --git a/tm2py/components/demand/temp.ipynb b/tm2py/components/demand/temp.ipynb deleted file mode 100644 index e69de29b..00000000 diff --git a/tm2py/components/network/transit/transit_assign.py b/tm2py/components/network/transit/transit_assign.py index 9dcb997b..c61cac53 100644 --- a/tm2py/components/network/transit/transit_assign.py +++ b/tm2py/components/network/transit/transit_assign.py @@ -452,14 +452,22 @@ def run(self): self.transit_network.update_auto_times(time_period) if self.controller.iteration == 0: + # iteration = 0 : run uncongested transit assignment use_ccr = False congested_transit_assignment = False print("running uncongested transit assignment with warmstart demand") self.run_transit_assign( time_period, use_ccr, congested_transit_assignment ) - - else: # iteration >=1 + elif (self.controller.iteration == 1) & (self.controller.config.warmstart.use_warmstart_skim): + # iteration = 1 and use_warmstart_skim = True : run uncongested transit assignment + use_ccr = False + congested_transit_assignment = False + self.run_transit_assign( + time_period, use_ccr, congested_transit_assignment + ) + else: + # iteration >= 1 and use_warmstart_skim = False : run congested transit assignment use_ccr = self.config.use_ccr if time_period in ["EA", "EV", "MD"]: congested_transit_assignment = False diff --git a/tm2py/config.py b/tm2py/config.py index f64d2fdc..05b1695a 100644 --- a/tm2py/config.py +++ b/tm2py/config.py @@ -322,6 +322,7 @@ class HouseholdConfig(ConfigItem): highway_demand_file: pathlib.Path transit_demand_file: pathlib.Path active_demand_file: pathlib.Path + highway_maz_ctramp_output_file: pathlib.Path OwnedAV_ZPV_factor: float TNC_ZPV_factor: float ctramp_indiv_trip_file: str @@ -1418,7 +1419,7 @@ class EmmeConfig(ConfigItem): active_north_database_path: pathlib.Path active_south_database_path: pathlib.Path transit_database_path: pathlib.Path - num_processors: str = Field(regex=r"^MAX$|^MAX-\d+$|^\d+$") + num_processors: str = Field(pattern=r"^MAX$|^MAX-\d+$|^\d+$") @dataclass(frozen=True) @@ -1477,21 +1478,21 @@ def maz_skim_period_exists(cls, value, values): @validator("highway", always=True) def relative_gap_length(cls, value, values): - """Validate highway.relative_gaps is a list of the same length as global iterations.""" + """Validate highway.relative_gaps is a list of length greater or equal to global iterations.""" if "run" in values: - assert len(value.relative_gaps) == ( + assert len(value.relative_gaps) >= ( values["run"]["end_iteration"] + 1 - ), f"'highway.relative_gaps must be the same length as end_iteration+1,\ + ), f"'highway.relative_gaps must be the same or greater length as end_iteration+1,\ that includes global iteration 0 to {values['run']['end_iteration']}'" return value @validator("transit", always=True) def transit_stop_criteria_length(cls, value, values): - """Validate transit.congested.stop_criteria is a list of the same length as global iterations.""" + """Validate transit.congested.stop_criteria is a list of length greater or equal to global iterations.""" if ("run" in values) & (value.congested_transit_assignment): - assert len(value.congested.stop_criteria) == ( + assert len(value.congested.stop_criteria) >= ( values["run"]["end_iteration"] - ), f"'transit.relative_gaps must be the same length as end_iteration,\ + ), f"'transit.stop_criteria must be the same or greater length as end_iteration,\ that includes global iteration 1 to {values['run']['end_iteration']}'" return value diff --git a/tm2py/controller.py b/tm2py/controller.py index 11bdc019..97dc22d0 100644 --- a/tm2py/controller.py +++ b/tm2py/controller.py @@ -265,21 +265,21 @@ def run_next(self): elif self.config.warmstart.use_warmstart_skim: highway_skim_file = self.get_abs_path( self.config["highway"].output_skim_path - + self.config["highway"].output_skim_filename_tmpl + / self.config["highway"].output_skim_filename_tmpl ).__str__() for time in self.config["time_periods"]: - path = highway_skim_file.format(period=time.name) + path = highway_skim_file.format(time_period=time.name) assert os.path.isfile( path ), f"{path} required as warmstart skim does not exist" transit_skim_file = self.get_abs_path( self.config["transit"].output_skim_path - + self.config["transit"].output_skim_filename_tmpl + / self.config["transit"].output_skim_filename_tmpl ).__str__() for time in self.config["time_periods"]: for tclass in self.config["transit"]["classes"]: path = transit_skim_file.format( - period=time.name, iter=tclass.name + time_period=time.name, tclass=tclass.name ) assert os.path.isfile( path diff --git a/tm2py/setup_model/setup.py b/tm2py/setup_model/setup.py new file mode 100644 index 00000000..7860cd40 --- /dev/null +++ b/tm2py/setup_model/setup.py @@ -0,0 +1,394 @@ +import os +import shutil +import requests +import zipfile +import io +import logging +import toml + + +class SetupModel: + """ + Main operational interface for setup model process. + + + """ + + def __init__(self, config_file): + self.config_file = config_file + # self.logger = logger + + def _setup_logging(self, log_file): + logging.basicConfig( + filename=log_file, + level=logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s", + datefmt="%d-%b-%Y (%H:%M:%S)", + ) + return logging.getLogger() + + def _load_toml(self): + """ + Load config from toml file. + + Args: + toml_path: path for toml file to read + """ + with open(self.config_file, "r", encoding="utf-8") as toml_file: + data = toml.load(toml_file) + return data + + def run_setup(self): + # Read setup configs + config_dict = self._load_toml() + configs = Config(config_dict) + + # Validate configs + configs.validate() + + # Create model directory + model_dir = os.path.join(configs.ROOT_DIR, configs.MODEL_FOLDER_NAME) + model_dir = model_dir.replace("\\", "/") + # TODO: what to do if the directory already exists? + os.mkdir(model_dir) + + # Initialize logging + log_file = os.path.join(model_dir, "setup_log.log") + logger = self._setup_logging(log_file) + + logger.info(f"Starting process to setup MTC model in directory: {model_dir}") + + # List of folders to create + folders_to_create = [ + "acceptance", + "CTRAMP", + "ctramp_output", + "demand_matrices", + "emme_project", + "inputs", + "logs", + "warmstart", + "notebooks", + "output_summaries", + "skim_matrices", + "skim_matrices/highway", + "skim_matrices/transit", + "skim_matrices/non_motorized", + ] + + # Create folder structure + self._create_folder_structure(folders_to_create, model_dir, logger) + + # Copy model inputs + self._copy_model_inputs(configs, model_dir, logger) + + # Copy emme project and database + self._copy_emme_project_and_database(configs, model_dir, logger) + + # Download toml config files from GitHub + config_files_list = [ + "observed_data.toml", + "canonical_crosswalk.toml", + "model_config.toml", + "scenario_config.toml", + ] + acceptance_config_files_list = [ + "observed_data.toml", + "canonical_crosswalk.toml", + ] + + for file in config_files_list: + github_url = os.path.join(configs.CONFIGS_GITHUB_PATH, file) + github_url = github_url.replace("\\", "/") + + local_file = os.path.join( + model_dir, + "acceptance" if file in acceptance_config_files_list else "", + file, + ) + local_file = local_file.replace("\\", "/") + + self._download_file_from_github(github_url, local_file, logger) + + # Fetch required folders from travel model two github release (zip file) + org = "BayAreaMetro" + repo = "travel-model-two" + tag = configs.TRAVEL_MODEL_TWO_RELEASE_TAG + folders_to_extract = ["runtime", "uec"] + + self._download_github_release( + org, + repo, + tag, + folders_to_extract, + os.path.join(model_dir, "CTRAMP"), + logger, + ) + + # Rename 'uec' folder to 'model' + os.rename( + os.path.join(model_dir, "CTRAMP", "uec"), + os.path.join(model_dir, "CTRAMP", "model"), + ) + + logger.info(f"Setup process completed successfully!") + + # Close logging + logging.shutdown() + + def _create_folder_structure(self, folder_names, root_dir, logger): + """ + Creates empty folder structure in the root directory + + Args: + folder_names: list of folders to create + root_dir: root directory for the model + logger: logger + """ + + logger.info(f"Creating folder structure in directory {root_dir}") + + if not os.path.exists(root_dir): + logger.error(f"Directory {root_dir} does not exists.") + raise FileNotFoundError(f"Directory {root_dir} does not exists.") + + for folder in folder_names: + path = os.path.join(root_dir, folder) + os.makedirs(path) + # logger.info(f" Created Empty Folder: {path}") + + def _copy_folder(self, src_dir, dest_dir, logger): + """ + Copies a folder from the source directory to the destination directory. + + Args: + src: source folder + dest: destination folder + logger: logger + """ + + src_dir = src_dir.replace("\\", "/") + dest_dir = dest_dir.replace("\\", "/") + + if os.path.exists(src_dir): + # Copy the entire folder and its contents + try: + # Check if the destination directory exists + if os.path.exists(dest_dir): + # delete the existing destination directory + # Newer versions supports `dirs_exist_ok` but with this version, + # the destination directory must not already exist + shutil.rmtree(dest_dir) + + shutil.copytree(src_dir, dest_dir) + + logger.info(f"Copied folder from {src_dir} to {dest_dir}") + except Exception as e: + logger.error(f"Failed to copy {src_dir} to {dest_dir}: {str(e)}") + raise Exception(f"Failed to copy {src_dir} to {dest_dir}: {str(e)}") + else: + logger.error(f"Source directory {src_dir} to copy from does not exists.") + raise FileNotFoundError( + f"Source directory {src_dir} to copy from does not exists." + ) + + def _download_file_from_github(self, github_url, local_file, logger): + """ + Downloads a file from a GitHub URL. + + Args: + github_url: raw github link for the file to download + local_file: local path for the file to download + logger: logger + """ + try: + response = requests.get(github_url) + response.raise_for_status() + + with open(local_file, "wb") as f: + # write the content of the response (file content) to the local file + f.write(response.content) + logger.info(f"Downloaded file from {github_url} to {local_file}") + except Exception as e: + logger.error(f"Failed to download file {github_url} from GitHub: {str(e)}") + raise Exception( + f"Failed to download file {github_url} from GitHub: {str(e)}" + ) + + def _download_github_release( + self, org_name, repo_name, release_tag, folders_to_extract, local_dir, logger + ): + """ + download a release ZIP from a GitHub repository and extract specified sub-folders to a local directory. + + Args: + org_name: github organization name + repo_name: github repository name + release_tag: release tag + folders_to_extract: list of sub-folders to extract from the ZIP file + local_dir: local directory to save extracted folders + logger: logger + """ + release_url = f"https://github.com/{org_name}/{repo_name}/archive/refs/tags/{release_tag}.zip" + + try: + response = requests.get(release_url) + response.raise_for_status() + + root_folder = f"{repo_name}-{release_tag}" + copied_folder = set([]) + local_dir = local_dir.replace("\\", "/") + + z = zipfile.ZipFile(io.BytesIO(response.content)) + for file_info in z.infolist(): + if not file_info.is_dir(): + if file_info.filename.startswith(root_folder): + file_path = file_info.filename[len(root_folder) + 1 :] + else: + file_path = file_info.filename + + if any( + file_path.startswith(folder) for folder in folders_to_extract + ): + # create the local path to extract the file + extract_path = os.path.join(local_dir, file_path) + extract_path = extract_path.replace("\\", "/") + + # ensure the directory exists + os.makedirs(os.path.dirname(extract_path), exist_ok=True) + + # extract the file + with z.open(file_info.filename) as source, open( + extract_path, "wb" + ) as target: + target.write(source.read()) + + copied_folder.add(file_path.split("/")[0]) + + if copied_folder is not None: + logger.info( + f"Extracted folders {copied_folder} from GitHub release {release_url} and to directory {local_dir}" + ) + + except Exception as e: + logger.error(f"Failed to download GitHub release {release_url}: {str(e)}") + raise Exception( + f"Failed to download GitHub release {release_url}: {str(e)}" + ) + + def _copy_model_inputs(self, configs, model_dir, logger): + """ + copy required model inputs into their respective directories. + + Args: + configs: setup model configurations + model_dir: path to model directory + logger: logger + """ + # Copy hwy and trn networks + self._copy_folder( + os.path.join(configs.INPUT_NETWORK_DIR, "hwy"), + os.path.join(model_dir, "inputs", "hwy"), + logger, + ) + self._copy_folder( + os.path.join(configs.INPUT_NETWORK_DIR, "trn"), + os.path.join(model_dir, "inputs", "trn"), + logger, + ) + + # Copy popsyn and landuse inputs + self._copy_folder( + os.path.join(configs.INPUT_POPLU_DIR, "popsyn"), + os.path.join(model_dir, "inputs", "popsyn"), + logger, + ) + self._copy_folder( + os.path.join(configs.INPUT_POPLU_DIR, "landuse"), + os.path.join(model_dir, "inputs", "landuse"), + logger, + ) + + # Copy nonres inputs + self._copy_folder( + os.path.join(configs.INPUT_NONRES_DIR, "nonres"), + os.path.join(model_dir, "inputs", "nonres"), + logger, + ) + + # Copy warmstart demand + self._copy_folder( + configs.WARMSTART_DEMAND_DIR, os.path.join(model_dir, "warmstart"), logger + ) + + def _copy_emme_project_and_database(self, configs, model_dir, logger): + """ + copy emme projects from template project and then copy the emme networks databases + + Args: + configs: setup model configurations + model_dir: path to model directory + ogger: logger + """ + # copy template emme project + self._copy_folder( + configs.EMME_TEMPLATE_PROJECT_DIR, + os.path.join(model_dir, "emme_project"), + logger, + ) + + # copy emme network database + self._copy_folder( + os.path.join( + configs.INPUT_EMME_NETWORK_DIR, "emme_drive_network", "Database" + ), + os.path.join(model_dir, "emme_project", "Database_highway"), + logger, + ) + self._copy_folder( + os.path.join( + configs.INPUT_EMME_NETWORK_DIR, "emme_taz_transit_network", "Database" + ), + os.path.join(model_dir, "emme_project", "Database_transit"), + logger, + ) + self._copy_folder( + os.path.join( + configs.INPUT_EMME_NETWORK_DIR, + "emme_maz_active_modes_network_subregion_north", + "Database", + ), + os.path.join(model_dir, "emme_project", "Database_active_north"), + logger, + ) + self._copy_folder( + os.path.join( + configs.INPUT_EMME_NETWORK_DIR, + "emme_maz_active_modes_network_subregion_south", + "Database", + ), + os.path.join(model_dir, "emme_project", "Database_active_south"), + logger, + ) + + +class Config: + def __init__(self, config_dict): + for key, value in config_dict.items(): + setattr(self, key, value) + + def validate(self): + # validate setup configuration + required_attrs = [ + "ROOT_DIR", + "MODEL_FOLDER_NAME", + "INPUT_NETWORK_DIR", + "INPUT_POPLU_DIR", + "WARMSTART_DEMAND_DIR", + "CONFIGS_GITHUB_PATH", + "TRAVEL_MODEL_TWO_RELEASE_TAG", + ] + + for attr in required_attrs: + if not getattr(self, attr, None): + raise ValueError(f"{attr} is required in the setup configuration!")