From 40137fdb1ba32b7e5eaac9323f76328028e4af2e Mon Sep 17 00:00:00 2001 From: pesap Date: Tue, 21 Jan 2025 20:39:14 -0700 Subject: [PATCH] fix: Infrasys json serialization and compatibiility fixes (#106) This PR bumps version of infrasys to 0.2.1 and fixes some of the incompatibility with the new version. It also adds some fixes for #104 and #105. --- pyproject.toml | 2 +- src/r2x/cli_functions.py | 7 +++-- src/r2x/config_utils.py | 4 +++ src/r2x/defaults/reeds_input.json | 14 ++++----- src/r2x/enums.py | 1 + src/r2x/exporter/handler.py | 6 ++-- src/r2x/exporter/plexos.py | 17 +++++++---- src/r2x/parser/handler.py | 8 ++++-- src/r2x/parser/plexos.py | 2 +- src/r2x/parser/reeds.py | 48 ++++++++++++++++++++++++------- src/r2x/runner.py | 4 +++ 11 files changed, 82 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bfb096f9..a59aaf0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "infrasys~=0.1.1", + "infrasys~=0.2.3", "jsonschema~=4.23", "loguru~=0.7.2", "pandas~=2.2", diff --git a/src/r2x/cli_functions.py b/src/r2x/cli_functions.py index 1b1884e2..039b4584 100644 --- a/src/r2x/cli_functions.py +++ b/src/r2x/cli_functions.py @@ -63,7 +63,10 @@ def get_additional_arguments( raise NotImplementedError(msg) _, package_name, script_name = package_str - package_script = importlib.import_module(f"{package}") + try: + package_script = importlib.import_module(f"{package}") + except ImportError: + continue if hasattr(package_script, "cli_arguments"): script_cli_group = parser.add_argument_group(f"{package_name.upper()}: {script_name}") package_script.cli_arguments(script_cli_group) @@ -125,7 +128,7 @@ def base_cli() -> argparse.ArgumentParser: group_cli.add_argument( "--output-model", dest="output_model", - choices=["plexos", "sienna", "pras"], + choices=["plexos", "sienna", "infrasys"], help="Output model to convert to", ) group_cli.add_argument( diff --git a/src/r2x/config_utils.py b/src/r2x/config_utils.py index 28421ff8..234b390b 100644 --- a/src/r2x/config_utils.py +++ b/src/r2x/config_utils.py @@ -38,6 +38,10 @@ def get_input_defaults(model_enum: Models) -> dict: def get_output_defaults(model_enum: Models) -> dict: """Return configuration dicitonary based on the output model.""" match model_enum: + case Models.INFRASYS: + # NOTE: Here we will add any infrasys configuration if we need in the future. + defaults_dict = None + pass case Models.PLEXOS: defaults_dict = ( read_json("r2x/defaults/plexos_output.json") diff --git a/src/r2x/defaults/reeds_input.json b/src/r2x/defaults/reeds_input.json index 82ae3ccd..b1de04ec 100644 --- a/src/r2x/defaults/reeds_input.json +++ b/src/r2x/defaults/reeds_input.json @@ -141,14 +141,14 @@ "SPINNING": 3 }, "reserve_time_frame": { - "FLEXIBILITY": 600, - "REGULATION": 1200, - "SPINNING": 300 + "FLEXIBILITY": 3600, + "REGULATION": 300, + "SPINNING": 600 }, "reserve_vors": { - "FLEXIBILITY": 390000, - "REGULATION": 410000, - "SPINNING": 400000 + "FLEXIBILITY": 3900, + "REGULATION": 4100, + "SPINNING": 4000 }, "season_map": { "fall": "fall", @@ -389,7 +389,7 @@ ], "wind_reserves": { "FLEXIBILITY": 0.1, - "REGULATION": 0.05, + "REGULATION": 0.005, "SPINNING": 0 } } diff --git a/src/r2x/enums.py b/src/r2x/enums.py index edbfe394..3a3ee6d0 100644 --- a/src/r2x/enums.py +++ b/src/r2x/enums.py @@ -31,6 +31,7 @@ class ACBusTypes(StrEnum): PQ = "PQ" REF = "REF" SLACK = "SLACK" + ISOLATED = "ISOLATED" class PrimeMoversType(StrEnum): diff --git a/src/r2x/exporter/handler.py b/src/r2x/exporter/handler.py index 5a8b614b..0258b69c 100644 --- a/src/r2x/exporter/handler.py +++ b/src/r2x/exporter/handler.py @@ -11,6 +11,7 @@ import numpy as np import infrasys from loguru import logger +from pint import Quantity from r2x.api import System from r2x.config_scenario import Scenario @@ -161,8 +162,9 @@ def export_data_files(self, year: int, time_series_folder: str = "Data") -> None string_template = string.Template(csv_fname) for component_type, (datetime_array, time_series) in datetime_arrays.items(): - time_series_arrays = list(map(lambda x: x.data.to_numpy(), time_series)) - + time_series_arrays = list( + map(lambda x: x.data.magnitude if isinstance(x.data, Quantity) else x.data, time_series) + ) config_dict["component_type"] = component_type csv_fname = string_template.safe_substitute(config_dict) csv_table = np.column_stack([datetime_array, *time_series_arrays]) diff --git a/src/r2x/exporter/plexos.py b/src/r2x/exporter/plexos.py index 5579fc3f..7641dcfc 100644 --- a/src/r2x/exporter/plexos.py +++ b/src/r2x/exporter/plexos.py @@ -49,7 +49,7 @@ from r2x.units import get_magnitude from r2x.utils import custom_attrgetter, get_enum_from_string, read_json -NESTED_ATTRIBUTES = ["ext", "bus", "services"] +NESTED_ATTRIBUTES = {"ext", "bus", "services"} TIME_SERIES_PROPERTIES = ["Min Provision", "Static Risk"] DEFAULT_XML_TEMPLATE = "master_9.2R6_btu.xml" EXT_PROPERTIES = {"UoS Charge", "Fixed Load"} @@ -226,7 +226,7 @@ def insert_component_properties( filter_func: Callable | None = None, scenario: str | None = None, records: list[dict] | None = None, - exclude_fields: list[str] | None = NESTED_ATTRIBUTES, + exclude_fields: set[str] | None = NESTED_ATTRIBUTES, ) -> None: """Bulk insert properties from selected component type.""" logger.debug("Adding {} table properties...", component_type.__name__) @@ -437,6 +437,7 @@ def add_topology(self) -> None: # Add node memberships to zone and regions. # On our default Plexos translation, both Zones and Regions are child of the Node class. for bus in self.system.get_components(ACBus): + bus_load_zone = bus.load_zone self._db_mgr.add_membership( bus.name, bus.name, # Zone has the same name @@ -444,9 +445,11 @@ def add_topology(self) -> None: child_class=ClassEnum.Region, collection=CollectionEnum.Region, ) + if bus_load_zone is None: + continue self._db_mgr.add_membership( bus.name, - bus.load_zone.name, + bus_load_zone.name, parent_class=ClassEnum.Node, child_class=ClassEnum.Zone, collection=CollectionEnum.Zone, @@ -663,7 +666,7 @@ def add_reserves(self) -> None: Reserve, parent_class=ClassEnum.System, collection=CollectionEnum.Reserves, - exclude_fields=[*NESTED_ATTRIBUTES, "max_requirement"], + exclude_fields=NESTED_ATTRIBUTES | {"max_requirement"}, ) for reserve in self.system.get_components(Reserve): properties: dict[str, Any] = {} @@ -700,13 +703,15 @@ def add_reserves(self) -> None: # Add Regions properties. Currently, we only add the load_risk component_dict = reserve.model_dump( - exclude_none=True, exclude=[*NESTED_ATTRIBUTES, "max_requirement"] + exclude_none=True, exclude=NESTED_ATTRIBUTES | {"max_requirement"} ) if not reserve.region: return + reserve_region = reserve.region + assert reserve_region is not None regions = self.system.get_components( - ACBus, filter_func=lambda x: x.load_zone.name == reserve.region.name + ACBus, filter_func=lambda x: x.load_zone.name == reserve_region.name ) collection_properties = self._db_mgr.get_valid_properties( diff --git a/src/r2x/parser/handler.py b/src/r2x/parser/handler.py index 3aae5f82..aabef042 100644 --- a/src/r2x/parser/handler.py +++ b/src/r2x/parser/handler.py @@ -4,6 +4,7 @@ """ # System packages +import json from copy import deepcopy import inspect from dataclasses import dataclass, field @@ -153,18 +154,21 @@ def file_handler( logger.warning("Skipping optional file {}", fpath) return None + logger.trace("Reading {}", fpath) match fpath.suffix: case ".csv": - logger.trace("Reading {}", fpath) return csv_handler(fpath, **kwargs) case ".h5": - logger.trace("Reading {}", fpath) return pl.LazyFrame(pd.read_hdf(fpath).reset_index()) # type: ignore case ".xml": class_kwargs = { key: value for key, value in kwargs.items() if key in inspect.signature(XMLHandler).parameters } return XMLHandler.parse(fpath=fpath, **class_kwargs) + case ".json": + with open(fpath) as json_file: + data = json.load(json_file) + return data case _: raise NotImplementedError(f"File {fpath.suffix = } not yet supported.") diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 28f93f12..50068b82 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -1468,7 +1468,7 @@ def _parse_value(self, value: Any, variable_name: str | None = None, unit: str | resolution = timedelta(hours=1) return SingleTimeSeries( - data=ureg.Quantity(value, unit) if unit else value, + data=ureg.Quantity(value, unit) if unit else value, # type: ignore variable_name=variable_name, initial_time=initial_time, resolution=resolution, diff --git a/src/r2x/parser/reeds.py b/src/r2x/parser/reeds.py index cacdeafb..6d799b83 100644 --- a/src/r2x/parser/reeds.py +++ b/src/r2x/parser/reeds.py @@ -174,7 +174,7 @@ def _construct_branches(self): for idx, branch in enumerate(branch_data.iter_rows(named=True)): from_bus = self.system.get_component(ACBus, branch["from_bus"]) to_bus = self.system.get_component(ACBus, branch["to_bus"]) - branch_name = f"{idx+1:>04}-{branch['from_bus']}-{branch['to_bus']}" + branch_name = f"{idx + 1:>04}-{branch['from_bus']}-{branch['to_bus']}" reverse_key = (branch["kind"], branch["from_bus"], branch["to_bus"]) if reverse_key in reverse_lines: continue @@ -440,13 +440,15 @@ def _construct_generators(self) -> None: # noqa: C901 ) bus = self.system.get_component(ACBus, name=row["region"]) row["bus"] = bus + bus_load_zone = bus.load_zone + assert bus_load_zone is not None # Add reserves/services to generator if they are not excluded if row["tech"] not in self.reeds_config.defaults["excluded_reserve_techs"]: row["services"] = list( self.system.get_components( Reserve, - filter_func=lambda x: x.region.name == bus.load_zone.name, + filter_func=lambda x: x.region.name == bus_load_zone.name, ) ) reserve_map = self.system.get_component(ReserveMap, name="reserve_map") @@ -633,22 +635,42 @@ def _construct_reserve_provision(self): solar_reserves = list( map( getattr, - map(self.system.get_time_series, provision_objects["solar"]), - repeat("data"), + map( + getattr, + map(self.system.get_time_series, provision_objects["solar"]), + repeat("data"), + ), + repeat("magnitude"), + ) + ) + solar_capacity = list( + map( + lambda component: self.system.get_component_by_label( + component.label + ).active_power.magnitude, + provision_objects["solar"], ) ) wind_reserves = list( map( getattr, - map(self.system.get_time_series, provision_objects["wind"]), - repeat("data"), + map( + getattr, + map(self.system.get_time_series, provision_objects["wind"]), + repeat("data"), + ), + repeat("magnitude"), ) ) load_reserves = list( map( getattr, - map(self.system.get_time_series, provision_objects["load"]), - repeat("data"), + map( + getattr, + map(self.system.get_time_series, provision_objects["load"]), + repeat("data"), + ), + repeat("magnitude"), ) ) wind_provision = ( @@ -661,6 +683,8 @@ def _construct_reserve_provision(self): pa.Table.from_arrays(solar_reserves, names=solar_names) .to_pandas() .sum(axis=1) + .apply(lambda x: 1 if x != 0 else 0) + .mul(sum(solar_capacity)) .mul(self.reeds_config.defaults["solar_reserves"].get(reserve.reserve_type.name, 1)) ) load_provision = ( @@ -715,7 +739,9 @@ def _construct_hydro_budgets(self) -> None: if generator.category == "can-imports": continue tech = generator.ext["reeds_tech"] - region = generator.bus.name + generator_bus = generator.bus + assert generator_bus + region = generator_bus.name hydro_ratings = hydro_data.filter((pl.col("tech") == tech) & (pl.col("region") == region)) hourly_time_series = np.zeros(len(month_of_day), dtype=float) @@ -761,7 +787,9 @@ def _construct_hydro_rating_profiles(self) -> None: initial_time = datetime(self.weather_year, 1, 1) for generator in self.system.get_components(HydroEnergyReservoir): tech = generator.ext["reeds_tech"] - region = generator.bus.name + generator_bus = generator.bus + assert generator_bus is not None + region = generator_bus.name hourly_time_series = np.zeros(len(month_of_hour), dtype=float) hydro_ratings = hydro_data.filter((pl.col("tech") == tech) & (pl.col("region") == region)) diff --git a/src/r2x/runner.py b/src/r2x/runner.py index 7c622963..a7aea792 100644 --- a/src/r2x/runner.py +++ b/src/r2x/runner.py @@ -127,6 +127,10 @@ def run_single_scenario(scenario: Scenario, **kwargs) -> None: system.to_json(output_fpath, overwrite=True) system = System.from_json(output_fpath) + if scenario.output_model == "infrasys": + logger.info("Serialize system to {}", output_fpath) + system.to_json(output_fpath, overwrite=True) + return run_exporter(config=scenario, system=system) return