From c3beb83cd7984c158bf6f1d6ffa407bd94b91567 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:47:16 -0700 Subject: [PATCH 01/15] Cost Function definition and various fixes for internal merge --- src/r2x/api.py | 1 + src/r2x/defaults/config.json | 2 +- src/r2x/defaults/plexos_input.json | 3 +- src/r2x/exporter/sienna.py | 18 ++++----- src/r2x/models/generators.py | 2 +- src/r2x/parser/plexos.py | 60 +++++++++++++++++++----------- src/r2x/utils.py | 5 ++- 7 files changed, 54 insertions(+), 37 deletions(-) diff --git a/src/r2x/api.py b/src/r2x/api.py index edc6c27e..f73af20b 100644 --- a/src/r2x/api.py +++ b/src/r2x/api.py @@ -78,6 +78,7 @@ def _add_operation_cost_data( # noqa: C901 continue operation_cost = sub_dict["operation_cost"] + for cost_field_key, cost_field_value in operation_cost.items(): if isinstance(cost_field_value, dict): assert ( diff --git a/src/r2x/defaults/config.json b/src/r2x/defaults/config.json index 591887a2..48dac4a6 100644 --- a/src/r2x/defaults/config.json +++ b/src/r2x/defaults/config.json @@ -106,7 +106,7 @@ "type": "HA" } ], - "RenewableFix": [ + "RenewableNonDispatch": [ { "fuel": "SOLAR", "type": "RTPV" diff --git a/src/r2x/defaults/plexos_input.json b/src/r2x/defaults/plexos_input.json index 765508d6..ec012b40 100644 --- a/src/r2x/defaults/plexos_input.json +++ b/src/r2x/defaults/plexos_input.json @@ -1,4 +1,5 @@ { + "device_name_inference_map": {}, "plexos_device_map": {}, "plexos_fuel_map": {}, "plexos_property_map": { @@ -17,7 +18,7 @@ "Load Risk": "load_risk", "Loss Incr": "losses", "Maintenance Rate": "planned_outage_rate", - "Max Capacity": "base_power", + "Max Capacity": "active_power", "Max Flow": "max_power_flow", "Max Ramp Down": "ramp_down", "Max Ramp Up": "ramp_up", diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index 64973e09..b5d3d9ac 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -120,7 +120,7 @@ def process_load_data(self, fname: str = "load.csv") -> None: "reactive_power", "max_active_power", "max_reeactive_power", - "base_power", + "active_power", ] self.system.export_component_to_csv( PowerLoad, @@ -230,9 +230,7 @@ def process_gen_data(self, fname="gen.csv"): "fuel", "rating", "unit_type", - "base_power", - # "active_power_limits_max", - # "active_power_limits_min", + "active_power", "min_rated_capacity", "min_down_time", "min_up_time", @@ -333,7 +331,7 @@ def process_storage_data(self, fname="storage.csv") -> None: "available", "generator_name", "bus_id", - "base_power", + "active_power", "rating", "input_efficiency", "output_efficiency", @@ -363,13 +361,13 @@ def process_storage_data(self, fname="storage.csv") -> None: for storage in storage_list: output_dict = storage output_dict["generator_name"] = storage["name"] - output_dict["input_active_power_limit_max"] = output_dict["base_power"] - output_dict["output_active_power_limit_max"] = output_dict["base_power"] + output_dict["input_active_power_limit_max"] = output_dict["active_power"] + output_dict["output_active_power_limit_max"] = output_dict["active_power"] # NOTE: If we need to change this in the future, we could probably # use the function max to check if the component has the field. - output_dict["input_active_power_limit_min"] = 0 # output_dict["base_power"] - output_dict["output_active_power_limit_min"] = 0 # output_dict["base_power"] - output_dict["base_power"] = output_dict["base_power"] + output_dict["input_active_power_limit_min"] = 0 # output_dict["active_power"] + output_dict["output_active_power_limit_min"] = 0 # output_dict["active_power"] + output_dict["active_power"] = output_dict["active_power"] output_dict["bus_id"] = getattr(self.system.get_component_by_label(output_dict["bus"]), "number") output_dict["rating"] = output_dict["rating"] diff --git a/src/r2x/models/generators.py b/src/r2x/models/generators.py index 5402874a..6e0bc52b 100644 --- a/src/r2x/models/generators.py +++ b/src/r2x/models/generators.py @@ -27,7 +27,7 @@ class Generator(Device): bus: Annotated[ACBus, Field(description="Bus where the generator is connected.")] | None = None rating: Annotated[ ApparentPower | None, - Field(gt=0, description="Maximum output power rating of the unit (MVA)."), + Field(ge=0, description="Maximum output power rating of the unit (MVA)."), ] = None active_power: Annotated[ ActivePower, diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index f2d17471..39dfd027 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -7,13 +7,14 @@ from argparse import ArgumentParser import pandas as pd -from pint import UndefinedUnitError +from pint import UndefinedUnitError, Quantity import polars as pl import numpy as np from loguru import logger from infrasys.exceptions import ISNotStored from infrasys.time_series_models import SingleTimeSeries from infrasys.value_curves import InputOutputCurve, AverageRateCurve +from infrasys.cost_curves import FuelCurve from infrasys.function_data import ( LinearFunctionData, QuadraticFunctionData, @@ -26,6 +27,7 @@ from r2x.exceptions import ModelError, ParserError from plexosdb import PlexosSQLite from plexosdb.enums import ClassEnum, CollectionEnum +from r2x.models.costs import ThermalGenerationCost from r2x.models import ( ACBus, Generator, @@ -132,11 +134,11 @@ def __init__(self, *args, xml_file: str | None = None, **kwargs) -> None: super().__init__(*args, **kwargs) assert self.config.run_folder self.run_folder = Path(self.config.run_folder) - self.system = System(name=self.config.name) + self.system = System(name=self.config.name, auto_add_composed_components=True) self.property_map = self.config.defaults["plexos_property_map"] self.device_map = self.config.defaults["plexos_device_map"] self.fuel_map = self.config.defaults["plexos_fuel_map"] - self.device_match_string = self.config.defaults["device_inference_string"] + self.device_match_string = self.config.defaults["device_name_inference_map"] # TODO(pesap): Rename exceptions to include R2X # https://github.com/NREL/R2X/issues/5 @@ -353,7 +355,7 @@ def _construct_buses(self, default_model=ACBus) -> None: valid_fields["base_voltage"] = ( 230.0 if not valid_fields.get("base_voltage") else valid_fields["base_voltage"] ) - self.system.add_component(default_model(id=idx + 1, **valid_fields)) + self.system.add_component(default_model(number=idx + 1, **valid_fields)) return def _construct_reserves(self, default_model=Reserve): @@ -423,6 +425,7 @@ def _construct_branches(self, default_model=MonitoredLine): ) for line in lines_pivot.iter_rows(named=True): line_properties_mapped = {self.property_map.get(key, key): value for key, value in line.items()} + line_properties_mapped["rating"] = line_properties_mapped.pop("max_power_flow", None) line_properties_mapped["rating_up"] = line_properties_mapped.pop("max_power_flow", None) line_properties_mapped["rating_down"] = line_properties_mapped.pop("min_power_flow", None) @@ -455,7 +458,7 @@ def _construct_branches(self, default_model=MonitoredLine): return def _infer_model_type(self, generator_name): - inference_mapper = self.config.device_name_inference_map + inference_mapper = self.device_match_string generator_name_lower = generator_name.lower() for key, device_info in inference_mapper.items(): if key in generator_name_lower: @@ -474,8 +477,8 @@ def _get_fuel_pmtype(self, generator_name, generator_fuel_map): logger.trace("Parsing generator = {} with fuel type = {}", generator_name, plexos_fuel_name) fuel_pmtype = ( - self.config.device_map.get(generator_name) - or self.config.plexos_fuel_map.get(plexos_fuel_name) + self.device_map.get(generator_name) + or self.fuel_map.get(plexos_fuel_name) or self._infer_model_type(generator_name) ) @@ -565,9 +568,9 @@ def _construct_generators(self): mapped_records["prime_mover_type"] = PrimeMoversType.PS # Pumped Storage generators are not required to have Max Capacity property - if "base_power" not in mapped_records and "pump_load" in mapped_records: - mapped_records["base_power"] = mapped_records["pump_load"] - + if "active_power" not in mapped_records and "pump_load" in mapped_records: + mapped_records["active_power"] = mapped_records["pump_load"] + # NOTE print which missing fields if not all(key in mapped_records for key in required_fields): logger.warning( "Skipping Generator {} since it does not have all the required fields", generator_name @@ -578,9 +581,9 @@ def _construct_generators(self): if mapped_records is None: continue # Pass if not available + mapped_records["fuel_price"] = fuel_prices.get(generator_fuel_map.get(generator_name), 0) mapped_records = self._construct_value_curves(mapped_records, generator_name) - mapped_records["fuel_price"] = fuel_prices.get(generator_fuel_map.get(generator_name)) - + mapped_records = self._construct_operating_costs(mapped_records, generator_name) valid_fields, ext_data = self._field_filter(mapped_records, model_map.model_fields) ts_fields = {k: v for k, v in mapped_records.items() if isinstance(v, SingleTimeSeries)} @@ -684,7 +687,7 @@ def _construct_batteries(self): mapped_records["name"] = battery_name if "Max Power" in mapped_records: - mapped_records["base_power"] = mapped_records["Max Power"] + mapped_records["active_power"] = mapped_records["Max Power"] if "Capacity" in mapped_records: mapped_records["storage_capacity"] = mapped_records["Capacity"] @@ -881,7 +884,20 @@ def _construct_value_curves(self, mapped_records, generator_name): fn = None if not vc: vc = InputOutputCurve(name=f"{generator_name}_HR", function_data=fn) - mapped_records["heat_rate"] = vc + mapped_records["hr_value_curve"] = vc + return mapped_records + + def _construct_operating_costs(self, mapped_records, generator_name): + """Construct operating costs from Value Curves and Operating Costs.""" + if mapped_records.get("hr_value_curve"): + hr_curve = mapped_records["hr_value_curve"] + fuel_cost = mapped_records["fuel_price"] + if isinstance(fuel_cost, SingleTimeSeries): + fuel_cost = np.mean(fuel_cost.data) + elif isinstance(fuel_cost, Quantity): + fuel_cost = fuel_cost.magnitude + fuel_curve = FuelCurve(value_curve=hr_curve, fuel_cost=fuel_cost) + mapped_records["operation_cost"] = ThermalGenerationCost(variable=fuel_curve) return mapped_records def _select_model_name(self): @@ -943,17 +959,17 @@ def _set_unit_availability(self, records): """Set availability and active power limit TS for generators.""" availability = records.get("available", None) if availability is not None and availability > 0: - # Set availability, base_power, storage_capacity as multiplier of availability + # Set availability, active_power, storage_capacity as multiplier of availability if records.get("storage_capacity") is not None: records["storage_capacity"] *= records.get("available") - records["base_power"] = records.get("base_power") * records.get("available") + records["active_power"] = records.get("active_power") * records.get("available") records["available"] = 1 # Set active power limits rating_factor = records.get("Rating Factor", 100) rating_factor = self._apply_action(np.divide, rating_factor, 100) rating = records.get("rating", None) - base_power = records.get("base_power", None) + active_power = records.get("active_power", None) min_energy_hour = records.get("Min Energy Hour", None) # Hack temporary until Hydro Max Monthly Rating is corrected @@ -964,9 +980,9 @@ def _set_unit_availability(self, records): if rating is not None: units = rating.units val = rating_factor * rating.magnitude - elif base_power is not None: - units = base_power.units - val = self._apply_action(np.multiply, rating_factor, base_power.magnitude) + elif active_power is not None: + units = active_power.units + val = self._apply_action(np.multiply, rating_factor, active_power.magnitude) else: return records val = self._apply_unit(val, units) @@ -975,8 +991,8 @@ def _set_unit_availability(self, records): records["min_active_power"] = records.pop("Min Energy Hour") if not isinstance(val, SingleTimeSeries): - # temp solution until Infrasys.model update to have both base_power and a rating field - records["base_power"] = val + # temp solution until Infrasys.model update to have both active_power and a rating field + records["active_power"] = val else: val.variable_name = "max_active_power" records["max_active_power"] = val diff --git a/src/r2x/utils.py b/src/r2x/utils.py index 6487cf1d..c96f2af9 100644 --- a/src/r2x/utils.py +++ b/src/r2x/utils.py @@ -162,8 +162,9 @@ def update_dict(base_dict: dict, override_dict: ChainMap | dict | None = None) - "static_models", "tech_map", "model_map", - "tech_fuel_pm_map", - "device_map", + "plexos_fuel_map", + "device_name_inference_map", + "plexos_device_map", ] for key, value in override_dict.items(): if key in base_dict and all(replace_key not in key for replace_key in _replace_keys): From 6e56af7805e03e07e9731706eab7aa31aa8c8abb Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:53:52 -0700 Subject: [PATCH 02/15] Makes placeholders for cost functions of each model type. Output VOM. --- src/r2x/api.py | 1 - src/r2x/exporter/sienna.py | 1 + src/r2x/parser/plexos.py | 54 ++++++++++++++++++++++++-------------- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/r2x/api.py b/src/r2x/api.py index f73af20b..edc6c27e 100644 --- a/src/r2x/api.py +++ b/src/r2x/api.py @@ -78,7 +78,6 @@ def _add_operation_cost_data( # noqa: C901 continue operation_cost = sub_dict["operation_cost"] - for cost_field_key, cost_field_value in operation_cost.items(): if isinstance(cost_field_value, dict): assert ( diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index b5d3d9ac..d5986f3c 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -242,6 +242,7 @@ def process_gen_data(self, fname="gen.csv"): "category", "must_run", "pump_load", + "vom_price", "operation_cost", ] diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 39dfd027..52ccd8fd 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -27,12 +27,11 @@ from r2x.exceptions import ModelError, ParserError from plexosdb import PlexosSQLite from plexosdb.enums import ClassEnum, CollectionEnum -from r2x.models.costs import ThermalGenerationCost +from r2x.models.costs import ThermalGenerationCost, RenewableGenerationCost, HydroGenerationCost from r2x.models import ( ACBus, Generator, GenericBattery, - HydroPumpedStorage, MonitoredLine, PowerLoad, Reserve, @@ -40,6 +39,10 @@ ReserveMap, TransmissionInterface, TransmissionInterfaceMap, + RenewableGen, + ThermalGen, + HydroDispatch, + HydroPumpedStorage, ) from r2x.utils import validate_string @@ -277,7 +280,6 @@ def _get_fuel_prices(self): ] ].to_dicts() - logger.debug("Parsing fuel = {}", fuel_name) for property in property_records: property.update({"property_unit": "$/MMBtu"}) @@ -582,8 +584,8 @@ def _construct_generators(self): continue # Pass if not available mapped_records["fuel_price"] = fuel_prices.get(generator_fuel_map.get(generator_name), 0) - mapped_records = self._construct_value_curves(mapped_records, generator_name) - mapped_records = self._construct_operating_costs(mapped_records, generator_name) + mapped_records = self._construct_operating_costs(mapped_records, generator_name, model_map) + valid_fields, ext_data = self._field_filter(mapped_records, model_map.model_fields) ts_fields = {k: v for k, v in mapped_records.items() if isinstance(v, SingleTimeSeries)} @@ -887,17 +889,29 @@ def _construct_value_curves(self, mapped_records, generator_name): mapped_records["hr_value_curve"] = vc return mapped_records - def _construct_operating_costs(self, mapped_records, generator_name): + def _construct_operating_costs(self, mapped_records, generator_name, model_map): """Construct operating costs from Value Curves and Operating Costs.""" - if mapped_records.get("hr_value_curve"): - hr_curve = mapped_records["hr_value_curve"] - fuel_cost = mapped_records["fuel_price"] - if isinstance(fuel_cost, SingleTimeSeries): - fuel_cost = np.mean(fuel_cost.data) - elif isinstance(fuel_cost, Quantity): - fuel_cost = fuel_cost.magnitude - fuel_curve = FuelCurve(value_curve=hr_curve, fuel_cost=fuel_cost) - mapped_records["operation_cost"] = ThermalGenerationCost(variable=fuel_curve) + mapped_records = self._construct_value_curves(mapped_records, generator_name) + hr_curve = mapped_records.get("hr_value_curve") + if issubclass(model_map, RenewableGen): + mapped_records["operation_cost"] = RenewableGenerationCost() + elif issubclass(model_map, ThermalGen): + if hr_curve: + fuel_cost = mapped_records["fuel_price"] + if isinstance(fuel_cost, SingleTimeSeries): + fuel_cost = np.mean(fuel_cost.data) + elif isinstance(fuel_cost, Quantity): + fuel_cost = fuel_cost.magnitude + fuel_curve = FuelCurve(value_curve=hr_curve, fuel_cost=fuel_cost) + mapped_records["operation_cost"] = ThermalGenerationCost(variable=fuel_curve) + else: + logger.warning("No heat rate curve found for generator={}", generator_name) + elif issubclass(model_map, HydroDispatch): + mapped_records["operation_cost"] = HydroGenerationCost() + else: + logger.warning( + "Operating Cost not implemented for generator={} model map={}", generator_name, model_map + ) return mapped_records def _select_model_name(self): @@ -990,12 +1004,14 @@ def _set_unit_availability(self, records): if min_energy_hour is not None: records["min_active_power"] = records.pop("Min Energy Hour") - if not isinstance(val, SingleTimeSeries): - # temp solution until Infrasys.model update to have both active_power and a rating field - records["active_power"] = val - else: + if isinstance(val, SingleTimeSeries): val.variable_name = "max_active_power" records["max_active_power"] = val + records["rating"] = active_power + else: + # Assume reactive power not modeled + records["active_power"] = val + records["rating"] = val if isinstance(rating_factor, SingleTimeSeries): records.pop("Rating Factor") From 98f8b9a3cca21565a6bca45cf4a7f2927cf2ffe9 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Thu, 5 Sep 2024 08:42:57 -0700 Subject: [PATCH 03/15] address comments --- src/r2x/exporter/sienna.py | 4 ++-- src/r2x/parser/plexos.py | 28 +++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index d5986f3c..0e8e4950 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -425,8 +425,8 @@ def create_timeseries_pointers(self) -> None: "normalization_factor": 1.0, "resolution": resolution, "name": variable_name, - "scaling_factor_multiplier_module": "PowerSystems", - "scaling_factor_multiplier": "get_" + variable_name, + "scaling_factor_multiplier_module": None, + "scaling_factor_multiplier": None, } ts_pointers_list.append(ts_pointers) diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 52ccd8fd..da3f0a8e 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -581,7 +581,8 @@ def _construct_generators(self): mapped_records = self._set_unit_availability(mapped_records) if mapped_records is None: - continue # Pass if not available + # When unit availability is not set, we skip the generator + continue mapped_records["fuel_price"] = fuel_prices.get(generator_fuel_map.get(generator_name), 0) mapped_records = self._construct_operating_costs(mapped_records, generator_name, model_map) @@ -893,6 +894,31 @@ def _construct_operating_costs(self, mapped_records, generator_name, model_map): """Construct operating costs from Value Curves and Operating Costs.""" mapped_records = self._construct_value_curves(mapped_records, generator_name) hr_curve = mapped_records.get("hr_value_curve") + + # match model_map: + # case RenewableDispatch: + # mapped_records["operation_cost"] = RenewableGenerationCost() + # case RenewableNonDispatch(): + # mapped_records["operation_cost"] = RenewableGenerationCost() + # case ThermalStandard(): + # if hr_curve: + # fuel_cost = mapped_records["fuel_price"] + # if isinstance(fuel_cost, SingleTimeSeries): + # fuel_cost = np.mean(fuel_cost.data) + # elif isinstance(fuel_cost, Quantity): + # fuel_cost = fuel_cost.magnitude + # fuel_curve = FuelCurve(value_curve=hr_curve, fuel_cost=fuel_cost) + # mapped_records["operation_cost"] = ThermalGenerationCost(variable=fuel_curve) + # else: + # logger.warning("No heat rate curve found for generator={}", generator_name) + # case HydroDispatch(): + # mapped_records["operation_cost"] = HydroGenerationCost() + # case _: + # logger.warning( + # "Operating Cost not implemented for generator={} model map={}", + # generator_name, model_map + # ) + if issubclass(model_map, RenewableGen): mapped_records["operation_cost"] = RenewableGenerationCost() elif issubclass(model_map, ThermalGen): From 683001635833a807040c436dde8d96ca7abf5678 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:07:29 -0700 Subject: [PATCH 04/15] rename rating and active_power --- src/r2x/defaults/plexos_input.json | 4 +- src/r2x/exporter/sienna.py | 1 + src/r2x/models/generators.py | 12 ++-- src/r2x/parser/plexos.py | 89 ++++++++++++------------------ 4 files changed, 44 insertions(+), 62 deletions(-) diff --git a/src/r2x/defaults/plexos_input.json b/src/r2x/defaults/plexos_input.json index ec012b40..e4ac5f13 100644 --- a/src/r2x/defaults/plexos_input.json +++ b/src/r2x/defaults/plexos_input.json @@ -18,7 +18,7 @@ "Load Risk": "load_risk", "Loss Incr": "losses", "Maintenance Rate": "planned_outage_rate", - "Max Capacity": "active_power", + "Max Capacity": "rating", "Max Flow": "max_power_flow", "Max Ramp Down": "ramp_down", "Max Ramp Up": "ramp_up", @@ -34,7 +34,7 @@ "Production Rate": "rate", "Pump Efficiency": "pump_efficiency", "Pump Load": "pump_load", - "Rating": "rating", + "Rating": "max_active_power", "Reactance": "reactance", "Resistance": "resistance", "Start Cost": "startup_cost", diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index 0e8e4950..110d7f87 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -228,6 +228,7 @@ def process_gen_data(self, fname="gen.csv"): "prime_mover_type", "bus_id", "fuel", + # "base_power", "rating", "unit_type", "active_power", diff --git a/src/r2x/models/generators.py b/src/r2x/models/generators.py index 6e0bc52b..df6023ac 100644 --- a/src/r2x/models/generators.py +++ b/src/r2x/models/generators.py @@ -26,18 +26,18 @@ class Generator(Device): bus: Annotated[ACBus, Field(description="Bus where the generator is connected.")] | None = None rating: Annotated[ - ApparentPower | None, + ApparentPower, Field(ge=0, description="Maximum output power rating of the unit (MVA)."), - ] = None + ] active_power: Annotated[ - ActivePower, + ActivePower | None, Field( description=( "Initial active power set point of the unit in MW. For power flow, this is the steady " "state operating point of the system." ), ), - ] + ] = None operation_cost: ( ThermalGenerationCost | RenewableGenerationCost | HydroGenerationCost | StorageCost | None ) = None @@ -229,7 +229,7 @@ class HydroPumpedStorage(HydroGen): def example(cls) -> "HydroPumpedStorage": return HydroPumpedStorage( name="HydroPumpedStorage", - active_power=ActivePower(100, "MW"), + rating=ActivePower(100, "MW"), pump_load=ActivePower(100, "MW"), bus=ACBus.example(), prime_mover_type=PrimeMoversType.PS, @@ -257,7 +257,7 @@ def example(cls) -> "ThermalStandard": name="ThermalStandard", bus=ACBus.example(), fuel="gas", - active_power=100.0 * ureg.MW, + rating=100.0 * ureg.MW, ext={"Additional data": "Additional value"}, ) diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index da3f0a8e..130be396 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -540,6 +540,7 @@ def _construct_generators(self): required_fields = { key: value for key, value in model_map.model_fields.items() if value.is_required() } + # breakpoint() property_records = generator_data[ [ @@ -556,7 +557,6 @@ def _construct_generators(self): mapped_records, multi_band_records = self._parse_property_data(property_records, generator_name) mapped_records["name"] = generator_name - # NOTE: Add logic to create Function data here # if multi_band_records: # pass @@ -571,7 +571,7 @@ def _construct_generators(self): # Pumped Storage generators are not required to have Max Capacity property if "active_power" not in mapped_records and "pump_load" in mapped_records: - mapped_records["active_power"] = mapped_records["pump_load"] + mapped_records["rating"] = mapped_records["pump_load"] # NOTE print which missing fields if not all(key in mapped_records for key in required_fields): logger.warning( @@ -587,6 +587,8 @@ def _construct_generators(self): mapped_records["fuel_price"] = fuel_prices.get(generator_fuel_map.get(generator_name), 0) mapped_records = self._construct_operating_costs(mapped_records, generator_name, model_map) + mapped_records["base_power"] = 1 + valid_fields, ext_data = self._field_filter(mapped_records, model_map.model_fields) ts_fields = {k: v for k, v in mapped_records.items() if isinstance(v, SingleTimeSeries)} @@ -690,7 +692,7 @@ def _construct_batteries(self): mapped_records["name"] = battery_name if "Max Power" in mapped_records: - mapped_records["active_power"] = mapped_records["Max Power"] + mapped_records["rating"] = mapped_records["Max Power"] if "Capacity" in mapped_records: mapped_records["storage_capacity"] = mapped_records["Capacity"] @@ -895,30 +897,6 @@ def _construct_operating_costs(self, mapped_records, generator_name, model_map): mapped_records = self._construct_value_curves(mapped_records, generator_name) hr_curve = mapped_records.get("hr_value_curve") - # match model_map: - # case RenewableDispatch: - # mapped_records["operation_cost"] = RenewableGenerationCost() - # case RenewableNonDispatch(): - # mapped_records["operation_cost"] = RenewableGenerationCost() - # case ThermalStandard(): - # if hr_curve: - # fuel_cost = mapped_records["fuel_price"] - # if isinstance(fuel_cost, SingleTimeSeries): - # fuel_cost = np.mean(fuel_cost.data) - # elif isinstance(fuel_cost, Quantity): - # fuel_cost = fuel_cost.magnitude - # fuel_curve = FuelCurve(value_curve=hr_curve, fuel_cost=fuel_cost) - # mapped_records["operation_cost"] = ThermalGenerationCost(variable=fuel_curve) - # else: - # logger.warning("No heat rate curve found for generator={}", generator_name) - # case HydroDispatch(): - # mapped_records["operation_cost"] = HydroGenerationCost() - # case _: - # logger.warning( - # "Operating Cost not implemented for generator={} model map={}", - # generator_name, model_map - # ) - if issubclass(model_map, RenewableGen): mapped_records["operation_cost"] = RenewableGenerationCost() elif issubclass(model_map, ThermalGen): @@ -995,55 +973,58 @@ def _process_scenarios(self, model_name: str | None = None) -> None: self.scenarios = [scenario[0] for scenario in valid_scenarios] # Flatten list of tuples return None - def _set_unit_availability(self, records): + def _set_unit_availability(self, mapped_records): """Set availability and active power limit TS for generators.""" - availability = records.get("available", None) + availability = mapped_records.get("available", None) if availability is not None and availability > 0: - # Set availability, active_power, storage_capacity as multiplier of availability - if records.get("storage_capacity") is not None: - records["storage_capacity"] *= records.get("available") - records["active_power"] = records.get("active_power") * records.get("available") - records["available"] = 1 + # breakpoint() + # Set availability, rating, storage_capacity as multiplier of availability/'units' + if mapped_records.get("storage_capacity") is not None: + mapped_records["storage_capacity"] *= mapped_records.get("available") + mapped_records["rating"] = mapped_records.get("rating") * mapped_records.get("available") + mapped_records["available"] = 1 # Set active power limits - rating_factor = records.get("Rating Factor", 100) + rating_factor = mapped_records.get("Rating Factor", 100) rating_factor = self._apply_action(np.divide, rating_factor, 100) - rating = records.get("rating", None) - active_power = records.get("active_power", None) - min_energy_hour = records.get("Min Energy Hour", None) + rating = mapped_records.get("rating", None) + active_power = mapped_records.get("active_power", None) + min_energy_hour = mapped_records.get("Min Energy Hour", None) # Hack temporary until Hydro Max Monthly Rating is corrected - max_energy_month = records.get("Max Energy Month", None) + max_energy_month = mapped_records.get("Max Energy Month", None) if max_energy_month is not None: - rating = None + active_power = None - if rating is not None: - units = rating.units - val = rating_factor * rating.magnitude - elif active_power is not None: + if active_power is not None: + # Need to fix rating timeslice handler. + # to convert the timeslice strings to max active power series units = active_power.units - val = self._apply_action(np.multiply, rating_factor, active_power.magnitude) + val = rating_factor * active_power.magnitude + elif rating is not None: + units = rating.units + val = self._apply_action(np.multiply, rating_factor, rating.magnitude) else: - return records + return mapped_records val = self._apply_unit(val, units) if min_energy_hour is not None: - records["min_active_power"] = records.pop("Min Energy Hour") + mapped_records["min_active_power"] = mapped_records.pop("Min Energy Hour") if isinstance(val, SingleTimeSeries): val.variable_name = "max_active_power" - records["max_active_power"] = val - records["rating"] = active_power + mapped_records["max_active_power"] = val + mapped_records["active_power"] = rating else: # Assume reactive power not modeled - records["active_power"] = val - records["rating"] = val + mapped_records["rating"] = val + mapped_records["active_power"] = val if isinstance(rating_factor, SingleTimeSeries): - records.pop("Rating Factor") + mapped_records.pop("Rating Factor") else: # if unit field not activated in model, skip generator - records = None - return records + mapped_records = None + return mapped_records def _plexos_table_data(self) -> list[tuple]: # Get objects table/membership table From 8ea579d908b850807094f7e748c7a50ec3aa05c3 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Mon, 9 Sep 2024 14:37:50 -0700 Subject: [PATCH 05/15] Changes max_active_power to PU --- src/r2x/exporter/sienna.py | 2 +- src/r2x/models/generators.py | 12 +++---- src/r2x/parser/plexos.py | 62 ++++++++++++++++-------------------- 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index 110d7f87..9a6fcce0 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -228,7 +228,7 @@ def process_gen_data(self, fname="gen.csv"): "prime_mover_type", "bus_id", "fuel", - # "base_power", + "base_mva", "rating", "unit_type", "active_power", diff --git a/src/r2x/models/generators.py b/src/r2x/models/generators.py index df6023ac..6e0bc52b 100644 --- a/src/r2x/models/generators.py +++ b/src/r2x/models/generators.py @@ -26,18 +26,18 @@ class Generator(Device): bus: Annotated[ACBus, Field(description="Bus where the generator is connected.")] | None = None rating: Annotated[ - ApparentPower, + ApparentPower | None, Field(ge=0, description="Maximum output power rating of the unit (MVA)."), - ] + ] = None active_power: Annotated[ - ActivePower | None, + ActivePower, Field( description=( "Initial active power set point of the unit in MW. For power flow, this is the steady " "state operating point of the system." ), ), - ] = None + ] operation_cost: ( ThermalGenerationCost | RenewableGenerationCost | HydroGenerationCost | StorageCost | None ) = None @@ -229,7 +229,7 @@ class HydroPumpedStorage(HydroGen): def example(cls) -> "HydroPumpedStorage": return HydroPumpedStorage( name="HydroPumpedStorage", - rating=ActivePower(100, "MW"), + active_power=ActivePower(100, "MW"), pump_load=ActivePower(100, "MW"), bus=ACBus.example(), prime_mover_type=PrimeMoversType.PS, @@ -257,7 +257,7 @@ def example(cls) -> "ThermalStandard": name="ThermalStandard", bus=ACBus.example(), fuel="gas", - rating=100.0 * ureg.MW, + active_power=100.0 * ureg.MW, ext={"Additional data": "Additional value"}, ) diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 130be396..a10e9511 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -98,7 +98,6 @@ "data_file_tag": pl.String, "data_file": pl.String, "variable_tag": pl.String, - # "variable": pl.String, "timeslice_tag": pl.String, "timeslice": pl.String, } @@ -112,13 +111,7 @@ "date_to", "text", ] -DEFAULT_INDEX = [ - "object_id", - "name", - "category", - # "band", -] -PROPERTIES_WITH_TEXT_TO_SKIP = ["Units Out", "Forced Outage Rate", "Commit", "Rating", "Maintenance Rate"] +DEFAULT_INDEX = ["object_id", "name", "category"] def cli_arguments(parser: ArgumentParser): @@ -540,7 +533,6 @@ def _construct_generators(self): required_fields = { key: value for key, value in model_map.model_fields.items() if value.is_required() } - # breakpoint() property_records = generator_data[ [ @@ -570,8 +562,14 @@ def _construct_generators(self): mapped_records["prime_mover_type"] = PrimeMoversType.PS # Pumped Storage generators are not required to have Max Capacity property - if "active_power" not in mapped_records and "pump_load" in mapped_records: + if "max_active_power" not in mapped_records and "pump_load" in mapped_records: mapped_records["rating"] = mapped_records["pump_load"] + + mapped_records = self._set_unit_availability(mapped_records) + if mapped_records is None: + # When unit availability is not set, we skip the generator + continue + # NOTE print which missing fields if not all(key in mapped_records for key in required_fields): logger.warning( @@ -579,15 +577,10 @@ def _construct_generators(self): ) continue - mapped_records = self._set_unit_availability(mapped_records) - if mapped_records is None: - # When unit availability is not set, we skip the generator - continue - mapped_records["fuel_price"] = fuel_prices.get(generator_fuel_map.get(generator_name), 0) mapped_records = self._construct_operating_costs(mapped_records, generator_name, model_map) - mapped_records["base_power"] = 1 + mapped_records["base_mva"] = 1 valid_fields, ext_data = self._field_filter(mapped_records, model_map.model_fields) @@ -701,6 +694,10 @@ def _construct_batteries(self): valid_fields, ext_data = self._field_filter(mapped_records, GenericBattery.model_fields) + valid_fields = self._set_unit_availability(valid_fields) + if valid_fields is None: + continue + if not all(key in valid_fields for key in required_fields): logger.warning( "Skipping battery {} since it does not have all the required fields", battery_name @@ -711,10 +708,6 @@ def _construct_batteries(self): logger.warning("Skipping battery {} since it has zero capacity", battery_name) continue - valid_fields = self._set_unit_availability(valid_fields) - if valid_fields is None: - continue - self.system.add_component(GenericBattery(**valid_fields)) return @@ -974,10 +967,12 @@ def _process_scenarios(self, model_name: str | None = None) -> None: return None def _set_unit_availability(self, mapped_records): - """Set availability and active power limit TS for generators.""" + """ + Set availability and active power limit TS for generators. + Note: variables use infrasys naming scheme, rating != plexos rating. + """ availability = mapped_records.get("available", None) if availability is not None and availability > 0: - # breakpoint() # Set availability, rating, storage_capacity as multiplier of availability/'units' if mapped_records.get("storage_capacity") is not None: mapped_records["storage_capacity"] *= mapped_records.get("available") @@ -988,19 +983,19 @@ def _set_unit_availability(self, mapped_records): rating_factor = mapped_records.get("Rating Factor", 100) rating_factor = self._apply_action(np.divide, rating_factor, 100) rating = mapped_records.get("rating", None) - active_power = mapped_records.get("active_power", None) + max_active_power = mapped_records.get("max_active_power", None) min_energy_hour = mapped_records.get("Min Energy Hour", None) # Hack temporary until Hydro Max Monthly Rating is corrected max_energy_month = mapped_records.get("Max Energy Month", None) if max_energy_month is not None: - active_power = None + max_active_power = None - if active_power is not None: + if max_active_power is not None: # Need to fix rating timeslice handler. # to convert the timeslice strings to max active power series - units = active_power.units - val = rating_factor * active_power.magnitude + units = max_active_power.units + val = rating_factor * max_active_power.magnitude elif rating is not None: units = rating.units val = self._apply_action(np.multiply, rating_factor, rating.magnitude) @@ -1013,15 +1008,13 @@ def _set_unit_availability(self, mapped_records): if isinstance(val, SingleTimeSeries): val.variable_name = "max_active_power" + self._apply_action(np.divide, val, rating.magnitude) mapped_records["max_active_power"] = val - mapped_records["active_power"] = rating - else: - # Assume reactive power not modeled - mapped_records["rating"] = val - mapped_records["active_power"] = val + mapped_records["active_power"] = rating if isinstance(rating_factor, SingleTimeSeries): - mapped_records.pop("Rating Factor") + mapped_records.pop("Rating Factor") # rm for ext exporter + else: # if unit field not activated in model, skip generator mapped_records = None return mapped_records @@ -1141,10 +1134,11 @@ def _construct_load_profiles(self): ts = self._csv_file_handler( property_name="max_active_power", property_data=region_data["variable"][0] ) + max_load = np.max(ts.data.to_numpy()) + ts = self._apply_action(np.divide, ts, max_load) if not ts: continue - max_load = np.max(ts.data.to_numpy()) bus_region_membership = self.db.get_memberships( region[0], object_class=ClassEnum.Region, From 1b3b8608ac5b707fb40861bc48bcb747e50e9157 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Tue, 10 Sep 2024 08:32:50 -0700 Subject: [PATCH 06/15] WIP ext data to json --- src/r2x/exporter/sienna.py | 55 +++++++++++++++++++++++--------------- src/r2x/parser/plexos.py | 1 + 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index 9a6fcce0..3e016dd5 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -9,8 +9,6 @@ from loguru import logger # Local imports -from infrasys.time_series_models import SingleTimeSeries -from pint import Quantity from r2x.exporter.handler import BaseExporter from r2x.models import ( ACBranch, @@ -78,6 +76,7 @@ def run(self, *args, path=None, **kwargs) -> "SiennaExporter": self.process_storage_data() self.export_data() self.create_timeseries_pointers() + # self.create_extra_data_json() return self def process_bus_data(self, fname: str = "bus.csv") -> None: @@ -437,26 +436,38 @@ def create_timeseries_pointers(self) -> None: logger.info("File timeseries_pointers.json created.") return - def create_extra_data_json(self) -> None: - """Create extra_data.json file.""" - extra_data = [] - for model in self.system.get_component_types(): - model_type_name = model.__name__ - component_dict = { - component.name: { - item: value.to_tuple() if isinstance(value, Quantity) else value - for item, value in component.ext.items() - if not isinstance(value, SingleTimeSeries) - } - for component in self.system.get_components(model) - } - extra_data.append({model_type_name: component_dict}) - - with open(os.path.join(self.output_folder, "extra_data.json"), mode="w") as f: - json.dump(extra_data, f) - - logger.info("File extra_data.json created.") - return + # def create_extra_data_json(self) -> None: + # """Create extra_data.json file.""" + # extra_data = [] + # model_types = [ + # ACBranch, + # Bus, + # DCBranch, + # Generator, + # HydroPumpedStorage, + # PowerLoad, + # Reserve, + # ReserveMap, + # Storage, + # ] + # for model in model_types: + # model_type_name = model.__name__ + # logger.debug(f"Processing {model_type_name}") + # component_dict = { + # component.name: { + # item: value.to_tuple() if isinstance(value, Quantity) else value + # for item, value in component.ext.items() + # if not isinstance(value, SingleTimeSeries) + # } + # for component in self.system.get_components(model) + # } + # extra_data.append({model_type_name: component_dict}) + + # with open(os.path.join(self.output_folder, "extra_data.json"), mode="w") as f: + # json.dump(extra_data, f) + + # logger.info("File extra_data.json created.") + # return def export_data(self) -> None: """Export csv data to specified folder from output_data attribute.""" diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index a10e9511..e184277f 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -901,6 +901,7 @@ def _construct_operating_costs(self, mapped_records, generator_name, model_map): fuel_cost = fuel_cost.magnitude fuel_curve = FuelCurve(value_curve=hr_curve, fuel_cost=fuel_cost) mapped_records["operation_cost"] = ThermalGenerationCost(variable=fuel_curve) + mapped_records.pop("hr_value_curve") else: logger.warning("No heat rate curve found for generator={}", generator_name) elif issubclass(model_map, HydroDispatch): From fd3df4467deafec9d8e7607410fd0669e88eb705 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Tue, 10 Sep 2024 08:33:46 -0700 Subject: [PATCH 07/15] remove ext data export function. to use each component ext field --- src/r2x/exporter/sienna.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/src/r2x/exporter/sienna.py b/src/r2x/exporter/sienna.py index 3e016dd5..0a22f9d4 100644 --- a/src/r2x/exporter/sienna.py +++ b/src/r2x/exporter/sienna.py @@ -76,7 +76,6 @@ def run(self, *args, path=None, **kwargs) -> "SiennaExporter": self.process_storage_data() self.export_data() self.create_timeseries_pointers() - # self.create_extra_data_json() return self def process_bus_data(self, fname: str = "bus.csv") -> None: @@ -436,39 +435,6 @@ def create_timeseries_pointers(self) -> None: logger.info("File timeseries_pointers.json created.") return - # def create_extra_data_json(self) -> None: - # """Create extra_data.json file.""" - # extra_data = [] - # model_types = [ - # ACBranch, - # Bus, - # DCBranch, - # Generator, - # HydroPumpedStorage, - # PowerLoad, - # Reserve, - # ReserveMap, - # Storage, - # ] - # for model in model_types: - # model_type_name = model.__name__ - # logger.debug(f"Processing {model_type_name}") - # component_dict = { - # component.name: { - # item: value.to_tuple() if isinstance(value, Quantity) else value - # for item, value in component.ext.items() - # if not isinstance(value, SingleTimeSeries) - # } - # for component in self.system.get_components(model) - # } - # extra_data.append({model_type_name: component_dict}) - - # with open(os.path.join(self.output_folder, "extra_data.json"), mode="w") as f: - # json.dump(extra_data, f) - - # logger.info("File extra_data.json created.") - # return - def export_data(self) -> None: """Export csv data to specified folder from output_data attribute.""" logger.debug("Saving Sienna data and timeseries files.") From f815430b9e6cb17a20719ed0050471d34c33b335 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:00:54 -0700 Subject: [PATCH 08/15] Fix ordering of date_from date_to filtering in _get_model_data --- src/r2x/parser/parser_helpers.py | 21 +++++++++++++++++++++ src/r2x/parser/plexos.py | 30 +++++++++++++----------------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/r2x/parser/parser_helpers.py b/src/r2x/parser/parser_helpers.py index acee5b63..e9840080 100644 --- a/src/r2x/parser/parser_helpers.py +++ b/src/r2x/parser/parser_helpers.py @@ -4,6 +4,7 @@ from loguru import logger import polars as pl import pandas as pd +from datetime import datetime def pl_filter_year(df, year: int | None = None, year_columns=["t", "year"], **kwargs): @@ -22,6 +23,26 @@ def pl_filter_year(df, year: int | None = None, year_columns=["t", "year"], **kw return df.filter(pl.col(matching_names[0]) == year) +def filter_property_dates(system_data: pl.DataFrame, study_year: int): + """filters query by date_from and date_to""" + # Remove Property by study year & date_from/to + study_year_date = datetime(study_year, 1, 1) + date_filter = ((pl.col("date_from").is_null()) | (pl.col("date_from") <= study_year_date)) & ( + (pl.col("date_to").is_null()) | (pl.col("date_to") >= study_year_date) + ) + + # Convert date_from and date_to to datetime + system_data = system_data.with_columns( + [ + pl.col("date_from").str.strptime(pl.Datetime, "%Y-%m-%dT%H:%M:%S").cast(pl.Date), + pl.col("date_to").str.strptime(pl.Datetime, "%Y-%m-%dT%H:%M:%S").cast(pl.Date), + ] + ) + + system_data = system_data.filter(date_filter) + return system_data + + def pl_lowercase(df: pl.DataFrame, **kwargs): logger.trace("Lowercase columns: {}", df.collect_schema().names()) result = df.with_columns(pl.col(pl.String).str.to_lowercase()).rename( diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index e184277f..2a8ae059 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -47,7 +47,12 @@ from r2x.utils import validate_string from .handler import PCMParser -from .parser_helpers import handle_leap_year_adjustment, fill_missing_timestamps, resample_data_to_hourly +from .parser_helpers import ( + handle_leap_year_adjustment, + fill_missing_timestamps, + resample_data_to_hourly, + filter_property_dates, +) models = importlib.import_module("r2x.models") @@ -100,6 +105,7 @@ "variable_tag": pl.String, "timeslice_tag": pl.String, "timeslice": pl.String, + "timeslice_value": pl.Float32, } COLUMNS = [ "name", @@ -194,6 +200,7 @@ def build_system(self) -> System: return self.system def _collect_horizon_data(self, model_name: str) -> dict[str, float]: + """Collect horizon data (Date From/To) from Plexos database.""" horizon_query = f""" SELECT atr.name AS attribute_name, @@ -1037,12 +1044,15 @@ def _get_model_data(self, data_filter) -> pl.DataFrame: "Check that the model name exists on the xml file." ) raise ModelError(msg) + + base_case_filter = pl.col("scenario").is_null() scenario_filter = pl.col("scenario").is_in(self.scenarios) scenario_specific_data = self.plexos_data.filter(data_filter & scenario_filter) + scenario_specific_data = filter_property_dates(scenario_specific_data, self.study_year) - base_case_filter = pl.col("scenario").is_null() if scenario_specific_data.is_empty(): system_data = self.plexos_data.filter(data_filter & base_case_filter) + system_data = filter_property_dates(system_data, self.study_year) else: # include both scenario specific and basecase data combined_key_base = pl.col("name") + "_" + pl.col("property_name") @@ -1054,6 +1064,7 @@ def _get_model_data(self, data_filter) -> pl.DataFrame: ~combined_key_base.is_in(combined_key_scenario) | pl.col("property_name").is_null() ) base_case_data = self.plexos_data.filter(data_filter & base_case_filter) + base_case_data = filter_property_dates(base_case_data, self.study_year) system_data = pl.concat([scenario_specific_data, base_case_data]) @@ -1100,21 +1111,6 @@ def _get_model_data(self, data_filter) -> pl.DataFrame: system_data = system_data.join( variables_filtered, left_on="variable_tag", right_on="name", how="left" ) - - # Convert date_from and date_to to datetime - system_data = system_data.with_columns( - [ - pl.col("date_from").str.strptime(pl.Datetime, "%Y-%m-%dT%H:%M:%S").cast(pl.Date), - pl.col("date_to").str.strptime(pl.Datetime, "%Y-%m-%dT%H:%M:%S").cast(pl.Date), - ] - ) - - # Remove Property by study year & date_from/to - study_year_date = datetime(self.study_year, 1, 1) - system_data = system_data.filter( - ((pl.col("date_from").is_null()) | (pl.col("date_from") <= study_year_date)) - & ((pl.col("date_to").is_null()) | (pl.col("date_to") >= study_year_date)) - ) return system_data def _construct_load_profiles(self): From dbdd0c8f0fc5cbbc72c252d109b59bfc693a26d2 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:12:58 -0700 Subject: [PATCH 09/15] Adds timeslice data. WIP parsing timeslice data properly --- src/r2x/parser/plexos.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 2a8ae059..8d30bacb 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -277,6 +277,8 @@ def _get_fuel_prices(self): "variable", "action", "variable_tag", + "timeslice", + "timeslice_value", ] ].to_dicts() @@ -551,6 +553,8 @@ def _construct_generators(self): "variable", "action", "variable_tag", + "timeslice", + "timeslice_value", ] ].to_dicts() mapped_records, multi_band_records = self._parse_property_data(property_records, generator_name) @@ -684,10 +688,11 @@ def _construct_batteries(self): "variable", "action", "variable_tag", + "timeslice", + "timeslice_value", ] ].to_dicts() - # logger.debug("Parsing battery = {}", battery_name) mapped_records, _ = self._parse_property_data(property_records, battery_name) mapped_records["name"] = battery_name @@ -1389,6 +1394,7 @@ def _parse_property_data(self, record_data, record_name): for record in record_data: band = record["band"] + timeslice = record["timeslice"] prop_name = record["property_name"] prop_value = record["property_value"] unit = record["property_unit"].replace("$", "usd") @@ -1400,19 +1406,30 @@ def _parse_property_data(self, record_data, record_name): unit = ureg[unit] except UndefinedUnitError: unit = None - if prop_name not in property_counts: - value = self._get_value(prop_value, unit, record, record_name) + value = self._get_value(prop_value, unit, record, record_name) + + # if record_name == "Hoover Dam (NV)": + # logger.debug("property name: {}", prop_name) + # breakpoint() + if mapped_property_name not in property_counts: + # First time reading property mapped_properties[mapped_property_name] = value property_counts[mapped_property_name] = {band} + property_counts[mapped_property_name].add(timeslice) else: + # Multi-band or Timeslice properties if band not in property_counts[mapped_property_name]: new_prop_name = f"{mapped_property_name}_{band}" - value = self._get_value(prop_value, unit, record, record_name) mapped_properties[new_prop_name] = value property_counts[mapped_property_name].add(band) multi_band_properties.add(mapped_property_name) - # If it's the same property and band, update the value + elif timeslice not in property_counts[mapped_property_name]: + new_prop_name = f"{mapped_property_name}_{timeslice}" + mapped_properties[new_prop_name] = value + property_counts[mapped_property_name].add(timeslice) + multi_band_properties.add(mapped_property_name) else: + # If it's the same property and band, update the value value = self._get_value(prop_value, unit, record, record_name) mapped_properties[mapped_property_name] = value return mapped_properties, multi_band_properties From fac75b0dc967858cc9875948ad9c5ae20e282ac0 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:32:31 -0700 Subject: [PATCH 10/15] Adds timeslice data parsing and logic. refactor _get_value --- src/r2x/parser/plexos.py | 194 ++++++++++++++++++++++++++------------- 1 file changed, 130 insertions(+), 64 deletions(-) diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 8d30bacb..692bd395 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -6,6 +6,7 @@ from pathlib import Path, PureWindowsPath from argparse import ArgumentParser import pandas as pd +import re from pint import UndefinedUnitError, Quantity import polars as pl @@ -1000,15 +1001,16 @@ def _set_unit_availability(self, mapped_records): min_energy_hour = mapped_records.get("Min Energy Hour", None) # Hack temporary until Hydro Max Monthly Rating is corrected - max_energy_month = mapped_records.get("Max Energy Month", None) - if max_energy_month is not None: - max_active_power = None + # max_energy_month = mapped_records.get("Max Energy Month", None) + # if max_energy_month is not None: + # breakpoint() + + if isinstance(max_active_power, dict): + max_active_power = self._time_slice_handler("max_active_power", max_active_power) if max_active_power is not None: - # Need to fix rating timeslice handler. - # to convert the timeslice strings to max active power series units = max_active_power.units - val = rating_factor * max_active_power.magnitude + val = self._apply_action(np.multiply, rating_factor, max_active_power) elif rating is not None: units = rating.units val = self._apply_action(np.multiply, rating_factor, rating.magnitude) @@ -1310,7 +1312,7 @@ def _retrieve_time_series_data(self, property_name, data_file): return def _time_slice_handler(self, property_name, property_data): - # Deconstruct pattern + """Deconstructs dict of timeslices into SingleTimeSeries objects.""" resolution = timedelta(hours=1) initial_time = datetime(self.study_year, 1, 1) date_time_array = np.arange( @@ -1319,21 +1321,33 @@ def _time_slice_handler(self, property_name, property_data): dtype="datetime64[h]", ) # Removing 1 day to match ReEDS convention and converting into a vector months = np.array([dt.astype("datetime64[M]").astype(int) % 12 + 1 for dt in date_time_array]) - month_datetime_series = np.zeros(len(date_time_array), dtype=float) - if not len(property_data) == 12: - logger.warning("Partial time slices is not yet supported for {}", property_name) - return - property_records = property_data[["text", "property_value"]].to_dicts() - variable_name = property_name # Change with property mapping - for record in property_records: - month = int(record["text"].strip("M")) - month_indices = np.where(months == month) - month_datetime_series[month_indices] = record["property_value"] + # Helper function to parse the key patterns + def parse_key(key): + # Split by semicolons for multiple ranges + ranges = key.split(";") + month_list = [] + for rng in ranges: + # Match ranges like 'M5-10' and single months like 'M1' + match = re.match(r"M(\d+)(?:-(\d+))?", rng) + if match: + start_month = int(match.group(1)) + end_month = int(match.group(2)) if match.group(2) else start_month + # Generate the list of months from the range + month_list.extend(range(start_month, end_month + 1)) + return month_list + + # Fill the month_datetime_series with the property data values + for key, value in property_data.items(): + months_in_key = parse_key(key) + # Set the value in the array for the corresponding months + for month in months_in_key: + month_datetime_series[months == month] = value.magnitude + return SingleTimeSeries.from_array( month_datetime_series, - variable_name, + property_name, initial_time=initial_time, resolution=resolution, ) @@ -1358,35 +1372,69 @@ def _apply_action(self, action, val_a, val_b): return results def _get_value(self, prop_value, unit, record, record_name): - data_file = variable = action = None - if record.get("data_file"): - data_file = self._csv_file_handler(record_name, record.get("data_file")) - if record.get("variable"): - variable = self._csv_file_handler(record.get("variable_tag"), record.get("variable")) - if variable is None: - variable = self._csv_file_handler(record_name, record.get("variable")) - - if record.get("action"): - actions = { - "×": np.multiply, # noqa - "+": np.add, - "-": np.subtract, - "/": np.divide, - "=": lambda x, y: y, - } - action = actions[record.get("action")] - if variable is not None and record.get("action") == "=": - return self._apply_unit(variable, unit) - if variable is not None and data_file is not None: - return self._apply_unit(self._apply_action(action, variable, data_file), unit) - if variable is not None and prop_value is not None: - return self._apply_unit(self._apply_action(action, variable, prop_value), unit) - elif variable is not None: + """Parse Property value from record csv, timeslice, and datafiles.""" + data_file = ( + self._csv_file_handler(record_name, record.get("data_file")) if record.get("data_file") else None + ) + if data_file is None and record.get("data_file"): + logger.warning("Assigned datafile is missing data. Skipping property.") + return None + + variable = ( + self._csv_file_handler(record.get("variable_tag"), record.get("variable")) + if record.get("variable") + else None + ) + + actions = { + "×": np.multiply, # noqa + "+": np.add, + "-": np.subtract, + "/": np.divide, + "=": lambda x, y: y, + } + action = actions[record.get("action")] if record.get("action") else None + timeslice_value = record.get("timeslice_value") + + if variable is not None: + if record.get("action") == "=": + return self._apply_unit(variable, unit) + if data_file is not None: + return self._apply_unit(self._apply_action(action, variable, data_file), unit) + if prop_value is not None: + return self._apply_unit(self._apply_action(action, variable, prop_value), unit) return self._apply_unit(variable, unit) - elif data_file is not None: + + if data_file is not None: + if timeslice_value is not None or timeslice_value == -1: + return self._apply_unit(data_file, unit) + if timeslice_value is not None: + return self._apply_unit(self._apply_action(action, data_file, timeslice_value), unit) return self._apply_unit(data_file, unit) + + if timeslice_value is not None and timeslice_value != -1: + return self._apply_unit(timeslice_value, unit) + return self._apply_unit(prop_value, unit) + # if variable is not None and record.get("action") == "=": + # return self._apply_unit(variable, unit) + # if variable is not None and data_file is not None: + # return self._apply_unit(self._apply_action(action, variable, data_file), unit) + # if variable is not None and prop_value is not None: + # return self._apply_unit(self._apply_action(action, variable, prop_value), unit) + # elif variable is not None: + # return self._apply_unit(variable, unit) + # if data_file is not None and (timeslice_value is not None or timeslice_value == -1): + # return self._apply_unit(data_file, unit) + # elif data_file is not None and timeslice_value is not None: + # return self._apply_unit(self._apply_action(action, data_file, timeslice_value), unit) + # elif data_file is not None: + # return self._apply_unit(data_file, unit) + # elif timeslice_value is not None and timeslice_value != -1: + # return self._apply_unit(timeslice_value, unit) + # return self._apply_unit(prop_value, unit) + def _parse_property_data(self, record_data, record_name): mapped_properties = {} property_counts = {} @@ -1406,32 +1454,50 @@ def _parse_property_data(self, record_data, record_name): unit = ureg[unit] except UndefinedUnitError: unit = None - value = self._get_value(prop_value, unit, record, record_name) - # if record_name == "Hoover Dam (NV)": - # logger.debug("property name: {}", prop_name) - # breakpoint() - if mapped_property_name not in property_counts: - # First time reading property - mapped_properties[mapped_property_name] = value - property_counts[mapped_property_name] = {band} - property_counts[mapped_property_name].add(timeslice) - else: - # Multi-band or Timeslice properties - if band not in property_counts[mapped_property_name]: - new_prop_name = f"{mapped_property_name}_{band}" - mapped_properties[new_prop_name] = value - property_counts[mapped_property_name].add(band) - multi_band_properties.add(mapped_property_name) + value = self._get_value( + prop_value, unit, record, record_name + ) # need to modify to include timeslice logic + if value is None: + logger.warning("Property {} missing record {} data. Skipping it.", prop_name, record) + continue + # logger.debug("record name : {}", record_name) + # logger.debug("Property: {} Value: {}", mapped_property_name, value) + + if timeslice is not None: + # Timeslice Properties + if mapped_property_name not in property_counts: + # First Time reading timeslice + nested_dict = {} + nested_dict[timeslice] = value + mapped_properties[mapped_property_name] = nested_dict + property_counts[mapped_property_name] = {timeslice} elif timeslice not in property_counts[mapped_property_name]: - new_prop_name = f"{mapped_property_name}_{timeslice}" - mapped_properties[new_prop_name] = value + mapped_properties[mapped_property_name][timeslice] = value property_counts[mapped_property_name].add(timeslice) multi_band_properties.add(mapped_property_name) - else: - # If it's the same property and band, update the value - value = self._get_value(prop_value, unit, record, record_name) + else: + # Standard Properties + if mapped_property_name not in property_counts: + # First time reading basic property mapped_properties[mapped_property_name] = value + property_counts[mapped_property_name] = {band} + else: + # Multi-band properties + if band not in property_counts[mapped_property_name]: + new_prop_name = f"{mapped_property_name}_{band}" + mapped_properties[new_prop_name] = value + property_counts[mapped_property_name].add(band) + multi_band_properties.add(mapped_property_name) + else: + # If it's the same property and band, update the value + logger.debug( + "Property {} has multiple values specified. Using the last one.", + mapped_property_name, + ) + # breakpoint() + value = self._get_value(prop_value, unit, record, record_name) + mapped_properties[mapped_property_name] = value return mapped_properties, multi_band_properties From 1631664443df56181eb13867a733e492865ba1e4 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:41:44 -0700 Subject: [PATCH 11/15] further simplify get_value logic --- src/r2x/parser/plexos.py | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 692bd395..99fbb629 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -1294,7 +1294,6 @@ def _retrieve_time_series_data(self, property_name, data_file): logger.debug("Weather year doesn't exist in {}. Skipping it.", property_name) return - # assert not data_file.is_empty() # Format to SingleTimeSeries if data_file.columns == output_columns: resolution = timedelta(hours=1) @@ -1406,9 +1405,7 @@ def _get_value(self, prop_value, unit, record, record_name): return self._apply_unit(variable, unit) if data_file is not None: - if timeslice_value is not None or timeslice_value == -1: - return self._apply_unit(data_file, unit) - if timeslice_value is not None: + if timeslice_value is not None and timeslice_value != -1: return self._apply_unit(self._apply_action(action, data_file, timeslice_value), unit) return self._apply_unit(data_file, unit) @@ -1417,24 +1414,6 @@ def _get_value(self, prop_value, unit, record, record_name): return self._apply_unit(prop_value, unit) - # if variable is not None and record.get("action") == "=": - # return self._apply_unit(variable, unit) - # if variable is not None and data_file is not None: - # return self._apply_unit(self._apply_action(action, variable, data_file), unit) - # if variable is not None and prop_value is not None: - # return self._apply_unit(self._apply_action(action, variable, prop_value), unit) - # elif variable is not None: - # return self._apply_unit(variable, unit) - # if data_file is not None and (timeslice_value is not None or timeslice_value == -1): - # return self._apply_unit(data_file, unit) - # elif data_file is not None and timeslice_value is not None: - # return self._apply_unit(self._apply_action(action, data_file, timeslice_value), unit) - # elif data_file is not None: - # return self._apply_unit(data_file, unit) - # elif timeslice_value is not None and timeslice_value != -1: - # return self._apply_unit(timeslice_value, unit) - # return self._apply_unit(prop_value, unit) - def _parse_property_data(self, record_data, record_name): mapped_properties = {} property_counts = {} @@ -1455,12 +1434,11 @@ def _parse_property_data(self, record_data, record_name): except UndefinedUnitError: unit = None - value = self._get_value( - prop_value, unit, record, record_name - ) # need to modify to include timeslice logic + value = self._get_value(prop_value, unit, record, record_name) if value is None: logger.warning("Property {} missing record {} data. Skipping it.", prop_name, record) continue + # logger.debug("record name : {}", record_name) # logger.debug("Property: {} Value: {}", mapped_property_name, value) From 7ebfffacb685529c616b7b70cc3430429f72fe6c Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:43:31 -0700 Subject: [PATCH 12/15] Fixes logic on parsing date_from date_to with default data --- src/r2x/parser/parser_helpers.py | 1 + src/r2x/parser/plexos.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/r2x/parser/parser_helpers.py b/src/r2x/parser/parser_helpers.py index e9840080..683bf3b8 100644 --- a/src/r2x/parser/parser_helpers.py +++ b/src/r2x/parser/parser_helpers.py @@ -25,6 +25,7 @@ def pl_filter_year(df, year: int | None = None, year_columns=["t", "year"], **kw def filter_property_dates(system_data: pl.DataFrame, study_year: int): """filters query by date_from and date_to""" + # note this only filters by first day of year, at some point revisit this to include partial years # Remove Property by study year & date_from/to study_year_date = datetime(study_year, 1, 1) date_filter = ((pl.col("date_from").is_null()) | (pl.col("date_from") <= study_year_date)) & ( diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 99fbb629..38d2d1a0 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -1075,7 +1075,16 @@ def _get_model_data(self, data_filter) -> pl.DataFrame: system_data = pl.concat([scenario_specific_data, base_case_data]) - # get system variables + # If date_from / date_to is specified, override the base_case_value + rows_to_keep = system_data.filter(pl.col("date_from").is_not_null() | pl.col("date_to").is_not_null()) + rtk_key = rows_to_keep["name"] + "_" + rows_to_keep["property_name"] + sys_data_key = system_data["name"] + "_" + system_data["property_name"] + if not rows_to_keep.is_empty(): + # remove if name and property_name are the same + system_data = system_data.filter(~sys_data_key.is_in(rtk_key)) + system_data = pl.concat([system_data, rows_to_keep]) + + # Get System Variables variable_filter = ( (pl.col("child_class_name") == ClassEnum.Variable.name) & (pl.col("parent_class_name") == ClassEnum.System.name) @@ -1089,7 +1098,7 @@ def _get_model_data(self, data_filter) -> pl.DataFrame: variable_base_data = self.plexos_data.filter(variable_filter & base_case_filter) variable_data = pl.concat([variable_scenario_data, variable_base_data]) - # Filter Variables + # Filter and Join Variables results = [] grouped = variable_data.group_by("name") for group_name, group_df in grouped: From cccf109952d718f37b7f807804cd6a8f0123ced1 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:55:30 -0700 Subject: [PATCH 13/15] Improves logger warning, cleans up old comments --- src/r2x/parser/plexos.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 38d2d1a0..a2cd04c8 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -1000,11 +1000,6 @@ def _set_unit_availability(self, mapped_records): max_active_power = mapped_records.get("max_active_power", None) min_energy_hour = mapped_records.get("Min Energy Hour", None) - # Hack temporary until Hydro Max Monthly Rating is corrected - # max_energy_month = mapped_records.get("Max Energy Month", None) - # if max_energy_month is not None: - # breakpoint() - if isinstance(max_active_power, dict): max_active_power = self._time_slice_handler("max_active_power", max_active_power) @@ -1385,7 +1380,6 @@ def _get_value(self, prop_value, unit, record, record_name): self._csv_file_handler(record_name, record.get("data_file")) if record.get("data_file") else None ) if data_file is None and record.get("data_file"): - logger.warning("Assigned datafile is missing data. Skipping property.") return None variable = ( @@ -1445,12 +1439,9 @@ def _parse_property_data(self, record_data, record_name): value = self._get_value(prop_value, unit, record, record_name) if value is None: - logger.warning("Property {} missing record {} data. Skipping it.", prop_name, record) + logger.warning("Property {} missing record data for {}. Skipping it.", prop_name, record_name) continue - # logger.debug("record name : {}", record_name) - # logger.debug("Property: {} Value: {}", mapped_property_name, value) - if timeslice is not None: # Timeslice Properties if mapped_property_name not in property_counts: @@ -1477,13 +1468,11 @@ def _parse_property_data(self, record_data, record_name): property_counts[mapped_property_name].add(band) multi_band_properties.add(mapped_property_name) else: - # If it's the same property and band, update the value - logger.debug( - "Property {} has multiple values specified. Using the last one.", + logger.warning( + "Property {} for {} has multiple values specified. Using the last one.", mapped_property_name, + record_name, ) - # breakpoint() - value = self._get_value(prop_value, unit, record, record_name) mapped_properties[mapped_property_name] = value return mapped_properties, multi_band_properties From a5741fccec262338a54042bba6d2f393f1c776f7 Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:26:23 -0700 Subject: [PATCH 14/15] Fixes #25 --- src/r2x/parser/plexos.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index a2cd04c8..f258c792 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -540,9 +540,6 @@ def _construct_generators(self): continue fuel_type, pm_type = fuel_pmtype["fuel"], fuel_pmtype["type"] model_map = getattr(R2X_MODELS, model_map) - required_fields = { - key: value for key, value in model_map.model_fields.items() if value.is_required() - } property_records = generator_data[ [ @@ -582,10 +579,13 @@ def _construct_generators(self): # When unit availability is not set, we skip the generator continue - # NOTE print which missing fields + required_fields = { + key: value for key, value in model_map.model_fields.items() if value.is_required() + } if not all(key in mapped_records for key in required_fields): + missing_fields = [key for key in required_fields if key not in mapped_records] logger.warning( - "Skipping Generator {} since it does not have all the required fields", generator_name + "Skipping Generator {}. Missing Required Fields: {}", generator_name, missing_fields ) continue @@ -673,9 +673,7 @@ def _construct_batteries(self): (pl.col("child_class_name") == ClassEnum.Battery.name) & (pl.col("parent_class_name") == ClassEnum.System.name) ) - required_fields = { - key: value for key, value in GenericBattery.model_fields.items() if value.is_required() - } + for battery_name, battery_data in system_batteries.group_by("name"): battery_name = battery_name[0] logger.trace("Parsing battery = {}", battery_name) @@ -711,9 +709,13 @@ def _construct_batteries(self): if valid_fields is None: continue - if not all(key in valid_fields for key in required_fields): + required_fields = { + key: value for key, value in GenericBattery.model_fields.items() if value.is_required() + } + if not all(key in mapped_records for key in required_fields): + missing_fields = [key for key in required_fields if key not in mapped_records] logger.warning( - "Skipping battery {} since it does not have all the required fields", battery_name + "Skipping battery {}. Missing required fields: {}", battery_name, missing_fields ) continue @@ -820,13 +822,16 @@ def _construct_interfaces(self, default_model=TransmissionInterface): valid_fields["ext"] = ext_data # Check that the interface has all the required fields of the model. - if not all( - k in valid_fields for k, field in default_model.model_fields.items() if field.is_required() - ): + required_fields = { + key: value for key, value in default_model.model_fields.items() if value.is_required() + } + if not all(key in mapped_interface for key in required_fields): + missing_fields = [key for key in required_fields if key not in mapped_interface] logger.warning( - "{}:{} does not have all the required fields. Skipping it.", + "{}:{} missing required fields: {}. Skipping it.", default_model.__name__, interface["name"], + missing_fields, ) continue From 6d18a1fde5e01d51e3b671d60aefb2c502cb60ff Mon Sep 17 00:00:00 2001 From: ktehranchi <83722342+ktehranchi@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:17:35 -0700 Subject: [PATCH 15/15] Fixes #31 --- src/r2x/exporter/handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/r2x/exporter/handler.py b/src/r2x/exporter/handler.py index 955705f3..56d73d39 100644 --- a/src/r2x/exporter/handler.py +++ b/src/r2x/exporter/handler.py @@ -145,10 +145,14 @@ def export_data_files(self, time_series_folder: str = "Data") -> None: config_dict["component_type"] = component_type csv_fname = string_template.safe_substitute(config_dict) csv_table = np.column_stack([date_time_column, *time_series_arrays]) + header = '"DateTime",' + ",".join( + [f'"{name}"' for name in self.time_series_name_by_type[component_type]] + ) + np.savetxt( csv_fpath / csv_fname, csv_table, - header="DateTime," + ",".join(self.time_series_name_by_type[component_type]), + header=header, delimiter=",", comments="", fmt="%s",