From 91865d7c94649ad37dc0497a2685ce417246107b Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Mon, 25 Sep 2023 18:18:32 +0200 Subject: [PATCH 01/18] Added check for s4 cloud height --- data/3.txt | 9 +++++ data/decoded_20230825T150000.csv | 2 + data/station_list.csv | 2 + example.py | 31 -------------- synop2bufr/__init__.py | 69 ++++++++++++++++++-------------- 5 files changed, 51 insertions(+), 62 deletions(-) create mode 100644 data/3.txt create mode 100644 data/decoded_20230825T150000.csv create mode 100644 data/station_list.csv delete mode 100644 example.py diff --git a/data/3.txt b/data/3.txt new file mode 100644 index 0000000..a6e4ad5 --- /dev/null +++ b/data/3.txt @@ -0,0 +1,9 @@ +SISQ01 LZIB 251500 + +AAXX 25151 + +15001 41/02 29901 10131 20108 38108 48587 52001 74100 82101 + + 333 828// + + 444 28//0 = \ No newline at end of file diff --git a/data/decoded_20230825T150000.csv b/data/decoded_20230825T150000.csv new file mode 100644 index 0000000..cf27bca --- /dev/null +++ b/data/decoded_20230825T150000.csv @@ -0,0 +1,2 @@ +report_type,year,month,day,hour,minute,block_no,station_no,station_id,region,WMO_station_type,lowest_cloud_base,visibility,cloud_cover,wind_indicator,template,wind_time_period,wind_direction,wind_speed,air_temperature,dewpoint_temperature,relative_humidity,station_pressure,isobaric_surface,geopotential_height,sea_level_pressure,3hr_pressure_change,pressure_tendency_characteristic,precipitation_s1,ps1_time_period,present_weather,past_weather_1,past_weather_2,past_weather_time_period,cloud_vs_s1,cloud_amount_s1,low_cloud_type,middle_cloud_type,high_cloud_type,maximum_temperature,minimum_temperature,maximum_temperature_period_start,maximum_temperature_period_end,minimum_temperature_period_start,minimum_temperature_period_end,ground_state,ground_temperature,snow_depth,evapotranspiration,evaporation_instrument,temperature_change,sunshine_amount_1hr,sunshine_amount_24hr,low_cloud_drift_direction,low_cloud_drift_vs,middle_cloud_drift_direction,middle_cloud_drift_vs,high_cloud_drift_direction,high_cloud_drift_vs,e_cloud_genus,e_cloud_direction,e_cloud_elevation,24hr_pressure_change,net_radiation_1hr,net_radiation_24hr,global_solar_radiation_1hr,global_solar_radiation_24hr,diffuse_solar_radiation_1hr,diffuse_solar_radiation_24hr,long_wave_radiation_1hr,long_wave_radiation_24hr,short_wave_radiation_1hr,short_wave_radiation_24hr,net_short_wave_radiation_1hr,net_short_wave_radiation_24hr,direct_solar_radiation_1hr,direct_solar_radiation_24hr,precipitation_s3,ps3_time_period,precipitation_24h,highest_gust_1,highest_gust_2,cloud_genus_s3_1,vs_s3_1,cloud_amount_s3_1,cloud_amount_s4_1,cloud_genus_s4_1,cloud_top_s4_1,_wsi_series,_wsi_issuer,_wsi_issue_number,_wsi_local,_latitude,_longitude,_station_height,_barometer_height +AAXX,2023,8,25,15,0,15,001,15001,VI,1,,200,25,8,307096,-10,,1,286.25,283.95,85.93925107124394,81080.0,85000,1587,,10.0,2,,,41,0,0,-3,7,2,31,20,11,,,-12,0,-12,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,1,2,2,8,0,0,20000,0,15001,0,0,503,0 diff --git a/data/station_list.csv b/data/station_list.csv new file mode 100644 index 0000000..03713c8 --- /dev/null +++ b/data/station_list.csv @@ -0,0 +1,2 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name +X,0-20000-0-15001,15001,Land (fixed),0,0,503,0,Switzerland diff --git a/example.py b/example.py deleted file mode 100644 index 0f5fbea..0000000 --- a/example.py +++ /dev/null @@ -1,31 +0,0 @@ -############################################################################### -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -############################################################################### - -# import synop2bufr -from synop2bufr import to_bufr - -# write the file name -file = "data/A_SMRO01YRBK211200_C_EDZW_20220321120500_12524785.txt" - -metadata = "data/metadata.csv" - -# Convert the file to BUFR -to_bufr(file, metadata) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index d7ed07e..40860fc 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1152,6 +1152,7 @@ def rad_convert(rad, time): # Name the array of section 4 items genus_array = decoded['section4'] + print("genus_array", genus_array) # Get the number of section 4 groups in the SYNOP message num_s4_clouds = len(genus_array) @@ -1539,38 +1540,44 @@ def transform(data: str, metadata: str, year: int, # significance code 10. # Clouds with bases and tops below station level # have vertical significance code 11. - cloud_top_height = msg[f'cloud_height_s4_{idx+1}'] - if cloud_top_height > int(station_height): - vs_s4 = 10 - else: - vs_s4 = 11 - - # NOTE: Some of the ecCodes keys are used in - # the above, so we must add 'num_s3_clouds' - s4_mappings = [ - {"eccodes_key": ( - f"#{idx+num_s3_clouds+8}" - "#verticalSignificanceSurfaceObservations" - ), - "value": f"const:{vs_s4}"}, - {"eccodes_key": - f"#{idx+num_s3_clouds+3}#cloudAmount", - "value": f"data:cloud_amount_s4_{idx+1}", - "valid_min": "const:0", - "valid_max": "const:8"}, - {"eccodes_key": - f"#{idx+num_s3_clouds+5}#cloudType", - "value": f"data:cloud_genus_s4_{idx+1}"}, - {"eccodes_key": - f"#{idx+1}#heightOfTopOfCloud", - "value": f"data:cloud_height_s4_{idx+1}"}, - {"eccodes_key": - f"#{idx+1}#cloudTopDescription", - "value": f"data:cloud_top_s4_{idx+1}"} - ] - for m in s4_mappings: - mapping.update(m) + cloud_top_height = msg.get(f'cloud_height_s4_{idx+1}') + + # Sometimes in section 4 the cloud height is omitted, + # so we need to check it exists before comparing it to + # the station height below + if cloud_top_height is not None: + + if cloud_top_height > int(station_height): + vs_s4 = 10 + else: + vs_s4 = 11 + + # NOTE: Some of the ecCodes keys are used in + # the above, so we must add 'num_s3_clouds' + s4_mappings = [ + {"eccodes_key": ( + f"#{idx+num_s3_clouds+8}" + "#verticalSignificanceSurfaceObservations" + ), + "value": f"const:{vs_s4}"}, + {"eccodes_key": + f"#{idx+num_s3_clouds+3}#cloudAmount", + "value": f"data:cloud_amount_s4_{idx+1}", + "valid_min": "const:0", + "valid_max": "const:8"}, + {"eccodes_key": + f"#{idx+num_s3_clouds+5}#cloudType", + "value": f"data:cloud_genus_s4_{idx+1}"}, + {"eccodes_key": + f"#{idx+1}#heightOfTopOfCloud", + "value": f"data:cloud_height_s4_{idx+1}"}, + {"eccodes_key": + f"#{idx+1}#cloudTopDescription", + "value": f"data:cloud_top_s4_{idx+1}"} + ] + for m in s4_mappings: + mapping.update(m) except Exception as e: LOGGER.error(e) LOGGER.error(f"Missing station height for station {tsi}") From 64c62037e1a26b5b364019718c3883dc63edf7cf Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Tue, 26 Sep 2023 09:33:42 +0200 Subject: [PATCH 02/18] Removed leftover test files --- data/3.txt | 9 --------- data/decoded_20230825T150000.csv | 2 -- data/station_list.csv | 2 -- 3 files changed, 13 deletions(-) delete mode 100644 data/3.txt delete mode 100644 data/decoded_20230825T150000.csv delete mode 100644 data/station_list.csv diff --git a/data/3.txt b/data/3.txt deleted file mode 100644 index a6e4ad5..0000000 --- a/data/3.txt +++ /dev/null @@ -1,9 +0,0 @@ -SISQ01 LZIB 251500 - -AAXX 25151 - -15001 41/02 29901 10131 20108 38108 48587 52001 74100 82101 - - 333 828// - - 444 28//0 = \ No newline at end of file diff --git a/data/decoded_20230825T150000.csv b/data/decoded_20230825T150000.csv deleted file mode 100644 index cf27bca..0000000 --- a/data/decoded_20230825T150000.csv +++ /dev/null @@ -1,2 +0,0 @@ -report_type,year,month,day,hour,minute,block_no,station_no,station_id,region,WMO_station_type,lowest_cloud_base,visibility,cloud_cover,wind_indicator,template,wind_time_period,wind_direction,wind_speed,air_temperature,dewpoint_temperature,relative_humidity,station_pressure,isobaric_surface,geopotential_height,sea_level_pressure,3hr_pressure_change,pressure_tendency_characteristic,precipitation_s1,ps1_time_period,present_weather,past_weather_1,past_weather_2,past_weather_time_period,cloud_vs_s1,cloud_amount_s1,low_cloud_type,middle_cloud_type,high_cloud_type,maximum_temperature,minimum_temperature,maximum_temperature_period_start,maximum_temperature_period_end,minimum_temperature_period_start,minimum_temperature_period_end,ground_state,ground_temperature,snow_depth,evapotranspiration,evaporation_instrument,temperature_change,sunshine_amount_1hr,sunshine_amount_24hr,low_cloud_drift_direction,low_cloud_drift_vs,middle_cloud_drift_direction,middle_cloud_drift_vs,high_cloud_drift_direction,high_cloud_drift_vs,e_cloud_genus,e_cloud_direction,e_cloud_elevation,24hr_pressure_change,net_radiation_1hr,net_radiation_24hr,global_solar_radiation_1hr,global_solar_radiation_24hr,diffuse_solar_radiation_1hr,diffuse_solar_radiation_24hr,long_wave_radiation_1hr,long_wave_radiation_24hr,short_wave_radiation_1hr,short_wave_radiation_24hr,net_short_wave_radiation_1hr,net_short_wave_radiation_24hr,direct_solar_radiation_1hr,direct_solar_radiation_24hr,precipitation_s3,ps3_time_period,precipitation_24h,highest_gust_1,highest_gust_2,cloud_genus_s3_1,vs_s3_1,cloud_amount_s3_1,cloud_amount_s4_1,cloud_genus_s4_1,cloud_top_s4_1,_wsi_series,_wsi_issuer,_wsi_issue_number,_wsi_local,_latitude,_longitude,_station_height,_barometer_height -AAXX,2023,8,25,15,0,15,001,15001,VI,1,,200,25,8,307096,-10,,1,286.25,283.95,85.93925107124394,81080.0,85000,1587,,10.0,2,,,41,0,0,-3,7,2,31,20,11,,,-12,0,-12,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,1,2,2,8,0,0,20000,0,15001,0,0,503,0 diff --git a/data/station_list.csv b/data/station_list.csv deleted file mode 100644 index 03713c8..0000000 --- a/data/station_list.csv +++ /dev/null @@ -1,2 +0,0 @@ -station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name -X,0-20000-0-15001,15001,Land (fixed),0,0,503,0,Switzerland From 562b50c92a549cc3eaf01678a480cc63e5c70b4e Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:10:49 +0200 Subject: [PATCH 03/18] Corrected encoding of station name --- synop2bufr/__init__.py | 2 ++ synop2bufr/resources/synop-mappings-307080.json | 2 +- synop2bufr/resources/synop-mappings-307096.json | 2 +- tests/test_synop2bufr.py | 12 ++++++------ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 40860fc..3b19d19 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1471,6 +1471,7 @@ def transform(data: str, metadata: str, year: int, wsi_series, wsi_issuer, wsi_issue_number, wsi_local = wsi.split("-") # noqa # get other required metadata + station_name = metadata_dict[wsi]["station_name"] latitude = metadata_dict[wsi]["latitude"] longitude = metadata_dict[wsi]["longitude"] station_height = metadata_dict[wsi]["elevation"] @@ -1481,6 +1482,7 @@ def transform(data: str, metadata: str, year: int, msg['_wsi_issuer'] = wsi_issuer msg['_wsi_issue_number'] = wsi_issue_number msg['_wsi_local'] = wsi_local + msg['_station_name'] = station_name msg['_latitude'] = latitude msg['_longitude'] = longitude msg['_station_height'] = station_height diff --git a/synop2bufr/resources/synop-mappings-307080.json b/synop2bufr/resources/synop-mappings-307080.json index c498366..58a8fb5 100644 --- a/synop2bufr/resources/synop-mappings-307080.json +++ b/synop2bufr/resources/synop-mappings-307080.json @@ -35,7 +35,7 @@ {"eccodes_key": "#1#heightOfBarometerAboveMeanSeaLevel", "value":"data:_barometer_height", "valid_min": "const:-400.0", "valid_max": "const:12707.1"}, {"eccodes_key": "#1#blockNumber", "value": "data:block_no", "valid_min": "const:0", "valid_max": "const:127"}, {"eccodes_key": "#1#stationNumber", "value": "data:station_no", "valid_min": "const:0", "valid_max": "const:1023"}, - {"eccodes_key": "#1#stationOrSiteName", "value": "data:station_id"}, + {"eccodes_key": "#1#stationOrSiteName", "value": "data:_station_name"}, {"eccodes_key": "#1#stationType", "value": "data:WMO_station_type", "valid_min": "const:0", "valid_max": "const:3"}, {"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:0", "valid_max": "const:4095"}, {"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:0", "valid_max": "const:15"}, diff --git a/synop2bufr/resources/synop-mappings-307096.json b/synop2bufr/resources/synop-mappings-307096.json index 33b6c1b..575fd5e 100644 --- a/synop2bufr/resources/synop-mappings-307096.json +++ b/synop2bufr/resources/synop-mappings-307096.json @@ -35,7 +35,7 @@ {"eccodes_key": "#1#heightOfBarometerAboveMeanSeaLevel", "value":"data:_barometer_height", "valid_min": "const:-400.0", "valid_max": "const:12707.1"}, {"eccodes_key": "#1#blockNumber", "value": "data:block_no", "valid_min": "const:0", "valid_max": "const:127"}, {"eccodes_key": "#1#stationNumber", "value": "data:station_no", "valid_min": "const:0", "valid_max": "const:1023"}, - {"eccodes_key": "#1#stationOrSiteName", "value": "data:station_id"}, + {"eccodes_key": "#1#stationOrSiteName", "value": "data:_station_name"}, {"eccodes_key": "#1#stationType", "value": "data:WMO_station_type", "valid_min": "const:0", "valid_max": "const:3"}, {"eccodes_key": "#1#year", "value": "data:year", "valid_min": "const:0", "valid_max": "const:4095"}, {"eccodes_key": "#1#month", "value": "data:month", "valid_min": "const:0", "valid_max": "const:15"}, diff --git a/tests/test_synop2bufr.py b/tests/test_synop2bufr.py index 64cf34f..b28d47d 100644 --- a/tests/test_synop2bufr.py +++ b/tests/test_synop2bufr.py @@ -149,9 +149,9 @@ def test_bufr_307080(multiple_reports_307080, metadata_string): for item in result: msgs[item['_meta']['id']] = item # Test the md5 keys - assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'f1595e9f82880b650de227fa007eb770' # noqa - assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '21cd8741f8615cc7b0df70060c3a98ff' # noqa - assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == 'f0b736dba245b34985f757b0597e3d54' # noqa + assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'f5874a7e7da9f6132018690f73017a10' # noqa + assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '230d74d29fba410e38cbb6fae1d6fb3f' # noqa + assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '15cefdcd27eded01db05cd5acf7abe02' # noqa # Test the bufr template used for all the reports # (they should be the same for every report) @@ -168,9 +168,9 @@ def test_bufr_307096(multiple_reports_307096, metadata_string): for item in result: msgs[item['_meta']['id']] = item # Test the md5 keys - assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '27c990045879acc2eedddb7fdc70db4d' # noqa - assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '9db622c40d53aae4ce4f38a658f36d86' # noqa - assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '89f424b9fc38a6db69c7b195bd71d92f' # noqa + assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '3611e3fd99a441f6b80bfa5bea607fee' # noqa + assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '1948789be15abd74b9312d0be1e179d6' # noqa + assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '19fa379ecf284c6b17f51baa36cfc473' # noqa # Test the bufr template used for all the reports # (they should be the same for every report) From 342fb6603127143db45997db1f5ed7dd1d047927 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Thu, 12 Oct 2023 09:45:14 +0200 Subject: [PATCH 04/18] Missing delimiter error is now stored --- synop2bufr/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 3b19d19..1c23efa 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1227,11 +1227,10 @@ def extract_individual_synop(data: str) -> list: s0 = d else: if not d.__contains__("="): - LOGGER.error(( + raise ValueError(( "Delimiters (=) are not present in the string," " thus unable to identify separate SYNOP reports." - )) # noqa - raise ValueError + )) d = re.sub(r"\n+", " ", d) d = re.sub(r"\x03", "", d) From ee2c01d6a444aade70d4f418ee648e1bcaa4f40f Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:05:14 +0200 Subject: [PATCH 05/18] AAXX YYGGiw typo verification --- synop2bufr/__init__.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 1c23efa..351d032 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1206,10 +1206,7 @@ def extract_individual_synop(data: str) -> list: :returns: `list` of messages """ - - # Check for abbreviated header line TTAAii etc. - - # Now split based as section 0 of synop, beginning AAXX YYGGi_w + # Split string based on section 0 of FM-12, beginning with AAXX start_position = data.find("AAXX") # Start position is -1 if AAXX is not present in the message @@ -1218,7 +1215,18 @@ def extract_individual_synop(data: str) -> list: "Invalid SYNOP message: AAXX could not be found." ) - data = re.split('(AAXX [0-9]{5})', data[start_position:]) + # Split the string by AAXX YYGGiw + data = re.split('(AAXX\s+[0-9]{5})', data[start_position:]) + + # Check if the beginning of the message, that we're about to throw + # away (data[0]), also contains AAXX and thus there must be a + # typo present at the AAXX YYGGiw part of the report + if data[0].__contains__("AAXX"): + raise ValueError(( + f"The following SYNOP message is invalid: {data[0]}" + " Please check again for typos." + )) + data = data[1:] # Drop first null element # Iterate over messages processing messages = [] From 7119bbe6fe7b39b5eae8eef3da956cd0242ddccb Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:57:54 +0200 Subject: [PATCH 06/18] Regex raw string literal fix --- synop2bufr/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 351d032..4634c49 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1216,7 +1216,7 @@ def extract_individual_synop(data: str) -> list: ) # Split the string by AAXX YYGGiw - data = re.split('(AAXX\s+[0-9]{5})', data[start_position:]) + data = re.split(r'(AAXX\s+[0-9]{5})', data[start_position:]) # Check if the beginning of the message, that we're about to throw # away (data[0]), also contains AAXX and thus there must be a From d8ddbc6eb550960d1eeede2e589c090c10f0688b Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:25:36 +0100 Subject: [PATCH 07/18] Mapping update fix attempt 1 --- synop2bufr/__init__.py | 90 ++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 4634c49..0cc6e38 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1511,6 +1511,20 @@ def transform(data: str, metadata: str, year: int, if conversion_success[tsi]: try: + # Define a new method which handles the updating of + # the mapping file with section 3 and 4 cloud data + def update_data_mapping(mapping: list, update: dict): + match = False + for idx in range(len(mapping)): + if mapping[idx]['eccodes_key'] == update['eccodes_key']: # noqa + match = True + break + if match: + mapping[idx] = update + else: + mapping.append(update) + return mapping + for idx in range(num_s3_clouds): # Build the dictionary of mappings for section 3 # group 8NsChshs @@ -1523,10 +1537,8 @@ def transform(data: str, metadata: str, year: int, # - verticalSignificance: used 7 times (for N, # low-high cloud amount, low-high cloud drift) s3_mappings = [ - {"eccodes_key": ( - f"#{idx+8}" - "#verticalSignificanceSurfaceObservations" - ), + {"eccodes_key": + f"#{idx+8}#verticalSignificanceSurfaceObservations", # noqa "value": f"data:vs_s3_{idx+1}"}, {"eccodes_key": f"#{idx+3}#cloudAmount", "value": f"data:cloud_amount_s3_{idx+1}", @@ -1538,7 +1550,8 @@ def transform(data: str, metadata: str, year: int, "value": f"data:cloud_height_s3_{idx+1}"} ] for m in s3_mappings: - mapping.update(m) + mapping['data'] = update_data_mapping( + mapping=mapping['data'], update=m) for idx in range(num_s4_clouds): # Based upon the station height metadata, the @@ -1549,44 +1562,37 @@ def transform(data: str, metadata: str, year: int, # significance code 10. # Clouds with bases and tops below station level # have vertical significance code 11. + cloud_top_height = msg[f'cloud_height_s4_{idx+1}'] - cloud_top_height = msg.get(f'cloud_height_s4_{idx+1}') - - # Sometimes in section 4 the cloud height is omitted, - # so we need to check it exists before comparing it to - # the station height below - if cloud_top_height is not None: - - if cloud_top_height > int(station_height): - vs_s4 = 10 - else: - vs_s4 = 11 - - # NOTE: Some of the ecCodes keys are used in - # the above, so we must add 'num_s3_clouds' - s4_mappings = [ - {"eccodes_key": ( - f"#{idx+num_s3_clouds+8}" - "#verticalSignificanceSurfaceObservations" - ), - "value": f"const:{vs_s4}"}, - {"eccodes_key": - f"#{idx+num_s3_clouds+3}#cloudAmount", - "value": f"data:cloud_amount_s4_{idx+1}", - "valid_min": "const:0", - "valid_max": "const:8"}, - {"eccodes_key": - f"#{idx+num_s3_clouds+5}#cloudType", - "value": f"data:cloud_genus_s4_{idx+1}"}, - {"eccodes_key": - f"#{idx+1}#heightOfTopOfCloud", - "value": f"data:cloud_height_s4_{idx+1}"}, - {"eccodes_key": - f"#{idx+1}#cloudTopDescription", - "value": f"data:cloud_top_s4_{idx+1}"} - ] - for m in s4_mappings: - mapping.update(m) + if cloud_top_height > int(station_height): + vs_s4 = 10 + else: + vs_s4 = 11 + + # NOTE: Some of the ecCodes keys are used in + # the above, so we must add 'num_s3_clouds' + s4_mappings = [ + {"eccodes_key": + f"#{idx+num_s3_clouds+8}#verticalSignificanceSurfaceObservations", # noqa + "value": f"const:{vs_s4}"}, + {"eccodes_key": + f"#{idx+num_s3_clouds+3}#cloudAmount", + "value": f"data:cloud_amount_s4_{idx+1}", + "valid_min": "const:0", + "valid_max": "const:8"}, + {"eccodes_key": + f"#{idx+num_s3_clouds+5}#cloudType", + "value": f"data:cloud_genus_s4_{idx+1}"}, + {"eccodes_key": + f"#{idx+1}#heightOfTopOfCloud", + "value": f"data:cloud_height_s4_{idx+1}"}, + {"eccodes_key": + f"#{idx+1}#cloudTopDescription", + "value": f"data:cloud_top_s4_{idx+1}"} + ] + for m in s4_mappings: + mapping['data'] = update_data_mapping( + mapping=mapping['data'], update=m) except Exception as e: LOGGER.error(e) LOGGER.error(f"Missing station height for station {tsi}") From 0004cc82ef8914a417a38202a6bebcb97da57b5a Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Wed, 14 Feb 2024 18:00:23 +0100 Subject: [PATCH 08/18] Temporary s3,s4 mappings fix + automatic s3 prec time period --- .github/workflows/tests.yml | 2 +- .gitignore | 2 ++ Dockerfile | 4 ++-- data/reinstall.sh | 16 +++++++++++++++ requirements.txt | 4 ++-- synop2bufr/__init__.py | 41 ++++++++++++++++++++++--------------- 6 files changed, 48 insertions(+), 21 deletions(-) create mode 100644 data/reinstall.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 434ffd5..23ef295 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.10, 3.11, 3.12] env: BUFR_ORIGINATING_CENTRE: 123 BUFR_ORIGINATING_SUBCENTRE: 123 diff --git a/.gitignore b/.gitignore index 189ff22..1ad7822 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,8 @@ logs .vscode/settings.json # Ignore decoded CSV files decoded_*.csv +# Ignore bash scripts in data folder +data/*.sh # pycharm .idea diff --git a/Dockerfile b/Dockerfile index f8f2940..f433b67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM wmoim/dim_eccodes_baseimage:2.28.0 +FROM wmoim/dim_eccodes_baseimage:2.31.0 ENV TZ="Etc/UTC" \ DEBIAN_FRONTEND="noninteractive" \ @@ -11,7 +11,7 @@ RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";" && apt-get update -y \ && apt-get install -y ${DEBIAN_PACKAGES} \ && apt-get install -y python3 python3-pip libeccodes-tools \ - && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.4.zip \ + && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.8.0.zip \ && pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip # Environment variables diff --git a/data/reinstall.sh b/data/reinstall.sh new file mode 100644 index 0000000..2b6910c --- /dev/null +++ b/data/reinstall.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Navigate to the pymetdecoder directory +cd /local/pymetdecoder + +# Uninstall the pymetdecoder package +pip uninstall -y pymetdecoder + +# Install the pymetdecoder package from the local setup.py file +python3 setup.py install + +# Navigate to the data directory +cd /local/data + +# Clear the terminal screen +clear diff --git a/requirements.txt b/requirements.txt index b2992b9..884b54d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ attrs==22.2.0 -click==8.1.3 -numpy==1.21.6 +numpy==1.24.0 +click csv2bufr diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 5cdcfe5..837b397 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1014,7 +1014,11 @@ def rad_convert(rad, time): # The time period is expected to be in hours output['ps3_time_period'] = -1 * decoded['precipitation_s3']['time_before_obs']['value'] # noqa except Exception: - output['ps3_time_period'] = None + # Regional manual (1/12.11, 2/12.12, 3/12.10, etc.) states that + # the precipitation time period is 3 hours, + # or another period required for regional exchange. + # This means that if tR is not given, it is assumed to be 3 hours. + output['ps3_time_period'] = -3 # Precipitation indicator iR is needed to determine whether the # section 1 and section 3 precipitation groups are missing because there @@ -1580,13 +1584,13 @@ def update_data_mapping(mapping: list, update: dict): # - verticalSignificance: used 7 times (for N, # low-high cloud amount, low-high cloud drift) s3_mappings = [ - {"eccodes_key": - f"#{idx+8}#verticalSignificanceSurfaceObservations", # noqa - "value": f"data:vs_s3_{idx+1}"}, - {"eccodes_key": f"#{idx+3}#cloudAmount", - "value": f"data:cloud_amount_s3_{idx+1}", - "valid_min": "const:0", - "valid_max": "const:8"}, + # {"eccodes_key": + # f"#{idx+8}#verticalSignificanceSurfaceObservations", # noqa + # "value": f"data:vs_s3_{idx+1}"}, + # {"eccodes_key": f"#{idx+3}#cloudAmount", + # "value": f"data:cloud_amount_s3_{idx+1}", + # "valid_min": "const:0", + # "valid_max": "const:8"}, {"eccodes_key": f"#{idx+5}#cloudType", "value": f"data:cloud_genus_s3_{idx+1}"}, {"eccodes_key": f"#{idx+2}#heightOfBaseOfCloud", @@ -1615,14 +1619,14 @@ def update_data_mapping(mapping: list, update: dict): # NOTE: Some of the ecCodes keys are used in # the above, so we must add 'num_s3_clouds' s4_mappings = [ - {"eccodes_key": - f"#{idx+num_s3_clouds+8}#verticalSignificanceSurfaceObservations", # noqa - "value": f"const:{vs_s4}"}, - {"eccodes_key": - f"#{idx+num_s3_clouds+3}#cloudAmount", - "value": f"data:cloud_amount_s4_{idx+1}", - "valid_min": "const:0", - "valid_max": "const:8"}, + # {"eccodes_key": + # f"#{idx+num_s3_clouds+8}#verticalSignificanceSurfaceObservations", # noqa + # "value": f"const:{vs_s4}"}, + # {"eccodes_key": + # f"#{idx+num_s3_clouds+3}#cloudAmount", + # "value": f"data:cloud_amount_s4_{idx+1}", + # "valid_min": "const:0", + # "valid_max": "const:8"}, {"eccodes_key": f"#{idx+num_s3_clouds+5}#cloudType", "value": f"data:cloud_genus_s4_{idx+1}"}, @@ -1636,6 +1640,11 @@ def update_data_mapping(mapping: list, update: dict): for m in s4_mappings: mapping['data'] = update_data_mapping( mapping=mapping['data'], update=m) + # Now section 3 and 4 cloud groups have been + # added to the mapping file, write the file + # for debugging purposes + with open('updated_mappings.json', 'w') as f: + json.dump(mapping, f, indent=2) except Exception as e: LOGGER.error(e) LOGGER.error(f"Missing station height for station {tsi}") From 070b3fd90c9d1c997704f804dbff940cb93bfbbf Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Thu, 15 Feb 2024 18:22:00 +0100 Subject: [PATCH 09/18] Attempted delayed replication factor fix + removal of Dockerfile + new csv2bufr/pymetdecoder versions --- .github/workflows/tests.yml | 2 +- .gitignore | 2 + Dockerfile | 35 ------------------ README.md | 16 +++----- data/reinstall.sh | 16 -------- docs/source/installation.rst | 15 ++++---- requirements.txt | 5 +-- synop2bufr/__init__.py | 71 ++++++++++++++++++++++++------------ tests/test_synop2bufr.py | 12 +++--- 9 files changed, 72 insertions(+), 102 deletions(-) delete mode 100644 Dockerfile delete mode 100644 data/reinstall.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 23ef295..e658e5a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.10, 3.11, 3.12] + python-version: [3.9, 3.10] env: BUFR_ORIGINATING_CENTRE: 123 BUFR_ORIGINATING_SUBCENTRE: 123 diff --git a/.gitignore b/.gitignore index 1ad7822..7848265 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,8 @@ logs .vscode/settings.json # Ignore decoded CSV files decoded_*.csv +# Ignore extra mapping files in data folders generated by synop2bufr +data/**/*.json # Ignore bash scripts in data folder data/*.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f433b67..0000000 --- a/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM wmoim/dim_eccodes_baseimage:2.31.0 - -ENV TZ="Etc/UTC" \ - DEBIAN_FRONTEND="noninteractive" \ - DEBIAN_PACKAGES="gnupg2 cron bash vim git libffi-dev libeccodes0 python3-eccodes python3-cryptography libssl-dev libudunits2-0 python3-paho-mqtt python3-dateparser python3-tz python3-setuptools" \ - ECCODES_DIR=/opt/eccodes \ - PATH="$PATH;/opt/eccodes/bin" - - -RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";" | cat > /etc/apt/apt.conf.d/10no--check-valid-until \ - && apt-get update -y \ - && apt-get install -y ${DEBIAN_PACKAGES} \ - && apt-get install -y python3 python3-pip libeccodes-tools \ - && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.8.0.zip \ - && pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip - -# Environment variables - -ENV LOG_LEVEL=INFO -# The following need to changed to the correct values for your centre! -ENV BUFR_ORIGINATING_CENTRE=65535 -ENV BUFR_ORIGINATING_SUBCENTRE=65535 - -# copy the app -COPY . /build - -# install pymetdecoder and synop2bufr -RUN cd /build \ - && python3 setup.py install \ - # delete the build folder that is no longer needed after installing the modules - && rm -r /build - -RUN adduser wis2user -USER wis2user -WORKDIR /home/wis2user \ No newline at end of file diff --git a/README.md b/README.md index 6b0cc49..1a9d5d2 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,7 @@ Dependencies are listed in [requirements.txt](https://github.com/wmo-im/synop2bu Before using synop2bufr, we highly encourage you to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, they will default to missing (255).** -It is recommended that you set these environment variables in the Dockerfile, by editing the following lines with your originating centre and subcentre values: - -```bash -ENV BUFR_ORIGINATING_CENTRE= -ENV BUFR_ORIGINATING_SUBCENTRE= -``` - -Alternatively, you can set these environment variables in your shell if you want to run synop2bufr on your local machine. Here's how you can do it in a Bash shell: +You can set these environment variables in your shell if you want to run synop2bufr on your local machine. Here's how you can do it in a Bash shell: ```bash export BUFR_ORIGINATING_CENTRE= @@ -36,8 +29,11 @@ export BUFR_ORIGINATING_SUBCENTRE= To run synop2bufr from a Docker container: ```console -docker build -t synop2bufr:local . -docker run -it -v ${pwd}:/local synop2bufr +docker run -it -v ${pwd}:/local wmoim/dim_eccodes_baseimage:2.34.0 bash +apt-get update && apt-get install -y git +cd /local +python3 setup.py install +synop2bufr --help ``` Example data can be found in `data` directory, with the corresponding reference BUFR4 in `data/bufr`. diff --git a/data/reinstall.sh b/data/reinstall.sh deleted file mode 100644 index 2b6910c..0000000 --- a/data/reinstall.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Navigate to the pymetdecoder directory -cd /local/pymetdecoder - -# Uninstall the pymetdecoder package -pip uninstall -y pymetdecoder - -# Install the pymetdecoder package from the local setup.py file -python3 setup.py install - -# Navigate to the data directory -cd /local/data - -# Clear the terminal screen -clear diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 51298df..02b8370 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -45,21 +45,20 @@ Alternatively, synop2bufr can be installed from source. First clone the reposito git clone https://github.com/wmo-im/synop2bufr.git cd synop2bufr -If running in a Docker environment, build the Docker image and run the container: +You can then run synop2bufr from an ecCodes base image as follows: .. code-block:: bash - docker build -t synop2bufr . - docker run -it -v ${pwd}:/app synop2bufr - cd /app + docker run -it -v ${pwd}:/local wmoim/dim_eccodes_baseimage:2.34.0 bash + apt-get update && apt-get install -y git + cd /local + python3 setup.py install + synop2bufr --help The above step can be skipped if not using Docker. If not using Docker the module and dependencies needs to be installed: .. code-block:: bash - - pip3 install -r requirements.txt - pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.3.1.zip - pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.0.zip + python3 setup.py install synop2bufr --help diff --git a/requirements.txt b/requirements.txt index 884b54d..09857f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -attrs==22.2.0 -numpy==1.24.0 click -csv2bufr +pymetdecoder @ git+https://github.com/wmo-im/pymetdecoder.git@v0.1.11 +csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1 \ No newline at end of file diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 837b397..3cff899 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1156,7 +1156,6 @@ def rad_convert(rad, time): # Name the array of section 4 items genus_array = decoded['section4'] - print("genus_array", genus_array) # Get the number of section 4 groups in the SYNOP message num_s4_clouds = len(genus_array) @@ -1242,7 +1241,7 @@ def extract_individual_synop(data: str) -> list: raise ValueError(( "Delimiters (=) are not present in the string," " thus unable to identify separate SYNOP reports." - )) + )) d = re.sub(r"\n+", " ", d) d = re.sub(r"\x03", "", d) @@ -1563,7 +1562,7 @@ def transform(data: str, metadata: str, year: int, def update_data_mapping(mapping: list, update: dict): match = False for idx in range(len(mapping)): - if mapping[idx]['eccodes_key'] == update['eccodes_key']: # noqa + if mapping[idx]['eccodes_key'] == update['eccodes_key']: # noqa match = True break if match: @@ -1572,6 +1571,19 @@ def update_data_mapping(mapping: list, update: dict): mapping.append(update) return mapping + # Add delayed descriptor replication factor (0 31 001) + # to represent the number of section 3 cloud groups + if num_s3_clouds > 0: + s3_delayed_replication = { + "eccodes_key": + "#1#delayedDescriptorReplicationFactor", + "value": f"const:{num_s3_clouds}" + } + mapping['data'] = update_data_mapping( + mapping=mapping['data'], + update=s3_delayed_replication) + + # Now add the rest of the mappings for section 3 clouds for idx in range(num_s3_clouds): # Build the dictionary of mappings for section 3 # group 8NsChshs @@ -1584,13 +1596,13 @@ def update_data_mapping(mapping: list, update: dict): # - verticalSignificance: used 7 times (for N, # low-high cloud amount, low-high cloud drift) s3_mappings = [ - # {"eccodes_key": - # f"#{idx+8}#verticalSignificanceSurfaceObservations", # noqa - # "value": f"data:vs_s3_{idx+1}"}, - # {"eccodes_key": f"#{idx+3}#cloudAmount", - # "value": f"data:cloud_amount_s3_{idx+1}", - # "valid_min": "const:0", - # "valid_max": "const:8"}, + {"eccodes_key": + f"#{idx+8}#verticalSignificanceSurfaceObservations", # noqa + "value": f"data:vs_s3_{idx+1}"}, + {"eccodes_key": f"#{idx+3}#cloudAmount", + "value": f"data:cloud_amount_s3_{idx+1}", + "valid_min": "const:0", + "valid_max": "const:8"}, {"eccodes_key": f"#{idx+5}#cloudType", "value": f"data:cloud_genus_s3_{idx+1}"}, {"eccodes_key": f"#{idx+2}#heightOfBaseOfCloud", @@ -1600,6 +1612,19 @@ def update_data_mapping(mapping: list, update: dict): mapping['data'] = update_data_mapping( mapping=mapping['data'], update=m) + # Add delayed descriptor replication factor (0 31 001) + # to represent the number of section 4 cloud groups + if num_s4_clouds > 0: + s4_delayed_replication = { + "eccodes_key": + "#2#delayedDescriptorReplicationFactor", + "value": f"const:{num_s4_clouds}" + } + mapping['data'] = update_data_mapping( + mapping=mapping['data'], + update=s4_delayed_replication) + + # Now add the rest of the mappings for section 4 clouds for idx in range(num_s4_clouds): # Based upon the station height metadata, the # value of vertical significance for section 4 @@ -1619,14 +1644,14 @@ def update_data_mapping(mapping: list, update: dict): # NOTE: Some of the ecCodes keys are used in # the above, so we must add 'num_s3_clouds' s4_mappings = [ - # {"eccodes_key": - # f"#{idx+num_s3_clouds+8}#verticalSignificanceSurfaceObservations", # noqa - # "value": f"const:{vs_s4}"}, - # {"eccodes_key": - # f"#{idx+num_s3_clouds+3}#cloudAmount", - # "value": f"data:cloud_amount_s4_{idx+1}", - # "valid_min": "const:0", - # "valid_max": "const:8"}, + {"eccodes_key": + f"#{idx+num_s3_clouds+8}#verticalSignificanceSurfaceObservations", # noqa + "value": f"const:{vs_s4}"}, + {"eccodes_key": + f"#{idx+num_s3_clouds+3}#cloudAmount", + "value": f"data:cloud_amount_s4_{idx+1}", + "valid_min": "const:0", + "valid_max": "const:8"}, {"eccodes_key": f"#{idx+num_s3_clouds+5}#cloudType", "value": f"data:cloud_genus_s4_{idx+1}"}, @@ -1640,17 +1665,17 @@ def update_data_mapping(mapping: list, update: dict): for m in s4_mappings: mapping['data'] = update_data_mapping( mapping=mapping['data'], update=m) - # Now section 3 and 4 cloud groups have been - # added to the mapping file, write the file - # for debugging purposes - with open('updated_mappings.json', 'w') as f: - json.dump(mapping, f, indent=2) except Exception as e: LOGGER.error(e) LOGGER.error(f"Missing station height for station {tsi}") error_msgs.append( f"Missing station height for station {tsi}") conversion_success[tsi] = False + # Now section 3 and 4 cloud groups have been + # added to the mapping file, write the file + # for debugging purposes + with open('updated_mappings.json', 'w') as f: + json.dump(mapping, f, indent=2) if conversion_success[tsi]: # At this point we have a dictionary for the data, a diff --git a/tests/test_synop2bufr.py b/tests/test_synop2bufr.py index 81f5852..d7b0e92 100644 --- a/tests/test_synop2bufr.py +++ b/tests/test_synop2bufr.py @@ -149,9 +149,9 @@ def test_bufr_307080(multiple_reports_307080, metadata_string): for item in result: msgs[item['_meta']['id']] = item # Test the md5 keys - assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '1e564e1ec2d679bbc120141ba031ab7a' # noqa - assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == 'db62277233118df3f1cf7b6a073f1cbe' # noqa - assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '538db43645fb4b2459edfcb467048b7a' # noqa + assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'db1d075059a70e978647eb1cb0a3f4d2' # noqa + assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '82035d667ce986cb528d0b11c2c8bd77' # noqa + assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '598aaead55964cc01ba5a58e53a59e9f' # noqa # Test the bufr template used for all the reports # (they should be the same for every report) @@ -168,9 +168,9 @@ def test_bufr_307096(multiple_reports_307096, metadata_string): for item in result: msgs[item['_meta']['id']] = item # Test the md5 keys - assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '5f1744ec26875630efca0e1583cddca9' # noqa - assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == 'e2dc1199d4e38fae25d26ded815597da' # noqa - assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '7c352acb43530946f2445a95eb349e68' # noqa + assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '30200eed11629d03562aafb899bb7729' # noqa + assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '7a368736fb403aa75408633fa17366e3' # noqa + assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '91fef5babfcba6af9358b7b7e38f6960' # noqa # Test the bufr template used for all the reports # (they should be the same for every report) From e45092aa1006ec126a7aa25e41d424a039240834 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Thu, 15 Feb 2024 19:08:42 +0100 Subject: [PATCH 10/18] Fixed wrong Python version in tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e658e5a..a8e0b83 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9, 3.10] + python-version: ['3.9', '3.10'] env: BUFR_ORIGINATING_CENTRE: 123 BUFR_ORIGINATING_SUBCENTRE: 123 From 75c8e5399d13c2e17c248f16e9b2cd7291fbd6b1 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:25:02 +0100 Subject: [PATCH 11/18] Mapping update fix final (hopefully) --- README.md | 2 +- docs/source/installation.rst | 2 +- setup.py | 4 ++ synop2bufr/__init__.py | 40 +++---------------- .../resources/synop-mappings-307080.json | 13 +++--- .../resources/synop-mappings-307096.json | 13 +++--- tests/test_synop2bufr.py | 12 +++--- 7 files changed, 27 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 1a9d5d2..4cabb81 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ export BUFR_ORIGINATING_SUBCENTRE= To run synop2bufr from a Docker container: ```console -docker run -it -v ${pwd}:/local wmoim/dim_eccodes_baseimage:2.34.0 bash +docker run -it -v /$(pwd):/local wmoim/dim_eccodes_baseimage:2.34.0 bash apt-get update && apt-get install -y git cd /local python3 setup.py install diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 02b8370..ec87d24 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -49,7 +49,7 @@ You can then run synop2bufr from an ecCodes base image as follows: .. code-block:: bash - docker run -it -v ${pwd}:/local wmoim/dim_eccodes_baseimage:2.34.0 bash + docker run -it -v /$(pwd):/local wmoim/dim_eccodes_baseimage:2.34.0 bash apt-get update && apt-get install -y git cd /local python3 setup.py install diff --git a/setup.py b/setup.py index 92fd4d9..cf276c7 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ import os import re from setuptools import Command, find_packages, setup +import subprocess class PyTest(Command): @@ -73,6 +74,9 @@ def get_package_version(): if (os.path.exists('MANIFEST')): os.unlink('MANIFEST') +# Install dependencies not on PyPI +subprocess.check_call("pip install 'pymetdecoder @ git+https://github.com/wmo-im/pymetdecoder.git@v0.1.11'", shell=True) +subprocess.check_call("pip install 'csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1'", shell=True) setup( name='synop2bufr', diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 3cff899..36ab268 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1571,18 +1571,6 @@ def update_data_mapping(mapping: list, update: dict): mapping.append(update) return mapping - # Add delayed descriptor replication factor (0 31 001) - # to represent the number of section 3 cloud groups - if num_s3_clouds > 0: - s3_delayed_replication = { - "eccodes_key": - "#1#delayedDescriptorReplicationFactor", - "value": f"const:{num_s3_clouds}" - } - mapping['data'] = update_data_mapping( - mapping=mapping['data'], - update=s3_delayed_replication) - # Now add the rest of the mappings for section 3 clouds for idx in range(num_s3_clouds): # Build the dictionary of mappings for section 3 @@ -1597,9 +1585,9 @@ def update_data_mapping(mapping: list, update: dict): # low-high cloud amount, low-high cloud drift) s3_mappings = [ {"eccodes_key": - f"#{idx+8}#verticalSignificanceSurfaceObservations", # noqa + f"#{idx+6}#verticalSignificanceSurfaceObservations", # noqa "value": f"data:vs_s3_{idx+1}"}, - {"eccodes_key": f"#{idx+3}#cloudAmount", + {"eccodes_key": f"#{idx+2}#cloudAmount", "value": f"data:cloud_amount_s3_{idx+1}", "valid_min": "const:0", "valid_max": "const:8"}, @@ -1612,18 +1600,6 @@ def update_data_mapping(mapping: list, update: dict): mapping['data'] = update_data_mapping( mapping=mapping['data'], update=m) - # Add delayed descriptor replication factor (0 31 001) - # to represent the number of section 4 cloud groups - if num_s4_clouds > 0: - s4_delayed_replication = { - "eccodes_key": - "#2#delayedDescriptorReplicationFactor", - "value": f"const:{num_s4_clouds}" - } - mapping['data'] = update_data_mapping( - mapping=mapping['data'], - update=s4_delayed_replication) - # Now add the rest of the mappings for section 4 clouds for idx in range(num_s4_clouds): # Based upon the station height metadata, the @@ -1645,10 +1621,10 @@ def update_data_mapping(mapping: list, update: dict): # the above, so we must add 'num_s3_clouds' s4_mappings = [ {"eccodes_key": - f"#{idx+num_s3_clouds+8}#verticalSignificanceSurfaceObservations", # noqa + f"#{idx+num_s3_clouds+6}#verticalSignificanceSurfaceObservations", # noqa "value": f"const:{vs_s4}"}, {"eccodes_key": - f"#{idx+num_s3_clouds+3}#cloudAmount", + f"#{idx+num_s3_clouds+2}#cloudAmount", "value": f"data:cloud_amount_s4_{idx+1}", "valid_min": "const:0", "valid_max": "const:8"}, @@ -1671,11 +1647,6 @@ def update_data_mapping(mapping: list, update: dict): error_msgs.append( f"Missing station height for station {tsi}") conversion_success[tsi] = False - # Now section 3 and 4 cloud groups have been - # added to the mapping file, write the file - # for debugging purposes - with open('updated_mappings.json', 'w') as f: - json.dump(mapping, f, indent=2) if conversion_success[tsi]: # At this point we have a dictionary for the data, a @@ -1684,8 +1655,7 @@ def update_data_mapping(mapping: list, update: dict): unexpanded_descriptors = [301150, bufr_template] short_delayed_replications = [] # update replications - delayed_replications = [max(1, num_s3_clouds), - max(1, num_s4_clouds)] + delayed_replications = [num_s3_clouds, num_s4_clouds] extended_delayed_replications = [] table_version = 37 try: diff --git a/synop2bufr/resources/synop-mappings-307080.json b/synop2bufr/resources/synop-mappings-307080.json index 800feba..066c361 100644 --- a/synop2bufr/resources/synop-mappings-307080.json +++ b/synop2bufr/resources/synop-mappings-307080.json @@ -11,10 +11,10 @@ "id": "b3abe03b-e502-48c5-a225-133b189207ee" }, "inputShortDelayedDescriptorReplicationFactor": [], - "inputDelayedDescriptorReplicationFactor": [1,1], + "inputDelayedDescriptorReplicationFactor": [0,0], "inputExtendedDelayedDescriptorReplicationFactor": [], "number_header_rows": 1, - "names_on_row": 1, + "column_names_row": 1, "wigos_station_identifier": "data:Station_ID", "header":[ {"eccodes_key": "edition", "value": "const:4"}, @@ -72,15 +72,12 @@ {"eccodes_key": "#1#cloudType", "value": "data:low_cloud_type", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#2#cloudType", "value": "data:middle_cloud_type", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#3#cloudType", "value": "data:high_cloud_type", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#1#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:low_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"}, {"eccodes_key": "#2#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:middle_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"}, {"eccodes_key": "#3#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:high_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"}, - {"eccodes_key": "#5#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#6#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#7#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"}, + {"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"}, + {"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"}, + {"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#1#stateOfGround", "value": "data:ground_state", "valid_min": "const:0", "valid_max": "const:31"}, {"eccodes_key": "#1#totalSnowDepth", "value": "data:snow_depth", "valid_min": "const:0.0", "valid_max": "const:25"}, {"eccodes_key": "#1#groundMinimumTemperaturePast12Hours", "value": "data:ground_temperature", "valid_min": "const:0.0", "valid_max": "const:655.35"}, diff --git a/synop2bufr/resources/synop-mappings-307096.json b/synop2bufr/resources/synop-mappings-307096.json index fef0c26..716c36e 100644 --- a/synop2bufr/resources/synop-mappings-307096.json +++ b/synop2bufr/resources/synop-mappings-307096.json @@ -11,10 +11,10 @@ "id": "f7b5e865-fd82-4e93-bdc3-40b705d73860" }, "inputShortDelayedDescriptorReplicationFactor": [], - "inputDelayedDescriptorReplicationFactor": [1,1], + "inputDelayedDescriptorReplicationFactor": [0,0], "inputExtendedDelayedDescriptorReplicationFactor": [], "number_header_rows": 1, - "names_on_row": 1, + "column_names_row": 1, "wigos_station_identifier": "data:Station_ID", "header":[ {"eccodes_key": "edition", "value": "const:4"}, @@ -71,15 +71,12 @@ {"eccodes_key": "#1#verticalSignificanceSurfaceObservations", "value": "data:cloud_vs_s1", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#1#cloudAmount", "value": "data:cloud_amount_s1", "valid_min": "const:0", "valid_max": "const:8"}, {"eccodes_key": "#1#heightOfBaseOfCloud", "value": "data:lowest_cloud_base", "valid_min": "const:0", "valid_max": "const:20070"}, - {"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#1#cloudType", "value": "data:low_cloud_type", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#2#cloudType", "value": "data:middle_cloud_type", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#3#cloudType", "value": "data:high_cloud_type", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#5#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#6#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"}, - {"eccodes_key": "#7#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"}, + {"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"}, + {"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"}, + {"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"}, {"eccodes_key": "#1#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:low_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"}, {"eccodes_key": "#2#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:middle_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"}, {"eccodes_key": "#3#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:high_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"}, diff --git a/tests/test_synop2bufr.py b/tests/test_synop2bufr.py index d7b0e92..d0d633f 100644 --- a/tests/test_synop2bufr.py +++ b/tests/test_synop2bufr.py @@ -149,9 +149,9 @@ def test_bufr_307080(multiple_reports_307080, metadata_string): for item in result: msgs[item['_meta']['id']] = item # Test the md5 keys - assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'db1d075059a70e978647eb1cb0a3f4d2' # noqa - assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '82035d667ce986cb528d0b11c2c8bd77' # noqa - assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '598aaead55964cc01ba5a58e53a59e9f' # noqa + assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'deb294033aee19f090aabc63660f273c' # noqa + assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == 'ef62c7b58ddc99724a585d6cd9b16628' # noqa + assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '7fc119bab009baf45bbbb49e0b3dd5fd' # noqa # Test the bufr template used for all the reports # (they should be the same for every report) @@ -168,9 +168,9 @@ def test_bufr_307096(multiple_reports_307096, metadata_string): for item in result: msgs[item['_meta']['id']] = item # Test the md5 keys - assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '30200eed11629d03562aafb899bb7729' # noqa - assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '7a368736fb403aa75408633fa17366e3' # noqa - assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '91fef5babfcba6af9358b7b7e38f6960' # noqa + assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '9e84ebe4283f114353c454e37b052076' # noqa + assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '8d6fb0d39ffee750f3da3228bf673862' # noqa + assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '8d70449d5f9f5021ecedd445a9a0b90a' # noqa # Test the bufr template used for all the reports # (they should be the same for every report) From e1531ca5b9cad230ba690e6d968bc4bda46d4c31 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Mon, 19 Feb 2024 17:21:05 +0100 Subject: [PATCH 12/18] Changed requirements to use PyPI pymetdecoder --- requirements.txt | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 09857f7..d351651 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ click -pymetdecoder @ git+https://github.com/wmo-im/pymetdecoder.git@v0.1.11 +pymetdecoder-wmo csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1 \ No newline at end of file diff --git a/setup.py b/setup.py index cf276c7..fbbb54d 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,6 @@ def get_package_version(): os.unlink('MANIFEST') # Install dependencies not on PyPI -subprocess.check_call("pip install 'pymetdecoder @ git+https://github.com/wmo-im/pymetdecoder.git@v0.1.11'", shell=True) subprocess.check_call("pip install 'csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1'", shell=True) setup( From 4550c86fabb8b2a2e76b30f4af1f2ce95db7b51d Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Mon, 19 Feb 2024 17:25:28 +0100 Subject: [PATCH 13/18] Flake 8... --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fbbb54d..13cd4e7 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def get_package_version(): os.unlink('MANIFEST') # Install dependencies not on PyPI -subprocess.check_call("pip install 'csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1'", shell=True) +subprocess.check_call("pip install 'csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1'", shell=True) # noqa setup( name='synop2bufr', From f546ee6c1da7923945b3bee6ae604b53540bcf3d Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Tue, 5 Mar 2024 15:35:40 +0100 Subject: [PATCH 14/18] Changed csv2bufr version to main and updated GitHub actions --- .github/workflows/tests.yml | 6 +++--- .gitignore | 2 ++ setup.py | 2 +- .../conftest.cpython-310-pytest-7.2.0.pyc | Bin 117 -> 0 bytes .../test_csv2bufr.cpython-310-pytest-7.2.0.pyc | Bin 7944 -> 0 bytes ...test_synop2bufr.cpython-310-pytest-7.2.0.pyc | Bin 12265 -> 0 bytes 6 files changed, 6 insertions(+), 4 deletions(-) delete mode 100644 tests/__pycache__/conftest.cpython-310-pytest-7.2.0.pyc delete mode 100644 tests/__pycache__/test_csv2bufr.cpython-310-pytest-7.2.0.pyc delete mode 100644 tests/__pycache__/test_synop2bufr.cpython-310-pytest-7.2.0.pyc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a8e0b83..2009b02 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,13 +7,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10'] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] env: BUFR_ORIGINATING_CENTRE: 123 BUFR_ORIGINATING_SUBCENTRE: 123 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Setup Python ${{ matrix.python-version }} with: python-version: ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index 7848265..a9dd3ea 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,8 @@ nosetests.xml logs .vscode/ .vscode/settings.json +# Pytest cache files +tests/__pycache__/ # Ignore decoded CSV files decoded_*.csv # Ignore extra mapping files in data folders generated by synop2bufr diff --git a/setup.py b/setup.py index 13cd4e7..3572f5e 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def get_package_version(): os.unlink('MANIFEST') # Install dependencies not on PyPI -subprocess.check_call("pip install 'csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1'", shell=True) # noqa +subprocess.check_call("pip install 'csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git'", shell=True) # noqa setup( name='synop2bufr', diff --git a/tests/__pycache__/conftest.cpython-310-pytest-7.2.0.pyc b/tests/__pycache__/conftest.cpython-310-pytest-7.2.0.pyc deleted file mode 100644 index bac58b2d76afea64fa89b1d5562eb10905248162..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 117 zcmd1j<>g`k0@>&}$&ZaRB0C79f$r5X_*-=(m!g2qcUkeo5%(TQ m=$E7xmlW$K=jWw?Sb7DOw>WHa^HWN5Qtd!GikW}}3j+Wo(H1KJ diff --git a/tests/__pycache__/test_csv2bufr.cpython-310-pytest-7.2.0.pyc b/tests/__pycache__/test_csv2bufr.cpython-310-pytest-7.2.0.pyc deleted file mode 100644 index 716c6ab8305a8424b3acb5f67d7ade2b102399c3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7944 zcma)BTW}o5b)A`=ot>S1VDTjQhD1u_S^|X!Ns)=Ezyd*;F|~{dFfw7wqs8`MHR8V1 zGfNWCY|1f}*p>L9gsDWSLMni&^1=M%CzX7rlAlyk$(Q{#6<3nVLxnHrVObVpSvmL4 z>|y~?Dlag%U$=X@Z{I%m_MK)fXDRqy`o`OpznxH&f1^VGFM+}w9&ZOkC_=3%%POZ@ zJ)tS8QrDN0YSdyZ8)(sLsk*sr#&zlCH0lyHtDafT)U(UkxIVX>L%m+h*9Vpd>V@S( zeQ%PT|USqQ>3|ch)b5p zaOp6YvLeT&BO)&b-c**4ih>wKdQ1$7VWh{!h!{mWA;!c3q>qVlaS-X_;*dCu^n^Gf zjv_rN)J3IuY#ZxWN<}p^7k%kAs#mWUweWDIDICvUX|GDV;oP-XWwUO3<$9~;cwsJT zlupeldrmPK7Vea5t|H9~%6gbuyg5I= zu(%i|FE2cQ^Mwk{rvEpF!aN@D-$AO_C;Vw5A@mJRBq23QQ*mm>X*y1=IL(MG^W`8h z;vc{mlosN2Fy@2�cs~AvsD9Kys8Ggybka49QV?6!SRt4v$eC6vyLUByVaZc~eUA zrj+DO=}GYfr%#GgoSqg%PCp}_qO_V8Pj4i#Ba@|V6>AMsj&Oaq+1S=m!k%QDN+kH( z&RUp^1UXnQd%h!=$}2Ue)UK~MQgYt{?cC>`vT)?Q)9|Ge4)4}1wpZfHk#>u=ZP9tX z?KCR!2w@g=<$2Usn{q8Y;x_PxMwzBku3bhG^^aS(4Vvk}zFBy`>PQc)(QLL)Qu{!I zr*?f6Jw3+{Vv#(I@#uG`UVw0$Qu>QOq{ZaDW^<%Ym4 z9IqnXmfw`#!=(FzKhPETdqloZWd4+*xC_rH%1-LU^a+~Xj)@Gvf5$)$ujM9s*-o08 z_ys!|WNQ$k-xf}2c5^#fs%|!_QGL3X-I+iGi={1{8;)pK9I@1V(J40;o$@v3j#CTG zZu8C{c+0L_a_TKdmVNXH2T->C-3px3Q*NWx_AfiN@>>5QZ#b{nTXY}vevJ!_QffP`j)ilGrTg*xfVLq+}~wr7{@9DSzN#11~=JD%?`wXt!r6}7OdSo=4PlzZ$kKQbMQ;Qok_Jhn`)9l_ zX(1n5VHTCV!fztu@){qg1d*UgrhrN0Q%p^!X{InNsE0uTg)spwFdgJpz#(d2Oh92u zK*yN^4j-t1z{7yRVSwcU{2cyF>!pu`V%rN6D3I#lQ|*qqIL=F=<0#e1JD8Ki|bMknvq&zt%AKNX> z%*hid$tQ@+5jjodmq57pg#5{zd=90eB~MXHk;rF=JVoSbB9lbU5Sbz}L*y)xy^0?d zc}w%WrGC=}vci~t1(>-(t z=rQQ|Kwd@Fo|Pp~k;KC*n?uIO+U~0z#a9D3QgFw#jvA=m5jqLJwwVajE!9=Wl#b?8 zIT>hM;QuXqEwyrFL`g>MSk=91n87%9^s=d_5OtX8 z6Bnyfmfse;01q$)k4!JXQkzoc60#LKUfmxF25=P#=l7BE6%q;w#8+=?9WBtj7f2{V z4YXCL1l6x z+b~6BX3qwljW*~k&M&Q-nI++9IB3HN)oj`t@-fOVi&+x*O53gZZo>-=Hd9_WZhJ7- zHNmWQwbooI*KAUDS)viI6ZsX8Vv>!F{1T1cw|CS1dpFx(IJbWz5AjC!9OAAeY(WV& zFseuDWvbs%rGv~Kji<%+u>MCVR3nF2BZqhcS{_?9b3+$t0c*wUdfD>5Fj0Ng zPO_^)XMVWtT&#~F4uF`Ml^*I0m0hFY{$8V@!bjq$;I z@nF|Jj5D}>V)o~$_eCNEmRE{643AwZ_8o@DdWV5^F4FB=)PZb?d>N#eVYLs_3->Bc zi;U^+36NhUE}O_JM9M_=t2C}-gk+4_q}XzW2&tYFM0QOaBhe=)@M_tuy#sdd2#BU@ zs-7t;BkB@CVhcvChf|ZNgQ7#}W~h~NP&zn&J&xyy zgA9V4=l+pe!a3}5TxSj%V~IJUF>uU!9Ir43p`pVZ5gJ;Y2kwc?h5=MG=uieQyU3?} zU?z@9s*8#`P!mtGqcb|@ORVLQ3Mps1zUaxW3Vn4>-h{}YFVDw@stcYMik|fb)gdzx zsnBfEJ54n}%cmJ&&vi5Bi(b*{NWRKhU36+YvWc--ouTm8kSFAwWtF4cIU?tYTv#9I zHRZ_JZ<8hFn}Vt!H3Gm4a&YdpuwT9zB(X;t?6naXmzB5FmuF$^(m~3%?rR-0FufKz zt$t=R%dmFQ&u!-YGp<=#BAZ9>W61D?A2r;7BsJ z2@NndblBXqUqH=ZfH%Y-Y>HIJ3alV?N9kmOjQ6K#9|8nt7=lsHUIo1qBjPIAozVmE zMKgMv-_f0s#Xy@S_dCck(B|TI=DP2Mjkh`(givzts2z+UQU{gJKrrx{PO^c-3;u9W zz{)->1~;b&;e2l*}aEmbC%4|$A%%L98a!*qoRhIv&Z@v27pRyBZ?nRp6#1~F@1 znHa_DELzDFz>;`Msm@d+`+|+Ch>g zo=nGl{RG45;S$2-sn^?0-0`ZRwdy)G!O_e0bA65q9HGcZLMHZ+;JUDnge!mKBjFMX z*C@1H54T0Xj2L;Sy9lZx?up1H5SNcQIY;u#M93cxlQxAw7$6#5fWU|oZlenjzy|I^ z*dTw42mwMd73Mf-gjj48dym6{D28U$X<}Fen1jD=zoGYlDvttTeQCiiT`!exzPNDX>U^lvBErmFYLCO^?N27e%o(10PM=HDCS?y7arov(dec+-J#_ zy zab4s~Zek=@aiwCJN*U+B(=1`o~d7l|lp#8(w|uz2I>J{qR6nb$B9 z)WEbk9%S30aqapGFD%>$O#~aF?=Y!V_a1$7iDEy?o1%G^asGoNW@=d%zOCy7^L^Er|vDY)m!2uZ_zlw?R2?gf$~ zdAN^}QBr{WIFU}rieo+C#pa4}cdTi0ZY-I)dC_4;-Eqh2wspZ|j_%kxp%5?vo)vCe zb<8sOjerNQ<8bjg2`VpjV@|9SyCij`jzr}Bv93(w-FQdt#Lvs!L?^+kj&xNf;9HbP z62lkeYY7O0G}k0?k6)8W>WaKK)=gqFI!P*lKjmPmWG8h^x+J{}-IIW_;YvX163GZs zHb7d5gp?Dce1P=*ohOkP4AAJ@`p=FXJ9kbgX( zr&XmqU7M^bmC|&zq*QBaNmV9QRn=xhK6t`>HKkP6)C%S+tC+7=Rm!SXt0@(=3~qI5 zm`|C4g#Fx5x|h2Oxl1DV^e}g+reW^M3g#|r9#xqtsg=nY29qMTKD*HfWm|2yF5Qr4 zX0A)?yY4>RTGNVLnVC`0TPgDEvKxmmbSgIr2InwsMcibOW8h>@)Zt{X4%_XI9xm4P zP+{Bco??n&ac*|DwdR=I(GBKK=yjvf(3#m{b-URznCWJ8ZY-J@;uemH_EFn9!I;h5 z-TI=k72|?QBCta5KSR}07g0&~aIuQx$Ho@8(V!ap%>vqA~7h?T5 z64rjfx+BE;Yb2}#f^{&!lKGvHu*tZMJ|?v9D336X2EB2IJ ze^1GU_Y~MylE9G2{ad#1ijTK*d)Nblb!5vH{?MDhpO6)Q9gx!m0{ZQ1&PIUm{^m3Rd7$bveloM8c{G)>NPm zll-YjSkr>_V2Jg4B&<&g)$G6Sx9H6@f5J!H+1`9w zP|t)~_^OY$bG`Y6Eqn9bNLXiwy}5DTN%Ox&!a65d&jk7~!|#oRHNQo>GW@wnSYH&Z zXG86JD-zaog7tie^{Yr&x?lyqSzXTXn~|`-Bv`?-Rkp8N*y(-75G<4+8*@I(&qva( zg)RCp%dbVkstXngwd?nhu*@y`Fc;DL-=bgz-;;6?z5gw3$%^LvZ&|QtV7ziXn)kmI z!3yrsxhUTMNCm#v%bgT_WyyE)m-GCd{`||sm;3V=gI`%l!%1BoxfZ(+gA_SPkw$Id z^3fj? zbkI=({gfbrALy-Z&{4!VrUm>aK!rxvJDu z2verYTBQoAz$%nBt*Po17O!bawo(BtqR{u1N@}%Ce*?-IHVR4`H_IKN=Fur}lcG5+ z3E>R2IV_D%4xKzWZk!lvz1z@NSfv)x;i7fJu~`ojy5nb`IHeDYbJkj>&?mYR4SN;U zW%Y*LCi;TGO?nO#X>Rrc<#d5IsIx}@2uwHUrKsDW%kJH?fgzNcb<~y7L!caVvx3cA zrb&8tVX85MGTkwo05qI7Gnb$<%jKWEFzfCi=4#8P7HGzO(Tn(dxi#yK4=USYpizgM zlyxO&2j%vH!DwBFutlS8(x)i|?_M(PmhG69Lyd;MY|^D=hyE!66;KBG0b>!a)HtD5AuOJC3tFM}dvkX1D2SW=j1BRj>q-&w)5||Q*Q8Jo# z8?thZp=Qr5z!3Jw562ZWmXr|CS9*y>(AYBchcBRqDnmE1Y`0ktXONp}QmgHly&693 zZWrfrOGm+kKt6>Wtoo73_`snAR?~524j6-sR2`

w-ouy6{s4OtHoy13Zp=GIA+ zIY_dvVOvZ3s&T;#moR=WMDORzHrQ-z6+e3g;===7?!~dKFxqARXuA`^2?34hl`!E% za45^V>W+IsOQzN4IuxT05_z*(1ksbY7B^+qHf7d&C@2OgH%idxdr(ogfD;1O{Zbno zm}!CM(jebprd2jo4@3IZ7!(dc^nM(;LVxDjEuH()Z4?%+ZUt(vsW6THeRE+NtVUtD zpl%p0lwt9WS}9-n~@+Z^$F%@A=;5i4eyfI-m;29uskkMrb-pt z1SOvZtE~k5FU6&pmDXZ>@2Vt$MVAzN2Cj~@0_ICjvJ(e`sRWVzM?22(&_VDBBB8mI zXTm^|PWoyF4?_H5h|fA;#=sr{Uk-h63<3MEVTva)MMr`ZXD|i&AVo^};m{)PXbypK zXbv%QB?*RBU~)w=Lvx5c#Ey3S<`8i-y7EjSgppEqZvEYj1I#v!ls=S=lE}PW8=Up$ zA2A!WNnvB<%Hf%!0v0!lWjkUAhRsf)mI9ld!W{;S<9_hBxze_bg@&m(wnAZZcIe`o z(l>80mDA76oqAFk*xeivXCJU!dK|Z4SjyQK`yR!!U;>$N7(+lHcv0Z#M}eKwkqbtn zZDza0YnI$ zGblH?n>lf@Znp3+hoi39PtV zt$P*Xy>(y3mAl2dKZYs9x<832c7EOar5efrkZY@cxi*jsXNBFOl2!V@9eTH!SwH%y zH|oK~_Mjr?QPn8J(}7gXci@ zRdoK(wXkQf76vZ^c$^i>0T!Lf!8Fg?LN8(2ub}fPI6~XK_k4PnFQtV#>ozfuD5v`=R4*%6n#P&}6Qe&yA7pBx38=vt5h Date: Tue, 5 Mar 2024 15:54:30 +0100 Subject: [PATCH 15/18] Requirements fix --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d351651..d19c2c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ click pymetdecoder-wmo -csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1 \ No newline at end of file +csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git \ No newline at end of file From 88acf25d78c6a4c18a3f182fd3997250a1810bfa Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Wed, 6 Mar 2024 10:14:06 +0100 Subject: [PATCH 16/18] Moved closure out of unnecessary try block --- synop2bufr/__init__.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 36ab268..4c4f7bc 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1554,22 +1554,23 @@ def transform(data: str, metadata: str, year: int, # Stop duplicated warnings can_var_warning_be_displayed = False - # Now we need to add the mappings for the cloud groups + # Define a new method which handles the updating of + # the mapping file with section 3 and 4 cloud data + def update_data_mapping(mapping: list, update: dict): + match = False + for idx in range(len(mapping)): + if mapping[idx]['eccodes_key'] == update['eccodes_key']: # noqa + match = True + break + if match: + mapping[idx] = update + else: + mapping.append(update) + return mapping + + # Now we add the mappings for the cloud groups # of section 3 and 4 try: - # Define a new method which handles the updating of - # the mapping file with section 3 and 4 cloud data - def update_data_mapping(mapping: list, update: dict): - match = False - for idx in range(len(mapping)): - if mapping[idx]['eccodes_key'] == update['eccodes_key']: # noqa - match = True - break - if match: - mapping[idx] = update - else: - mapping.append(update) - return mapping # Now add the rest of the mappings for section 3 clouds for idx in range(num_s3_clouds): From 9d3071ad00aa507a09f8787e879a8673cbcbbe59 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Thu, 7 Mar 2024 11:15:54 +0100 Subject: [PATCH 17/18] Improved SYNOP typo check explanation --- synop2bufr/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 4c4f7bc..94bd4fb 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1221,9 +1221,10 @@ def extract_individual_synop(data: str) -> list: # Split the string by AAXX YYGGiw data = re.split(r'(AAXX\s+[0-9]{5})', data[start_position:]) - # Check if the beginning of the message, that we're about to throw - # away (data[0]), also contains AAXX and thus there must be a - # typo present at the AAXX YYGGiw part of the report + # Check if the beginning of the message (e.g. ZCZC 123 etc.) + # that we're about to throw away (data[0]) also contains AAXX. + # If this is true, there must be a typo present at the AAXX YYGGiw + # part and thus we can't process the message. if data[0].__contains__("AAXX"): raise ValueError(( f"The following SYNOP message is invalid: {data[0]}" From 9ddbc2db0c52ae746ec23f07e160ec203c6b8c70 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Thu, 7 Mar 2024 13:57:06 +0100 Subject: [PATCH 18/18] Replaced '__contains__' with 'in' --- synop2bufr/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 94bd4fb..a68c370 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1225,7 +1225,7 @@ def extract_individual_synop(data: str) -> list: # that we're about to throw away (data[0]) also contains AAXX. # If this is true, there must be a typo present at the AAXX YYGGiw # part and thus we can't process the message. - if data[0].__contains__("AAXX"): + if "AAXX" in data[0]: raise ValueError(( f"The following SYNOP message is invalid: {data[0]}" " Please check again for typos." @@ -1238,7 +1238,7 @@ def extract_individual_synop(data: str) -> list: if "AAXX" in d: s0 = d else: - if not d.__contains__("="): + if "=" not in d: raise ValueError(( "Delimiters (=) are not present in the string," " thus unable to identify separate SYNOP reports."