Merge pull request #49 from globaldothealth/fhirflat-ini

Create fhirflat.ini

.readthedocs.yaml

@@ -22,4 +22,4 @@ sphinx:
 python:
    install:
      - method: pip
-       path: .[docs]
+       path: .[docs]

README.md

@@ -13,7 +13,7 @@ for the [ISARIC](https://isaric.org) 3.0 project and data pipelines. Portions of
 the code are specific to the ISARIC project, such as ISARIC specific FHIR
 extensions.
 
-The FHIRflat FHIR resources are derived from the 
+The FHIRflat FHIR resources are derived from the
 [fhir.resources](https://github.com/nazrulworld/fhir.resources) package.
 
 For more information, howtos and tutorials, see the
@@ -35,8 +35,8 @@ https://github.com/globaldothealth/fhirflat/main.tar.gz
 ## Development
 
 To test and develop fhirflat, from a cloned version of fhirflat use an editable install
-including the development dependencies(`pip install -e ".[dev]"`). This will allow you 
+including the development dependencies(`pip install -e ".[dev]"`). This will allow you
 to test the packages, and installs formatting and linting tools, and
 [pre-commit](https://pre-commit.com).
 
-Setup pre-commit hooks (`pre-commit install`) which will do linting checks before commit.
+Setup pre-commit hooks (`pre-commit install`) which will do linting checks before commit.

docs/conf.py

@@ -37,12 +37,12 @@
 
 html_theme = "better"
 html_static_path = ["_static"]
-html_theme_path=[better.better_theme_path]
+html_theme_path = [better.better_theme_path]
 html_short_title = "Home"
 
 html_theme_options = {
     "rightsidebar": True,
     "sidebarwidth": "25rem",
     "cssfiles": ["_static/style.css"],
     "showheader": False,
-}
+}

docs/howto/conversion-data.md

@@ -11,14 +11,14 @@ fhirflat transform data-file google-sheet-id date-format timezone-name
 
 Here *data-file* data file that fhirflat will transform, and *google-sheet-id* is the unique
 ID of the google sheet containing the mapping information (found in the url; the format
-if usually https://docs.google.com/spreadsheets/d/{spreadsheet_id}/edit#gid={sheet_id}, 
+if usually https://docs.google.com/spreadsheets/d/{spreadsheet_id}/edit#gid={sheet_id},
 you want the spreadsheet_id. The sheet has to be public, i.e. share settings must be set
 to 'Anyone with the link' for this to work). *date-format* is the format dates follow in
 the raw data, e.g. a "2020-04-20" date has a date format of "%Y-%m-%d", and *timezone*
-is the time zone the data was recorded in, e.g. "America/New_York". A full list of 
+is the time zone the data was recorded in, e.g. "America/New_York". A full list of
 timezones can be found [here](https://nodatime.org/timezones).
 
-Further information on the structure of the mapping file can be found 
+Further information on the structure of the mapping file can be found
 [in the specification](../spec/mapping.md)
 
 ## Library

docs/howto/conversion-fhir.md

@@ -38,7 +38,7 @@ from fhirflat import Patient
 Patient.fhir_file_to_flat("patient_export.ndjson")
 ```
 creates a "patient_export.parquet" FHIRflat file.
-This first initialises a `Patient` data class for each row to make use of the Pydantic 
+This first initialises a `Patient` data class for each row to make use of the Pydantic
 data validation, then creates a FHIRflat file.
 
 ## From FHIRflat
@@ -47,4 +47,4 @@ FHIR resources can also be created directly from FHIRflat files
 Patient.from_flat("patient_flat.parquet")
 ```
 which returns either a single Patient resource, or a list of Patient resources if
-the Parquet file contains multiple rows of data.
+the Parquet file contains multiple rows of data.

docs/index.md

@@ -100,4 +100,4 @@ maxdepth: 1
 fhirflat
 resources_base
 resources
-```
+```

docs/resources.rst

@@ -1,2 +1,2 @@
 .. automodule:: fhirflat.resources
-    :members:
+    :members:

docs/resources_base.rst

@@ -2,4 +2,4 @@ FHIRFlat Base Class
 ===================
 
 .. autoclass:: fhirflat.resources.base.FHIRFlatBase
-    :members:
+    :members:

docs/spec/fhirflat.md

@@ -51,7 +51,7 @@ normalising a FHIR structure, noted below.
         "display": "Donald Duck"
     }
     ```
-    becomes 
+    becomes
     | subject.reference |
     |-------------------|
     |"Patient/f001"     |
@@ -154,7 +154,7 @@ normalising a FHIR structure, noted below.
         },
     ]
     ```
-    becomes 
+    becomes
     | encounter.diagnosis_dense            |
     |--------------------------------------|
-    |"[{"condition": [{"reference"...}]}]" |
+    |"[{"condition": [{"reference"...}]}]" |

docs/spec/isaric-fhir-extensions.rst

@@ -1,3 +1,3 @@
 .. automodule:: fhirflat.resources.extensions
     :members:
-    :exclude-members: elements_sequence
+    :exclude-members: elements_sequence

docs/spec/mapping.md

@@ -114,4 +114,4 @@ Conditional assignments are made using the **`if not`** statement. A value of
 FHIR resource entries can be lists, such as a set of codes. This is handled in
 the mapping specification by allowing multiple matches for a (`raw_variable`,
 `raw_response`) tuple. The corresponding resource assignments (such as
-`coding.code`) are then collected into lists.
+`coding.code`) are then collected into lists.

fhirflat/__init__.py

@@ -18,4 +18,6 @@
 )
 from .ingest import convert_data_to_flat
 
-__all__ = ["convert_data_to_flat"]
+# Update this when bumping version in pyproject.toml!
+__version__ = "0.1.0"
+__all__ = ["convert_data_to_flat"]

fhirflat/fhir2flat.py

@@ -213,7 +213,6 @@ def flattenExtensions(df: pd.DataFrame, extension: str) -> pd.DataFrame:
     """
 
     def expand_and_redefine(df, extension):
-
         def redefine(
             row: pd.Series | pd.DataFrame, extension: str
         ) -> pd.Series | pd.DataFrame:

fhirflat/ingest.py

@@ -4,17 +4,22 @@
 """
 
 import argparse
+import hashlib
 import os
 import timeit
 import warnings
 from datetime import datetime
+from glob import glob
 from math import isnan
+from pathlib import Path
+from typing import Literal, TypedDict
 from zoneinfo import ZoneInfo
 
 import dateutil.parser
 import numpy as np
 import pandas as pd
 
+import fhirflat
 from fhirflat.util import get_local_resource, group_keys
 
 # 1:1 (single row, single resource) mapping: Patient, Encounter
@@ -31,6 +36,13 @@
 """
 
 
+class FlatMetadata(TypedDict):
+    N: int | Literal["NA"]
+    generator: str
+    checksum: str
+    checksum_file: str
+
+
 def find_field_value(
     row, response, fhir_attr, mapp, date_format, timezone, raw_data=None
 ):
@@ -357,6 +369,60 @@ def condense(x):
         return melted_data["flat_dict"].to_frame()
 
 
+def checksum(file: str) -> str:
+    "Calculate the SHA-256 checksum of a file"
+    h = hashlib.sha256()
+    with open(file, "rb") as fp:
+        while True:
+            data = fp.read(4096)
+            if len(data) == 0:
+                break
+            h.update(data)
+    return h.hexdigest()
+
+
+def checksum_text(checksums: dict[str, str]) -> str:
+    return "\n".join(f"{checksums[k]}  {k}" for k in sorted(checksums)) + "\n"
+
+
+def generate_metadata(folder_name: str) -> tuple[FlatMetadata, dict[str, str]]:
+    "Generate metadata for a FHIRFlat folder"
+
+    patient_file = os.path.join(folder_name, "patient.parquet")
+    if not os.path.exists(patient_file):
+        N = "NA"
+    else:
+        N = len(pd.read_parquet(patient_file).id.unique())
+    if isinstance(N, int):
+        assert N > 0, "patient.parquet file is empty"
+    checksums = {
+        os.path.basename(f): checksum(f) for f in glob(f"{folder_name}/*.parquet")
+    }
+    m = hashlib.sha256()
+    m.update(checksum_text(checksums).encode("utf-8"))
+
+    # write checksums file
+    return {
+        "N": N,
+        "generator": f"fhirflat/{fhirflat.__version__}",
+        "checksum": m.hexdigest(),
+        "checksum_file": "sha256sums.txt",
+    }, checksums
+
+
+def write_metadata(
+    metadata: FlatMetadata, checksums: dict[str, str], metadata_path: Path
+):
+    metadata_text = f"""[metadata]
+N = {metadata['N']}
+generator = "{metadata['generator']}"
+checksum = "{metadata['checksum']}"
+checksum_file = "{metadata['checksum_file']}"
+"""
+    metadata_path.write_text(metadata_text)
+    (metadata_path.parent / "sha256sums.txt").write_text(checksum_text(checksums))
+
+
 def convert_data_to_flat(
     data: str,
     date_format: str,
@@ -458,7 +524,6 @@ def convert_data_to_flat(
             f"{resource.__name__} took {total_time:.2f} seconds to convert"
             f" {len(df)} rows. "
         )
-
         if errors is not None:
             errors.to_csv(
                 os.path.join(folder_name, f"{resource.__name__.lower()}_errors.csv"),
@@ -470,6 +535,8 @@ def convert_data_to_flat(
                 f"Errors saved to {resource.__name__.lower()}_errors.csv"
             )
 
+    write_metadata(*generate_metadata(folder_name), Path(folder_name) / "fhirflat.toml")
+
 
 def main():
     parser = argparse.ArgumentParser(

fhirflat/resources/extension_validators.py

@@ -96,9 +96,9 @@ def fhir_model_validator(
         self, model_name: str, v: Union[StrBytes, dict, Path, FHIRAbstractModel]
     ):
         """ """
-        model_class: Type[BaseModel] | Type[FHIRAbstractModel] = (
-            self.get_fhir_model_class(model_name)
-        )
+        model_class: Type[BaseModel] | Type[
+            FHIRAbstractModel
+        ] = self.get_fhir_model_class(model_name)
 
         if isinstance(v, (str, bytes)):
             try:

pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
     "pydantic==2.6.1",
     "pydantic_core==2.16.2",
     "tzdata",
-    "python-dateutil"
+    "python-dateutil",
 ]
 
 [project.optional-dependencies]
@@ -35,6 +35,7 @@ dev = [
   "pytest-cov",
   "pytest-unordered",
   "ruff",
+  "tomli==2.*; python_version < '3.11'",
   "pre-commit"
 ]
 docs = [
@@ -73,4 +74,7 @@ extend-select = [
     "RUF",      # Ruff-specific
     "YTT",      # flake8-2020
 ]
-ignore = ["C901"]
+ignore = [
+  "C901",       # function is too complex
+  "C408",       # unnecessary `dict` call (rewrite as a literal)
+]

tests/data/patient.ndjson

@@ -1,3 +1,3 @@
 {"resourceType":"Patient","id":"ewnMwMK-UNvVvM.bakFSlkw3","identifier":[{"use":"usual","type":{"text":"EPIC"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.5.737384.0","value":"E9254"},{"use":"usual","type":{"text":"EXTERNAL"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.2.698084","value":"Z11363"},{"use":"usual","type":{"text":"FHIR"},"system":"http://open.epic.com/FHIR/StructureDefinition/patient-dstu2-fhir-id","value":"TYO3ktvhYAUhbae7JuBwDdpyIbUZc8kZG.bMW2ZwVnwgB"},{"use":"usual","type":{"text":"FHIR STU3"},"system":"http://open.epic.com/FHIR/StructureDefinition/patient-fhir-id","value":"ewnMwMK-UNvVvM.bakFSlkw3"},{"use":"usual","type":{"text":"PAS"},"system":"urn:oid:2.16.840.1.113883.2.1.3.12.1.1","value":"2005294"},{"use":"usual","type":{"text":"INTERNAL"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.2.698084","value":"    Z11363"}],"active":true,"name":[{"extension":[{"valueCode":"NL4","url":"http://hl7.org/fhir/StructureDefinition/humanname-assembly-order"}],"use":"official","text":"Test ADVANCEPREPTWO","family":"ADVANCEPREPTWO","given":["Test"],"_family":{"extension":[{"valueString":"Advancepreptwo","url":"http://hl7.org/fhir/StructureDefinition/humanname-own-name"}]}},{"extension":[{"valueCode":"NL4","url":"http://hl7.org/fhir/StructureDefinition/humanname-assembly-order"}],"use":"usual","text":"Test ADVANCEPREPTWO","family":"ADVANCEPREPTWO","given":["Test"],"_family":{"extension":[{"valueString":"Advancepreptwo","url":"http://hl7.org/fhir/StructureDefinition/humanname-own-name"}]}}],"gender":"female","birthDate":"2006-10-07","deceasedBoolean":false,"maritalStatus":{"text":"Single"}}
 {"resourceType":"Patient","id":"exU8JSL0p8npSw5g1QYAyOw3","identifier":[{"use":"usual","type":{"text":"EPIC"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.5.737384.0","value":"E9340"},{"use":"usual","type":{"text":"EXTERNAL"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.2.698084","value":"Z11449"},{"use":"usual","type":{"text":"FHIR"},"system":"http://open.epic.com/FHIR/StructureDefinition/patient-dstu2-fhir-id","value":"TwzgMHMPt5OhnHkFA2H2DudL3FU8qcRZkZyn0F3extkAB"},{"use":"usual","type":{"text":"FHIR STU3"},"system":"http://open.epic.com/FHIR/StructureDefinition/patient-fhir-id","value":"exU8JSL0p8npSw5g1QYAyOw3"},{"use":"usual","type":{"text":"PAS"},"system":"urn:oid:2.16.840.1.113883.2.1.3.12.1.1","value":"2005380"},{"use":"usual","type":{"text":"INTERNAL"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.2.698084","value":"    Z11449"}],"active":true,"name":[{"extension":[{"valueCode":"NL4","url":"http://hl7.org/fhir/StructureDefinition/humanname-assembly-order"}],"use":"official","text":"BcnScTenAugTwentyOne UPGRADETEST","family":"UPGRADETEST","given":["BcnScTenAugTwentyOne"],"_family":{"extension":[{"valueString":"UpgradeTest","url":"http://hl7.org/fhir/StructureDefinition/humanname-own-name"}]}},{"extension":[{"valueCode":"NL4","url":"http://hl7.org/fhir/StructureDefinition/humanname-assembly-order"}],"use":"usual","text":"BcnScTenAugTwentyOne UPGRADETEST","family":"UPGRADETEST","given":["BcnScTenAugTwentyOne"],"_family":{"extension":[{"valueString":"UpgradeTest","url":"http://hl7.org/fhir/StructureDefinition/humanname-own-name"}]}}],"gender":"female","birthDate":"2019-09-21","deceasedBoolean":false,"maritalStatus":{"text":"Single"}}
-{"resourceType":"Patient","id":"ezER-U3fAMP-WvI-Fc8V9wQ3","identifier":[{"use":"usual","type":{"text":"EPIC"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.5.737384.0","value":"E9411"},{"use":"usual","type":{"text":"EXTERNAL"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.2.698084","value":"Z11522"},{"use":"usual","type":{"text":"FHIR"},"system":"http://open.epic.com/FHIR/StructureDefinition/patient-dstu2-fhir-id","value":"T2oDrcOUw0w.1ZUdthu24c7V95lTOTEOUPRqBqGn99KEB"},{"use":"usual","type":{"text":"FHIR STU3"},"system":"http://open.epic.com/FHIR/StructureDefinition/patient-fhir-id","value":"ezER-U3fAMP-WvI-Fc8V9wQ3"},{"use":"usual","type":{"text":"PAS"},"system":"urn:oid:2.16.840.1.113883.2.1.3.12.1.1","value":"2005452"},{"use":"usual","type":{"text":"INTERNAL"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.2.698084","value":"    Z11522"}],"active":true,"name":[{"extension":[{"valueCode":"NL4","url":"http://hl7.org/fhir/StructureDefinition/humanname-assembly-order"}],"use":"official","text":"Fred TEST","family":"TEST","given":["Fred"],"_family":{"extension":[{"valueString":"Test","url":"http://hl7.org/fhir/StructureDefinition/humanname-own-name"}]}},{"extension":[{"valueCode":"NL4","url":"http://hl7.org/fhir/StructureDefinition/humanname-assembly-order"}],"use":"usual","text":"Fred TEST","family":"TEST","given":["Fred"],"_family":{"extension":[{"valueString":"Test","url":"http://hl7.org/fhir/StructureDefinition/humanname-own-name"}]}}],"telecom":[{"system":"phone","value":"07594 832748","use":"mobile"},{"system":"email","value":"fred@email.com"}],"gender":"male","birthDate":"1967-01-19","deceasedBoolean":false,"address":[{"use":"old","line":["123 Anywhere"],"city":"Bristol","postalCode":"BS1 6JY","country":"ENG"},{"use":"home","line":["123 Anywhere"],"city":"Bristol","postalCode":"BS1 6JY","country":"ENG","period":{"start":"2022-01-19"}}],"maritalStatus":{"text":"Single"},"communication":[{"language":{"coding":[{"system":"urn:ietf:bcp:47","code":"en","display":"English"}],"text":"English"},"preferred":true}]}
+{"resourceType":"Patient","id":"ezER-U3fAMP-WvI-Fc8V9wQ3","identifier":[{"use":"usual","type":{"text":"EPIC"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.5.737384.0","value":"E9411"},{"use":"usual","type":{"text":"EXTERNAL"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.2.698084","value":"Z11522"},{"use":"usual","type":{"text":"FHIR"},"system":"http://open.epic.com/FHIR/StructureDefinition/patient-dstu2-fhir-id","value":"T2oDrcOUw0w.1ZUdthu24c7V95lTOTEOUPRqBqGn99KEB"},{"use":"usual","type":{"text":"FHIR STU3"},"system":"http://open.epic.com/FHIR/StructureDefinition/patient-fhir-id","value":"ezER-U3fAMP-WvI-Fc8V9wQ3"},{"use":"usual","type":{"text":"PAS"},"system":"urn:oid:2.16.840.1.113883.2.1.3.12.1.1","value":"2005452"},{"use":"usual","type":{"text":"INTERNAL"},"system":"urn:oid:1.2.840.114350.1.13.520.3.7.2.698084","value":"    Z11522"}],"active":true,"name":[{"extension":[{"valueCode":"NL4","url":"http://hl7.org/fhir/StructureDefinition/humanname-assembly-order"}],"use":"official","text":"Fred TEST","family":"TEST","given":["Fred"],"_family":{"extension":[{"valueString":"Test","url":"http://hl7.org/fhir/StructureDefinition/humanname-own-name"}]}},{"extension":[{"valueCode":"NL4","url":"http://hl7.org/fhir/StructureDefinition/humanname-assembly-order"}],"use":"usual","text":"Fred TEST","family":"TEST","given":["Fred"],"_family":{"extension":[{"valueString":"Test","url":"http://hl7.org/fhir/StructureDefinition/humanname-own-name"}]}}],"telecom":[{"system":"phone","value":"07594 832748","use":"mobile"},{"system":"email","value":"fred@email.com"}],"gender":"male","birthDate":"1967-01-19","deceasedBoolean":false,"address":[{"use":"old","line":["123 Anywhere"],"city":"Bristol","postalCode":"BS1 6JY","country":"ENG"},{"use":"home","line":["123 Anywhere"],"city":"Bristol","postalCode":"BS1 6JY","country":"ENG","period":{"start":"2022-01-19"}}],"maritalStatus":{"text":"Single"},"communication":[{"language":{"coding":[{"system":"urn:ietf:bcp:47","code":"en","display":"English"}],"text":"English"},"preferred":true}]}

tests/dummy_data/combined_dummy_data.csv

@@ -2,4 +2,4 @@
 1,10,2020-05-01,0,,,,,,,cough,,,7,2020-01-01,36.2,120,30,70,120,5,,75,1,1,1,150
 2,11,,1,2021-04-01,18:00,fish,1,,2,,,2021-04-10,1,2021-02-02,37,100,40,80,130,6,10,85,0,2,1,200
 3,12,,1,2021-05-10,17:30,,1,,1,flu,,2021-05-15,4,2022-03-03,35.5,70,50,90,140,7,,95,0,3,1,
-4,13,,1,2022-06-15,21:00,dolphin,0,Malaria,,,,2022-06-20,2,,,,,,,,,,,,,
+4,13,,1,2022-06-15,21:00,dolphin,0,Malaria,,,,2022-06-20,2,,,,,,,,,,,,,

tests/dummy_data/encounter_dummy_data_multi_patient.csv

@@ -2,4 +2,4 @@
 1,10,2020-05-01,0,,,,,,,cough,,,7
 2,11,,1,2021-04-01,18:00,fish,1,,2,,,2021-04-10,1
 3,12,,1,2021-05-10,17:30,,1,,1,flu,,2021-05-15,4
-4,13,,1,2022-06-15,21:00,dolphin,0,Malaria,,,,2022-06-20,2
+4,13,,1,2022-06-15,21:00,dolphin,0,Malaria,,,,2022-06-20,2

tests/dummy_data/encounter_dummy_data_single.csv

@@ -1,2 +1,2 @@
 subjid,visitid,dates_enrolment,dates_adm,dates_admdate,dates_admtime,non_encounter_field,outco_denguediag,outco_denguediag_main,outco_denguediag_class,outco_not_dengue,outco_secondiag_oth,outco_date,outco_outcome
-2,11,2021-04-02,1,2021-04-01,18:00,fish,1,,2,,,2021-04-10,1
+2,11,2021-04-02,1,2021-04-01,18:00,fish,1,,2,,,2021-04-10,1

tests/dummy_data/observation_dummy_mapping.csv

@@ -17,4 +17,4 @@ vital_avpu,"1, Alert",,http://terminology.hl7.org/CodeSystem/observation-categor
 ,"3, Pain",,http://terminology.hl7.org/CodeSystem/observation-category,vital-signs,Vital Signs,<daily_date>,https://snomed.info/sct,1.10444E+15,Alert Confusion Voice Pain Unresponsive scale score (observable entity),Patient/+<subjid>,Encounter/+<visitid>,,,,,https://snomed.info/sct,450847001,Responds to pain (finding),,
 ,"4, Unresponsive",,http://terminology.hl7.org/CodeSystem/observation-category,vital-signs,Vital Signs,<daily_date>,https://snomed.info/sct,1.10444E+15,Alert Confusion Voice Pain Unresponsive scale score (observable entity),Patient/+<subjid>,Encounter/+<visitid>,,,,,https://snomed.info/sct,422768004,Unresponsive (finding),,
 vital_gcs,,,http://terminology.hl7.org/CodeSystem/observation-category,vital-signs,Vital Signs,<daily_date>,https://snomed.info/sct,9269-2,Glasgow coma score total,Patient/+<subjid>,Encounter/+<visitid>,,,,,,,,,<FIELD>
-vital_urineflow,,,http://terminology.hl7.org/CodeSystem/observation-category,vital-signs,Vital Signs,<daily_date>,https://loinc.org,9192-6,Urine output 24 hour,Patient/+<subjid>,Encounter/+<visitid>,<FIELD>,https://snomed.info/sct,258861009,Millilitre/24 hours (qualifier value),,,,,
+vital_urineflow,,,http://terminology.hl7.org/CodeSystem/observation-category,vital-signs,Vital Signs,<daily_date>,https://loinc.org,9192-6,Urine output 24 hour,Patient/+<subjid>,Encounter/+<visitid>,<FIELD>,https://snomed.info/sct,258861009,Millilitre/24 hours (qualifier value),,,,,

tests/dummy_data/vital_signs_dummy_data.csv

@@ -1,4 +1,4 @@
 subjid,visitid,daily_date,vital_highesttem_c,vital_hr,vital_rr,vital_systolicbp,vital_diastolicbp,vital_spo2,vital_fio2spo2_02110,vital_fio2spo2_pcnt,vital_capillaryr,vital_avpu,vital_gcs,vital_urineflow
 1,10,2020-01-01,36.2,120,30,70,120,5,,75,1,1,1,150
 2,11,2021-02-02,37,100,40,80,130,6,10,85,0,2,1,200
-3,12,2022-03-03,35.5,70,50,90,140,7,,95,0,3,1,
+3,12,2022-03-03,35.5,70,50,90,140,7,,95,0,3,1,