completed nxdict class

Author: Bing Li

src/tavi/data/nxdict.py

@@ -270,11 +270,11 @@ def spice_scan_to_nxdict(
         name=NXdataset(ds=metadata.get("samplename"), type="NX_CHAR", EX_required="true"),
         type=NXdataset(ds=metadata.get("sampletype"), type="NX_CHAR", EX_required="true"),
         unit_cell=NXdataset(ds=metadata.get("latticeconstants"), type="NX_FLOAT", EX_required="true"),
-        qh=NXdataset(ds=spicelogs.get("h"), type="NX_FLOAT", EX_required="true"),
-        qk=NXdataset(ds=spicelogs.get("k"), type="NX_FLOAT", EX_required="true"),
-        ql=NXdataset(ds=spicelogs.get("l"), type="NX_FLOAT", EX_required="true"),
+        qh=NXdataset(ds=spicelogs.get("h"), type="NX_FLOAT", EX_required="true", units="r.l.u."),
+        qk=NXdataset(ds=spicelogs.get("k"), type="NX_FLOAT", EX_required="true", units="r.l.u."),
+        ql=NXdataset(ds=spicelogs.get("l"), type="NX_FLOAT", EX_required="true", units="r.l.u."),
         en=NXdataset(ds=spicelogs.get("e"), type="NX_FLOAT", EX_required="true", units="meV"),
-        q=NXdataset(ds=spicelogs.get("q"), type="NX_FLOAT"),
+        q=NXdataset(ds=spicelogs.get("q"), type="NX_FLOAT", units="Angstrom^-1"),
         sense=NXdataset(ds=metadata["sense"][1], type="NX_CHAR"),
         NX_class="NXsample",
         EX_required="true",
@@ -379,15 +379,16 @@ def spice_data_to_nxdict(
 
     nexus_dict = {}
     for path_to_scan_file in scan_list:
-        pass
+        *_, file_name = path_to_scan_file.split("/")
+        *_, scan_dat = file_name.split("_")
+        scan_num, _ = scan_dat.split(".")
 
-        scan_name, scan_dict = spice_scan_to_nxdict(
+        nxentry = spice_scan_to_nxdict(
             path_to_scan_file,
             path_to_instrument_json,
             path_to_sample_json,
         )
-
-        nexus_dict.update({scan_name: scan_dict})
-        nexus_dict[scan_name]["attrs"].update({"dataset_name": dataset_name})
+        nxentry["attrs"].update({"dataset_name": dataset_name})
+        nexus_dict.update({scan_num: nxentry})
 
     return nexus_dict

src/tavi/data/nxentry.py

@@ -44,22 +44,28 @@ def _getitem_recursively(obj: dict, key: str, ATTRS: bool):
         value = None
         if key.split("/")[0] in obj:
             for key_item in key.split("/"):
-                obj = obj[key_item]
-            if ATTRS:
+                # you could be looking in the wrong folder
+                # e.g. looking for detector/data in data/detector
+                # return None when this happens
+                try:
+                    obj = obj[key_item]
+                except KeyError:
+                    return None
+            if ATTRS:  # get attributes
                 try:
                     value = obj["attrs"]
                 except KeyError:
                     print(f"Attribute of {key} does not exist.")
-            else:
+            else:  # get dataset
                 try:
                     value = obj["dataset"]
                 except KeyError:
                     print(f"Dataset of {key} does not exist.")
 
         for k, v in obj.items():
-            if value is not None:
+            if value is not None:  # value found
                 break
-            if k == "attrs" or k == "dataset":
+            if k == "attrs" or k == "dataset":  # gone too far
                 continue
 
             if isinstance(v, dict):
@@ -107,10 +113,11 @@ def format_dataset(value):
                 if "dataset" in value.keys():
                     dv = format_dataset(value)
                     if key in nexus_entry:  # dataset exists
-                        ds = nexus_entry[key]
-                        ds[...] = dv
-                    else:  # create dataset
-                        ds = nexus_entry.create_dataset(name=key, data=dv, maxshape=None)
+                        del nexus_entry[key]
+                        # ds = nexus_entry[key]
+                        # ds[...] = dv
+                    # else:  # create dataset
+                    ds = nexus_entry.create_dataset(name=key, data=dv, maxshape=None)
                     NexusEntry._write_recursively(value, ds)
                 else:
                     grp = nexus_entry.require_group(key + "/")
@@ -137,7 +144,10 @@ def _read_recursively(nexus_entry, items=None):
                 except TypeError:  # arrays instead
                     value = value[...]
 
-                items.update({key: {"attrs": attr_dict, "dataset": value}})
+                if not attr_dict:  # empty attributes
+                    items.update({key: {"dataset": value}})
+                else:
+                    items.update({key: {"attrs": attr_dict, "dataset": value}})
 
         return items
 
@@ -152,7 +162,7 @@ def _dict_to_nexus_entry(nexus_dict):
             nexus_entries.update({scan_num: NexusEntry(content_list)})
         return nexus_entries
 
-    # TODO read in instrument and sample configuratio json files
+    # TODO read in instrument and sample configuration json files
     @classmethod
     def from_spice(
         cls,
@@ -211,6 +221,7 @@ def to_nexus(self, path_to_nexus: str, name="scan") -> None:
             def_x = nexus_file["scan0034"]["data"].attrs["axes"]
             path_y = _find_val_path(def_y, nexus_file)
             path_x = _find_val_path(def_x, nexus_file)
+
             if path_y is not None:
                 def_y = "data/" + def_y
                 if isinstance(scan_grp.get(def_y), h5py.Dataset):

src/tavi/data/spice_reader.py

@@ -62,7 +62,12 @@ def read_spice_datafile(file_name: str):
 
 
 def read_spice_ubconf(ub_file_name: str):
-    """Reads ub info from UBConf"""
+    """Reads ub info from UBConf
+    Args:
+        ub_file_name (str): a string containing the filename
+
+    Returns:
+    """
     ubconf = {}
     with open(ub_file_name, encoding="utf-8") as f:
         all_content = f.readlines()
@@ -143,7 +148,8 @@ def _create_spicelogs(path_to_scan_file: str) -> dict:
     if len(spice_data_shape) == 1:  # 1 row ony
         if spice_data_shape[0] != 0:
             for idx, col_header in enumerate(col_names):
-                dataset_dict.update({col_header: data[idx]})
+                # convert to ndarray if one point only
+                dataset_dict.update({col_header: np.array([data[idx]])})
         else:  # ignore if empty
             pass
     elif len(spice_data_shape) == 2:  # nomarl data with mutiple rows

test_data/scan_to_nexus_test.h5

@@ -10,17 +10,27 @@ def test_get_dataset(nexus_entries):
     scan0034 = nexus_entries["scan0034"]
     assert scan0034.get("definition") == "NXtas"
     assert scan0034.get("title") == "scan_title_34"
-    assert np.allclose(scan0034.get("a2"), np.array([242.0, 242.1, 242.2]))
+    assert np.allclose(
+        scan0034.get("a2"),
+        np.array([242.0, 242.1, 242.2]),
+    )
     assert scan0034.get("data", ATTRS=True) == {
         "EX_required": "true",
         "NX_class": "NXdata",
-        "axes": "en",
+        "axes": "a1",
         "signal": "detector",
     }
     assert scan0034.get("a3") is None
     assert scan0034.get("detector") is None
-    assert scan0034.get("detector/data", ATTRS=True) == {"EX_required": "true", "type": "NX_INT", "units": "counts"}
-    assert np.allclose(scan0034.get("instrument/analyser/a2"), np.array([242.0, 242.1, 242.2]))
+    assert scan0034.get("detector/data", ATTRS=True) == {
+        "EX_required": "true",
+        "type": "NX_INT",
+        "units": "counts",
+    }
+    assert np.allclose(
+        scan0034.get("instrument/analyser/a2"),
+        np.array([242.0, 242.1, 242.2]),
+    )
 
     scan0035 = nexus_entries["scan0035"]
     assert scan0035.get("title") == "scan_title_35"
@@ -36,24 +46,37 @@ def test_to_nexus(nexus_entries):
         assert nexus_file["scan0034"].attrs["NX_class"] == "NXentry"
 
 
-def test_from_nexus():
+def test_from_nexus(nexus_entries):
     path_to_nexus_entry = "./test_data/scan_to_nexus_test.h5"
-    nexus_entries = NexusEntry.from_nexus(path_to_nexus_entry)
-    scan0034 = nexus_entries["scan0034"]
-    assert scan0034.get("definition") == "NXtas"
+    entries = NexusEntry.from_nexus(path_to_nexus_entry)
+
+    scan0034 = entries["scan0034"]
+    assert scan0034["definition"] == nexus_entries["scan0034"]["definition"]
+    assert np.allclose(
+        scan0034["instrument"]["analyser"]["a2"]["dataset"],
+        nexus_entries["scan0034"]["instrument"]["analyser"]["a2"]["dataset"],
+    )
     assert np.allclose(scan0034.get("a2"), np.array([242.0, 242.1, 242.2]))
     assert scan0034.get("data", ATTRS=True) == {
         "EX_required": "true",
         "NX_class": "NXdata",
-        "axes": "en",
+        "axes": "a1",
         "signal": "detector",
     }
     assert scan0034.get("a3") is None
-    assert scan0034.get("detector") is None
-    assert scan0034.get("detector/data", ATTRS=True) == {"EX_required": "true", "type": "NX_INT", "units": "counts"}
-    assert np.allclose(scan0034.get("instrument/analyser/a2"), np.array([242.0, 242.1, 242.2]))
+    assert np.allclose(scan0034.get("detector"), np.array([1, 2, 3]))
+    assert scan0034.get("detector/data", ATTRS=True) == {
+        "EX_required": "true",
+        "type": "NX_INT",
+        "units": "counts",
+        "target": "/scan0034/instrument/detector/data",
+    }
+    assert np.allclose(
+        scan0034.get("instrument/analyser/a2"),
+        np.array([242.0, 242.1, 242.2]),
+    )
 
-    scan0035 = nexus_entries["scan0035"]
+    scan0035 = entries["scan0035"]
     assert scan0035.get("title") == "scan_title_35"
 
 
@@ -73,19 +96,27 @@ def test_from_nexus_IPTS32124_CG4C_exp0424():
     assert np.allclose(scan0034.get("s1")[0:3], [36.14, 36.5025, 36.855])
 
 
-def test_from_spice_IPTS32124_CG4C_exp0424():
+def test_from_spice_IPTS32124_CG4C_exp0424_single_scan():
     path_to_spice_entry = "./test_data/exp424"
-    nexus_entries = NexusEntry.from_spice(path_to_spice_entry, 34)
+    scan0034 = NexusEntry.from_spice(path_to_spice_entry, 34)["scan0034"]
 
-    path_to_nexus_entry = "./test_data/spice_to_nexus_test_scan34.h5"
-    for scan_num, nexus_entry in nexus_entries.items():
-        nexus_entry.to_nexus(path_to_nexus_entry, scan_num)
+    assert scan0034.get("definition") == "NXtas"
+    assert scan0034.get("end_time") == "2024-07-03T02:41:28"
+    assert np.allclose(scan0034.get("s1")[0:3], np.array([36.14, 36.5025, 36.855]))
 
-    scan0034 = NexusEntry.from_nexus(path_to_nexus_entry, 34)["scan0034"]
 
+def test_from_spice_IPTS32124_CG4C_exp0424_all_scan():
+    path_to_spice_entry = "./test_data/exp424"
+    scans = NexusEntry.from_spice(path_to_spice_entry)
+
+    scan0034 = scans["scan0034"]
     assert scan0034.get("definition") == "NXtas"
     assert scan0034.get("end_time") == "2024-07-03T02:41:28"
-    # assert np.allclose(scan0034.get("s1")[0:3], np.array([36.14, 36.5025, 36.855]))
+    assert np.allclose(scan0034.get("s1")[0:3], np.array([36.14, 36.5025, 36.855]))
+
+    # scan 41 contains only one data point
+    scan0041 = scans["scan0041"]
+    assert scan0041.get("Pt.") == 3
 
 
 def test_get_from_daslogs():
@@ -132,6 +163,7 @@ def nexus_entries():
         "attrs": {"EX_required": "true", "NX_class": "NXdetector"},
         "data": {
             "attrs": {"EX_required": "true", "type": "NX_INT", "units": "counts"},
+            "dataset": np.array([1, 2, 3]),
         },
     }
 
@@ -169,7 +201,7 @@ def nexus_entries():
                 "attrs": {
                     "EX_required": "true",
                     "NX_class": "NXdata",
-                    "axes": "en",
+                    "axes": "a1",
                     "signal": "detector",
                 }
             },