Skip to content

Commit 542bcd5

Browse files
author
Bing Li
committed
completed spice_scan_to_nxdict
1 parent 76109a0 commit 542bcd5

File tree

6 files changed

+254
-20
lines changed

6 files changed

+254
-20
lines changed

‎src/tavi/data/nxdict.py

+159-14
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,42 @@
1010
from tavi.data.spice_reader import _create_spicelogs, read_spice_datafile
1111

1212

13+
def _find_val(val, grp, prefix=""):
14+
"""Find value in hdf5 groups"""
15+
for obj_name, obj in grp.items():
16+
if obj_name in ("SPICElogs", "data"):
17+
continue
18+
else:
19+
path = f"{prefix}/{obj_name}"
20+
if val == obj_name:
21+
return path
22+
# test for group (go down)
23+
elif isinstance(obj, h5py.Group):
24+
gpath = _find_val(val, obj, path)
25+
if gpath:
26+
return gpath
27+
28+
29+
def _recast_type(ds, dtype):
30+
if type(ds) is str:
31+
if "," in ds: # vector
32+
if dtype == "NX_FLOAT":
33+
dataset = np.array([float(d) for d in ds.split(",")])
34+
else:
35+
dataset = np.array([int(d) for d in ds.split(",")])
36+
elif ds.replace(".", "").isnumeric(): # numebrs only
37+
if dtype == "NX_FLOAT":
38+
dataset = float(ds)
39+
else:
40+
dataset = int(ds)
41+
else: # expect np.ndarray
42+
if dtype == "NX_FLOAT":
43+
dataset = np.array([float(d) for d in ds])
44+
else:
45+
dataset = np.array([int(d) for d in ds])
46+
return dataset
47+
48+
1349
class NXdataset(dict):
1450
"""Dataset in a format consistent with NeXus, containg attrs and dataset"""
1551

@@ -24,10 +60,10 @@ def __init__(self, ds, **kwargs):
2460
match kwargs:
2561
case {"type": "NX_CHAR"}:
2662
dataset = str(ds)
27-
case {"type": "NX_INT"}:
28-
dataset = np.array([int(d) for d in ds])
29-
case {"type": "NX_FLOAT"}:
30-
dataset = np.array([float(d) for d in ds])
63+
case {"type": "NX_INT"} | {"type": "NX_FLOAT"}:
64+
dataset = _recast_type(ds, kwargs["type"])
65+
case {"type": "NX_DATE_TIME"}:
66+
dataset = datetime.strptime(ds, "%m/%d/%Y %I:%M:%S %p").isoformat()
3167
case _:
3268
dataset = ds
3369

@@ -64,13 +100,36 @@ def add_dataset(self, key: str, ds: NXdataset):
64100
else:
65101
self.update({key: ds})
66102

103+
def add_attribute(self, key: str, attr):
104+
if not attr: # ignore if empty
105+
pass
106+
else:
107+
self["attrs"].update({key: attr})
108+
67109

110+
def _formatted_spicelogs(spicelogs: dict) -> NXentry:
111+
"""Format SPICE logs into NeXus dict"""
112+
formatted_spicelogs = NXentry(NX_class="NXcollection", EX_required="false")
113+
metadata = spicelogs.pop("metadata")
114+
for attr_key, attr_entry in metadata.items():
115+
formatted_spicelogs.add_attribute(attr_key, attr_entry)
116+
117+
for entry_key, entry_data in spicelogs.items():
118+
formatted_spicelogs.add_dataset(key=entry_key, ds=NXdataset(ds=entry_data))
119+
return formatted_spicelogs
120+
121+
122+
# TODO json support
68123
def spice_scan_to_nxdict(
69124
path_to_scan_file: str,
70125
path_to_instrument_json: Optional[str] = None,
71126
path_to_sample_json: Optional[str] = None,
72127
) -> NXentry:
73-
"""Format SPICE data in a nested dictionary format"""
128+
"""Format SPICE data in a nested dictionary format
129+
130+
Note:
131+
json files can overwrite the parameters in SPICE
132+
"""
74133

75134
# parse instruemnt and sample json files
76135
instrument_config_params = None
@@ -92,17 +151,20 @@ def spice_scan_to_nxdict(
92151
spicelogs = _create_spicelogs(path_to_scan_file)
93152
metadata = spicelogs["metadata"]
94153

154+
# ---------------------------------------- source ----------------------------------------
155+
95156
nxsource = NXentry(
96157
name=NXdataset(ds="HFIR", type="NX_CHAR", EX_required="true"),
97158
probe=NXdataset(ds="neutron", type="NX_CHAR", EX_required="true"),
98159
NX_class="NXsource",
99160
EX_required="true",
100161
)
162+
# ------------------------------------- monochromator ----------------------------------------
101163

102164
nxmono = NXentry(
103165
ei=NXdataset(ds=spicelogs.get("ei"), type="NX_FLOAT", EX_required="true", units="meV"),
104166
type=NXdataset(ds=metadata.get("monochromator"), type="NX_CHAR"),
105-
sense=NXdataset(ds=metadata.get["sense"][0], type="NX_CHAR"),
167+
sense=NXdataset(ds=metadata["sense"][0], type="NX_CHAR"),
106168
m1=NXdataset(ds=spicelogs.get("m1"), type="NX_FLOAT", units="degrees"),
107169
m2=NXdataset(ds=spicelogs.get("m2"), type="NX_FLOAT", units="degrees"),
108170
NX_class="NXcrystal",
@@ -113,6 +175,8 @@ def spice_scan_to_nxdict(
113175
nxmono.add_dataset("mtrans", NXdataset(ds=spicelogs.get("mtrans"), type="NX_FLOAT"))
114176
nxmono.add_dataset("focal_length", NXdataset(ds=spicelogs.get("focal_length"), type="NX_FLOAT"))
115177

178+
# ------------------------------------- analyzer ----------------------------------------
179+
116180
nxana = NXentry(
117181
ef=NXdataset(ds=spicelogs.get("ef"), type="NX_FLOAT", EX_required="true", units="meV"),
118182
type=NXdataset(ds=metadata.get("analyzer"), type="NX_CHAR"),
@@ -123,25 +187,60 @@ def spice_scan_to_nxdict(
123187
NX_class="NXcrystal",
124188
EX_required="true",
125189
)
126-
for i in range(8):
190+
for i in range(8): # CG4C horizontal focusing
127191
nxana.add_dataset(key=f"qm{i+1}", ds=NXdataset(ds=spicelogs.get(f"qm{i+1}"), type="NX_FLOAT"))
128192
nxana.add_dataset(key=f"xm{i+1}", ds=NXdataset(ds=spicelogs.get(f"xm{i+1}"), type="NX_FLOAT"))
129193

194+
# ------------------------------------- detector ----------------------------------------
195+
130196
nxdet = NXentry(
131197
data=NXdataset(ds=spicelogs.get("detector"), type="NX_INT", EX_required="true", units="counts"),
132198
NX_class="NXdetector",
133199
EX_required="true",
134200
)
135201

202+
# ------------------------------------- collimators ----------------------------------------
203+
204+
div_x = [float(v) for v in list(metadata["collimation"].split("-"))]
205+
nxcoll = NXentry(
206+
type=NXdataset(ds="Soller", type="NX_CHAR"),
207+
NX_class="NXcollimator",
208+
)
209+
nxcoll.add_dataset(key="divergence_x", ds=NXdataset(ds=div_x, type="NX_ANGLE", units="minutes of arc"))
210+
211+
# ------------------------------------- slits ----------------------------------------
212+
213+
nxslits = NXentry(NX_class="NXslit")
214+
slits_str1 = tuple([f"b{idx}{loc}" for idx in ("a", "b") for loc in ("t", "b", "l", "r")])
215+
slits_str2 = tuple([f"slit{idx}_{loc}" for idx in ("a", "b") for loc in ("lf", "rt", "tp", "bt")])
216+
slits_str3 = tuple([f"slit_{idx}_{loc}" for idx in ("pre",) for loc in ("lf", "rt", "tp", "bt")])
217+
slits_str = (slits_str1, slits_str2, slits_str3)
218+
for slit_str in slits_str:
219+
for st in slit_str:
220+
nxslits.add_dataset(key=st, ds=NXdataset(ds=spicelogs.get(st), type="NX_FLOAT", units="cm"))
221+
# ------------------------------------- flipper ----------------------------------------
222+
223+
nxflipper = NXentry(NX_class="NXflipper")
224+
nxflipper.add_dataset(key="fguide", ds=NXdataset(ds=spicelogs.get("fguide"), type="NX_FLOAT"))
225+
nxflipper.add_dataset(key="hguide", ds=NXdataset(ds=spicelogs.get("hguide"), type="NX_FLOAT"))
226+
nxflipper.add_dataset(key="vguide", ds=NXdataset(ds=spicelogs.get("vguide"), type="NX_FLOAT"))
227+
228+
# ---------------------------------------- instrument ---------------------------------------------
229+
136230
nxinstrument = NXentry(
137231
source=nxsource,
138232
monochromator=nxmono,
233+
collimator=nxcoll,
139234
analyzer=nxana,
140235
detector=nxdet,
236+
slits=nxslits,
237+
flipper=nxflipper,
141238
name=NXdataset(ds=metadata.get("instrument"), type="NX_CHAR"),
142239
NX_class="NXinstrument",
143240
EX_required="true",
144241
)
242+
# ---------------------------------------- monitor ---------------------------------------------
243+
145244
preset_type = metadata.get("preset_type")
146245
if preset_type == "normal":
147246
preset_channel = metadata.get("preset_channel")
@@ -156,6 +255,7 @@ def spice_scan_to_nxdict(
156255
NX_class="NXmonitor",
157256
EX_required="true",
158257
)
258+
159259
# TODO polarized exp at HB1
160260
elif preset_type == "countfile":
161261
print("Polarization data, not yet supported.")
@@ -164,34 +264,79 @@ def spice_scan_to_nxdict(
164264
print(f"Unrecogonized preset type {preset_type}.")
165265
nxmonitor = NXentry(NX_class="NXmonitor", EX_required="true")
166266

167-
# ------------------------------------------------------------------
168-
nxsample = NXentry(NX_class="NXsample", EX_required="true")
267+
# ---------------------------------------- sample ---------------------------------------------
268+
269+
nxsample = NXentry(
270+
name=NXdataset(ds=metadata.get("samplename"), type="NX_CHAR", EX_required="true"),
271+
type=NXdataset(ds=metadata.get("sampletype"), type="NX_CHAR", EX_required="true"),
272+
unit_cell=NXdataset(ds=metadata.get("latticeconstants"), type="NX_FLOAT", EX_required="true"),
273+
qh=NXdataset(ds=spicelogs.get("h"), type="NX_FLOAT", EX_required="true"),
274+
qk=NXdataset(ds=spicelogs.get("k"), type="NX_FLOAT", EX_required="true"),
275+
ql=NXdataset(ds=spicelogs.get("l"), type="NX_FLOAT", EX_required="true"),
276+
en=NXdataset(ds=spicelogs.get("e"), type="NX_FLOAT", EX_required="true", units="meV"),
277+
q=NXdataset(ds=spicelogs.get("q"), type="NX_FLOAT"),
278+
sense=NXdataset(ds=metadata["sense"][1], type="NX_CHAR"),
279+
NX_class="NXsample",
280+
EX_required="true",
281+
)
282+
nxsample.add_dataset(key="Pt.", ds=NXdataset(ds=spicelogs.get("Pt."), type="NX_INT"))
283+
284+
# Motor angles
285+
nxsample.add_dataset(key="s1", ds=NXdataset(ds=spicelogs.get("s1"), type="NX_FLOAT", units="degrees"))
286+
nxsample.add_dataset(key="s2", ds=NXdataset(ds=spicelogs.get("s2"), type="NX_FLOAT", units="degrees"))
287+
nxsample.add_dataset(key="sgu", ds=NXdataset(ds=spicelogs.get("sgu"), type="NX_FLOAT", units="degrees"))
288+
nxsample.add_dataset(key="sgl", ds=NXdataset(ds=spicelogs.get("sgl"), type="NX_FLOAT", units="degrees"))
289+
nxsample.add_dataset(key="stu", ds=NXdataset(ds=spicelogs.get("stu"), type="NX_FLOAT", units="degrees"))
290+
nxsample.add_dataset(key="stl", ds=NXdataset(ds=spicelogs.get("stl"), type="NX_FLOAT", units="degrees"))
291+
292+
# UB info
293+
nxsample.add_dataset(
294+
key="orientation_matrix",
295+
ds=NXdataset(ds=metadata.get("ubmatrix"), type="NX_FLOAT", EX_required="true", units="NX_DIMENSIONLESS"),
296+
)
297+
nxsample.add_dataset(key="ub_conf", ds=NXdataset(ds=metadata["ubconf"].split(".")[0], type="NX_CHAR"))
298+
nxsample.add_dataset(key="plane_normal", ds=NXdataset(ds=metadata.get("plane_normal"), type="NX_FLOAT"))
299+
300+
# ---------------------------------------- sample environment ---------------------------------------------
169301

170302
# TODO all sample environment variable names needed!
171303
temperatue_str = (
172304
("temp", "temp_a", "temp_2", "coldtip", "tsample", "sample")
173305
+ ("vti", "dr_tsample", "dr_temp")
174306
+ ("lt", "ht", "sorb_temp", "sorb", "sample_ht")
175307
)
308+
for te in temperatue_str:
309+
nxsample.add_dataset(key=te, ds=NXdataset(ds=spicelogs.get(te), type="NX_FLOAT", units="Kelvin"))
176310

177311
field_str = ("persistent_field", "mag_i")
312+
for fi in field_str:
313+
nxsample.add_dataset(key=fi, ds=NXdataset(ds=spicelogs.get(fi), type="NX_FLOAT", units="Tesla"))
314+
# ---------------------------------------- data ---------------------------------------------
315+
nexus_keywork_conversion_dict = {"h": "qh", "k": "qk", "l": "ql", "e": "en"}
316+
def_x = metadata.get("def_x")
317+
def_y = metadata.get("def_y")
318+
if def_x in nexus_keywork_conversion_dict:
319+
def_x = nexus_keywork_conversion_dict[def_x]
320+
321+
nxdata = NXentry(NX_class="NXdata", EX_required="true", signal=def_y, axes=def_x)
322+
# ---------------------------------------- scan ---------------------------------------------
178323

179324
# TODO timezone
180325
start_date_time = "{} {}".format(metadata.get("date"), metadata.get("time"))
181-
start_time = datetime.strptime(start_date_time, "%m/%d/%Y %I:%M:%S %p").isoformat()
182326
# TODO what is last scan never finished?
183327
# if "end_time" in das_logs.attrs:
184328
end_date_time = metadata.get("end_time")
185-
end_time = datetime.strptime(end_date_time, "%m/%d/%Y %I:%M:%S %p").isoformat()
186329

187330
nxscan = NXentry(
188-
SPICElogs=spicelogs,
331+
SPICElogs=_formatted_spicelogs(spicelogs),
332+
data=nxdata,
189333
definition=NXdataset(ds="NXtas", type="NX_CHAR", EX_required="true"),
190334
title=NXdataset(ds=metadata.get("scan_title"), type="NX_CHAR", EX_required="true"),
191-
start_time=NXdataset(ds=start_time, type="NX_DATE_TIME", EX_required="true"),
192-
end_time=NXdataset(ds=end_time, type="NX_DATE_TIME"),
335+
start_time=NXdataset(ds=start_date_time, type="NX_DATE_TIME", EX_required="true"),
336+
end_time=NXdataset(ds=end_date_time, type="NX_DATE_TIME"),
193337
instrument=nxinstrument,
194338
monitor=nxmonitor,
339+
sample=nxsample,
195340
NX_class="NXentry",
196341
EX_required="true",
197342
)

‎src/tavi/data/nxentry.py

+47-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,32 @@
1+
import os
2+
from datetime import datetime
13
from typing import Optional
24

35
import h5py
46
import numpy as np
57

6-
from tavi.data.spice_reader import spice_data_to_nxdict
8+
from tavi.data.nxdict import spice_data_to_nxdict
9+
10+
11+
def _find_val_path(val, grp, prefix=""):
12+
"""Find value in hdf5 groups"""
13+
for obj_name, obj in grp.items():
14+
if obj_name in ("SPICElogs", "data"):
15+
continue
16+
else:
17+
path = f"{prefix}/{obj_name}"
18+
if val == obj_name:
19+
if val == "detector":
20+
return path + "/data"
21+
elif val == "monitor":
22+
return path + "/monitor"
23+
else:
24+
return path
25+
# test for group (go down)
26+
elif isinstance(obj, h5py.Group):
27+
gpath = _find_val_path(val, obj, path)
28+
if gpath:
29+
return gpath
730

831

932
class NexusEntry(dict):
@@ -183,6 +206,29 @@ def to_nexus(self, path_to_nexus: str, name="scan") -> None:
183206
with h5py.File(path_to_nexus, "a") as nexus_file:
184207
scan_grp = nexus_file.require_group(name + "/")
185208
NexusEntry._write_recursively(self, scan_grp)
209+
# create soft link for data
210+
def_y = nexus_file["scan0034"]["data"].attrs["signal"]
211+
def_x = nexus_file["scan0034"]["data"].attrs["axes"]
212+
path_y = _find_val_path(def_y, nexus_file)
213+
path_x = _find_val_path(def_x, nexus_file)
214+
if path_y is not None:
215+
def_y = "data/" + def_y
216+
if isinstance(scan_grp.get(def_y), h5py.Dataset):
217+
del scan_grp[def_y]
218+
scan_grp[def_y] = h5py.SoftLink(path_y)
219+
scan_grp[def_y + "/"].attrs["target"] = path_y
220+
if path_x is not None:
221+
def_x = "data/" + def_x
222+
if isinstance(scan_grp.get(def_x), h5py.Dataset):
223+
del scan_grp[def_x]
224+
scan_grp[def_x] = h5py.SoftLink(path_x)
225+
scan_grp[def_x + "/"].attrs["target"] = path_x
226+
227+
# Create the ATTRIBUTES
228+
scan_grp.attrs["file_name"] = os.path.abspath(path_to_nexus)
229+
scan_grp.attrs["file_time"] = datetime.now().isoformat()
230+
scan_grp.attrs["h5py_version"] = h5py.version.version
231+
scan_grp.attrs["HDF5_Version"] = h5py.version.hdf5_version
186232

187233
def get(self, key, ATTRS=False, default=None):
188234
"""

‎src/tavi/data/spice_reader.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -124,11 +124,12 @@ def format_spice_header(headers):
124124

125125
def _create_spicelogs(path_to_scan_file: str) -> dict:
126126
"""read in SPICE data, return a dictionary containing metadata and data columns"""
127-
(data, col_names, metadata, others, error_messages) = read_spice_datafile(path_to_scan_file)
128127

128+
(data, col_names, metadata, others, error_messages) = read_spice_datafile(path_to_scan_file)
129+
*_, file_name = path_to_scan_file.split("/")
130+
instrument_str, *_ = file_name.split("_")
129131
# write SPICElogs attributes
130-
# attrs_dict = {"NX_class": "NXcollection", "EX_required": "false"}
131-
attrs_dict = {}
132+
attrs_dict = {"instrument": instrument_str}
132133
for k, v in metadata.items():
133134
attrs_dict.update({k: v})
134135
if len(error_messages) != 0:

‎test_data/spice_to_nxdict_test_scan0034.h5

109 KB
Binary file not shown.

‎tests/test_nxdict.py

+26-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
from tavi.data.nxdict import NXdataset, NXentry
1+
import numpy as np
2+
3+
from tavi.data.nxdict import NXdataset, NXentry, spice_scan_to_nxdict
4+
from tavi.data.nxentry import NexusEntry
25
from tavi.data.spice_reader import _create_spicelogs
36

47

@@ -68,4 +71,25 @@ def test_add_nonexisting_dataset():
6871

6972
def test_spice_scan_to_nxdict():
7073
path_to_spice_data = "./test_data/exp424/Datafiles/CG4C_exp0424_scan0034.dat"
71-
spicelogs = _create_spicelogs(path_to_spice_data)
74+
nxdict = spice_scan_to_nxdict(path_to_spice_data)
75+
76+
assert nxdict["SPICElogs"]["attrs"]["scan"] == "34"
77+
assert nxdict["start_time"]["dataset"] == "2024-07-03T01:44:46"
78+
assert np.allclose(nxdict["instrument"]["monochromator"]["ei"]["dataset"][0:3], [4.9, 5, 5.1])
79+
# assert nxdict["data"]
80+
81+
entries = {"scan0034": nxdict}
82+
nexus_entries = {}
83+
for scan_num, scan_content in entries.items():
84+
content_list = []
85+
for key, val in scan_content.items():
86+
content_list.append((key, val))
87+
nexus_entries.update({scan_num: NexusEntry(content_list)})
88+
89+
path_to_nexus_entry = "./test_data/spice_to_nxdict_test_scan0034.h5"
90+
for scan_num, nexus_entry in nexus_entries.items():
91+
nexus_entry.to_nexus(path_to_nexus_entry, scan_num)
92+
93+
94+
def test_spice_data_to_nxdict():
95+
pass

0 commit comments

Comments
 (0)