First draft of fast featurizer preset and fix initial scripts

experiments/1_ingesting_refractive_index/requirements.txt

@@ -1,3 +1,3 @@
 pandas
-optimade[http_client]
+optimade[http-client]
 tqdm

experiments/1_ingesting_refractive_index/run.py

@@ -15,7 +15,7 @@
 
 data_path = Path(__file__).parent.parent / "data"
 db_path = data_path / "db.csv"
-structures_path = data_path / "structures.json"
+structures_path = data_path / "structures-2024.json"
 
 if not db_path.exists():
     import urllib.request
@@ -42,6 +42,9 @@
         continue
 
     filter_ = f'id = "{mp_id}"'
+    structure = {}
+    structure["attributes"] = {}
+
 
     try:
         # redirect output from stdout to dev null
@@ -51,6 +54,12 @@
             ]["data"][0]
     except Exception:
         print(f"No structure found for {mp_id}")
+        continue
+
+    if structure["id"]  != mp_id:
+        print(f"Structure ID {structure['id']} does not match MP ID {mp_id}")
+        continue
+
     structure["attributes"]["_naccarato_refractive_index"] = np.mean(
         [
             row["Ref_index (1)"],

experiments/2_featurizing_refractive_index_data/requirements.txt

@@ -1,2 +1,5 @@
 modnet==0.2.1
-optimade
+optimade~=0.25
+matminer==0.8.0
+pymatgen==2023.9.2
+ruamel.yaml<0.18

experiments/2_featurizing_refractive_index_data/run.py

@@ -1,38 +1,52 @@
 import json
 from pathlib import Path
+from modnet.featurizers.presets import Matminer2023Featurizer
 
 import tqdm
 from modnet.preprocessing import MODData
 from optimade.adapters import Structure
 from optimade.models.utils import reduce_formula
 
+N_CPU = 6
+
 data_path = Path(__file__).absolute().parent.parent / "data"
 db_path = data_path / "db.csv"
-structures_path = data_path / "structures.json"
-
-
-with open(structures_path) as f:
-    structures = json.load(f)
+structures_path = data_path / "structures-2024.json"
+moddata_path = Path("./mod.data")
+feature_selected_path = Path("./mod-feature-selected.data")
+
+if not moddata_path.exists():
+
+    with open(structures_path) as f:
+        structures = json.load(f)
+
+    pmg_structures = []
+    targets = []
+
+    for ind, s in tqdm.tqdm(enumerate(structures.items())):
+        s[1]["attributes"]["chemical_formula_reduced"] = reduce_formula(
+            s[1]["attributes"]["chemical_formula_reduced"]
+        )
+        targets.append(
+            s[1]["attributes"]["_naccarato_refractive_index"],
+        )
+        if s[0] == "mp-505722":
+            breakpoint()
+        pmg_structures.append(Structure(s[1]).as_pymatgen)
+
+    moddata = MODData(
+        materials=pmg_structures,
+        targets=targets,
+        featurizer=Matminer2023Featurizer(fast_oxid=True),
+        target_names=["refractive_index"],
+        structure_ids=structures.keys(),
+    )
+    moddata.featurizer.featurizer_mode = "single"
+    moddata.featurize(n_jobs=N_CPU)
 
-pmg_structures = []
-targets = []
+    moddata.save("mod.data")
 
-for ind, s in tqdm.tqdm(enumerate(structures.items())):
-    s[1]["attributes"]["chemical_formula_reduced"] = reduce_formula(
-        s[1]["attributes"]["chemical_formula_reduced"]
-    )
-    targets.append(
-        s[1]["attributes"]["_naccarato_refractive_index"],
-    )
-    pmg_structures.append(Structure(s[1]).as_pymatgen)
-
-moddata = MODData(
-    materials=pmg_structures,
-    targets=targets,
-    target_names=["refractive_index"],
-    structure_ids=structures.keys(),
-)
-moddata.featurizer.featurizer_mode = "single"
-moddata.featurize(n_jobs=6)
-
-moddata.save("mod.data")
+if not feature_selected_path.exists():
+    moddata = MODData.load("mod.data")
+    moddata.feature_selection(n=-1, n_jobs=N_CPU)
+    moddata.save("mod-feature-selected.data")

experiments/7_scraping_other_databases/requirements.txt

@@ -1 +1,3 @@
-optimade[http-client]~=1.0
+optimade[http-client]<1.0
+modnet
+pymatgen==2023.9.2