From bd584ed99c92518bd982ada48704c98cb66eff7a Mon Sep 17 00:00:00 2001
From: Sara Lambert <lambert8@illinois.edu>
Date: Wed, 19 Feb 2025 17:10:40 -0600
Subject: [PATCH 1/6] feat: first-pass at handling download/<label> ->
 downloadUrl array

---
 manager/registry.py                            | 16 ++++++++++++++--
 manager/templates/crud/widgets/references.html |  4 ++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/manager/registry.py b/manager/registry.py
index 4b122f6..95092ba 100644
--- a/manager/registry.py
+++ b/manager/registry.py
@@ -234,8 +234,20 @@ def to_solr(self):
             value = self.data.get(key)
             if value is not None and str(value).lower() != "none":
                 if key == "references":
-                    value = json.dumps(value)
-                solr_doc[field.uri] = value
+                    non_download_refs = {k:v for k, v in value.items() if not str.startswith(k, "download/")}
+                    download_refs = [{
+                        'label': k[9:],
+                        'url': v,
+                    } for k, v in value.items() if str.startswith(k, "download/")]
+                    download_ref_formatted = {'http://schema.org/downloadUrl': download_refs}
+                    #print(non_download_refs)
+                    if len(download_refs) > 0:
+                        #print(download_ref_formatted)
+                        value = {**non_download_refs, **download_ref_formatted}
+                    else:
+                        value = non_download_refs
+                    print(json.dumps(value))
+                solr_doc[field.uri] = json.dumps(value)
         return solr_doc
 
     def index(self, solr_instance=None):
diff --git a/manager/templates/crud/widgets/references.html b/manager/templates/crud/widgets/references.html
index daee741..2860fc2 100644
--- a/manager/templates/crud/widgets/references.html
+++ b/manager/templates/crud/widgets/references.html
@@ -4,6 +4,10 @@
 put multiple references on separate lines, with this format: <code>&lt;reference uri&gt;:: &lt;value&gt;</code> where <code>reference uri</code> is one of <a href="https://opengeometadata.org/reference-uris/" target="_blank">these options &nearr;</a>
 {% endblock %}
 
+{% block format %}
+specify multiple download formats using <code>download/&lt;label&gt;:: &lt;url&gt;</code> where <code>label</code> is <code>CSV</code>, <code>JSON</code>, <code>XML</code>, etc.
+{% endblock %}
+
 {% block input %}
 <textarea 
 style="white-space:pre; overflow-wrap:normal; overflow-x:scroll; width:100%;"

From 7b932f9faf609827f828c964272090743820362e Mon Sep 17 00:00:00 2001
From: Sara Lambert <lambert8@illinois.edu>
Date: Thu, 20 Feb 2025 15:27:03 -0600
Subject: [PATCH 2/6] fix: tested loading/editing/saving using new download/
 prefix, addd note in the UI

---
 manager/metadata/records/sdoh-indices.json    | 15 ++--
 manager/registry.py                           | 83 +++++++++++++++----
 .../templates/crud/widgets/references.html    |  6 +-
 3 files changed, 78 insertions(+), 26 deletions(-)

diff --git a/manager/metadata/records/sdoh-indices.json b/manager/metadata/records/sdoh-indices.json
index 40aba65..dc815f5 100644
--- a/manager/metadata/records/sdoh-indices.json
+++ b/manager/metadata/records/sdoh-indices.json
@@ -3,8 +3,8 @@
     "created_by": "",
     "last_modified_by": "Admin",
     "filled": 20,
-    "to_fill": 25,
-    "filled_pct": 80,
+    "to_fill": 26,
+    "filled_pct": 76,
     "progress_class": "warning"
   },
   "id": "herop-yvjqrx",
@@ -83,11 +83,11 @@
   "file_size": null,
   "references": {
     "http://schema.org/url": "https://sdohatlas.github.io/",
-    "http://schema.org/downloadUrl": "[{'url': 'https://geodacenter.github.io/data-and-lab/data/us-sdoh-2014.zip', 'label': 'ZIP Archive'}]"
+    "http://schema.org/downloadUrl": "[{\"label\": \"ZIP Archive\", \"url\": \"https://geodacenter.github.io/data-and-lab/data/us-sdoh-2014.zip\"}]"
   },
   "wxs_indentifier": null,
   "identifier": null,
-  "modified": "2025-02-03T21:05:47Z",
+  "modified": "2025-02-20T21:26:18Z",
   "metadata_version": "SDOH PlaceProject",
   "suppressed": false,
   "spatial_resolution": [
@@ -139,6 +139,9 @@
     "SDOH_CL - 1:rural affordable, 2:suburban affluent, 3:suburban affordable, 4:extreme poverty, 5:multilingual working, 6:urban core opportunity, 7:sparse areas"
   ],
   "highlight_ids": null,
-  "featured_variable": "Socioeconomic Advantage Index, Limited Mobility Index, Urban Core Opportunity Index, Mixed Immigrant, Cohesion, & Access Index, SDOH Cluster - 1:rural affordable, 2:suburban affluent, 3:suburban affordable, 4:extreme poverty, 5:multilingual working, 6:urban core opportunity, 7:sparse areas (see https://sdohatlas.github.io/)",
-  "data_usage_notes": null
+  "featured_variable": [
+    "Socioeconomic Advantage Index, Limited Mobility Index, Urban Core Opportunity Index, Mixed Immigrant, Cohesion, & Access Index, SDOH Cluster - 1:rural affordable, 2:suburban affluent, 3:suburban affordable, 4:extreme poverty, 5:multilingual working, 6:urban core opportunity, 7:sparse areas (see https://sdohatlas.github.io/)"
+  ],
+  "data_usage_notes": null,
+  "preferred_citation": null
 }
\ No newline at end of file
diff --git a/manager/registry.py b/manager/registry.py
index 95092ba..58e7d4d 100644
--- a/manager/registry.py
+++ b/manager/registry.py
@@ -1,4 +1,6 @@
 import json
+import logging
+from json import JSONDecodeError
 from pathlib import Path
 from datetime import datetime
 from dotenv import load_dotenv
@@ -133,11 +135,20 @@ def save_data(self, index=False):
 
         if self.data["references"]:
             cleaned_references = {}
+            download_refs = []
             for (
                 k,
                 v,
             ) in self.data["references"].items():
-                cleaned_references[k.rstrip().lstrip()] = v.rstrip().lstrip()
+                if k.startswith('download/'):
+                    label = k.rstrip().lstrip()[9:]
+                    url = v.rstrip().lstrip()
+                    download_refs.append({'label': label, 'url': url})
+                else:
+                    cleaned_references[k.rstrip().lstrip()] = v.rstrip().lstrip()
+
+            if len(download_refs) > 0:
+                cleaned_references['http://schema.org/downloadUrl'] = json.dumps(download_refs)
             self.data["references"] = cleaned_references
 
         coverages = (
@@ -204,8 +215,28 @@ def to_json(self):
 
         self.data["_meta"] = meta
 
+        # cleaned_references = {}
+        # download_refs = []
+        # refs = self.data["references"]
+        # if refs and len(refs) > 0:
+        #     for k, v in refs.items():
+        #         if k.startswith('download/'):
+        #             ref = {'label': k[9:], 'url': v}
+        #             #print(f'Adding references: {str(ref)}')
+        #             download_refs.append(ref)
+        #         else:
+        #             cleaned_references[k] = v
+        #
+        #         if len(download_refs) > 0:
+        #             cleaned_references['http://schema.org/downloadUrl'] = json.dumps(download_refs)
+        #
+        #         #if k.startswith('download/'):
+        #         #    print(cleaned_references)
+        #     self.data["references"] = cleaned_references
+
         return self.data
 
+
     def to_form(self):
         """Prepares the raw backend data to populate an html form."""
         form_data = {}
@@ -216,7 +247,23 @@ def to_form(self):
             if key == "references" and isinstance(value, dict):
                 lines = ""
                 for x, y in value.items():
-                    lines += f"{x}:: {y}\n"
+                    logging.warning(f'item - {x}:: {y}')
+                    if x == 'http://schema.org/downloadUrl':
+                        try:
+                            y_parsed = json.loads(y)
+                            if isinstance(y_parsed, list):
+                                print('Dictionary found')
+                                # downloadUrl is a list of objects defining label + url
+                                # break it up into multiple lines of download/<label>:: <url>
+                                for u in y_parsed:
+                                    lines += f"download/{u['label']}:: {u['url']}\n"
+                        except JSONDecodeError as ex:
+                            # downloadUrl is a single string
+                            print('String found')
+                            lines += f"{x}:: {y}\n"
+                    else:
+                        print('Typical key found')
+                        lines += f"{x}:: {y}\n"
                 value = lines
             if field.multiple and isinstance(value, list):
                 if field.widget == "text-area.html":
@@ -234,20 +281,24 @@ def to_solr(self):
             value = self.data.get(key)
             if value is not None and str(value).lower() != "none":
                 if key == "references":
-                    non_download_refs = {k:v for k, v in value.items() if not str.startswith(k, "download/")}
-                    download_refs = [{
-                        'label': k[9:],
-                        'url': v,
-                    } for k, v in value.items() if str.startswith(k, "download/")]
-                    download_ref_formatted = {'http://schema.org/downloadUrl': download_refs}
-                    #print(non_download_refs)
-                    if len(download_refs) > 0:
-                        #print(download_ref_formatted)
-                        value = {**non_download_refs, **download_ref_formatted}
-                    else:
-                        value = non_download_refs
-                    print(json.dumps(value))
-                solr_doc[field.uri] = json.dumps(value)
+                    value = json.dumps(value)
+                    # non_download_refs = {k:v for k, v in value.items() if not k.startswith("download/")}
+                    # download_refs = [{
+                    #     'label': k[9:],
+                    #     'url': v,
+                    # } for k, v in value.items() if k.startswith("download/")]
+                    # download_ref_formatted = {'http://schema.org/downloadUrl': download_refs}
+                    # if len(non_download_refs) > 0:
+                    #     print('Non Download Refs:')
+                    #     print(non_download_refs)
+                    #
+                    # if len(download_refs) > 0:
+                    #     print('Download Refs:')
+                    #     print(download_ref_formatted)
+                    #     value = {**non_download_refs, **download_ref_formatted}
+                    # else:
+                    #     value = non_download_refs
+                solr_doc[field.uri] = value
         return solr_doc
 
     def index(self, solr_instance=None):
diff --git a/manager/templates/crud/widgets/references.html b/manager/templates/crud/widgets/references.html
index 2860fc2..b2da6dc 100644
--- a/manager/templates/crud/widgets/references.html
+++ b/manager/templates/crud/widgets/references.html
@@ -2,10 +2,8 @@
 
 {% block note %}
 put multiple references on separate lines, with this format: <code>&lt;reference uri&gt;:: &lt;value&gt;</code> where <code>reference uri</code> is one of <a href="https://opengeometadata.org/reference-uris/" target="_blank">these options &nearr;</a>
-{% endblock %}
-
-{% block format %}
-specify multiple download formats using <code>download/&lt;label&gt;:: &lt;url&gt;</code> where <code>label</code> is <code>CSV</code>, <code>JSON</code>, <code>XML</code>, etc.
+<br>
+specify multiple download formats using <code>download/&lt;label&gt;:: &lt;url&gt;</code> where <code>label</code> is a unique string, such as <code>CSV</code>, <code>JSON</code>, <code>XML</code>, etc.
 {% endblock %}
 
 {% block input %}

From e4c4c8e2f0bf368ab346183662256b125be79449 Mon Sep 17 00:00:00 2001
From: Sara Lambert <lambert8@illinois.edu>
Date: Thu, 20 Feb 2025 16:58:50 -0600
Subject: [PATCH 3/6] fix: remove temp commented-out code block

---
 manager/registry.py | 38 --------------------------------------
 1 file changed, 38 deletions(-)

diff --git a/manager/registry.py b/manager/registry.py
index 58e7d4d..0d619b6 100644
--- a/manager/registry.py
+++ b/manager/registry.py
@@ -215,25 +215,6 @@ def to_json(self):
 
         self.data["_meta"] = meta
 
-        # cleaned_references = {}
-        # download_refs = []
-        # refs = self.data["references"]
-        # if refs and len(refs) > 0:
-        #     for k, v in refs.items():
-        #         if k.startswith('download/'):
-        #             ref = {'label': k[9:], 'url': v}
-        #             #print(f'Adding references: {str(ref)}')
-        #             download_refs.append(ref)
-        #         else:
-        #             cleaned_references[k] = v
-        #
-        #         if len(download_refs) > 0:
-        #             cleaned_references['http://schema.org/downloadUrl'] = json.dumps(download_refs)
-        #
-        #         #if k.startswith('download/'):
-        #         #    print(cleaned_references)
-        #     self.data["references"] = cleaned_references
-
         return self.data
 
 
@@ -252,17 +233,14 @@ def to_form(self):
                         try:
                             y_parsed = json.loads(y)
                             if isinstance(y_parsed, list):
-                                print('Dictionary found')
                                 # downloadUrl is a list of objects defining label + url
                                 # break it up into multiple lines of download/<label>:: <url>
                                 for u in y_parsed:
                                     lines += f"download/{u['label']}:: {u['url']}\n"
                         except JSONDecodeError as ex:
                             # downloadUrl is a single string
-                            print('String found')
                             lines += f"{x}:: {y}\n"
                     else:
-                        print('Typical key found')
                         lines += f"{x}:: {y}\n"
                 value = lines
             if field.multiple and isinstance(value, list):
@@ -282,22 +260,6 @@ def to_solr(self):
             if value is not None and str(value).lower() != "none":
                 if key == "references":
                     value = json.dumps(value)
-                    # non_download_refs = {k:v for k, v in value.items() if not k.startswith("download/")}
-                    # download_refs = [{
-                    #     'label': k[9:],
-                    #     'url': v,
-                    # } for k, v in value.items() if k.startswith("download/")]
-                    # download_ref_formatted = {'http://schema.org/downloadUrl': download_refs}
-                    # if len(non_download_refs) > 0:
-                    #     print('Non Download Refs:')
-                    #     print(non_download_refs)
-                    #
-                    # if len(download_refs) > 0:
-                    #     print('Download Refs:')
-                    #     print(download_ref_formatted)
-                    #     value = {**non_download_refs, **download_ref_formatted}
-                    # else:
-                    #     value = non_download_refs
                 solr_doc[field.uri] = value
         return solr_doc
 

From afbca75ec3dcbe3e9d7d5a8c9822ccf089f46e8d Mon Sep 17 00:00:00 2001
From: Sara Lambert <lambert8@illinois.edu>
Date: Mon, 24 Feb 2025 11:31:36 -0600
Subject: [PATCH 4/6] fix: use raw JSON (unstringified) to store download map

---
 manager/registry.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/manager/registry.py b/manager/registry.py
index 0d619b6..82069b1 100644
--- a/manager/registry.py
+++ b/manager/registry.py
@@ -148,7 +148,7 @@ def save_data(self, index=False):
                     cleaned_references[k.rstrip().lstrip()] = v.rstrip().lstrip()
 
             if len(download_refs) > 0:
-                cleaned_references['http://schema.org/downloadUrl'] = json.dumps(download_refs)
+                cleaned_references['http://schema.org/downloadUrl'] = download_refs
             self.data["references"] = cleaned_references
 
         coverages = (
@@ -231,11 +231,10 @@ def to_form(self):
                     logging.warning(f'item - {x}:: {y}')
                     if x == 'http://schema.org/downloadUrl':
                         try:
-                            y_parsed = json.loads(y)
-                            if isinstance(y_parsed, list):
+                            if isinstance(y, list):
                                 # downloadUrl is a list of objects defining label + url
                                 # break it up into multiple lines of download/<label>:: <url>
-                                for u in y_parsed:
+                                for u in y:
                                     lines += f"download/{u['label']}:: {u['url']}\n"
                         except JSONDecodeError as ex:
                             # downloadUrl is a single string

From 6099227a79615267de439004bee3be4b21d7f187 Mon Sep 17 00:00:00 2001
From: Sara Lambert <lambert8@illinois.edu>
Date: Mon, 24 Feb 2025 11:35:05 -0600
Subject: [PATCH 5/6] fix: changed expected format for downloadUrl

---
 manager/metadata/records/sdoh-indices.json | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/manager/metadata/records/sdoh-indices.json b/manager/metadata/records/sdoh-indices.json
index dc815f5..19f8509 100644
--- a/manager/metadata/records/sdoh-indices.json
+++ b/manager/metadata/records/sdoh-indices.json
@@ -83,11 +83,16 @@
   "file_size": null,
   "references": {
     "http://schema.org/url": "https://sdohatlas.github.io/",
-    "http://schema.org/downloadUrl": "[{\"label\": \"ZIP Archive\", \"url\": \"https://geodacenter.github.io/data-and-lab/data/us-sdoh-2014.zip\"}]"
+    "http://schema.org/downloadUrl": [
+      {
+        "label": "ZIP Archive",
+        "url": "https://geodacenter.github.io/data-and-lab/data/us-sdoh-2014.zip"
+      }
+    ]
   },
   "wxs_indentifier": null,
   "identifier": null,
-  "modified": "2025-02-20T21:26:18Z",
+  "modified": "2025-02-24T17:30:29Z",
   "metadata_version": "SDOH PlaceProject",
   "suppressed": false,
   "spatial_resolution": [

From a2388d3429c4fa772b790dbe79435cca9d78b51e Mon Sep 17 00:00:00 2001
From: Sara Lambert <lambert8@illinois.edu>
Date: Mon, 24 Feb 2025 11:40:04 -0600
Subject: [PATCH 6/6] fix: revert changes to JSON record until migration path
 is determined

---
 manager/metadata/records/sdoh-indices.json | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/manager/metadata/records/sdoh-indices.json b/manager/metadata/records/sdoh-indices.json
index 19f8509..40aba65 100644
--- a/manager/metadata/records/sdoh-indices.json
+++ b/manager/metadata/records/sdoh-indices.json
@@ -3,8 +3,8 @@
     "created_by": "",
     "last_modified_by": "Admin",
     "filled": 20,
-    "to_fill": 26,
-    "filled_pct": 76,
+    "to_fill": 25,
+    "filled_pct": 80,
     "progress_class": "warning"
   },
   "id": "herop-yvjqrx",
@@ -83,16 +83,11 @@
   "file_size": null,
   "references": {
     "http://schema.org/url": "https://sdohatlas.github.io/",
-    "http://schema.org/downloadUrl": [
-      {
-        "label": "ZIP Archive",
-        "url": "https://geodacenter.github.io/data-and-lab/data/us-sdoh-2014.zip"
-      }
-    ]
+    "http://schema.org/downloadUrl": "[{'url': 'https://geodacenter.github.io/data-and-lab/data/us-sdoh-2014.zip', 'label': 'ZIP Archive'}]"
   },
   "wxs_indentifier": null,
   "identifier": null,
-  "modified": "2025-02-24T17:30:29Z",
+  "modified": "2025-02-03T21:05:47Z",
   "metadata_version": "SDOH PlaceProject",
   "suppressed": false,
   "spatial_resolution": [
@@ -144,9 +139,6 @@
     "SDOH_CL - 1:rural affordable, 2:suburban affluent, 3:suburban affordable, 4:extreme poverty, 5:multilingual working, 6:urban core opportunity, 7:sparse areas"
   ],
   "highlight_ids": null,
-  "featured_variable": [
-    "Socioeconomic Advantage Index, Limited Mobility Index, Urban Core Opportunity Index, Mixed Immigrant, Cohesion, & Access Index, SDOH Cluster - 1:rural affordable, 2:suburban affluent, 3:suburban affordable, 4:extreme poverty, 5:multilingual working, 6:urban core opportunity, 7:sparse areas (see https://sdohatlas.github.io/)"
-  ],
-  "data_usage_notes": null,
-  "preferred_citation": null
+  "featured_variable": "Socioeconomic Advantage Index, Limited Mobility Index, Urban Core Opportunity Index, Mixed Immigrant, Cohesion, & Access Index, SDOH Cluster - 1:rural affordable, 2:suburban affluent, 3:suburban affordable, 4:extreme poverty, 5:multilingual working, 6:urban core opportunity, 7:sparse areas (see https://sdohatlas.github.io/)",
+  "data_usage_notes": null
 }
\ No newline at end of file