From 5c2ee7efad4e09ef7c86ce7a3a8669003eae1506 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Fri, 5 Jul 2019 14:22:43 -0500 Subject: [PATCH 1/5] chore(sync): more debug logs for dbgap syncing --- fence/sync/sync_users.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index a07af318f..6ed6ec816 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -460,6 +460,13 @@ def _parse_csv(self, file_dict, sess, encrypted=True): if dbgap_project not in self.project_mapping: if dbgap_project not in self._projects: + + self.logger.debug( + "creating Project in fence from dbGaP study: {}".format( + dbgap_project + ) + ) + project = self._get_or_create( sess, Project, auth_id=dbgap_project ) @@ -858,6 +865,9 @@ def _init_projects(self, user_project, sess): if self.project_mapping: for projects in self.project_mapping.values(): for p in projects: + self.logger.debug( + "creating Project with info from project_mapping: {}".format(p) + ) project = self._get_or_create(sess, Project, **p) self._projects[p["auth_id"]] = project for _, projects in user_project.iteritems(): @@ -1028,6 +1038,9 @@ def _update_arborist(self, session, user_yaml): for path_list in self._dbgap_study_to_resources.values(): dbgap_resource_paths.extend(path_list) + self.logger.debug("user_yaml resources: {}".format(resources)) + self.logger.debug("dbgap resource paths: {}".format(dbgap_resource_paths)) + combined_resources = utils.combine_provided_and_dbgap_resources( resources, dbgap_resource_paths ) From d6d8e23af7e219146f7caa2c559791b63f3bf0a7 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Fri, 5 Jul 2019 14:42:22 -0500 Subject: [PATCH 2/5] fix(sync): default root level resource to have empty list for subresources if they don't exist --- fence/sync/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/sync/utils.py b/fence/sync/utils.py index d406a32a5..ceb6f8958 100644 --- a/fence/sync/utils.py +++ b/fence/sync/utils.py @@ -139,7 +139,7 @@ def insert_segment(current, segment): # In [3]: list(map(lambda x: x["name"] == "b", xs)).index(True) # Out[3]: 1 i = list(map(lambda c: c["name"] == segment, current)).index(True) - return current[i]["subresources"] + return current[i].get("subresources", []) reduce(insert_segment, segments, start) return root From 058cbe188d8e7b3bed42f75dbb5496fcd3b66520 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Fri, 5 Jul 2019 14:44:52 -0500 Subject: [PATCH 3/5] fix(sync): add subresources to reference, don't just get an empty list --- fence/sync/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fence/sync/utils.py b/fence/sync/utils.py index ceb6f8958..c987507be 100644 --- a/fence/sync/utils.py +++ b/fence/sync/utils.py @@ -139,7 +139,11 @@ def insert_segment(current, segment): # In [3]: list(map(lambda x: x["name"] == "b", xs)).index(True) # Out[3]: 1 i = list(map(lambda c: c["name"] == segment, current)).index(True) - return current[i].get("subresources", []) + + if "subresources" not in current[i] + current[i]["subresources"] = [] + + return current[i]["subresources"] reduce(insert_segment, segments, start) return root From dee547f043086245bec7d50877f4450311be265c Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Fri, 5 Jul 2019 14:47:25 -0500 Subject: [PATCH 4/5] fix(sync): add missing colon --- fence/sync/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/sync/utils.py b/fence/sync/utils.py index c987507be..a1b4f8b6b 100644 --- a/fence/sync/utils.py +++ b/fence/sync/utils.py @@ -140,7 +140,7 @@ def insert_segment(current, segment): # Out[3]: 1 i = list(map(lambda c: c["name"] == segment, current)).index(True) - if "subresources" not in current[i] + if "subresources" not in current[i]: current[i]["subresources"] = [] return current[i]["subresources"] From cabc362dbf32b1450cd72cc5e96a89252e76e984 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Fri, 5 Jul 2019 14:58:23 -0500 Subject: [PATCH 5/5] chore(tests): regression test for bug --- tests/dbgap_sync/test_utils.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/dbgap_sync/test_utils.py b/tests/dbgap_sync/test_utils.py index 9b017bab3..15b1a14b2 100644 --- a/tests/dbgap_sync/test_utils.py +++ b/tests/dbgap_sync/test_utils.py @@ -11,6 +11,7 @@ def test_combine_arborist_resources(): yaml_string = """ rbac: resources: + - name: 'root_no_subresources' - name: 'gen3' subresources: - name: 'programs' @@ -36,6 +37,7 @@ def test_combine_arborist_resources(): useryaml_resources = useryaml.get("rbac", {}).get("resources") test_paths = [ + "/root_no_subresources", "/programs/phs000172", "/orgA/programs/phs000175", "/orgC/programs/phs000175", @@ -48,7 +50,14 @@ def test_combine_arborist_resources(): useryaml_resources, test_paths ) - expected_roots = ["orgA", "orgB", "orgC", "programs", "gen3"] + expected_roots = [ + "root_no_subresources", + "orgA", + "orgB", + "orgC", + "programs", + "gen3", + ] for item in combined: # ensure one of each of the items in expected roots assert item.get("name") in expected_roots @@ -57,7 +66,9 @@ def test_combine_arborist_resources(): subresources = _get_subresources(item) # ensure result has correct subresources - if item.get("name") == "orgA": + if item.get("name") == "root_no_subresources": + assert not subresources + elif item.get("name") == "orgA": program_subresources = _get_subresources(subresources["programs"]) assert "phs000179" in program_subresources assert "phs000175" in program_subresources @@ -97,4 +108,4 @@ def test_combine_arborist_resources(): def _get_subresources(item): - return {subr.get("name"): subr for subr in item.get("subresources")} + return {subr.get("name"): subr for subr in item.get("subresources", [])}