From 03f865120d6b0c329d2c88828d7a6e4b2f7f3bfc Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Fri, 14 Jun 2024 14:39:19 -0400 Subject: [PATCH 1/4] Autocorrelation alignment method. --- pyproject.toml | 1 + src/corrgi/alignment.py | 37 ++++++++++++++++++++++++++++++++++ tests/corrgi/conftest.py | 28 +++++++++++++++++++++++++ tests/corrgi/test_alignment.py | 8 ++++++++ tests/ruff.toml | 3 +++ 5 files changed, 77 insertions(+) create mode 100644 src/corrgi/alignment.py create mode 100644 tests/corrgi/test_alignment.py create mode 100644 tests/ruff.toml diff --git a/pyproject.toml b/pyproject.toml index 9389f62..65d0602 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dynamic = ["version"] requires-python = ">=3.9, <3.12" dependencies = [ "lsdb", + "hipscat", "gundam", ] diff --git a/src/corrgi/alignment.py b/src/corrgi/alignment.py new file mode 100644 index 0000000..9171627 --- /dev/null +++ b/src/corrgi/alignment.py @@ -0,0 +1,37 @@ +import itertools + +import pandas as pd +from hipscat.catalog import Catalog +from hipscat.pixel_tree.pixel_alignment import PixelAlignment +from hipscat.pixel_tree.pixel_alignment_types import PixelAlignmentType + +column_names = [ + PixelAlignment.PRIMARY_ORDER_COLUMN_NAME, + PixelAlignment.PRIMARY_PIXEL_COLUMN_NAME, + PixelAlignment.JOIN_ORDER_COLUMN_NAME, + PixelAlignment.JOIN_PIXEL_COLUMN_NAME, + PixelAlignment.ALIGNED_ORDER_COLUMN_NAME, + PixelAlignment.ALIGNED_PIXEL_COLUMN_NAME, +] + + +def autocorrelation_alignment(catalog: Catalog) -> PixelAlignment: + """TODO""" + upper_triangle = [ + [left.order, left.pixel, right.order, right.pixel, left.order, left.pixel] + for (left, right) in itertools.combinations(catalog.get_healpix_pixels(), 2) + ] + upper_triangle = pd.DataFrame(upper_triangle, columns=column_names) + diagonal = pd.DataFrame( + [ + [pix.order, pix.pixel, pix.order, pix.pixel, pix.order, pix.pixel] + for pix in catalog.get_healpix_pixels() + ] + ) + result_mapping = pd.concat([upper_triangle, diagonal]) + return PixelAlignment(catalog.pixel_tree, result_mapping, PixelAlignmentType.OUTER) + + +def crosscorrelation_alignment(catalog_left: Catalog, catalog_right: Catalog) -> PixelAlignment: + """TODO""" + pass diff --git a/tests/corrgi/conftest.py b/tests/corrgi/conftest.py index e69de29..cc9b047 100644 --- a/tests/corrgi/conftest.py +++ b/tests/corrgi/conftest.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import pytest + + +@pytest.fixture +def test_data_dir(): + return Path(__file__).parent.parent / "data" + + +@pytest.fixture +def data_catalog_dir(test_data_dir): + return test_data_dir / "DATA" + + +@pytest.fixture +def raw_catalog_dir(test_data_dir): + return test_data_dir / "RAW" + + +@pytest.fixture +def dr7_lrg_catalog_dir(test_data_dir): + return test_data_dir / "DR7-lrg" + + +@pytest.fixture +def dr7_lrg_rand_catalog_dir(test_data_dir): + return test_data_dir / "DR7-lrg-rand" diff --git a/tests/corrgi/test_alignment.py b/tests/corrgi/test_alignment.py new file mode 100644 index 0000000..5286496 --- /dev/null +++ b/tests/corrgi/test_alignment.py @@ -0,0 +1,8 @@ +import hipscat +from corrgi.alignment import autocorrelation_alignment + + +def test_autocorrelation_alignment(data_catalog_dir): + data_catalog = hipscat.read_from_hipscat(data_catalog_dir) + alignment = autocorrelation_alignment(data_catalog) + assert len(alignment.pixel_mapping) == 28 diff --git a/tests/ruff.toml b/tests/ruff.toml new file mode 100644 index 0000000..05abe95 --- /dev/null +++ b/tests/ruff.toml @@ -0,0 +1,3 @@ +lint.ignore = [ + "D103", # Allow Missing docstring in public function +] From 54541645f929f8792120528ab88b5c194a162493 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Fri, 14 Jun 2024 15:30:03 -0400 Subject: [PATCH 2/4] Cross-correlation alignment method and docstrings. --- src/corrgi/__init__.py | 3 --- src/corrgi/alignment.py | 31 ++++++++++++++++++++++++++--- src/corrgi/example_module.py | 23 --------------------- tests/corrgi/test_alignment.py | 12 ++++++++++- tests/corrgi/test_example_module.py | 13 ------------ 5 files changed, 39 insertions(+), 43 deletions(-) delete mode 100644 src/corrgi/example_module.py delete mode 100644 tests/corrgi/test_example_module.py diff --git a/src/corrgi/__init__.py b/src/corrgi/__init__.py index b564b85..e69de29 100644 --- a/src/corrgi/__init__.py +++ b/src/corrgi/__init__.py @@ -1,3 +0,0 @@ -from .example_module import greetings, meaning - -__all__ = ["greetings", "meaning"] diff --git a/src/corrgi/alignment.py b/src/corrgi/alignment.py index 9171627..4c4213b 100644 --- a/src/corrgi/alignment.py +++ b/src/corrgi/alignment.py @@ -16,7 +16,16 @@ def autocorrelation_alignment(catalog: Catalog) -> PixelAlignment: - """TODO""" + """Determine all pairs of partitions that should be correlated within the same catalog. + + This considers all combinations, without duplicates between the "primary" and "join" + pixels in the alignment. + + Args: + catalog (Catalog): catalog for autocorrelation + Returns: + alignment object where the `aligned` columns simply match the left pixel. + """ upper_triangle = [ [left.order, left.pixel, right.order, right.pixel, left.order, left.pixel] for (left, right) in itertools.combinations(catalog.get_healpix_pixels(), 2) @@ -33,5 +42,21 @@ def autocorrelation_alignment(catalog: Catalog) -> PixelAlignment: def crosscorrelation_alignment(catalog_left: Catalog, catalog_right: Catalog) -> PixelAlignment: - """TODO""" - pass + """Determine all pairs of partitions that should be correlated between two catalogs. + + This considers the full cross-product of pixels. + + Args: + catalog_left (Catalog): left side of the cross-correlation + catalog_right (Catalog): right side of the cross-correlation + Returns: + alignment object where the `aligned` columns simply match the left pixel. + """ + full_product = [ + [left.order, left.pixel, right.order, right.pixel, left.order, left.pixel] + for (left, right) in itertools.product( + catalog_left.get_healpix_pixels(), catalog_right.get_healpix_pixels() + ) + ] + result_mapping = pd.DataFrame(full_product, columns=column_names) + return PixelAlignment(catalog_left.pixel_tree, result_mapping, PixelAlignmentType.OUTER) diff --git a/src/corrgi/example_module.py b/src/corrgi/example_module.py deleted file mode 100644 index f76e837..0000000 --- a/src/corrgi/example_module.py +++ /dev/null @@ -1,23 +0,0 @@ -"""An example module containing simplistic functions.""" - - -def greetings() -> str: - """A friendly greeting for a future friend. - - Returns - ------- - str - A typical greeting from a software engineer. - """ - return "Hello from LINCC-Frameworks!" - - -def meaning() -> int: - """The meaning of life, the universe, and everything. - - Returns - ------- - int - The meaning of life. - """ - return 42 diff --git a/tests/corrgi/test_alignment.py b/tests/corrgi/test_alignment.py index 5286496..8840b44 100644 --- a/tests/corrgi/test_alignment.py +++ b/tests/corrgi/test_alignment.py @@ -1,8 +1,18 @@ import hipscat -from corrgi.alignment import autocorrelation_alignment +from corrgi.alignment import autocorrelation_alignment, crosscorrelation_alignment def test_autocorrelation_alignment(data_catalog_dir): data_catalog = hipscat.read_from_hipscat(data_catalog_dir) alignment = autocorrelation_alignment(data_catalog) assert len(alignment.pixel_mapping) == 28 + + +def test_crosscorrelation_alignment(dr7_lrg_catalog_dir, dr7_lrg_rand_catalog_dir): + dr7_catalog = hipscat.read_from_hipscat(dr7_lrg_catalog_dir) + dr7_rand_catalog = hipscat.read_from_hipscat(dr7_lrg_rand_catalog_dir) + alignment = crosscorrelation_alignment(dr7_catalog, dr7_rand_catalog) + ## dr7_catalog has 12 partitions + ## dr7_rand_catalog has 22 partitions + ## 12*21 = 252 + assert len(alignment.pixel_mapping) == 252 diff --git a/tests/corrgi/test_example_module.py b/tests/corrgi/test_example_module.py deleted file mode 100644 index 64cb10f..0000000 --- a/tests/corrgi/test_example_module.py +++ /dev/null @@ -1,13 +0,0 @@ -from corrgi import example_module - - -def test_greetings() -> None: - """Verify the output of the `greetings` function""" - output = example_module.greetings() - assert output == "Hello from LINCC-Frameworks!" - - -def test_meaning() -> None: - """Verify the output of the `meaning` function""" - output = example_module.meaning() - assert output == 42 From 04c0c2299d7edf0bf09fabbf6d0e8c2189f01c8c Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Mon, 17 Jun 2024 11:19:56 -0400 Subject: [PATCH 3/4] Use same column names. --- src/corrgi/alignment.py | 3 ++- tests/corrgi/test_alignment.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/corrgi/alignment.py b/src/corrgi/alignment.py index 4c4213b..ea8667b 100644 --- a/src/corrgi/alignment.py +++ b/src/corrgi/alignment.py @@ -35,7 +35,8 @@ def autocorrelation_alignment(catalog: Catalog) -> PixelAlignment: [ [pix.order, pix.pixel, pix.order, pix.pixel, pix.order, pix.pixel] for pix in catalog.get_healpix_pixels() - ] + ], + columns=column_names, ) result_mapping = pd.concat([upper_triangle, diagonal]) return PixelAlignment(catalog.pixel_tree, result_mapping, PixelAlignmentType.OUTER) diff --git a/tests/corrgi/test_alignment.py b/tests/corrgi/test_alignment.py index 8840b44..e4ff359 100644 --- a/tests/corrgi/test_alignment.py +++ b/tests/corrgi/test_alignment.py @@ -6,6 +6,7 @@ def test_autocorrelation_alignment(data_catalog_dir): data_catalog = hipscat.read_from_hipscat(data_catalog_dir) alignment = autocorrelation_alignment(data_catalog) assert len(alignment.pixel_mapping) == 28 + assert len(alignment.pixel_mapping.columns) == 6 def test_crosscorrelation_alignment(dr7_lrg_catalog_dir, dr7_lrg_rand_catalog_dir): @@ -16,3 +17,4 @@ def test_crosscorrelation_alignment(dr7_lrg_catalog_dir, dr7_lrg_rand_catalog_di ## dr7_rand_catalog has 22 partitions ## 12*21 = 252 assert len(alignment.pixel_mapping) == 252 + assert len(alignment.pixel_mapping.columns) == 6 From e3523a5fd55556f1780b36f25cc782e801efc2b7 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Mon, 17 Jun 2024 11:24:31 -0400 Subject: [PATCH 4/4] Pin numpy<2 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 65d0602..c8cabf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "lsdb", "hipscat", "gundam", + "numpy<2", ] [project.urls]