Merge pull request #166 from LSSTDESC/u/yymao/composite-reader

New experimental feature: composite catalog reader

GCRCatalogs/alphaq_addon.py

@@ -1,36 +1,25 @@
 """
 Add-on catalogs for alpha q.
 """
-from __future__ import division
 import os
-import numpy as np
-import h5py
 from itertools import product
+import h5py
 from GCR import BaseGenericCatalog
 
-from .alphaq import AlphaQGalaxyCatalog
-
 __all__ = ['AlphaQTidalCatalog', 'AlphaQAddonCatalog']
 
-
-class AlphaQAddonCatalog(AlphaQGalaxyCatalog):
+class AlphaQAddonCatalog(BaseGenericCatalog):
     """
     Addon to the AlphaQ catalog that can add extra quanities to the baseline
     catalog
     """
     def _subclass_init(self, **kwargs):
-        # Loads main catalog
-        super(self.__class__, self)._subclass_init(**kwargs)
-
         # Sets the filename of the addon
         self._addon_filename = kwargs['addon_filename']
         assert os.path.isfile(self._addon_filename), 'Addon file {} does not exist'.format(self._addon_filename)
         self._addon_group = kwargs['addon_group']
 
     def _generate_native_quantity_list(self):
-        # Generates the native quantity list for the parent catalog
-        native_quantities = super(self.__class__, self)._generate_native_quantity_list()
-
         # Loads the additional data provided by the addon file
         with h5py.File(self._addon_filename, 'r') as fh:
             hgroup = fh[self._addon_group]
@@ -40,35 +29,25 @@ def _generate_native_quantity_list(self):
             #filter out the group objects and keep the dataste objects
             hdatasets = [hobject for hobject in hobjects if type(hgroup[hobject]) == h5py.Dataset]
             addon_native_quantities = set(hdatasets)
-
-        self._addon_native_quantities = addon_native_quantities
-
-        return native_quantities.union(addon_native_quantities)
+        return addon_native_quantities
 
     def _iter_native_dataset(self, native_filters=None):
         """
         Caution, fully overiddes parent function
         """
         assert not native_filters, '*native_filters* is not supported'
-        with h5py.File(self._file, 'r') as fh:
-            with h5py.File(self._addon_filename, 'r') as fh_addon:
-                def native_quantity_getter(native_quantity):
-                    if native_quantity in self._addon_native_quantities:
-                        return fh_addon['{}/{}'.format(self._addon_group,native_quantity)].value
-                    else:
-                        return fh['galaxyProperties/{}'.format(native_quantity)].value
-                yield native_quantity_getter
+        with h5py.File(self._addon_filename, 'r') as fh_addon:
+            def native_quantity_getter(native_quantity):
+                return fh_addon['{}/{}'.format(self._addon_group,native_quantity)].value # pylint: disable=E1101
+            yield native_quantity_getter
 
 
 class AlphaQTidalCatalog(BaseGenericCatalog):
     """
     Alpha Q tidal catalog class. Uses generic quantity and filter mechanisms
     defined by BaseGenericCatalog class.
     """
-    is_addon = True
-
     def _subclass_init(self, **kwargs):
-
         self._filename = kwargs['filename']
         assert os.path.isfile(self._filename), 'Catalog file {} does not exist'.format(self._filename)
 
@@ -80,22 +59,20 @@ def _subclass_init(self, **kwargs):
         for i, j in product(range(3), repeat=2):
             self._quantity_modifiers['tidal_eigvects[{}][{}]'.format(i, j)] = 'eigvects/{}/{}'.format(i, j)
 
-
     def _generate_native_quantity_list(self):
         native_quantities = set()
         with h5py.File(self._filename, 'r') as fh:
-            data = fh['tidal'].value
+            data = fh['tidal'].value # pylint: disable=E1101
             for name, (dt, _) in data.dtype.fields.items():
                 native_quantities.add(name)
                 if dt.shape:
                     for indices in product(*map(range, dt.shape)):
                         native_quantities.add((name + '/' + '/'.join(map(str, indices))).strip('/'))
         return native_quantities
 
-
     def _iter_native_dataset(self, native_filters=None):
         with h5py.File(self._filename, 'r') as fh:
-            data = fh['tidal'].value
+            data = fh['tidal'].value # pylint: disable=E1101
             def native_quantity_getter(native_quantity):
                 if '/' not in native_quantity:
                     return data[native_quantity]

GCRCatalogs/catalog_configs/proto-dc2_v2.1.2_addon_knots.yaml

@@ -1,6 +1,8 @@
-subclass_name: alphaq_addon.AlphaQAddonCatalog
-filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/ANL_AlphaQ_v2.1.2.hdf5
-lightcone: true
-version: "2.1.2"
-addon_filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/protoDC2_knots_v2.1.2.hdf5
-addon_group: knots
+subclass_name: composite.CompositeReader
+catalogs:
+  - catalog_name: proto-dc2_v2.1.2
+  - catalog_name: knots
+    matching_method: MATCHING_FORMAT
+    subclass_name: alphaq_addon.AlphaQAddonCatalog
+    addon_filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/protoDC2_knots_v2.1.2.hdf5
+    addon_group: knots

GCRCatalogs/catalog_configs/proto-dc2_v2.1.2_addon_tidal.yaml

@@ -0,0 +1,8 @@
+subclass_name: composite.CompositeReader
+catalogs:
+  - catalog_name: proto-dc2_v2.1.2_test
+    matching_method: galaxy_id
+  - catalog_name: tidal
+    matching_method: galaxy_id
+    subclass_name: alphaq_addon.AlphaQTidalCatalog
+    filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/tidal_247_protoDC2_v2.1.hdf5

GCRCatalogs/catalog_configs/proto-dc2_v3.0_addon_knots.yaml

@@ -1,7 +1,9 @@
-subclass_name: alphaq_addon.AlphaQAddonCatalog
-filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/ANL_AlphaQ_v3.0.hdf5
-lightcone: true
-version: "3.0.0"
-md5: 1efca46756b59b75d23e5382ee97cd10
-addon_filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/protoDC2_knots_v3.0.hdf5
-addon_group: knots
+subclass_name: composite.CompositeReader
+catalogs:
+  - catalog_name: proto-dc2_v3.0
+  - catalog_name: knots
+    matching_method: MATCHING_FORMAT
+    subclass_name: alphaq_addon.AlphaQAddonCatalog
+    filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/ANL_AlphaQ_v3.0.hdf5
+    addon_filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/protoDC2_knots_v3.0.hdf5
+    addon_group: knots

GCRCatalogs/catalog_configs/proto-dc2_v3.0_addon_redmapper.yaml

@@ -0,0 +1,6 @@
+subclass_name: composite.CompositeReader
+catalogs:
+  - catalog_name: proto-dc2_v3.0_test
+    matching_method: galaxyID
+  - catalog_name: proto-dc2_v3.0_redmapper
+    matching_method: galaxy_id

GCRCatalogs/catalog_configs/proto-dc2_v4.3_addon_redmapper.yaml

@@ -0,0 +1,6 @@
+subclass_name: composite.CompositeReader
+catalogs:
+  - catalog_name: proto-dc2_v4.3_test
+    matching_method: galaxy_id
+  - catalog_name: proto-dc2_v4.3_redmapper
+    matching_method: galaxy_id

GCRCatalogs/catalog_configs/proto-dc2_v4.3_test.yaml

@@ -0,0 +1,13 @@
+subclass_name: alphaq.AlphaQGalaxyCatalog
+
+filename: /global/projecta/projectdirs/lsst/groups/CS/descqa/catalog/v4.3.full.all.hdf5
+
+lightcone: true
+
+version: "4.0.0"
+
+creators: [ 'Andrew Benson', 'Andrew Hearin',  'Katrin Heitmann', 'Danila Korytov', 'Eve Kovacs']
+
+description: |
+    ProtoDC2 is a down-scaled version of the catalog to be generated for LSST-DESC DC2.
+    For a description of the catalog and the methods, please see https://goo.gl/fXDQwP

GCRCatalogs/catalog_configs/um_v0.1_test_shear.yaml

@@ -0,0 +1,18 @@
+subclass_name: composite.CompositeReader
+
+catalogs:
+  - catalog_name: um_v0.1_test
+  - catalog_name: shear
+    matching_method: MATCHING_FORMAT
+    subclass_name: cosmodc2.UMShearCatalog
+    catalog_root_dir: /global/projecta/projectdirs/lsst/groups/CS/cosmoDC2/shear_addon
+    catalog_path_template: umachine_color_mock_z_{}_{}_cutout_{}.hdf5
+    healpix_pixels: [564]
+    zlo: 0
+    zhi: 1
+    cosmology:
+      H0: 71.0
+      Om0: 0.2648
+      Ob0: 0.0448
+    check_file_metadata: false
+    creators: [ 'Patricia Larsen']

GCRCatalogs/composite.py

@@ -0,0 +1,25 @@
+"""
+composite reader
+"""
+from GCR import CompositeCatalog, MATCHING_FORMAT, MATCHING_ORDER
+from .register import load_catalog, load_catalog_from_config_dict
+
+class CompositeReader(CompositeCatalog):
+    def __init__(self, **kwargs):
+        instances = []
+        identifiers = []
+        methods = []
+        for catalog_dict in kwargs['catalogs']:
+            if 'subclass_name' in catalog_dict:
+                catalog = load_catalog_from_config_dict(catalog_dict)
+            else:
+                catalog = load_catalog(catalog_dict['catalog_name'])
+            instances.append(catalog)
+            identifiers.append(catalog_dict.get('catalog_name'))
+            method = catalog_dict.get('matching_method', 'MATCHING_FORMAT')
+            if method == 'MATCHING_FORMAT':
+                method = MATCHING_FORMAT
+            elif method == 'MATCHING_ORDER':
+                method = MATCHING_ORDER
+            methods.append(method)
+        super(CompositeReader, self).__init__(instances, identifiers, methods, **kwargs)

GCRCatalogs/version.py

@@ -1 +1 @@
-__version__ = '0.8.0'
+__version__ = '0.8.1'