Merge pull request #1667 from bhilbert4/stop-using-preview-ig-invento…

…ry-files

Remove support for preview image and thumbnail listfiles

jwql/jwql_monitors/generate_preview_images.py

@@ -36,8 +36,11 @@
 import numpy as np
 
 from jwql.utils import permissions
-from jwql.utils.constants import IGNORED_SUFFIXES, JWST_INSTRUMENT_NAMES_SHORTHAND, NIRCAM_LONGWAVE_DETECTORS, \
-    NIRCAM_SHORTWAVE_DETECTORS, PREVIEW_IMAGE_LISTFILE, THUMBNAIL_LISTFILE
+from jwql.utils.constants import (IGNORED_SUFFIXES,
+                                  JWST_INSTRUMENT_NAMES_SHORTHAND,
+                                  NIRCAM_LONGWAVE_DETECTORS,
+                                  NIRCAM_SHORTWAVE_DETECTORS
+                                  )
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.protect_module import lock_module
 from jwql.utils.preview_image import PreviewImage
@@ -616,22 +619,6 @@ def generate_preview_images(overwrite, programs=None):
         full_preview_files.extend(r[0])
         full_thumbnail_files.extend(r[1])
 
-    # Filter the preview and thumbnail images by instrument and update the listfiles.
-    # We do this by looking for instrument abbreviations in the filenames. But will
-    # this work for level 3 files?? If an instrument abbreviation is not in the filename,
-    # then the preview/thubnail images won't be caught and added here.
-    for abbrev, inst_name in JWST_INSTRUMENT_NAMES_SHORTHAND.items():
-        inst_previews = [ele for ele in full_preview_files if re.search(abbrev, ele, re.IGNORECASE)]
-        inst_thumbs = [ele for ele in full_thumbnail_files if abbrev in ele]
-
-        # Read in the preview image listfile and the thumbnail image list file
-        # and add these new files to each
-        preview_image_listfile = os.path.join(SETTINGS['preview_image_filesystem'], f"{PREVIEW_IMAGE_LISTFILE}_{inst_name}.txt")
-        update_listfile(preview_image_listfile, inst_previews, 'preview')
-
-        thumbnail_image_listfile = os.path.join(SETTINGS['thumbnail_filesystem'], f"{THUMBNAIL_LISTFILE}_{inst_name}.txt")
-        update_listfile(thumbnail_image_listfile, inst_thumbs, 'thumbnail')
-
     # Complete logging:
     logging.info("Completed.")
 
@@ -842,43 +829,6 @@ def process_program(program, overwrite):
     return preview_image_files, thumbnail_files
 
 
-def update_listfile(filename, file_list, filetype):
-    """Add a list of files to a text file. Designed to add new files to the
-    file containing the list of all preview images and the file containing the
-    list of all thumbnail images.
-
-    Parameters
-    ----------
-    filename : str
-        Name, including path, of the file to be amended/created
-
-    file_list : list
-        List of filenames to be added to filename
-
-    filetype : str
-        Descriptor of the contents of the file being amended. Used only for
-        the logging statement
-    """
-    if len(file_list) > 0:
-        if not os.path.isfile(filename):
-            logging.warning(f"{filetype} image listfile not found!! Expected to be at {filename}. Creating a new file.")
-
-        with open(filename, 'a+') as fobj:
-            # Move read cursor to the start of file.
-            fobj.seek(0)
-
-            # If file is not empty then append '\n'
-            data = fobj.read(100)
-            if len(data) > 0:
-                fobj.write("\n")
-
-            # Append file_list at the end of file
-            for file_to_add in file_list:
-                fobj.write(f'{file_to_add}\n')
-
-        logging.info(f"{filetype} image listfile {filename} updated with new entries.")
-
-
 @lock_module
 def protected_code(overwrite, programs):
     """Protected code ensures only 1 instance of module will run at any given time

jwql/utils/constants.py

@@ -795,10 +795,6 @@
 # Determine if the code is being run as part of a Readthedocs build
 ON_READTHEDOCS = os.environ.get('READTHEDOCS', False)
 
-# Base name for the file listing the preview images for a given instrument.
-# The complete name will have "_{instrument.lower}.txt" added to the end of this.
-PREVIEW_IMAGE_LISTFILE = "preview_image_inventory"
-
 # All possible proposal categories
 PROPOSAL_CATEGORIES = ["AR", "CAL", "COM", "DD", "ENG", "GO", "GTO", "NASA", "SURVEY"]
 
@@ -1008,10 +1004,6 @@ class QueryConfigKeys:
 # boolean accessed according to a viewed flag
 THUMBNAIL_FILTER_LOOK = ["New", "Viewed"]
 
-# Base name for the file listing the thumbnail images for a given instrument.
-# The complete name will have "_{instrument.lower}.txt" added to the end of this.
-THUMBNAIL_LISTFILE = "thumbnail_inventory"
-
 # Possible suffix types for time-series exposures
 TIME_SERIES_SUFFIX_TYPES = ["phot", "whtlt"]
 

jwql/website/apps/jwql/data_containers.py

@@ -68,7 +68,6 @@
     SUFFIXES_TO_ADD_ASSOCIATION,
     SUFFIXES_WITH_AVERAGED_INTS,
     THUMBNAIL_FILTER_LOOK,
-    THUMBNAIL_LISTFILE,
     QueryConfigKeys,
 )
 from jwql.utils.credentials import get_mast_token
@@ -1860,43 +1859,6 @@ def get_rootnames_from_query(parameters):
     return filtered_rootnames
 
 
-def get_thumbnails_by_instrument(inst):
-    """Return a list of thumbnails available in the filesystem for the
-    given instrument.
-
-    Parameters
-    ----------
-    inst : str
-        The instrument of interest (e.g. ``NIRCam``).
-
-    Returns
-    -------
-    preview_images : list
-        A list of thumbnails available in the filesystem for the
-        given instrument.
-    """
-    # Get list of all thumbnails
-    thumb_inventory = f'{THUMBNAIL_LISTFILE}_{inst.lower()}.txt'
-    all_thumbnails = retrieve_filelist(os.path.join(THUMBNAIL_FILESYSTEM, thumb_inventory))
-
-    thumbnails = []
-    all_proposals = get_instrument_proposals(inst)
-    for proposal in all_proposals:
-        results = mast_query_filenames_by_instrument(inst, proposal)
-
-        # Parse the results to get the rootnames
-        filenames = [result['filename'].split('.')[0] for result in results]
-
-        if len(filenames) > 0:
-            # Get subset of preview images that match the filenames
-            prop_thumbnails = [os.path.basename(item) for item in all_thumbnails if
-                               os.path.basename(item).split('_integ')[0] in filenames]
-
-            thumbnails.extend(prop_thumbnails)
-
-    return thumbnails
-
-
 def get_thumbnails_by_proposal(proposal):
     """Return a list of thumbnails available in the filesystem for the
     given ``proposal``.