diff --git a/jwql/example_config.json b/jwql/example_config.json index adcf87555..80f920a76 100644 --- a/jwql/example_config.json +++ b/jwql/example_config.json @@ -43,6 +43,7 @@ "setup_file" : "", "test_data" : "", "test_dir" : "", + "wisp_finder_ML_model" : "", "thumbnail_filesystem" : "", "cores" : "", "redis_host": "", diff --git a/jwql/instrument_monitors/nircam_monitors/prepare_wisp_pngs.py b/jwql/instrument_monitors/nircam_monitors/prepare_wisp_pngs.py new file mode 100755 index 000000000..e0d2678e4 --- /dev/null +++ b/jwql/instrument_monitors/nircam_monitors/prepare_wisp_pngs.py @@ -0,0 +1,151 @@ +#! /usr/bin/env python + +""" +Given a fits file, prepare an image of the data that can be provided to the ML wisp +prediction model. +""" + +import argparse +import numpy as np +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +import os +from PIL import Image +import matplotlib.pyplot as plt + + +def create_figure(image, outfile): + """Create a figure of the scaled, resized image + + Parameters + ---------- + image : PIL.Image.Image + Image to be saved + + outfile : str + Name of file it save the image into + """ + plt.imshow(image, origin='lower') + plt.axis('off') + plt.savefig(outfile, bbox_inches='tight') + + +def rescale_array(arr): + """Rescales an array to the range 0-255. + + Parameters + ---------- + arr : nump.ndarray + 2D image array + + Returns + ------- + adjusted_image : numpy.ndarray + Rescaled image + """ + # Calculate basic stats on the image + mn, med, dev = sigma_clipped_stats(arr) + + # Don't worry about any pixels more than 2-sigma from the peak value + maximum_gray = med + dev * 1. + minimum_gray = med + + # Calculate scaling factor and contrast adjustment + alpha = 255 / (maximum_gray - minimum_gray) + beta = -minimum_gray * alpha + + # Rescale the image + adjusted_image = alpha * arr + beta + adjusted_image = np.clip(adjusted_image, 0, 255).astype(np.uint8) + + return adjusted_image + + +def resize_image(arr): + """Resize the input image to the size expected by the ML model + + Parameters + ---------- + arr : numpy.ndarray + 2D image to te resized + + Returns + ------- + resized_image : PIL.Image.Image + Resized image + """ + img = Image.fromarray(arr) + resized_image = img.resize(size=(256, 256)) + return resized_image + + +def add_options(parser=None, usage='', conflict_handler='resolve'): + """ + Add command line options + + Parrameters + ----------- + parser : argparse.parser + Parser object + + usage : str + Usage string + + conflict_handler : str + Conflict handling strategy + + Returns + ------- + parser : argparse.parser + Parser object with added options + """ + if parser is None: + parser = argparse.ArgumentParser(usage=usage, conflict_handler=conflict_handler) + + parser.add_argument('filename', type=str, default='', help='File from which to create image') + return parser + + +def run(filename, out_dir=None): + """Main function. Read in fits file, create scaled and resized image. Save + as png. + + Parameters + ---------- + filename : str + Name of fits file + + out_dir : str + Output directory in which to save the final png file + + Returns + ------- + output_file : str + Full path to the output png file + """ + data = fits.getdata(filename) + + # Get the basename of the input file. This will be used to create + # the output png file name + outfile_base = os.path.basename(filename).split('.')[0] + + # Rescale and adjust contrast of the image + adjusted_image = rescale_array(data) + + # Resize image to 256x256 pixels + shrunk_img = resize_image(adjusted_image) + + # Create output filename + output_file = f'{outfile_base}.png' + if out_dir is not None: + output_file = os.path.join(out_dir, output_file) + + # Create image and save + create_figure(shrunk_img, output_file) + return output_file + + +if __name__ == '__main__': + parser = add_options() + args = parser.parse_args() + run(args.filename) diff --git a/jwql/instrument_monitors/nircam_monitors/wisp_finder.py b/jwql/instrument_monitors/nircam_monitors/wisp_finder.py new file mode 100755 index 000000000..32ea78533 --- /dev/null +++ b/jwql/instrument_monitors/nircam_monitors/wisp_finder.py @@ -0,0 +1,491 @@ +#! /usr/bin/env python + +"""This module contains code for the wisp finder monitor. + +Author +------ + - Bryan Hilbert + +Use +--- + This module can be used from the command line as such: + + :: + + python wisp_monitor.py + + +Overall flow: + + +1. Look in database table for last successful run on the monitor. +2. Get the datetime of that run. +3. Query MAST for all NIRCam B4 full frame files (exclude coron?) since that datetime +4. Copy over rate files to working dir +5. Re-scale, and create png files using the same method that was used for the ML training +6. Load the trained model +7. Use the model to predict whether each png contains a wisp +8. For those files where the prediction is that a wisp is present, set the wisp flag in the anomalies database +9. Delete pngs +10. Update the database with the datetime of the current run + + +""" + +import argparse +import datetime +import logging +import os +import shutil +import warnings + +from astroquery.mast import Observations +from django import setup +from django.utils import timezone +import numpy as np +from PIL import Image +import torch +import torch.nn as nn +from torchvision import transforms +import torchvision.models as models + +from jwql.utils import monitor_utils +from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS +from jwql.utils.logging_functions import log_info, log_fail +from jwql.utils.utils import get_config +from jwql.website.apps.jwql.archive_database_update import files_in_filesystem +from jwql.instrument_monitors.nircam_monitors import prepare_wisp_pngs + +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + setup() + from jwql.website.apps.jwql.models import Anomalies, RootFileInfo + from jwql.website.apps.jwql.monitor_models.wisp_finder import WispFinderB4QueryHistory + + +def add_wisp_flag(basename): + """Add the wisps flag to the RootFileInfo entry for the given filename + + Parameters + ---------- + basename : str + Filename minus the suffix and ".fits". e.g. "jw01068004001_02101_00001_nrcb1" + """ + # Get the RootFileInfo instance + root_file_info = RootFileInfo.objects.get(root_name=basename) + + # Set user name and date + user_name = 'ML_wisp_finder' + entry_date = timezone.now() + + # Set the wisps flag, and add the current time, and say that the flag is coming from the wisp finder + anomalies_exist = hasattr(root_file_info, 'anomalies') + if anomalies_exist: + # If an Anomalies instance is already associated with the RootFileInfo instance, then + # set the wisps flag + root_file_info.anomalies.wisps = True + root_file_info.anomalies.flag_date = timezone.now() + root_file_info.anomalies.user = 'ML_wisp_finder' + root_file_info.anomalies.save(update_fields=['wisps', 'flag_date', 'user']) + else: + # If an Anomaly object is not associated with the RootFileInfo instance, create one + default_dict = {'flag_date': entry_date, + 'user': user_name} + for anomaly in Anomalies.get_all_anomalies(): + default_dict[anomaly] = (anomaly in ['wisps']) + Anomalies.objects.update_or_create(root_file_info=root_file_info, defaults=default_dict) + + +def copy_files_to_working_dir(filepaths): + """Copy files from MAST into a working directory + + Parameters + ---------- + filepaths : list + List of full paths of files to be copied + + Returns + ------- + copied_filepaths : list + List of new locations for the files + """ + working_dir = get_config()["working"] + copied_filepaths = [] + for filepath in filepaths: + shutil.copy2(filepath, working_dir) + copied_filepaths.append(os.path.join(working_dir, os.path.basename(filepath))) + logging.info(f'Copying {filepath} to {working_dir}') + + return copied_filepaths + + +def create_transform(): + """Create a transform function that will be used to modify images + and place them in the format expected by the ML model + + Returns + ------- + transform : torchvision.transforms.transforms.Compose + Image transform model + """ + transform = transforms.Compose([ + transforms.Resize((128, 128)), # Resize images to a fixed size + transforms.ToTensor(), # Convert images to Tensor + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # Normalize images + ]) + return transform + + +def define_model_architecture(): + """Define the basic architecture of the ML model. This will be the framework into which + the model parameters will be loaded, in order to fully define the function. + + Returns + ------- + model : torchvision.models.resnet.ResNet + ResNet model to use for wisp prediction + """ + # Load pre-trained ResNet-18 model + model = models.resnet18(weights='IMAGENET1K_V1') + + # Modify the final fully connected layer for binary classification + model.fc = nn.Linear(model.fc.in_features, 1) + + # Add a sigmoid activation after the final layer + model.add_module('sigmoid', nn.Sigmoid()) + return model + + +def define_options(parser=None, usage=None, conflict_handler='resolve'): + """Add command line options + + Parrameters + ----------- + parser : argparse.parser + Parser object + + usage : str + Usage string + + conflict_handler : str + Conflict handling strategy + + Returns + ------- + parser : argparse.parser + Parser object with added options + """ + if parser is None: + parser = argparse.ArgumentParser(usage=usage, conflict_handler=conflict_handler) + + parser.add_argument('-m', '--model_filename', type=str, default=None, + help='Filename of saved ML model. (default=%(default)s)') + parser.add_argument('-s', '--starting_date', type=float, default=None, + help='Earliest MJD to search for data. If None, date is retrieved from database.') + parser.add_argument('-e', '--ending_date', type=float, default=None, + help='Latest MJD to search for data. If None, the current date is used.') + parser.add_argument('-f', '--file_list', type=str, nargs='+', default=None, + help='List of full paths to files to run the monitor on.') + return parser + + +def get_latest_run(): + """Retrieve the ending time of the latest successful run from the database + + Returns + ------- + latset_date : float + MJD of the ending time of the latest successful run of the monitor + """ + filters = {"run_monitor": True} + record = WispFinderB4QueryHistory.objects.filter(**filters).order_by("-end_time_mjd").first() + + if record is None: + query_result = 59607.0 # a.k.a. Jan 28, 2022 == First JWST images + logging.info(f'\tNo successful previous runs found. Beginning search date will be set to {query_result}.') + else: + query_result = record.end_time_mjd + + return query_result + + +def load_ml_model(model_filename): + """Load the ML model for wisp prediction + + Parameters + ---------- + model_filename : str + Location of file containing the model. e.g. /path/to/my_best_model.pth + + Returns + ------- + model : torchvision.models.resnet.ResNet + ResNet model to use for wisp prediction + """ + model = define_model_architecture() + model.load_state_dict(torch.load(model_filename)) + model.eval() # Set model to evaluation mode + return model + + +def predict_wisp(model, image_path, transform): + """Use the model to predict whether there is a wisp in the image. The model returns + a probability. So we use a threshold to separate those predictions into 'wisp' and + 'no wisp' bins. + + Parameters + ---------- + model : torchvision.models.resnet.ResNet + ResNet model to use for wisp prediction + + image_path : str + Full path to the png file + + transform : torchvision.transforms.transforms.Compose + Image transform function used to modify the input images into the format + expected by the ML model. + + Returns + ------- + prediction_label : str + "wisp" or "no wisp" + """ + image_tensor = preprocess_image(image_path, transform) # Preprocess the image + + with torch.no_grad(): # Make prediction without gradients + output = model(image_tensor) + + # The model outputs a single probability (e.g., for "wisp"). So, use a threshold + # to determine whether the prediction is wisp or no_wisp. + probability = torch.sigmoid(output).item() + threshold = 0.5 + prediction_label = "wisp" if probability >= threshold else "no wisp" + return prediction_label + + +def preprocess_image(image_path, transform): + """Load the png file and prepare it for input to the model + + Parameters + ---------- + image_path : str + Path and filename of the png file + + transform : torchvision.transforms.transforms.Compose + Image transform function used to modify the input images into the format + expected by the ML model. + + Returns + ------- + image : torch.Tensor + Tensor on which the model will run + """ + image = Image.open(image_path).convert('RGB') # Ensure image is RGB + image = transform(image) # Apply transformations + image = image.unsqueeze(0) # Add batch dimension + return image + + +def query_mast(starttime, endtime): + """Query MAST between the given dates. Generate a list of NRCB4 files on which + the wisp model will be run + + Parameters + ---------- + starttime : float or str + MJD of the beginning of the search interval + + endtime : float or str + MJD of the end of the search interval + + Returns + ------- + rate_files : list + List of filenames + """ + sci_obs_id_table = Observations.query_criteria(instrument_name=["NIRCAM/IMAGE"], + provenance_name=["CALJWST"], # Executed observations + t_min=[starttime, endtime] + ) + + sci_files_to_download = [] + + # Loop over visits identifying uncalibrated files that are associated + # with them + for exposure in (sci_obs_id_table): + products = Observations.get_product_list(exposure) + filtered_products = Observations.filter_products(products, + productType='SCIENCE', + productSubGroupDescription='RATE', + calib_level=[2]) + sci_files_to_download.extend(filtered_products['dataURI']) + + # The current ML wisp finder model is only trained for the wisps on the B4 detector, + # so keep only those files. Also, keep only the filenames themselves. + rate_files = sorted([fname.replace('mast:JWST/product/', '') for fname in sci_files_to_download if 'nrcb4' in fname]) + return rate_files + + +def remove_duplicate_files(file_list): + """When running locally, it's possible to end up with duplicates of some filenames in + the list of files, because the files are present in both the public and proprietary + lists. This function will remove the duplicates. + + Parameters + ---------- + file_list : list + List of full paths to input files + + Returns + ------- + unique_files : list + List of files with unique basenames + """ + file_list = np.array(file_list) + unique_files = [] + basenames_only = set([os.path.basename(e) for e in file_list]) + for basename in basenames_only: + matches = np.array([basename in e for e in file_list]) + unique_files.append(file_list[matches][0]) + return unique_files + + +@log_fail +@log_info +def run(model_filename=None, starting_date=None, ending_date=None, file_list=None): + """Run the wisp finder monitor. From user-input dates or dates retrieved from + the database, query MAST for all NIRCam NRCB4 full-frame imaging mode data. For + each file, create a png file continaing an image of the rate file, scaled to a + consistent brightness/range as well as size. Use a trained neural network model + to predict whether the image contains a wisp. If so, set the wisps anomaly flag + for that file. + + Parameters + ---------- + model_filename : str + Name of a pytorch-generated model to load and use for prediction + + starting_date : float + Earliest MJD to use when querying MAST for data + + ending_date : float + Latest MJD to use when querying MAST for data + + file_list : list + List of filenames (e.g. ["jw01068004001_02101_00001_nrcb4_rate.fits"]) + to run the wisp prediction for. If this list is provided, the MAST query + is skipped. + """ + # If no model_filename is given, the retrieve the default model_filename + # from the config file + if model_filename is None: + model_filename = get_config()['wisp_finder_ML_model'] + + if os.path.isfile(model_filename): + logging.info(f'Using ML model saved in: {model_filename}') + else: + raise FileNotFoundError(f"WARNING: {model_filename} does not exist. Unable to load ML model.") + + if file_list is None: + + # If ending_date is not provided, set it equal to the current time + if ending_date is None: + ending_date = timezone.now() + + # If starting date is not provided, then query the database for the last + # successful run of this monitor. Use the ending date of that run for the + # starting_date of this run + if starting_date is None: + latest_run_end = get_latest_run() + starting_date = latest_run_end + + logging.info(f"Using MJD {starting_date} to {ending_date} to search for files") + + # Query MAST between starting_date and ending_date, and get a list of files + # to run the wisp prediction on. + rate_files = query_mast(starting_date, ending_date) + logging.info(f"Found {len(rate_files)} rate files") + + else: + rate_files = file_list + starting_date = 0.0 + ending_date = 0.0 + + if len(rate_files) > 0: + monitor_run = True + + # Find the location in the filesystem for all files + logging.info("Locating files in the filesystem") + filepaths_public = files_in_filesystem(rate_files, 'public') + filepaths_proprietary = files_in_filesystem(rate_files, 'proprietary') + filepaths = filepaths_public + filepaths_proprietary + filepaths = remove_duplicate_files(filepaths) + + logging.info("Copying files from the filesystem to the working directory.") + working_filepaths = copy_files_to_working_dir(filepaths) + + # Load the trained ML model + logging.info(f"Loading ML model from {model_filename}") + model = load_ml_model(model_filename) + + # Create transform to use when creating image tensor + transform = create_transform() + + # For each fits file, create a png file, and have the ML model predict if there is a wisp + for working_filepath in working_filepaths: + # Create png + working_dir = os.path.dirname(working_filepath) + png_filename = prepare_wisp_pngs.run(working_filepath, out_dir=working_dir) + + # Predict + prediction = predict_wisp(model, png_filename, transform) + + print(png_filename, prediction) # FOR DEVELOPMENT ONLY. REMOVE BEFORE MERGING + + # If a wisp is predicted, set the wisp flag in the anomalies database + if prediction == "wisp": + # Create the rootname. Strip off the path info, and remove '.fits' and the suffix + # (i.e. 'rate'') + rootfile = '_'.join(os.path.basename(working_filepath).split('.')[0].split('_')[0:-1]) + logging.info(f"Found wisp in {rootfile}") + + # Add the wisp flag to the RootFileInfo object for the rootfile + add_wisp_flag(rootfile) + else: + pass + + # Delete the png and fits files + os.remove(png_filename) + os.remove(working_filepath) + else: + # If no rate_files are found + logging.info(f"No rate files found. Ending monitor run.") + monitor_run = False + + # Update the database with info about this run of the monitor. We keep the + # staring and ending dates of the search. No need to keep the names of the files + # that are found to contain a wisp, because that info will be in the RootFileInfo + # instances. + new_entry = {'start_time_mjd': starting_date, + 'end_time_mjd': ending_date, + 'run_monitor': monitor_run, + 'entry_date': datetime.datetime.now(datetime.timezone.utc)} + entry = WispFinderB4QueryHistory(**new_entry) + entry.save() + + logging.info('Wisp Finder Monitor completed successfully.') + + +if __name__ == '__main__': + module = os.path.basename(__file__).strip('.py') + start_time, log_file = monitor_utils.initialize_instrument_monitor(module) + + parser = define_options() + args = parser.parse_args() + + run(args.model_filename, + file_list=args.file_list, + starting_date=args.starting_date, + ending_date=args.ending_date) + + monitor_utils.update_monitor_table(module, start_time, log_file) diff --git a/jwql/tests/test_wisp_finder.py b/jwql/tests/test_wisp_finder.py new file mode 100644 index 000000000..66ebc3a46 --- /dev/null +++ b/jwql/tests/test_wisp_finder.py @@ -0,0 +1,129 @@ +#! /usr/bin/env python + +"""Tests for the ``wisp_finder`` module. + +Authors +------- + + - Bryan Hilbert + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s test_wisp_finder.py +""" + +import datetime +import os +import pytest + +import numpy as np +import torch +import torchvision + +from jwql.instrument_monitors.nircam_monitors import wisp_finder, prepare_wisp_pngs +from jwql.utils.constants import ON_GITHUB_ACTIONS +from jwql.utils.utils import get_config +from jwql.website.apps.jwql.archive_database_update import files_in_filesystem + +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + setup() + from jwql.website.apps.jwql.models import RootFileInfo + + +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to database.') +def test_add_wisp_flag(): + """Test that the wisp flag is successfully set on a given rootfileinfo + """ + basename = 'jw01068001001_02102_00003_nrcb4' + wisp_finder.add_wisp_flag(basename) + + root_file_info = RootFileInfo.objects.get(root_name=basename) + assert root_file_info.anomalies.wisps is True + + if root_file_info.anomalies.wisps is True: + # If the flag was checked and successfully set, return it back + # to False for future tests + root_file_info.anomalies.wisps = False + root_file_info.anomalies.save(update_fields=['wisps']) + + +def test_create_transform(): + """Test that the pytorch transform is successfully created + """ + transform = wisp_finder.create_transform() + assert isinstance(transform, torchvision.transforms.transforms.Compose) + + +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central store.') +def test_load_ml_model(): + """Test that a file containing a saved ML model can be successfully loaded + """ + modelname = get_config()['wisp_finder_ML_model'] + model = wisp_finder.load_ml_model(modelname) + assert isinstance(model.fc, torch.nn.modules.linear.Linear) + + +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central store') +def test_predict_wisp(): + modelname = get_config()['wisp_finder_ML_model'] + model = wisp_finder.load_ml_model(modelname) + transform = wisp_finder.create_transform() + + fits_file = ['jw01068004001_02101_00001_nrcb4_rate.fits'] + filepath_public = files_in_filesystem(fits_file, 'public') + copied_file = wisp_finder.copy_files_to_working_dir(filepath_public) + working_dir = get_config()["working"] + png_filename = prepare_wisp_pngs.run(copied_file[0], out_dir=working_dir) + prediction = wisp_finder.predict_wisp(model, png_filename, transform) + assert isinstance(prediction, str) + assert prediction in ['wisp', 'no wisp'] + os.remove(png_filename) + os.remove(os.path.join(working_dir, copied_file[0])) + + +def test_query_mast(): + """Test that a MAST query returns the expected data + """ + results = wisp_finder.query_mast(59714.625, 59714.6458) + assert results == ['jw01068004001_02101_00001_nrcb4_rate.fits'] + + +def test_remove_duplicate_files(): + """Test that duplicate instances of a given file are removed + """ + files = ['/location/one/jw01068001001_02101_00001_nrcb4_rate.fits', + '/location/one/jw01068001001_05101_00001_nrcb4_rate.fits', + '/location/one/jw01068001001_03101_00001_nrcb4_rate.fits', + '/location/two/jw01068001001_03101_00001_nrcb4_rate.fits', + '/location/two/jw01068001001_02101_00001_nrcb4_rate.fits', + '/location/one/jw01068001001_09101_00001_nrcb4_rate.fits' + ] + unique_files = sorted(wisp_finder.remove_duplicate_files(files)) + assert unique_files == ['/location/one/jw01068001001_02101_00001_nrcb4_rate.fits', + '/location/one/jw01068001001_03101_00001_nrcb4_rate.fits', + '/location/one/jw01068001001_05101_00001_nrcb4_rate.fits', + '/location/one/jw01068001001_09101_00001_nrcb4_rate.fits' + ] + + +def test_rescale_array(): + """Test that an input array is correctly rescaled + """ + arr = np.random.normal(0, 1, size=(100, 100)) + arr[3, 3] = 10. + rescaled = prepare_wisp_pngs.rescale_array(arr) + assert rescaled[3, 3] == 255 + + +def test_resize_image(): + """Test image resizing + """ + img = np.zeros((500, 500)) + resized = prepare_wisp_pngs.resize_image(img) + assert resized.size == (256, 256) diff --git a/jwql/website/apps/jwql/apps.py b/jwql/website/apps.py similarity index 93% rename from jwql/website/apps/jwql/apps.py rename to jwql/website/apps.py index d8c347d31..b603db453 100644 --- a/jwql/website/apps/jwql/apps.py +++ b/jwql/website/apps.py @@ -28,3 +28,4 @@ def ready(self): import jwql.website.apps.jwql.monitor_models.dark_current import jwql.website.apps.jwql.monitor_models.readnoise import jwql.website.apps.jwql.monitor_models.ta + import jwql.website.apps.jwql.monitor_models.wisp_finder diff --git a/jwql/website/apps/jwql/migrations/0029_wispfinderb4queryhistory.py b/jwql/website/apps/jwql/migrations/0029_wispfinderb4queryhistory.py new file mode 100644 index 000000000..9ef047c67 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0029_wispfinderb4queryhistory.py @@ -0,0 +1,28 @@ +# Generated by Django 5.1.4 on 2025-01-21 20:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0028_alter_filesystemcharacteristics_filter_pupil_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='WispFinderB4QueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ], + options={ + 'db_table': 'wisp_finder_b4_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + ] diff --git a/jwql/website/apps/jwql/monitor_models/wisp_finder.py b/jwql/website/apps/jwql/monitor_models/wisp_finder.py new file mode 100644 index 000000000..2623409a0 --- /dev/null +++ b/jwql/website/apps/jwql/monitor_models/wisp_finder.py @@ -0,0 +1,41 @@ +"""Defines the models for the ``jwql`` monitors. + +In Django, "a model is the single, definitive source of information +about your data. It contains the essential fields and behaviors of the +data you’re storing. Generally, each model maps to a single database +table" (from Django documentation). Each model contains fields, such +as character fields or date/time fields, that function like columns in +a data table. This module defines models that are used to store data +related to the JWQL monitors. + +Authors +------- + - Bryan Hilbert +Use +--- + This module is used as such: + + :: + from monitor_models import MyModel + data = MyModel.objects.filter(name="JWQL") + +References +---------- + For more information please see: + ```https://docs.djangoproject.com/en/2.0/topics/db/models/``` +""" +# This is an auto-generated Django model module. +# Feel free to rename the models, but don't rename db_table values or field names. +from django.db import models + + +class WispFinderB4QueryHistory(models.Model): + entry_date = models.DateTimeField(blank=True, null=True) + start_time_mjd = models.FloatField(blank=True, null=True) + end_time_mjd = models.FloatField(blank=True, null=True) + run_monitor = models.BooleanField(blank=True, null=True) + + class Meta: + managed = True + db_table = 'wisp_finder_b4_query_history' + unique_together = (('id', 'entry_date'),) diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index ba9fc9d4a..ede985f9f 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -42,3 +42,5 @@ from jwql.website.apps.jwql.monitor_models.readnoise import NIRSpecReadnoiseQueryHistory, NIRSpecReadnoiseStats from jwql.website.apps.jwql.monitor_models.ta import MIRITaQueryHistory + + from jwql.website.apps.jwql.monitor_models.wisp_finder import WispFinderB4QueryHistory diff --git a/pyproject.toml b/pyproject.toml index 51a66c40b..826378d0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,8 @@ dependencies = [ "scipy>=1.12.0,<2", "selenium>=4.18.1,<5", "sqlalchemy>=2.0.29,<3", + "torch>=2.2.2,<2.5.2", + "torchvision>=0.17.2,<0.20.2", "wtforms>=3.1.2,<4", ] dynamic = [