From 038546ca7b2c2a0abc04c1f0e042e241926ab6d7 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 6 Jan 2025 12:24:33 -0500 Subject: [PATCH 1/5] Use different alg for two_byte_sum that fixes off-by-one error --- mica/archive/aca_hdr3.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mica/archive/aca_hdr3.py b/mica/archive/aca_hdr3.py index 8736afc5..f67f2983 100644 --- a/mica/archive/aca_hdr3.py +++ b/mica/archive/aca_hdr3.py @@ -24,11 +24,19 @@ def two_byte_sum(byte_msids, scale=1): def func(slot_data): - return ( - (slot_data[byte_msids[0]].astype("int") >> 7) * (-1 * 65535) - + (slot_data[byte_msids[0]].astype("int") << 8) - + (slot_data[byte_msids[1]].astype("int")) - ) * scale + # For each pair bytes0[i], bytes1[i], return the 16-bit signed integer + # corresponding to those two bytes. The input bytes are unsigned. + bytes0 = slot_data[byte_msids[0]] + bytes1 = slot_data[byte_msids[1]] + + # Make a 2xN array, then transpose to Nx2, then flatten to 2N, then copy to + # get values continous in memory. + bytes = np.array([bytes0, bytes1], dtype=np.uint8).transpose().flatten().copy() + + # Now view the 2N bytes as N 16-bit signed integers. + out = bytes.view(">i2") * scale + + return out return func From 351244a68df78aacdfdcc5ca74d23433f287d7c5 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 28 Jan 2025 12:02:53 -0500 Subject: [PATCH 2/5] ruff format 0.9 --- mica/archive/aca_dark/dark_cal.py | 2 +- mica/archive/cda/services.py | 5 ++--- mica/centroid_dashboard.py | 16 ++++++++-------- mica/report/report.py | 6 +++--- mica/stats/update_acq_stats.py | 3 +-- mica/vv/process.py | 2 +- mica/web/views.py | 2 +- scripts/update_agasc_supplement.py | 3 +-- 8 files changed, 18 insertions(+), 21 deletions(-) diff --git a/mica/archive/aca_dark/dark_cal.py b/mica/archive/aca_dark/dark_cal.py index b38ed209..e4db3efb 100644 --- a/mica/archive/aca_dark/dark_cal.py +++ b/mica/archive/aca_dark/dark_cal.py @@ -141,7 +141,7 @@ def _get_dark_cal_id_scalar(date, select="before", dark_cal_ids=None): if ii < 0: earliest = CxoTime(dark_cal_secs[0]).date[:8] raise MissingDataError( - f"No dark cal found before {earliest}" f"(requested dark cal on {date})" + f"No dark cal found before {earliest}(requested dark cal on {date})" ) try: diff --git a/mica/archive/cda/services.py b/mica/archive/cda/services.py index 09a4ae37..55cec32d 100644 --- a/mica/archive/cda/services.py +++ b/mica/archive/cda/services.py @@ -430,8 +430,7 @@ def _get_cda_service_text(service, timeout=60, **params): if not resp.ok: raise RuntimeError( - f"got error {resp.status_code} for {resp.url}\n" - f"{html_to_text(resp.text)}" + f"got error {resp.status_code} for {resp.url}\n{html_to_text(resp.text)}" ) return resp.text @@ -630,7 +629,7 @@ def get_ocat_local( # accurate enough for this application. where = ( f"arccos(sin({ra * d2r})*sin(ra*{d2r}) + " - f"cos({ra * d2r})*cos(ra*{d2r})*cos({dec*d2r}-dec*{d2r}))" + f"cos({ra * d2r})*cos(ra*{d2r})*cos({dec * d2r}-dec*{d2r}))" f"< {radius / 60 * d2r}" ) where_parts.append(where) diff --git a/mica/centroid_dashboard.py b/mica/centroid_dashboard.py index f3774e4d..2df78fa4 100755 --- a/mica/centroid_dashboard.py +++ b/mica/centroid_dashboard.py @@ -1182,7 +1182,7 @@ def make_html(row_obsid, rows_slot, obs_dir): for row in t_slot: string += f""" -{row['slot']} +{row["slot"]} """ if row["id"] < 100: id_ = "" @@ -1193,13 +1193,13 @@ def make_html(row_obsid, rows_slot, obs_dir): ) string += f"""{id_} -{row['type']} -{row['mag']} -{row['yang']} -{row['zang']} -{row['median_mag']:.3f} -{row['median_dy']:.2f} -{row['median_dz']:.2f} +{row["type"]} +{row["mag"]} +{row["yang"]} +{row["zang"]} +{row["median_mag"]:.3f} +{row["median_dy"]:.2f} +{row["median_dz"]:.2f} """ string += f""" diff --git a/mica/report/report.py b/mica/report/report.py index 9a148379..95fed986 100644 --- a/mica/report/report.py +++ b/mica/report/report.py @@ -920,9 +920,9 @@ def save_state_in_db(obsid, notes): del notes["last_sched"] idcheck = db.fetchone( - "select * from report_proc " - "where obsid = '{}' " - "and report_version = '{}'".format(obsid, REPORT_VERSION) + "select * from report_proc where obsid = '{}' and report_version = '{}'".format( + obsid, REPORT_VERSION + ) ) if idcheck is None: diff --git a/mica/stats/update_acq_stats.py b/mica/stats/update_acq_stats.py index 2e68be65..8e34202e 100755 --- a/mica/stats/update_acq_stats.py +++ b/mica/stats/update_acq_stats.py @@ -126,8 +126,7 @@ def get_options(): parser.add_argument( "--email", action="append", - help="email warning recipient, specify multiple times " - "for multiple recipients", + help="email warning recipient, specify multiple times for multiple recipients", ) opt = parser.parse_args() return opt diff --git a/mica/vv/process.py b/mica/vv/process.py index 5eb91633..f67103a0 100644 --- a/mica/vv/process.py +++ b/mica/vv/process.py @@ -158,7 +158,7 @@ def update(obsids=None): logger.warn(f"Skipping obs:ver {obsid}:{obs['revision']}. Missing data") continue update_str = f"""UPDATE aspect_1_proc set vv_complete = {VV_VERSION} - where obsid = {obsid} and revision = {obs['revision']}""" + where obsid = {obsid} and revision = {obs["revision"]}""" logger.info(update_str) with ska_dbi.DBI(dbi="sqlite", server=FILES["asp1_proc_table"]) as db: diff --git a/mica/web/views.py b/mica/web/views.py index 2b84c62b..79cebaae 100644 --- a/mica/web/views.py +++ b/mica/web/views.py @@ -83,7 +83,7 @@ def get_context_data(self, **kwargs): context["gui_table"] = gui_table reports_url = ( "https://cxc.cfa.harvard.edu/mta/ASPECT/agasc/supplement_reports/stars/" - f"{int(agasc_id//1e7):03d}/{agasc_id}/index.html" + f"{int(agasc_id // 1e7):03d}/{agasc_id}/index.html" ) context["reports_url"] = reports_url return context diff --git a/scripts/update_agasc_supplement.py b/scripts/update_agasc_supplement.py index 83943095..73ee246e 100644 --- a/scripts/update_agasc_supplement.py +++ b/scripts/update_agasc_supplement.py @@ -38,8 +38,7 @@ def get_options(args=None): "--bad-star-source", type=int, help=( - "Source identifier indicating provenance (default=max " - "existing source + 1)" + "Source identifier indicating provenance (default=max existing source + 1)" ), ) parser.add_argument( From b122f6e6e92ae8764de6f9945e72c23f9b55ac02 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 28 Jan 2025 12:27:38 -0500 Subject: [PATCH 3/5] Make fix that works for masked arrays --- mica/archive/aca_hdr3.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mica/archive/aca_hdr3.py b/mica/archive/aca_hdr3.py index f67f2983..b6af9c65 100644 --- a/mica/archive/aca_hdr3.py +++ b/mica/archive/aca_hdr3.py @@ -23,20 +23,21 @@ def two_byte_sum(byte_msids, scale=1): - def func(slot_data): + def func(slot_data) -> np.ma.MaskedArray: # For each pair bytes0[i], bytes1[i], return the 16-bit signed integer # corresponding to those two bytes. The input bytes are unsigned. - bytes0 = slot_data[byte_msids[0]] - bytes1 = slot_data[byte_msids[1]] + bytes0 = slot_data[byte_msids[0]].astype(np.uint8) + bytes1 = slot_data[byte_msids[1]].astype(np.uint8) # Make a 2xN array, then transpose to Nx2, then flatten to 2N, then copy to # get values continous in memory. - bytes = np.array([bytes0, bytes1], dtype=np.uint8).transpose().flatten().copy() + bytes8_2xN = np.ma.vstack([bytes0, bytes1], dtype=np.uint8) + bytes8 = bytes8_2xN.transpose().flatten().copy() # Now view the 2N bytes as N 16-bit signed integers. - out = bytes.view(">i2") * scale + ints16 = np.ma.array(bytes8.data.view(">i2"), mask=bytes8.mask[::2]) - return out + return ints16 * scale return func From 0ade721447e8b9681f496685fee98c68f653ba76 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 29 Jan 2025 06:43:34 -0500 Subject: [PATCH 4/5] Add a unit test --- mica/archive/tests/test_aca_hdr3.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mica/archive/tests/test_aca_hdr3.py b/mica/archive/tests/test_aca_hdr3.py index 0ac6d9b7..74af952c 100644 --- a/mica/archive/tests/test_aca_hdr3.py +++ b/mica/archive/tests/test_aca_hdr3.py @@ -63,3 +63,25 @@ def test_MSIDset(): 10679, ] ) + + +def test_two_byte_sum(): + bytes0 = np.ma.array([0x00, 0xF0, 0x0F, 0xFF, 0xFF], dtype=np.uint8) + bytes1 = np.ma.array([0x00, 0x0F, 0xF0, 0xFF, 0xFF], dtype=np.uint8) + bytes0[-1] = np.ma.masked + bytes1[-1] = np.ma.masked + + # Original code prior to PR #315 + out1 = ( + (bytes0.astype("int") >> 7) * (-1 * 65535) + + (bytes0.astype("int") << 8) + + (bytes1.astype("int")) + ) + assert np.all(out1 == np.ma.array([0, -4080, 4080, 0, 0], mask=[0, 0, 0, 0, 1])) + + # New code in PR #315 + bytes8_2xN = np.ma.vstack([bytes0, bytes1], dtype=np.uint8) + bytes8 = bytes8_2xN.transpose().flatten().copy() + ints16 = np.ma.array(bytes8.data.view(">i2"), mask=bytes8.mask[::2]) + + assert np.all(ints16 == np.ma.array([0, -4081, 4080, -1, 0], mask=[0, 0, 0, 0, 1])) From 812d3f079115c5601bb9ebd39123985085cb9f52 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 24 Feb 2025 08:45:02 -0500 Subject: [PATCH 5/5] Update test --- mica/archive/tests/test_aca_hdr3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mica/archive/tests/test_aca_hdr3.py b/mica/archive/tests/test_aca_hdr3.py index 74af952c..e74f3ca8 100644 --- a/mica/archive/tests/test_aca_hdr3.py +++ b/mica/archive/tests/test_aca_hdr3.py @@ -7,6 +7,7 @@ import numpy as np import pytest +from astropy.table import Table from mica.archive import aca_hdr3 @@ -80,8 +81,7 @@ def test_two_byte_sum(): assert np.all(out1 == np.ma.array([0, -4080, 4080, 0, 0], mask=[0, 0, 0, 0, 1])) # New code in PR #315 - bytes8_2xN = np.ma.vstack([bytes0, bytes1], dtype=np.uint8) - bytes8 = bytes8_2xN.transpose().flatten().copy() - ints16 = np.ma.array(bytes8.data.view(">i2"), mask=bytes8.mask[::2]) + slot_data = Table([bytes0, bytes1], names=["byte0", "byte1"]) + ints16 = aca_hdr3.two_byte_sum(["byte0", "byte1"])(slot_data) assert np.all(ints16 == np.ma.array([0, -4081, 4080, -1, 0], mask=[0, 0, 0, 0, 1]))