From 5be74c2b0afea6fc9ef8b66a433f4de9171ee53a Mon Sep 17 00:00:00 2001 From: Veronica Martinez Date: Tue, 24 Feb 2026 16:25:00 -0700 Subject: [PATCH 1/5] Add factory class method to check for good data before creating HistogramL2 object. Returns if no good data exists --- imap_processing/glows/l2/glows_l2.py | 6 +- imap_processing/glows/l2/glows_l2_data.py | 119 +++++++++++++--------- 2 files changed, 72 insertions(+), 53 deletions(-) diff --git a/imap_processing/glows/l2/glows_l2.py b/imap_processing/glows/l2/glows_l2.py index 29aa589d0..6f49b7f61 100644 --- a/imap_processing/glows/l2/glows_l2.py +++ b/imap_processing/glows/l2/glows_l2.py @@ -46,9 +46,9 @@ def glows_l2( pipeline_settings_dataset.sel(epoch=day, method="nearest") ) - l2 = HistogramL2(input_dataset, pipeline_settings) - - return [create_l2_dataset(l2, cdf_attrs)] + # TODO: log returning no data if l2 is None + l2 = HistogramL2.create(input_dataset, pipeline_settings) + return [create_l2_dataset(l2, cdf_attrs)] if l2 else [] def create_l2_dataset( diff --git a/imap_processing/glows/l2/glows_l2_data.py b/imap_processing/glows/l2/glows_l2_data.py index 52a7510b3..9eca82079 100644 --- a/imap_processing/glows/l2/glows_l2_data.py +++ b/imap_processing/glows/l2/glows_l2_data.py @@ -1,5 +1,7 @@ """Module containing the class definition for the HistogramL2 class.""" +from __future__ import annotations + from dataclasses import InitVar, dataclass, field import numpy as np @@ -123,10 +125,8 @@ class HistogramL2: Parameters ---------- - l1b_dataset : xr.Dataset - GLOWS histogram L1B dataset, as produced by glows_l1b.py. - pipeline_settings : PipelineSettings - Pipeline settings object read from ancillary file. + good_l1b_data : xr.Dataset + GLOWS histogram L1B dataset filtered by good times. Attributes ---------- @@ -210,123 +210,142 @@ class HistogramL2: spin_axis_orientation_average: np.ndarray[np.double] bad_time_flag_occurrences: np.ndarray - def __init__(self, l1b_dataset: xr.Dataset, pipeline_settings: PipelineSettings): + def __init__(self, good_l1b_data: xr.Dataset): """ Given an L1B dataset, process data into an output HistogramL2 object. Parameters ---------- - l1b_dataset : xr.Dataset - GLOWS histogram L1B dataset, as produced by glows_l1b.py. - pipeline_settings : PipelineSettings - Pipeline settings object read from ancillary file. + good_l1b_data : xr.Dataset + GLOWS histogram L1B dataset filtered by good times. """ - active_flags = np.array(pipeline_settings.active_bad_time_flags, dtype=float) - - # Select the good blocks (i.e. epoch values) according to the flags. Drop any - # bad blocks before processing. - good_data = l1b_dataset.isel( - epoch=self.return_good_times(l1b_dataset["flags"], active_flags) - ) - # todo: bad angle filter + # TODO: bad angle filter # TODO filter bad bins out. Needs to happen here while everything is still - # per-timestamp. + # per-timestamp. - self.daily_lightcurve = DailyLightcurve(good_data) + self.daily_lightcurve = DailyLightcurve(good_l1b_data) - self.total_l1b_inputs = len(good_data["epoch"]) - self.number_of_good_l1b_inputs = len(good_data["epoch"]) + self.total_l1b_inputs = len(good_l1b_data["epoch"]) + self.number_of_good_l1b_inputs = len(good_l1b_data["epoch"]) self.identifier = -1 # TODO: retrieve from spin table # TODO fill this in self.bad_time_flag_occurrences = np.zeros((1, FLAG_LENGTH)) - if len(good_data["epoch"]) != 0: - # Generate outputs that are passed in directly from L1B - self.start_time = good_data["epoch"].data[0] - self.end_time = good_data["epoch"].data[-1] - else: - # No good times in the file - self.start_time = l1b_dataset["imap_start_time"].data[0] - self.end_time = ( - l1b_dataset["imap_start_time"].data[0] - + l1b_dataset["imap_time_offset"].data[0] - ) + # Generate outputs that are passed in directly from L1B + self.start_time = good_l1b_data["epoch"].data[0] + self.end_time = good_l1b_data["epoch"].data[-1] self.filter_temperature_average = ( - good_data["filter_temperature_average"] + good_l1b_data["filter_temperature_average"] .mean(dim="epoch", keepdims=True) .data ) self.filter_temperature_std_dev = ( - good_data["filter_temperature_average"].std(dim="epoch", keepdims=True).data + good_l1b_data["filter_temperature_average"] + .std(dim="epoch", keepdims=True) + .data ) self.hv_voltage_average = ( - good_data["hv_voltage_average"].mean(dim="epoch", keepdims=True).data + good_l1b_data["hv_voltage_average"].mean(dim="epoch", keepdims=True).data ) self.hv_voltage_std_dev = ( - good_data["hv_voltage_average"].std(dim="epoch", keepdims=True).data + good_l1b_data["hv_voltage_average"].std(dim="epoch", keepdims=True).data ) self.spin_period_average = ( - good_data["spin_period_average"].mean(dim="epoch", keepdims=True).data + good_l1b_data["spin_period_average"].mean(dim="epoch", keepdims=True).data ) self.spin_period_std_dev = ( - good_data["spin_period_average"].std(dim="epoch", keepdims=True).data + good_l1b_data["spin_period_average"].std(dim="epoch", keepdims=True).data ) self.pulse_length_average = ( - good_data["pulse_length_average"].mean(dim="epoch", keepdims=True).data + good_l1b_data["pulse_length_average"].mean(dim="epoch", keepdims=True).data ) self.pulse_length_std_dev = ( - good_data["pulse_length_average"].std(dim="epoch", keepdims=True).data + good_l1b_data["pulse_length_average"].std(dim="epoch", keepdims=True).data ) self.spin_period_ground_average = ( - good_data["spin_period_ground_average"] + good_l1b_data["spin_period_ground_average"] .mean(dim="epoch", keepdims=True) .data ) self.spin_period_ground_std_dev = ( - good_data["spin_period_ground_average"].std(dim="epoch", keepdims=True).data + good_l1b_data["spin_period_ground_average"] + .std(dim="epoch", keepdims=True) + .data ) self.position_angle_offset_average = ( - good_data["position_angle_offset_average"] + good_l1b_data["position_angle_offset_average"] .mean(dim="epoch", keepdims=True) .data ) self.position_angle_offset_std_dev = ( - good_data["position_angle_offset_average"] + good_l1b_data["position_angle_offset_average"] .std(dim="epoch", keepdims=True) .data ) self.spacecraft_location_average = ( - good_data["spacecraft_location_average"] + good_l1b_data["spacecraft_location_average"] .mean(dim="epoch") .data[np.newaxis, :] ) self.spacecraft_location_std_dev = ( - good_data["spacecraft_location_average"] + good_l1b_data["spacecraft_location_average"] .std(dim="epoch") .data[np.newaxis, :] ) self.spacecraft_velocity_average = ( - good_data["spacecraft_velocity_average"] + good_l1b_data["spacecraft_velocity_average"] .mean(dim="epoch") .data[np.newaxis, :] ) self.spacecraft_velocity_std_dev = ( - good_data["spacecraft_velocity_average"] + good_l1b_data["spacecraft_velocity_average"] .std(dim="epoch") .data[np.newaxis, :] ) self.spin_axis_orientation_average = ( - good_data["spin_axis_orientation_average"] + good_l1b_data["spin_axis_orientation_average"] .mean(dim="epoch") .data[np.newaxis, :] ) self.spin_axis_orientation_std_dev = ( - good_data["spin_axis_orientation_average"] + good_l1b_data["spin_axis_orientation_average"] .std(dim="epoch") .data[np.newaxis, :] ) + @classmethod + def create( + cls, l1b_dataset: xr.Dataset, pipeline_settings: PipelineSettings + ) -> HistogramL2 | None: + """ + Create a HistogramL2 object from an L1B dataset, filtering out bad times. + + Parameters + ---------- + l1b_dataset : xr.Dataset + GLOWS histogram L1B dataset, as produced by glows_l1b.py. + pipeline_settings : PipelineSettings + Pipeline settings object read from ancillary file. + + Returns + ------- + HistogramL2 or None + A HistogramL2 object if there are good times in the dataset, + or None if there are no good times. + """ + # Check if dataset contains good + good_idx = cls.return_good_times( + l1b_dataset["flags"], + np.array(pipeline_settings.active_bad_time_flags, dtype=float), + ) + if len(good_idx) == 0: + return None + + # Proceed with creating the HistogramL2 object using only the good data + filtered_data = l1b_dataset.isel(epoch=good_idx) + return cls(filtered_data) + def filter_bad_bins(self, histograms: NDArray, bin_exclusions: NDArray) -> NDArray: """ Filter out bad bins from the histogram. From 334b50b60808c25949329c5ffcca205b5f6ddde3 Mon Sep 17 00:00:00 2001 From: Veronica Martinez Date: Tue, 24 Feb 2026 16:37:19 -0700 Subject: [PATCH 2/5] Add logging to log if an L1B dataset doesn't contain any good data and so an empty list is returned --- imap_processing/glows/l2/glows_l2.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/imap_processing/glows/l2/glows_l2.py b/imap_processing/glows/l2/glows_l2.py index 6f49b7f61..775a29f52 100644 --- a/imap_processing/glows/l2/glows_l2.py +++ b/imap_processing/glows/l2/glows_l2.py @@ -1,6 +1,7 @@ """Module for GLOWS Level 2 processing.""" import dataclasses +import logging import numpy as np import xarray as xr @@ -13,6 +14,8 @@ from imap_processing.glows.l2.glows_l2_data import HistogramL2 from imap_processing.spice.time import et_to_datetime64, ttj2000ns_to_et +logger = logging.getLogger(__name__) + def glows_l2( input_dataset: xr.Dataset, @@ -46,9 +49,12 @@ def glows_l2( pipeline_settings_dataset.sel(epoch=day, method="nearest") ) - # TODO: log returning no data if l2 is None l2 = HistogramL2.create(input_dataset, pipeline_settings) - return [create_l2_dataset(l2, cdf_attrs)] if l2 else [] + if l2 is None: + logger.warning("No good data found in L1B dataset. Returning empty list.") + return [] + else: + return [create_l2_dataset(l2, cdf_attrs)] def create_l2_dataset( From 9093860522409eb0a27809b042e56776d2a71b15 Mon Sep 17 00:00:00 2001 From: Veronica Martinez Date: Wed, 25 Feb 2026 14:10:37 -0700 Subject: [PATCH 3/5] Add unit test to check that the create method returns None if passed an L1B dataset with no good times --- imap_processing/glows/l2/glows_l2_data.py | 2 +- .../tests/glows/test_glows_l2_data.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/imap_processing/glows/l2/glows_l2_data.py b/imap_processing/glows/l2/glows_l2_data.py index 9eca82079..673ccaa67 100644 --- a/imap_processing/glows/l2/glows_l2_data.py +++ b/imap_processing/glows/l2/glows_l2_data.py @@ -391,7 +391,7 @@ def return_good_times(flags: xr.DataArray, active_flags: NDArray) -> NDArray: An array of indices for good times. """ if len(active_flags) != flags.shape[1]: - print("Active flags don't matched expected length") + print("Active flags don't match expected length") # A good time is where all the active flags are equal to one. # Here, we mask the active indices using active_flags, and then return the times diff --git a/imap_processing/tests/glows/test_glows_l2_data.py b/imap_processing/tests/glows/test_glows_l2_data.py index 63ac19baf..881b30376 100644 --- a/imap_processing/tests/glows/test_glows_l2_data.py +++ b/imap_processing/tests/glows/test_glows_l2_data.py @@ -41,8 +41,12 @@ def pipeline_settings(): ] pipeline_dataset = xr.Dataset( { - "active_bad_time_flags": xr.DataArray(active_bad_time_flags), - "active_bad_angle_flags": xr.DataArray(active_bad_angle_flags), + "active_bad_time_flags": xr.DataArray( + active_bad_time_flags, dims=["flags"] + ), + "active_bad_angle_flags": xr.DataArray( + active_bad_angle_flags, dims=["angle_flags"] + ), } ) return PipelineSettings(pipeline_dataset) @@ -151,3 +155,13 @@ def test_filter_good_times(): expected_good_times = [0, 2, 3] assert np.array_equal(good_times, expected_good_times) + + +def test_create_returns_none_when_no_good_times(pipeline_settings, l1b_dataset): + """L1B dataset with no good times, returns None for HistogramL2 dataset.""" + # All flags set to 0 -> all bad times + test_flags = np.zeros((l1b_dataset["epoch"].size, 17), dtype=bool) + l1b_dataset["flags"] = xr.DataArray(test_flags, dims=["epoch", "flags"]) + ds = HistogramL2.create(l1b_dataset, pipeline_settings) + expected_ds = None + assert np.array_equal(ds, expected_ds) From 7bfb489da7f7c2cb749d6b8b735da00296a75508 Mon Sep 17 00:00:00 2001 From: Veronica Martinez Date: Wed, 25 Feb 2026 16:25:00 -0700 Subject: [PATCH 4/5] Move unit test into existing test_generate_l2 unit test to use the same set up needed --- imap_processing/tests/glows/test_glows_l2.py | 10 +++++++++- .../tests/glows/test_glows_l2_data.py | 18 ++---------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/imap_processing/tests/glows/test_glows_l2.py b/imap_processing/tests/glows/test_glows_l2.py index e73233a77..371be673d 100644 --- a/imap_processing/tests/glows/test_glows_l2.py +++ b/imap_processing/tests/glows/test_glows_l2.py @@ -83,7 +83,9 @@ def test_generate_l2( pipeline_settings = PipelineSettings( mock_pipeline_settings.sel(epoch=day, method="nearest") ) - l2 = HistogramL2(l1b_hist_dataset, pipeline_settings) + + # Test case 1: L1B dataset has good times + l2 = HistogramL2.create(l1b_hist_dataset, pipeline_settings) expected_values = { "filter_temperature_average": [57.59], @@ -109,6 +111,12 @@ def test_generate_l2( l2.hv_voltage_std_dev, expected_values["hv_voltage_std_dev"], 0.01 ) + # Test case 2: L1B dataset has no good times (all flags 0) + l1b_hist_dataset["flags"].values = np.zeros(l1b_hist_dataset.flags.shape) + ds = HistogramL2.create(l1b_hist_dataset, pipeline_settings) + expected_ds = None + assert ds == expected_ds + def test_bin_exclusions(l1b_hists): # TODO test excluding bins as well diff --git a/imap_processing/tests/glows/test_glows_l2_data.py b/imap_processing/tests/glows/test_glows_l2_data.py index 881b30376..63ac19baf 100644 --- a/imap_processing/tests/glows/test_glows_l2_data.py +++ b/imap_processing/tests/glows/test_glows_l2_data.py @@ -41,12 +41,8 @@ def pipeline_settings(): ] pipeline_dataset = xr.Dataset( { - "active_bad_time_flags": xr.DataArray( - active_bad_time_flags, dims=["flags"] - ), - "active_bad_angle_flags": xr.DataArray( - active_bad_angle_flags, dims=["angle_flags"] - ), + "active_bad_time_flags": xr.DataArray(active_bad_time_flags), + "active_bad_angle_flags": xr.DataArray(active_bad_angle_flags), } ) return PipelineSettings(pipeline_dataset) @@ -155,13 +151,3 @@ def test_filter_good_times(): expected_good_times = [0, 2, 3] assert np.array_equal(good_times, expected_good_times) - - -def test_create_returns_none_when_no_good_times(pipeline_settings, l1b_dataset): - """L1B dataset with no good times, returns None for HistogramL2 dataset.""" - # All flags set to 0 -> all bad times - test_flags = np.zeros((l1b_dataset["epoch"].size, 17), dtype=bool) - l1b_dataset["flags"] = xr.DataArray(test_flags, dims=["epoch", "flags"]) - ds = HistogramL2.create(l1b_dataset, pipeline_settings) - expected_ds = None - assert np.array_equal(ds, expected_ds) From ecf3753faa23425a1b1c0424918c983721274d0d Mon Sep 17 00:00:00 2001 From: Veronica Martinez Date: Wed, 25 Feb 2026 16:50:04 -0700 Subject: [PATCH 5/5] Add test coverage for no good times in existing test_glows_l2 unit test --- imap_processing/glows/l2/glows_l2_data.py | 2 +- imap_processing/tests/glows/test_glows_l2.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/imap_processing/glows/l2/glows_l2_data.py b/imap_processing/glows/l2/glows_l2_data.py index 673ccaa67..d618157cc 100644 --- a/imap_processing/glows/l2/glows_l2_data.py +++ b/imap_processing/glows/l2/glows_l2_data.py @@ -334,7 +334,7 @@ def create( A HistogramL2 object if there are good times in the dataset, or None if there are no good times. """ - # Check if dataset contains good + # Check if dataset contains good times good_idx = cls.return_good_times( l1b_dataset["flags"], np.array(pipeline_settings.active_bad_time_flags, dtype=float), diff --git a/imap_processing/tests/glows/test_glows_l2.py b/imap_processing/tests/glows/test_glows_l2.py index 371be673d..8942ca554 100644 --- a/imap_processing/tests/glows/test_glows_l2.py +++ b/imap_processing/tests/glows/test_glows_l2.py @@ -42,6 +42,7 @@ def test_glows_l2( mock_ancillary_exclusions, mock_pipeline_settings, mock_conversion_table_dict, + caplog, ): mock_spice_function.side_effect = mock_update_spice_parameters @@ -54,11 +55,19 @@ def test_glows_l2( mock_pipeline_settings, mock_conversion_table_dict, ) + + # Test case 1: L1B dataset has good times l2 = glows_l2(l1b_hist_dataset, mock_pipeline_settings)[0] assert l2.attrs["Logical_source"] == "imap_glows_l2_hist" - assert np.allclose(l2["filter_temperature_average"].values, [57.6], rtol=0.1) + # Test case 2: L1B dataset has no good times (all flags 0) + l1b_hist_dataset["flags"].values = np.zeros(l1b_hist_dataset.flags.shape) + caplog.set_level("WARNING") + result = glows_l2(l1b_hist_dataset, mock_pipeline_settings) + assert result == [] + assert any(record.levelname == "WARNING" for record in caplog.records) + @patch.object(HistogramL1B, "update_spice_parameters", autospec=True) def test_generate_l2(