From bb68830e098d8a8af1a72f363ae3e64410a0b05c Mon Sep 17 00:00:00 2001 From: Sean Hoyt Date: Mon, 23 Feb 2026 09:46:44 -0700 Subject: [PATCH 1/8] added yaml metadata --- .../config/imap_glows_l1b_variable_attrs.yaml | 8 +------- .../cdf/config/imap_glows_l2_variable_attrs.yaml | 16 ++++++++++++++++ imap_processing/glows/l1b/glows_l1b.py | 3 +++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml index f93e56af9..06458d385 100644 --- a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml @@ -131,15 +131,9 @@ histogram: VAR_TYPE: data pkts_file_name: - <<: *support_data_defaults CATDESC: Name of input file with L0 CCSDS packets data - DISPLAY_TYPE: no_plot FIELDNAM: L0 data input filename - FILLVAL: # TBD: what is fillval for strings? - FORMAT: S256 # TBC - LABLAXIS: File name # MS: is this needed for no_plot? - VALIDMAX: # TBD: what is validmax for a string? - VALIDMIN: # TBD: what is validmin for a string? + FORMAT: A49 VAR_TYPE: metadata first_spin_id: diff --git a/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml index 05992dd3b..bb8d4eb6f 100644 --- a/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml @@ -129,6 +129,22 @@ identifier: VALIDMAX: 99999 DICT_KEY: SPASE>Support>SupportQuantity:Temporal +flight_software_version: + <<: *support_data_defaults + CATDESC: GLOWS flight software version + FIELDNAM: GLOWS flight software version + FILLVAL: *max_uint32 + FORMAT: I8 + LABLAXIS: FSW ver + VALIDMAX: 16777215 + +pkts_file_name: + VAR_TYPE: metadata + CATDESC: Name of L0 packets file data originated + FIELDNAM: Packets File Name + FILLVAL: ' ' + FORMAT: A49 + start_time: <<: *time_data_defaults CATDESC: Start time (UTC) of observational day diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index eb87d3404..a7685ff21 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -485,6 +485,9 @@ def create_l1b_de_output( fields[index].name ) + parents = input_dataset.attrs.get("Parent", "") + output_dataset["pkts_file_name"] = [parent for parent in parents if parent.endswith('pkts')] + output_dataset["within_the_second"] = within_the_second_data output_dataset.attrs["missing_packets_sequence"] = input_dataset.attrs.get( "missing_packets_sequence", "" From 5ab27f365a690d33317b9c302825ee4fd8de927d Mon Sep 17 00:00:00 2001 From: Sean Hoyt Date: Mon, 23 Feb 2026 09:51:43 -0700 Subject: [PATCH 2/8] variable updates --- imap_processing/glows/l1b/glows_l1b.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index a7685ff21..8ad549773 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -486,7 +486,9 @@ def create_l1b_de_output( ) parents = input_dataset.attrs.get("Parent", "") - output_dataset["pkts_file_name"] = [parent for parent in parents if parent.endswith('pkts')] + output_dataset["pkts_file_name"] = [ + parent for parent in parents if parent.endswith("pkts") + ] output_dataset["within_the_second"] = within_the_second_data output_dataset.attrs["missing_packets_sequence"] = input_dataset.attrs.get( From d75409f0ffbdf9f9a84d6ac07b122fa3b2f8d720 Mon Sep 17 00:00:00 2001 From: Sean Hoyt Date: Tue, 24 Feb 2026 16:16:53 -0700 Subject: [PATCH 3/8] creating L1b variable --- imap_processing/glows/l1b/glows_l1b.py | 7 ++++--- imap_processing/tests/glows/test_glows_l1b.py | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index 8ad549773..819202ae1 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -486,9 +486,10 @@ def create_l1b_de_output( ) parents = input_dataset.attrs.get("Parent", "") - output_dataset["pkts_file_name"] = [ - parent for parent in parents if parent.endswith("pkts") - ] + output_dataset["pkts_file_name"] = xr.DataArray( + [parent for parent in parents if parent.endswith("pkts")], + attrs=cdf_attrs.get_variable_attributes("pkts_file_name", check_schema=False), + ) output_dataset["within_the_second"] = within_the_second_data output_dataset.attrs["missing_packets_sequence"] = input_dataset.attrs.get( diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 3fec7ba0a..868e55a46 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -72,6 +72,8 @@ def hist_dataset(): coords={"epoch": epoch, "bins": bins}, ) + ds.attrs["Parent"] = ["test_packet_file.pkts", "test_spice_file.tls"] + for var, data in variables.items(): ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) @@ -147,6 +149,8 @@ def de_dataset(): }, ) + ds.attrs["Parent"] = ["test_packet_file.pkts", "test_spice_file.tls"] + for var, data in variables.items(): ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) From bfe8034eff1d316e3b619725156565744321fe80 Mon Sep 17 00:00:00 2001 From: David Gathright Date: Wed, 25 Feb 2026 13:10:04 -0700 Subject: [PATCH 4/8] Initial stab at adding FSW version and L1A file to the L1B products. All pytests pass, but still need to look at the actual CDF files to verify the new variables are correct. --- .../config/imap_glows_l1a_variable_attrs.yaml | 14 ++++++-- .../config/imap_glows_l1b_variable_attrs.yaml | 9 +++-- imap_processing/glows/l1a/glows_l1a.py | 13 +++++++ imap_processing/glows/l1a/glows_l1a_data.py | 10 +++--- imap_processing/glows/l1b/glows_l1b.py | 31 +++++++++++++--- imap_processing/glows/l1b/glows_l1b_data.py | 11 ++++-- imap_processing/tests/glows/test_glows_l1b.py | 36 +++++++++---------- 7 files changed, 87 insertions(+), 37 deletions(-) diff --git a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml index 526ffda52..b9bfa8877 100644 --- a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml @@ -101,9 +101,17 @@ pkts_file_name: CATDESC: Name of input file with CCSDS packets data DISPLAY_TYPE: no_plot FIELDNAM: Packets-data input filename - FILLVAL: # TBD: what is fillval for strings? - FORMAT: S256 # TBC - LABLAXIS: File name + FILLVAL: " " + FORMAT: A49 + VAR_TYPE: metadata + +ground_software_version: + <<: *support_data_defaults + CATDESC: Version of ground software used for processing + DISPLAY_TYPE: no_plot + FIELDNAM: Ground Software Version + FILLVAL: " " + FORMAT: A4 VAR_TYPE: metadata first_spin_id: diff --git a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml index 06458d385..94719520f 100644 --- a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml @@ -130,9 +130,12 @@ histogram: UNITS: '#' VAR_TYPE: data -pkts_file_name: - CATDESC: Name of input file with L0 CCSDS packets data - FIELDNAM: L0 data input filename +l1a_file_name: + <<: *support_data_defaults + CATDESC: Name of input file with l1a data + DISPLAY_TYPE: no_plot + FIELDNAM: L1a data input filename + FILLVAL: " " FORMAT: A49 VAR_TYPE: metadata diff --git a/imap_processing/glows/l1a/glows_l1a.py b/imap_processing/glows/l1a/glows_l1a.py index ba9f484a6..b9b0ff163 100644 --- a/imap_processing/glows/l1a/glows_l1a.py +++ b/imap_processing/glows/l1a/glows_l1a.py @@ -336,6 +336,10 @@ def generate_histogram_dataset( ) # First variable is the output data type, second is the list of values + support_data_str: dict = { + "ground_software_version": [object, []], + "pkts_file_name": [object, []], + } support_data: dict = { "flight_software_version": [np.uint32, []], "seq_count_in_pkts_file": [np.uint16, []], @@ -374,6 +378,8 @@ def generate_histogram_dataset( ) # Add support_data keys to the support_data dictionary + for key, support_val in support_data_str.items(): + support_val[1].append(hist.__getattribute__(key)[0]) for key, support_val in support_data.items(): if key not in ["flags_set_onboard", "is_generated_on_ground"]: support_val[1].append(hist.__getattribute__(key)) @@ -428,6 +434,13 @@ def generate_histogram_dataset( output["histogram"] = hist + for key, value in support_data_str.items(): + output[key] = xr.DataArray( + np.array([value[1][0]], dtype=value[0]), + attrs=glows_cdf_attributes.get_variable_attributes( + "pkts_file_name", check_schema=False + ), + ) for key, value in support_data.items(): output[key] = xr.DataArray( np.array(value[1], dtype=value[0]), diff --git a/imap_processing/glows/l1a/glows_l1a_data.py b/imap_processing/glows/l1a/glows_l1a_data.py index 814f08680..2cd0564ad 100644 --- a/imap_processing/glows/l1a/glows_l1a_data.py +++ b/imap_processing/glows/l1a/glows_l1a_data.py @@ -203,8 +203,6 @@ class HistogramL1A: histogram: list[int] = field(init=False) # next four are in block header flight_software_version: int = field(init=False) - ground_software_version: str = field(init=False) - pkts_file_name: str = field(init=False) seq_count_in_pkts_file: int = field(init=False) first_spin_id: int = field(init=False) last_spin_id: int = field(init=False) @@ -225,6 +223,8 @@ class HistogramL1A: pulse_length_average: int = field(init=False) pulse_length_variance: int = field(init=False) flags: dict = field(init=False) + ground_software_version: str = field(init=False) + pkts_file_name: str = field(init=False) def __post_init__(self, l0: HistogramL0) -> None: """ @@ -383,7 +383,7 @@ def merge_de_packets(self, second_l0: DirectEventL0) -> None: raise ValueError( f"Sequence for direct event L1A is out of order or " f"incorrect. Attempted to append sequence counter " - f"{second_l0.SEQ} after {self.most_recent_seq}." + f"{second_l0.SEQ} after {self.most_recent_seq}. " f"New DE time: {second_l0.SEC}, current time: {self.l0.SEC}." ) @@ -397,7 +397,7 @@ def merge_de_packets(self, second_l0: DirectEventL0) -> None: if not match: raise ValueError( f"While attempting to merge L0 packet {second_l0} " - f"with {self.l0} mismatched values" + f"with {self.l0} mismatched values " f"were found. " ) @@ -534,7 +534,7 @@ def _build_compressed_event( else: raise ValueError( - f"Incorrect length {len(raw)} for {raw}, expecting 2 or 3" + f"Incorrect length {len(raw)} for {raw}, expecting 2 or 3 " f"bit compressed direct event data" ) diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index 819202ae1..4bbbe5379 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -82,6 +82,18 @@ def glows_l1b( output_dataarrays, input_dataset["epoch"], input_dataset["bins"], cdf_attrs ) + output_dataset["flight_software_version"] = np.uint32( + input_dataset["flight_software_version"].data + ) + parents = input_dataset.attrs.get("Parent", "") + # output_dataset["l1a_file_name"] = xr.DataArray( + # [parent for parent in parents if parent.endswith("cdf")], + # attrs=cdf_attrs.get_variable_attributes("l1a_file_name", check_schema=False), + # ) + output_dataset["l1a_file_name"] = [ + parent for parent in parents if parent.endswith("cdf") + ] + return output_dataset @@ -114,6 +126,10 @@ def glows_l1b_de( input_dataset, cdf_attrs, ancillary_parameters ) + # output_dataset["flight_software_version"] = np.uint32( + # input_dataset["flight_software_version"].data + # ) + return output_dataset @@ -159,6 +175,7 @@ def process_de( "de_flags": ["flag_dim"], "direct_event_glows_times": ["within_the_second"], "direct_event_pulse_lengths": ["within_the_second"], + "flight_software_version": ["flight_software_version"], } # For each attribute, retrieve the dims from output_dimension_mapping or use an @@ -485,11 +502,15 @@ def create_l1b_de_output( fields[index].name ) - parents = input_dataset.attrs.get("Parent", "") - output_dataset["pkts_file_name"] = xr.DataArray( - [parent for parent in parents if parent.endswith("pkts")], - attrs=cdf_attrs.get_variable_attributes("pkts_file_name", check_schema=False), - ) + # TODO: Not sure if this is requested in this product... + # parents = input_dataset.attrs.get("Parent", "") + # output_dataset["l1a_file_name"] = xr.DataArray( + # [parent for parent in parents if parent.endswith("cdf")], + # attrs=cdf_attrs.get_variable_attributes("l1a_file_name", check_schema=False), + # ) + # output_dataset["l1a_file_name"] = [ + # parent for parent in parents if parent.endswith("cdf") + # ] output_dataset["within_the_second"] = within_the_second_data output_dataset.attrs["missing_packets_sequence"] = input_dataset.attrs.get( diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index a5409f187..42eae4dff 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -492,6 +492,8 @@ class DirectEventL1B: float. From direct_events. direct_event_pulse_lengths: ndarray array of pulse lengths [μs] for direct events. From direct_events + l1a_file_name + Name of the input L1a file """ direct_events: InitVar[np.ndarray] @@ -526,6 +528,7 @@ class DirectEventL1B: direct_event_glows_times: np.ndarray | None = field(init=False, default=None) # 3rd value is pulse length direct_event_pulse_lengths: np.ndarray | None = field(init=False, default=None) + # l1a_file_name: str = "" # TODO: where does the multi-event flag go? def __post_init__( @@ -662,7 +665,8 @@ class HistogramL1B: ---------- histogram array of block-accumulated count numbers - flight_software_version: str + flight_software_version + The version of the flight software, copied from L1A seq_count_in_pkts_file: int first_spin_id: int The start ID @@ -731,10 +735,12 @@ class HistogramL1B: flags flags for extra information, per histogram. This should be a human-readable structure. + l1a_file_name + The name of the input L1A file """ histogram: np.ndarray - flight_software_version: str + flight_software_version: np.uint32 seq_count_in_pkts_file: int first_spin_id: int last_spin_id: int @@ -777,6 +783,7 @@ class HistogramL1B: ancillary_exclusions: InitVar[AncillaryExclusions] ancillary_parameters: InitVar[AncillaryParameters] pipeline_settings: InitVar[PipelineSettings] + # l1a_file_name: str = "" # TODO: # - Determine a good way to output flags as "human readable" # - Bad angle algorithm using SPICE locations diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 868e55a46..fd70bade4 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -26,7 +26,6 @@ @pytest.fixture def hist_dataset(): variables = { - "flight_software_version": np.zeros((20,)), "seq_count_in_pkts_file": np.zeros((20,)), "first_spin_id": np.zeros((20,)), "last_spin_id": np.zeros((20,)), @@ -72,7 +71,9 @@ def hist_dataset(): coords={"epoch": epoch, "bins": bins}, ) - ds.attrs["Parent"] = ["test_packet_file.pkts", "test_spice_file.tls"] + ds["flight_software_version"] = np.uint32(67) + + ds.attrs["Parent"] = ["test_packet_file.cdf", "test_spice_file.tls"] for var, data in variables.items(): ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) @@ -149,7 +150,7 @@ def de_dataset(): }, ) - ds.attrs["Parent"] = ["test_packet_file.pkts", "test_spice_file.tls"] + ds.attrs["Parent"] = ["test_packet_file.cdf", "test_spice_file.tls"] for var, data in variables.items(): ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) @@ -205,14 +206,12 @@ def test_histogram_mapping( mock_pipeline_settings, ): mock_spice_function.side_effect = mock_update_spice_parameters - time_val = 1111111.11 - # A = 2.318 - # B = 69.5454 + time_val = np.double(1111111.11) expected_temp = 100 test_hists = np.zeros((200, 3600)) # For temp - encoded_val = expected_temp * 2.318 + 69.5454 + encoded_val = np.double(expected_temp * 2.3182 + 69.5455) # For now, testing types and number of inputs pipeline_settings = PipelineSettings( @@ -225,7 +224,7 @@ def test_histogram_mapping( dataclasses.asdict( HistogramL1B( test_hists, - "test", + np.uint32(67), 0, 0, 0, @@ -253,10 +252,12 @@ def test_histogram_mapping( ).values() ) - assert output[18] == time_val + # Correctly decoded temperatures + assert np.isclose(output[10], expected_temp, 0.1) - # Correctly decoded temperature - assert output[10] - expected_temp < 0.1 + # Ensure time values are correctly mapped + assert output[18] == time_val + assert output[21] == time_val @patch.object(HistogramL1B, "update_spice_parameters", autospec=True) @@ -269,14 +270,12 @@ def test_process_histogram( ): mock_spice_function.side_effect = mock_update_spice_parameters - time_val = np.single(1111111.11) - # A = 2.318 - # B = 69.5454 + time_val = np.double(1111111.11) expected_temp = 100 test_hists = np.zeros((200,)) # For temp - encoded_val = np.single(expected_temp * 2.318 + 69.5454) + encoded_val = np.double(expected_temp * 2.3182 + 69.5455) pipeline_settings = PipelineSettings( mock_pipeline_settings.sel( @@ -286,7 +285,7 @@ def test_process_histogram( test_l1b = HistogramL1B( test_hists, - "test", + np.uint32(67), 0, 0, 0, @@ -366,8 +365,6 @@ def test_glows_l1b( # This needs to be added eventually, but is skipped for now. expected_de_data = [ "flight_software_version", - "ground_software_version", - "pkts_file_name", "seq_count_in_pkts_file", "l1a_file_name", "ancillary_data_files", @@ -405,6 +402,7 @@ def test_glows_l1b( "spacecraft_velocity_average", "spacecraft_velocity_std_dev", "flags", + "flight_software_version", ] for key in expected_hist_data: @@ -494,7 +492,7 @@ def test_hist_spice_output( use_fake_spin_data_for_time(data_start_time) params = { "histogram": np.zeros((1, 3600)), - "flight_software_version": "v0.0.1", + "flight_software_version": np.uint32(67), "seq_count_in_pkts_file": 0, "first_spin_id": 0, "last_spin_id": 0, From c490bc5e259fab7b6c599163e89b09d8d8e24914 Mon Sep 17 00:00:00 2001 From: David Gathright Date: Wed, 25 Feb 2026 15:02:48 -0700 Subject: [PATCH 5/8] Fixed and verified CDF format (through Matlab) for L1a string/single element attributes: ground_software_version, pkts_file_name, and flight_software_version. --- .../config/imap_glows_l1a_variable_attrs.yaml | 2 - imap_processing/glows/l1a/glows_l1a.py | 41 ++++++++++++------- imap_processing/glows/l1a/glows_l1a_data.py | 4 +- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml index b9bfa8877..254c8442c 100644 --- a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml @@ -97,7 +97,6 @@ histogram: VAR_TYPE: data pkts_file_name: - <<: *support_data_defaults CATDESC: Name of input file with CCSDS packets data DISPLAY_TYPE: no_plot FIELDNAM: Packets-data input filename @@ -106,7 +105,6 @@ pkts_file_name: VAR_TYPE: metadata ground_software_version: - <<: *support_data_defaults CATDESC: Version of ground software used for processing DISPLAY_TYPE: no_plot FIELDNAM: Ground Software Version diff --git a/imap_processing/glows/l1a/glows_l1a.py b/imap_processing/glows/l1a/glows_l1a.py index b9b0ff163..6d5c0b807 100644 --- a/imap_processing/glows/l1a/glows_l1a.py +++ b/imap_processing/glows/l1a/glows_l1a.py @@ -336,12 +336,7 @@ def generate_histogram_dataset( ) # First variable is the output data type, second is the list of values - support_data_str: dict = { - "ground_software_version": [object, []], - "pkts_file_name": [object, []], - } support_data: dict = { - "flight_software_version": [np.uint32, []], "seq_count_in_pkts_file": [np.uint16, []], "first_spin_id": [np.uint32, []], "last_spin_id": [np.uint32, []], @@ -378,8 +373,6 @@ def generate_histogram_dataset( ) # Add support_data keys to the support_data dictionary - for key, support_val in support_data_str.items(): - support_val[1].append(hist.__getattribute__(key)[0]) for key, support_val in support_data.items(): if key not in ["flags_set_onboard", "is_generated_on_ground"]: support_val[1].append(hist.__getattribute__(key)) @@ -434,13 +427,33 @@ def generate_histogram_dataset( output["histogram"] = hist - for key, value in support_data_str.items(): - output[key] = xr.DataArray( - np.array([value[1][0]], dtype=value[0]), - attrs=glows_cdf_attributes.get_variable_attributes( - "pkts_file_name", check_schema=False - ), - ) + # These attributes are the same for each record, so we don't + # need to store them per epoch like most of the other fields + output["flight_software_version"] = xr.DataArray( + np.array([hist_l1a_list[0].flight_software_version], dtype=np.uint32), + name="flight_software_version", + dims=["flight_software_version"], + attrs=glows_cdf_attributes.get_variable_attributes( + "flight_software_version", check_schema=False + ), + ) + output["pkts_file_name"] = xr.DataArray( + np.array([hist_l1a_list[0].pkts_file_name], dtype=object), + name="pkts_file_name", + dims=["pkts_file_name"], + attrs=glows_cdf_attributes.get_variable_attributes( + "pkts_file_name", check_schema=False + ), + ) + output["ground_software_version"] = xr.DataArray( + np.array([hist_l1a_list[0].ground_software_version], dtype=object), + name="ground_software_version", + dims=["ground_software_version"], + attrs=glows_cdf_attributes.get_variable_attributes( + "ground_software_version", check_schema=False + ), + ) + for key, value in support_data.items(): output[key] = xr.DataArray( np.array(value[1], dtype=value[0]), diff --git a/imap_processing/glows/l1a/glows_l1a_data.py b/imap_processing/glows/l1a/glows_l1a_data.py index 2cd0564ad..ae569ebe2 100644 --- a/imap_processing/glows/l1a/glows_l1a_data.py +++ b/imap_processing/glows/l1a/glows_l1a_data.py @@ -223,8 +223,8 @@ class HistogramL1A: pulse_length_average: int = field(init=False) pulse_length_variance: int = field(init=False) flags: dict = field(init=False) - ground_software_version: str = field(init=False) - pkts_file_name: str = field(init=False) + ground_software_version: str = "" + pkts_file_name: str = "" def __post_init__(self, l0: HistogramL0) -> None: """ From f1fefc3e8e1f04d1b22dccdf31fdb95a288cd77f Mon Sep 17 00:00:00 2001 From: David Gathright Date: Wed, 25 Feb 2026 18:44:00 -0700 Subject: [PATCH 6/8] Successfully processed L1a and L1b histograms with new variables. Need to refactor tests after validating CDFs. --- .../config/imap_glows_l1a_variable_attrs.yaml | 1 - .../config/imap_glows_l1b_variable_attrs.yaml | 10 ++++- imap_processing/glows/l1a/glows_l1a.py | 7 ++-- imap_processing/glows/l1b/glows_l1b.py | 28 +++++++------ imap_processing/glows/l1b/glows_l1b_data.py | 5 ++- imap_processing/tests/glows/test_glows_l1b.py | 40 ++++++++++++------- 6 files changed, 54 insertions(+), 37 deletions(-) diff --git a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml index 254c8442c..af3275a4f 100644 --- a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml @@ -510,7 +510,6 @@ missing_packets_sequence: # Used to be missing_packets_sequence VAR_TYPE: metadata flight_software_version: - <<: *support_data_defaults CATDESC: GLOWS flight software version FIELDNAM: GLOWS flight software version FILLVAL: *max_uint32 diff --git a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml index 94719520f..353fa9b2c 100644 --- a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml @@ -131,7 +131,6 @@ histogram: VAR_TYPE: data l1a_file_name: - <<: *support_data_defaults CATDESC: Name of input file with l1a data DISPLAY_TYPE: no_plot FIELDNAM: L1a data input filename @@ -687,7 +686,6 @@ missing_packets_sequence: # Used to be missing_packets_sequence VAR_TYPE: metadata flight_software_version: - <<: *support_data_defaults CATDESC: GLOWS flight software version FIELDNAM: GLOWS flight software version FILLVAL: *max_uint32 @@ -695,6 +693,14 @@ flight_software_version: LABLAXIS: FSW ver VALIDMAX: 16777215 +ground_software_version: + CATDESC: Version of ground software used for processing + DISPLAY_TYPE: no_plot + FIELDNAM: Ground Software Version + FILLVAL: " " + FORMAT: A4 + VAR_TYPE: metadata + # Flags # MS: this needs to be thoroughly discussed bad_time_flag_hist_attrs: # MS: this should be different for DE and HIST diff --git a/imap_processing/glows/l1a/glows_l1a.py b/imap_processing/glows/l1a/glows_l1a.py index 6d5c0b807..d45eb197b 100644 --- a/imap_processing/glows/l1a/glows_l1a.py +++ b/imap_processing/glows/l1a/glows_l1a.py @@ -171,7 +171,6 @@ def generate_de_dataset( # First variable is the output data type, second is the list of values support_data: dict = { - # "flight_software_version": [], "seq_count_in_pkts_file": [np.uint16, []], "number_of_de_packets": [np.uint32, []], } @@ -432,7 +431,7 @@ def generate_histogram_dataset( output["flight_software_version"] = xr.DataArray( np.array([hist_l1a_list[0].flight_software_version], dtype=np.uint32), name="flight_software_version", - dims=["flight_software_version"], + dims=["scalar"], attrs=glows_cdf_attributes.get_variable_attributes( "flight_software_version", check_schema=False ), @@ -440,7 +439,7 @@ def generate_histogram_dataset( output["pkts_file_name"] = xr.DataArray( np.array([hist_l1a_list[0].pkts_file_name], dtype=object), name="pkts_file_name", - dims=["pkts_file_name"], + dims=["scalar"], attrs=glows_cdf_attributes.get_variable_attributes( "pkts_file_name", check_schema=False ), @@ -448,7 +447,7 @@ def generate_histogram_dataset( output["ground_software_version"] = xr.DataArray( np.array([hist_l1a_list[0].ground_software_version], dtype=object), name="ground_software_version", - dims=["ground_software_version"], + dims=["scalar"], attrs=glows_cdf_attributes.get_variable_attributes( "ground_software_version", check_schema=False ), diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index 4bbbe5379..04e1d7fd0 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -82,17 +82,19 @@ def glows_l1b( output_dataarrays, input_dataset["epoch"], input_dataset["bins"], cdf_attrs ) - output_dataset["flight_software_version"] = np.uint32( - input_dataset["flight_software_version"].data - ) - parents = input_dataset.attrs.get("Parent", "") - # output_dataset["l1a_file_name"] = xr.DataArray( - # [parent for parent in parents if parent.endswith("cdf")], - # attrs=cdf_attrs.get_variable_attributes("l1a_file_name", check_schema=False), - # ) - output_dataset["l1a_file_name"] = [ - parent for parent in parents if parent.endswith("cdf") - ] + # output_dataset["flight_software_version"] = xr.DataArray( + # input_dataset["flight_software_version"].data, + # ) + # + # parents = input_dataset.attrs.get("Parent", "") + # l1a_file_name = [parent for parent in parents if parent.endswith("cdf")] + # output_dataset["l1a_file_name"] = xr.DataArray( + # np.array([l1a_file_name], dtype=object), + # name="l1a_file_name", + # attrs=cdf_attrs.get_variable_attributes( + # "l1a_file_name", check_schema=False + # ), + # ) return output_dataset @@ -175,7 +177,6 @@ def process_de( "de_flags": ["flag_dim"], "direct_event_glows_times": ["within_the_second"], "direct_event_pulse_lengths": ["within_the_second"], - "flight_software_version": ["flight_software_version"], } # For each attribute, retrieve the dims from output_dimension_mapping or use an @@ -284,7 +285,7 @@ def process_histogram( # histograms is the only multi dimensional input variable, so we set the non-epoch # dimension ("bins"). - # The rest of the input vars are epoch only, so they have an empty list. + # The rest of the non-scalar input vars are epoch only, so they have an empty list. input_dims[0] = ["bins"] # Create a closure that captures the ancillary objects @@ -428,6 +429,7 @@ def create_l1b_hist_output( output_dataset[fields[index].name].attrs = cdf_attrs.get_variable_attributes( fields[index].name ) + print(f"Output dataarray {index}: {fields[index].name}") output_dataset["bins"] = bin_data return output_dataset diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index 42eae4dff..a09f95879 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -740,7 +740,6 @@ class HistogramL1B: """ histogram: np.ndarray - flight_software_version: np.uint32 seq_count_in_pkts_file: int first_spin_id: int last_spin_id: int @@ -780,10 +779,12 @@ class HistogramL1B: spacecraft_velocity_average: np.ndarray = field(init=False) # from SPICE spacecraft_velocity_std_dev: np.ndarray = field(init=False) # from SPICE flags: np.ndarray = field(init=False) + flight_software_version: np.ndarray + l1a_file_name: np.ndarray + ground_software_version: np.ndarray ancillary_exclusions: InitVar[AncillaryExclusions] ancillary_parameters: InitVar[AncillaryParameters] pipeline_settings: InitVar[PipelineSettings] - # l1a_file_name: str = "" # TODO: # - Determine a good way to output flags as "human readable" # - Bad angle algorithm using SPICE locations diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index fd70bade4..8d1a215ba 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -23,6 +23,7 @@ from imap_processing.tests.glows.conftest import mock_update_spice_parameters +# Fixture for L1a histogram dataset @pytest.fixture def hist_dataset(): variables = { @@ -46,23 +47,25 @@ def hist_dataset(): "imap_time_offset": np.zeros((20,)), "glows_start_time": np.zeros((20,)), "glows_time_offset": np.zeros((20,)), + "flight_software_version": np.array([67], dtype=np.uint32), + "pkts_file_name": np.array(["test_packet_file.pkts"], dtype=object), + "ground_software_version": np.array(["v999"], dtype=object), } cdf_attrs = ImapCdfAttributes() cdf_attrs.add_instrument_global_attrs("glows") - cdf_attrs.add_instrument_variable_attrs("glows", "l1b") + cdf_attrs.add_instrument_variable_attrs("glows", "l1a") epoch = xr.DataArray( np.arange(20), name="epoch", dims=["epoch"], - attrs=cdf_attrs.get_variable_attributes("epoch"), + attrs=cdf_attrs.get_variable_attributes("epoch", check_schema=False), ) bins = xr.DataArray(np.arange(3600), name="bins", dims=["bins"]) ds = xr.Dataset( - coords={"epoch": epoch}, - attrs=cdf_attrs.get_global_attributes("imap_glows_l1b_hist"), + attrs=cdf_attrs.get_global_attributes("imap_glows_l1a_hist"), ) ds["histogram"] = xr.DataArray( @@ -71,12 +74,11 @@ def hist_dataset(): coords={"epoch": epoch, "bins": bins}, ) - ds["flight_software_version"] = np.uint32(67) - - ds.attrs["Parent"] = ["test_packet_file.cdf", "test_spice_file.tls"] - for var, data in variables.items(): - ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) + if 1 == len(data): + ds[var] = xr.DataArray(data, dims=["scalar"]) + else: + ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) return ds @@ -224,7 +226,6 @@ def test_histogram_mapping( dataclasses.asdict( HistogramL1B( test_hists, - np.uint32(67), 0, 0, 0, @@ -245,6 +246,9 @@ def test_histogram_mapping( time_val, time_val, time_val, + np.array([67], dtype=np.uint32), + np.array(["test_packet_file.pkts"], dtype=object), + np.array(["v999"], dtype=object), mock_ancillary_exclusions, mock_ancillary_parameters, pipeline_settings, @@ -253,11 +257,11 @@ def test_histogram_mapping( ) # Correctly decoded temperatures - assert np.isclose(output[10], expected_temp, 0.1) + assert np.isclose(output[9], expected_temp, 0.1) # Ensure time values are correctly mapped - assert output[18] == time_val - assert output[21] == time_val + assert output[17] == time_val + assert output[20] == time_val @patch.object(HistogramL1B, "update_spice_parameters", autospec=True) @@ -285,7 +289,6 @@ def test_process_histogram( test_l1b = HistogramL1B( test_hists, - np.uint32(67), 0, 0, 0, @@ -306,6 +309,9 @@ def test_process_histogram( time_val, time_val, time_val, + np.array([67], dtype=np.uint32), + np.array(["test_packet_file.pkts"], dtype=object), + np.array(["v999"], dtype=object), mock_ancillary_exclusions, mock_ancillary_parameters, pipeline_settings, @@ -403,6 +409,8 @@ def test_glows_l1b( "spacecraft_velocity_std_dev", "flags", "flight_software_version", + "ground_software_version", + "l1a_file_name", ] for key in expected_hist_data: @@ -492,7 +500,6 @@ def test_hist_spice_output( use_fake_spin_data_for_time(data_start_time) params = { "histogram": np.zeros((1, 3600)), - "flight_software_version": np.uint32(67), "seq_count_in_pkts_file": 0, "first_spin_id": 0, "last_spin_id": 0, @@ -513,6 +520,9 @@ def test_hist_spice_output( "imap_time_offset": 200.0, "glows_start_time": 504975603.125, "glows_time_offset": 200.0, + "flight_software_version": np.array([67], dtype=np.uint32), + "pkts_file_name": np.array(["test_packet_file.pkts"], dtype=object), + "ground_software_version": np.array(["v999"], dtype=object), "ancillary_exclusions": mock_ancillary_exclusions, "ancillary_parameters": mock_ancillary_parameters, "pipeline_settings": PipelineSettings( From 3add98933f3390de0d57572e0ecfa32f46643790 Mon Sep 17 00:00:00 2001 From: David Gathright Date: Wed, 25 Feb 2026 21:51:28 -0700 Subject: [PATCH 7/8] Fixed CDF write failures for L1b Histogram product. Still need to refactor test code. Validated CDF file output with Matlab using this code to generate L1a and L1b Histogram data from 20260125. --- imap_processing/glows/l1a/glows_l1a_data.py | 6 +-- imap_processing/glows/l1b/glows_l1b.py | 41 +++++++++++-------- imap_processing/glows/l1b/glows_l1b_data.py | 12 +++--- imap_processing/tests/glows/test_glows_l1b.py | 6 +-- 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/imap_processing/glows/l1a/glows_l1a_data.py b/imap_processing/glows/l1a/glows_l1a_data.py index ae569ebe2..4f26cd793 100644 --- a/imap_processing/glows/l1a/glows_l1a_data.py +++ b/imap_processing/glows/l1a/glows_l1a_data.py @@ -201,8 +201,10 @@ class HistogramL1A: l0: InitVar[HistogramL0] histogram: list[int] = field(init=False) - # next four are in block header flight_software_version: int = field(init=False) + pkts_file_name: str = "" + ground_software_version: str = "" + # next four are in block header seq_count_in_pkts_file: int = field(init=False) first_spin_id: int = field(init=False) last_spin_id: int = field(init=False) @@ -223,8 +225,6 @@ class HistogramL1A: pulse_length_average: int = field(init=False) pulse_length_variance: int = field(init=False) flags: dict = field(init=False) - ground_software_version: str = "" - pkts_file_name: str = "" def __post_init__(self, l0: HistogramL0) -> None: """ diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index 04e1d7fd0..0cb2d4be0 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -82,19 +82,29 @@ def glows_l1b( output_dataarrays, input_dataset["epoch"], input_dataset["bins"], cdf_attrs ) - # output_dataset["flight_software_version"] = xr.DataArray( - # input_dataset["flight_software_version"].data, - # ) - # - # parents = input_dataset.attrs.get("Parent", "") - # l1a_file_name = [parent for parent in parents if parent.endswith("cdf")] - # output_dataset["l1a_file_name"] = xr.DataArray( - # np.array([l1a_file_name], dtype=object), - # name="l1a_file_name", - # attrs=cdf_attrs.get_variable_attributes( - # "l1a_file_name", check_schema=False - # ), - # ) + output_dataset["flight_software_version"] = xr.DataArray( + input_dataset["flight_software_version"].data, + name="flight_software_version", + dims=["scalar"], + attrs=cdf_attrs.get_variable_attributes( + "flight_software_version", check_schema=False + ), + ) + output_dataset["ground_software_version"] = xr.DataArray( + input_dataset["ground_software_version"].data, + name="ground_software_version", + dims=["scalar"], + attrs=cdf_attrs.get_variable_attributes( + "ground_software_version", check_schema=False + ), + ) + + output_dataset["l1a_file_name"] = xr.DataArray( + input_dataset["pkts_file_name"].data, + name="l1a_file_name", + dims=["scalar"], + attrs=cdf_attrs.get_variable_attributes("l1a_file_name", check_schema=False), + ) return output_dataset @@ -128,10 +138,6 @@ def glows_l1b_de( input_dataset, cdf_attrs, ancillary_parameters ) - # output_dataset["flight_software_version"] = np.uint32( - # input_dataset["flight_software_version"].data - # ) - return output_dataset @@ -429,7 +435,6 @@ def create_l1b_hist_output( output_dataset[fields[index].name].attrs = cdf_attrs.get_variable_attributes( fields[index].name ) - print(f"Output dataarray {index}: {fields[index].name}") output_dataset["bins"] = bin_data return output_dataset diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index a09f95879..bac5eab1b 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -665,8 +665,6 @@ class HistogramL1B: ---------- histogram array of block-accumulated count numbers - flight_software_version - The version of the flight software, copied from L1A seq_count_in_pkts_file: int first_spin_id: int The start ID @@ -735,11 +733,18 @@ class HistogramL1B: flags flags for extra information, per histogram. This should be a human-readable structure. + flight_software_version + The version of the flight software, copied from L1A l1a_file_name The name of the input L1A file + ground_software_version + The version of the ground software, copied from L1A """ histogram: np.ndarray + flight_software_version: np.ndarray + l1a_file_name: np.ndarray + ground_software_version: np.ndarray seq_count_in_pkts_file: int first_spin_id: int last_spin_id: int @@ -779,9 +784,6 @@ class HistogramL1B: spacecraft_velocity_average: np.ndarray = field(init=False) # from SPICE spacecraft_velocity_std_dev: np.ndarray = field(init=False) # from SPICE flags: np.ndarray = field(init=False) - flight_software_version: np.ndarray - l1a_file_name: np.ndarray - ground_software_version: np.ndarray ancillary_exclusions: InitVar[AncillaryExclusions] ancillary_parameters: InitVar[AncillaryParameters] pipeline_settings: InitVar[PipelineSettings] diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 8d1a215ba..6d7d186ec 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -75,10 +75,10 @@ def hist_dataset(): ) for var, data in variables.items(): - if 1 == len(data): - ds[var] = xr.DataArray(data, dims=["scalar"]) - else: + if 1 != len(data): ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) + # else: + # ds[var] = xr.DataArray(data, dims=["scalar"]) return ds From 53db6b20bca88d76b2a260648079ac89e52b5833 Mon Sep 17 00:00:00 2001 From: David Gathright Date: Wed, 25 Feb 2026 22:27:56 -0700 Subject: [PATCH 8/8] Cleaned up tests--all tests are now passing. Re-validated good CDF files with the updated code using MATLAB and flight 20260125 data. --- .../config/imap_glows_l1b_variable_attrs.yaml | 6 +-- imap_processing/glows/l1b/glows_l1b.py | 13 +++--- imap_processing/glows/l1b/glows_l1b_data.py | 12 +++--- imap_processing/tests/glows/test_glows_l1b.py | 42 +++++++++---------- 4 files changed, 37 insertions(+), 36 deletions(-) diff --git a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml index 353fa9b2c..c2f97c08a 100644 --- a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml @@ -130,10 +130,10 @@ histogram: UNITS: '#' VAR_TYPE: data -l1a_file_name: - CATDESC: Name of input file with l1a data +pkts_file_name: + CATDESC: Name of input file with CCSDS packet data DISPLAY_TYPE: no_plot - FIELDNAM: L1a data input filename + FIELDNAM: CCSDS packet data input filename FILLVAL: " " FORMAT: A49 VAR_TYPE: metadata diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index 0cb2d4be0..75d439bdd 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -90,6 +90,7 @@ def glows_l1b( "flight_software_version", check_schema=False ), ) + output_dataset["ground_software_version"] = xr.DataArray( input_dataset["ground_software_version"].data, name="ground_software_version", @@ -99,11 +100,11 @@ def glows_l1b( ), ) - output_dataset["l1a_file_name"] = xr.DataArray( + output_dataset["pkts_file_name"] = xr.DataArray( input_dataset["pkts_file_name"].data, - name="l1a_file_name", + name="pkts_file_name", dims=["scalar"], - attrs=cdf_attrs.get_variable_attributes("l1a_file_name", check_schema=False), + attrs=cdf_attrs.get_variable_attributes("pkts_file_name", check_schema=False), ) return output_dataset @@ -511,11 +512,11 @@ def create_l1b_de_output( # TODO: Not sure if this is requested in this product... # parents = input_dataset.attrs.get("Parent", "") - # output_dataset["l1a_file_name"] = xr.DataArray( + # output_dataset["pkts_file_name"] = xr.DataArray( # [parent for parent in parents if parent.endswith("cdf")], - # attrs=cdf_attrs.get_variable_attributes("l1a_file_name", check_schema=False), + # attrs=cdf_attrs.get_variable_attributes("pkts_file_name", check_schema=False), # ) - # output_dataset["l1a_file_name"] = [ + # output_dataset["pkts_file_name"] = [ # parent for parent in parents if parent.endswith("cdf") # ] diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index bac5eab1b..23fc4854a 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -492,8 +492,8 @@ class DirectEventL1B: float. From direct_events. direct_event_pulse_lengths: ndarray array of pulse lengths [μs] for direct events. From direct_events - l1a_file_name - Name of the input L1a file + pkts_file_name + Name of the input CCSDS packets file """ direct_events: InitVar[np.ndarray] @@ -528,7 +528,7 @@ class DirectEventL1B: direct_event_glows_times: np.ndarray | None = field(init=False, default=None) # 3rd value is pulse length direct_event_pulse_lengths: np.ndarray | None = field(init=False, default=None) - # l1a_file_name: str = "" + # pkts_file_name: str = "" # TODO: where does the multi-event flag go? def __post_init__( @@ -735,15 +735,15 @@ class HistogramL1B: structure. flight_software_version The version of the flight software, copied from L1A - l1a_file_name - The name of the input L1A file + pkts_file_name + The name of the input CCSDS packets file ground_software_version The version of the ground software, copied from L1A """ histogram: np.ndarray flight_software_version: np.ndarray - l1a_file_name: np.ndarray + pkts_file_name: np.ndarray ground_software_version: np.ndarray seq_count_in_pkts_file: int first_spin_id: int diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 6d7d186ec..1e38964a6 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -27,6 +27,9 @@ @pytest.fixture def hist_dataset(): variables = { + "flight_software_version": np.array([67], dtype=np.uint32), + "pkts_file_name": np.array(["test_packet_file.pkts"], dtype=object), + "ground_software_version": np.array(["v999"], dtype=object), "seq_count_in_pkts_file": np.zeros((20,)), "first_spin_id": np.zeros((20,)), "last_spin_id": np.zeros((20,)), @@ -47,9 +50,6 @@ def hist_dataset(): "imap_time_offset": np.zeros((20,)), "glows_start_time": np.zeros((20,)), "glows_time_offset": np.zeros((20,)), - "flight_software_version": np.array([67], dtype=np.uint32), - "pkts_file_name": np.array(["test_packet_file.pkts"], dtype=object), - "ground_software_version": np.array(["v999"], dtype=object), } cdf_attrs = ImapCdfAttributes() cdf_attrs.add_instrument_global_attrs("glows") @@ -77,8 +77,8 @@ def hist_dataset(): for var, data in variables.items(): if 1 != len(data): ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) - # else: - # ds[var] = xr.DataArray(data, dims=["scalar"]) + else: + ds[var] = xr.DataArray(data, dims="scalar") return ds @@ -226,6 +226,9 @@ def test_histogram_mapping( dataclasses.asdict( HistogramL1B( test_hists, + np.array([67], dtype=np.uint32), + np.array(["test_packet_file.pkts"], dtype=object), + np.array(["v999"], dtype=object), 0, 0, 0, @@ -246,9 +249,6 @@ def test_histogram_mapping( time_val, time_val, time_val, - np.array([67], dtype=np.uint32), - np.array(["test_packet_file.pkts"], dtype=object), - np.array(["v999"], dtype=object), mock_ancillary_exclusions, mock_ancillary_parameters, pipeline_settings, @@ -256,12 +256,12 @@ def test_histogram_mapping( ).values() ) - # Correctly decoded temperatures - assert np.isclose(output[9], expected_temp, 0.1) + # Correctly decoded temperature + assert np.isclose(output[12], expected_temp, 0.1) # Ensure time values are correctly mapped - assert output[17] == time_val assert output[20] == time_val + assert output[23] == time_val @patch.object(HistogramL1B, "update_spice_parameters", autospec=True) @@ -289,6 +289,9 @@ def test_process_histogram( test_l1b = HistogramL1B( test_hists, + np.array([67], dtype=np.uint32), + np.array(["test_packet_file.pkts"], dtype=object), + np.array(["v999"], dtype=object), 0, 0, 0, @@ -309,9 +312,6 @@ def test_process_histogram( time_val, time_val, time_val, - np.array([67], dtype=np.uint32), - np.array(["test_packet_file.pkts"], dtype=object), - np.array(["v999"], dtype=object), mock_ancillary_exclusions, mock_ancillary_parameters, pipeline_settings, @@ -365,14 +365,14 @@ def test_glows_l1b( mock_conversion_table_dict, ) - assert hist_output["histogram"].dims == ("epoch", "bins") - assert hist_output["histogram"].shape == (20, 3600) + assert hist_output["histogram"].dims == ("epoch", "scalar", "bins") + assert hist_output["histogram"].shape == (20, 1, 3600) # This needs to be added eventually, but is skipped for now. expected_de_data = [ "flight_software_version", "seq_count_in_pkts_file", - "l1a_file_name", + "pkts_file_name", "ancillary_data_files", ] @@ -410,7 +410,7 @@ def test_glows_l1b( "flags", "flight_software_version", "ground_software_version", - "l1a_file_name", + "pkts_file_name", ] for key in expected_hist_data: @@ -500,6 +500,9 @@ def test_hist_spice_output( use_fake_spin_data_for_time(data_start_time) params = { "histogram": np.zeros((1, 3600)), + "flight_software_version": np.array([67], dtype=np.uint32), + "pkts_file_name": np.array(["test_packet_file.pkts"], dtype=object), + "ground_software_version": np.array(["v999"], dtype=object), "seq_count_in_pkts_file": 0, "first_spin_id": 0, "last_spin_id": 0, @@ -520,9 +523,6 @@ def test_hist_spice_output( "imap_time_offset": 200.0, "glows_start_time": 504975603.125, "glows_time_offset": 200.0, - "flight_software_version": np.array([67], dtype=np.uint32), - "pkts_file_name": np.array(["test_packet_file.pkts"], dtype=object), - "ground_software_version": np.array(["v999"], dtype=object), "ancillary_exclusions": mock_ancillary_exclusions, "ancillary_parameters": mock_ancillary_parameters, "pipeline_settings": PipelineSettings(