diff --git a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml index 526ffda52..af3275a4f 100644 --- a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml @@ -97,13 +97,19 @@ histogram: VAR_TYPE: data pkts_file_name: - <<: *support_data_defaults CATDESC: Name of input file with CCSDS packets data DISPLAY_TYPE: no_plot FIELDNAM: Packets-data input filename - FILLVAL: # TBD: what is fillval for strings? - FORMAT: S256 # TBC - LABLAXIS: File name + FILLVAL: " " + FORMAT: A49 + VAR_TYPE: metadata + +ground_software_version: + CATDESC: Version of ground software used for processing + DISPLAY_TYPE: no_plot + FIELDNAM: Ground Software Version + FILLVAL: " " + FORMAT: A4 VAR_TYPE: metadata first_spin_id: @@ -504,7 +510,6 @@ missing_packets_sequence: # Used to be missing_packets_sequence VAR_TYPE: metadata flight_software_version: - <<: *support_data_defaults CATDESC: GLOWS flight software version FIELDNAM: GLOWS flight software version FILLVAL: *max_uint32 diff --git a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml index f93e56af9..c2f97c08a 100644 --- a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml @@ -131,15 +131,11 @@ histogram: VAR_TYPE: data pkts_file_name: - <<: *support_data_defaults - CATDESC: Name of input file with L0 CCSDS packets data + CATDESC: Name of input file with CCSDS packet data DISPLAY_TYPE: no_plot - FIELDNAM: L0 data input filename - FILLVAL: # TBD: what is fillval for strings? - FORMAT: S256 # TBC - LABLAXIS: File name # MS: is this needed for no_plot? - VALIDMAX: # TBD: what is validmax for a string? - VALIDMIN: # TBD: what is validmin for a string? + FIELDNAM: CCSDS packet data input filename + FILLVAL: " " + FORMAT: A49 VAR_TYPE: metadata first_spin_id: @@ -690,7 +686,6 @@ missing_packets_sequence: # Used to be missing_packets_sequence VAR_TYPE: metadata flight_software_version: - <<: *support_data_defaults CATDESC: GLOWS flight software version FIELDNAM: GLOWS flight software version FILLVAL: *max_uint32 @@ -698,6 +693,14 @@ flight_software_version: LABLAXIS: FSW ver VALIDMAX: 16777215 +ground_software_version: + CATDESC: Version of ground software used for processing + DISPLAY_TYPE: no_plot + FIELDNAM: Ground Software Version + FILLVAL: " " + FORMAT: A4 + VAR_TYPE: metadata + # Flags # MS: this needs to be thoroughly discussed bad_time_flag_hist_attrs: # MS: this should be different for DE and HIST diff --git a/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml index 2d56d2b5d..dffe1453d 100644 --- a/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml @@ -129,6 +129,22 @@ identifier: VALIDMAX: 99999 DICT_KEY: SPASE>Support>SupportQuantity:Temporal +flight_software_version: + <<: *support_data_defaults + CATDESC: GLOWS flight software version + FIELDNAM: GLOWS flight software version + FILLVAL: *max_uint32 + FORMAT: I8 + LABLAXIS: FSW ver + VALIDMAX: 16777215 + +pkts_file_name: + VAR_TYPE: metadata + CATDESC: Name of L0 packets file data originated + FIELDNAM: Packets File Name + FILLVAL: ' ' + FORMAT: A49 + start_time: <<: *time_data_defaults CATDESC: Start time (UTC) of observational day diff --git a/imap_processing/glows/l1a/glows_l1a.py b/imap_processing/glows/l1a/glows_l1a.py index ba9f484a6..d45eb197b 100644 --- a/imap_processing/glows/l1a/glows_l1a.py +++ b/imap_processing/glows/l1a/glows_l1a.py @@ -171,7 +171,6 @@ def generate_de_dataset( # First variable is the output data type, second is the list of values support_data: dict = { - # "flight_software_version": [], "seq_count_in_pkts_file": [np.uint16, []], "number_of_de_packets": [np.uint32, []], } @@ -337,7 +336,6 @@ def generate_histogram_dataset( # First variable is the output data type, second is the list of values support_data: dict = { - "flight_software_version": [np.uint32, []], "seq_count_in_pkts_file": [np.uint16, []], "first_spin_id": [np.uint32, []], "last_spin_id": [np.uint32, []], @@ -428,6 +426,33 @@ def generate_histogram_dataset( output["histogram"] = hist + # These attributes are the same for each record, so we don't + # need to store them per epoch like most of the other fields + output["flight_software_version"] = xr.DataArray( + np.array([hist_l1a_list[0].flight_software_version], dtype=np.uint32), + name="flight_software_version", + dims=["scalar"], + attrs=glows_cdf_attributes.get_variable_attributes( + "flight_software_version", check_schema=False + ), + ) + output["pkts_file_name"] = xr.DataArray( + np.array([hist_l1a_list[0].pkts_file_name], dtype=object), + name="pkts_file_name", + dims=["scalar"], + attrs=glows_cdf_attributes.get_variable_attributes( + "pkts_file_name", check_schema=False + ), + ) + output["ground_software_version"] = xr.DataArray( + np.array([hist_l1a_list[0].ground_software_version], dtype=object), + name="ground_software_version", + dims=["scalar"], + attrs=glows_cdf_attributes.get_variable_attributes( + "ground_software_version", check_schema=False + ), + ) + for key, value in support_data.items(): output[key] = xr.DataArray( np.array(value[1], dtype=value[0]), diff --git a/imap_processing/glows/l1a/glows_l1a_data.py b/imap_processing/glows/l1a/glows_l1a_data.py index 814f08680..4f26cd793 100644 --- a/imap_processing/glows/l1a/glows_l1a_data.py +++ b/imap_processing/glows/l1a/glows_l1a_data.py @@ -201,10 +201,10 @@ class HistogramL1A: l0: InitVar[HistogramL0] histogram: list[int] = field(init=False) - # next four are in block header flight_software_version: int = field(init=False) - ground_software_version: str = field(init=False) - pkts_file_name: str = field(init=False) + pkts_file_name: str = "" + ground_software_version: str = "" + # next four are in block header seq_count_in_pkts_file: int = field(init=False) first_spin_id: int = field(init=False) last_spin_id: int = field(init=False) @@ -383,7 +383,7 @@ def merge_de_packets(self, second_l0: DirectEventL0) -> None: raise ValueError( f"Sequence for direct event L1A is out of order or " f"incorrect. Attempted to append sequence counter " - f"{second_l0.SEQ} after {self.most_recent_seq}." + f"{second_l0.SEQ} after {self.most_recent_seq}. " f"New DE time: {second_l0.SEC}, current time: {self.l0.SEC}." ) @@ -397,7 +397,7 @@ def merge_de_packets(self, second_l0: DirectEventL0) -> None: if not match: raise ValueError( f"While attempting to merge L0 packet {second_l0} " - f"with {self.l0} mismatched values" + f"with {self.l0} mismatched values " f"were found. " ) @@ -534,7 +534,7 @@ def _build_compressed_event( else: raise ValueError( - f"Incorrect length {len(raw)} for {raw}, expecting 2 or 3" + f"Incorrect length {len(raw)} for {raw}, expecting 2 or 3 " f"bit compressed direct event data" ) diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index eb87d3404..75d439bdd 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -82,6 +82,31 @@ def glows_l1b( output_dataarrays, input_dataset["epoch"], input_dataset["bins"], cdf_attrs ) + output_dataset["flight_software_version"] = xr.DataArray( + input_dataset["flight_software_version"].data, + name="flight_software_version", + dims=["scalar"], + attrs=cdf_attrs.get_variable_attributes( + "flight_software_version", check_schema=False + ), + ) + + output_dataset["ground_software_version"] = xr.DataArray( + input_dataset["ground_software_version"].data, + name="ground_software_version", + dims=["scalar"], + attrs=cdf_attrs.get_variable_attributes( + "ground_software_version", check_schema=False + ), + ) + + output_dataset["pkts_file_name"] = xr.DataArray( + input_dataset["pkts_file_name"].data, + name="pkts_file_name", + dims=["scalar"], + attrs=cdf_attrs.get_variable_attributes("pkts_file_name", check_schema=False), + ) + return output_dataset @@ -267,7 +292,7 @@ def process_histogram( # histograms is the only multi dimensional input variable, so we set the non-epoch # dimension ("bins"). - # The rest of the input vars are epoch only, so they have an empty list. + # The rest of the non-scalar input vars are epoch only, so they have an empty list. input_dims[0] = ["bins"] # Create a closure that captures the ancillary objects @@ -485,6 +510,16 @@ def create_l1b_de_output( fields[index].name ) + # TODO: Not sure if this is requested in this product... + # parents = input_dataset.attrs.get("Parent", "") + # output_dataset["pkts_file_name"] = xr.DataArray( + # [parent for parent in parents if parent.endswith("cdf")], + # attrs=cdf_attrs.get_variable_attributes("pkts_file_name", check_schema=False), + # ) + # output_dataset["pkts_file_name"] = [ + # parent for parent in parents if parent.endswith("cdf") + # ] + output_dataset["within_the_second"] = within_the_second_data output_dataset.attrs["missing_packets_sequence"] = input_dataset.attrs.get( "missing_packets_sequence", "" diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index a5409f187..23fc4854a 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -492,6 +492,8 @@ class DirectEventL1B: float. From direct_events. direct_event_pulse_lengths: ndarray array of pulse lengths [μs] for direct events. From direct_events + pkts_file_name + Name of the input CCSDS packets file """ direct_events: InitVar[np.ndarray] @@ -526,6 +528,7 @@ class DirectEventL1B: direct_event_glows_times: np.ndarray | None = field(init=False, default=None) # 3rd value is pulse length direct_event_pulse_lengths: np.ndarray | None = field(init=False, default=None) + # pkts_file_name: str = "" # TODO: where does the multi-event flag go? def __post_init__( @@ -662,7 +665,6 @@ class HistogramL1B: ---------- histogram array of block-accumulated count numbers - flight_software_version: str seq_count_in_pkts_file: int first_spin_id: int The start ID @@ -731,10 +733,18 @@ class HistogramL1B: flags flags for extra information, per histogram. This should be a human-readable structure. + flight_software_version + The version of the flight software, copied from L1A + pkts_file_name + The name of the input CCSDS packets file + ground_software_version + The version of the ground software, copied from L1A """ histogram: np.ndarray - flight_software_version: str + flight_software_version: np.ndarray + pkts_file_name: np.ndarray + ground_software_version: np.ndarray seq_count_in_pkts_file: int first_spin_id: int last_spin_id: int diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 3fec7ba0a..1e38964a6 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -23,10 +23,13 @@ from imap_processing.tests.glows.conftest import mock_update_spice_parameters +# Fixture for L1a histogram dataset @pytest.fixture def hist_dataset(): variables = { - "flight_software_version": np.zeros((20,)), + "flight_software_version": np.array([67], dtype=np.uint32), + "pkts_file_name": np.array(["test_packet_file.pkts"], dtype=object), + "ground_software_version": np.array(["v999"], dtype=object), "seq_count_in_pkts_file": np.zeros((20,)), "first_spin_id": np.zeros((20,)), "last_spin_id": np.zeros((20,)), @@ -50,20 +53,19 @@ def hist_dataset(): } cdf_attrs = ImapCdfAttributes() cdf_attrs.add_instrument_global_attrs("glows") - cdf_attrs.add_instrument_variable_attrs("glows", "l1b") + cdf_attrs.add_instrument_variable_attrs("glows", "l1a") epoch = xr.DataArray( np.arange(20), name="epoch", dims=["epoch"], - attrs=cdf_attrs.get_variable_attributes("epoch"), + attrs=cdf_attrs.get_variable_attributes("epoch", check_schema=False), ) bins = xr.DataArray(np.arange(3600), name="bins", dims=["bins"]) ds = xr.Dataset( - coords={"epoch": epoch}, - attrs=cdf_attrs.get_global_attributes("imap_glows_l1b_hist"), + attrs=cdf_attrs.get_global_attributes("imap_glows_l1a_hist"), ) ds["histogram"] = xr.DataArray( @@ -73,7 +75,10 @@ def hist_dataset(): ) for var, data in variables.items(): - ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) + if 1 != len(data): + ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) + else: + ds[var] = xr.DataArray(data, dims="scalar") return ds @@ -147,6 +152,8 @@ def de_dataset(): }, ) + ds.attrs["Parent"] = ["test_packet_file.cdf", "test_spice_file.tls"] + for var, data in variables.items(): ds[var] = xr.DataArray(data, dims=["epoch"], coords={"epoch": epoch}) @@ -201,14 +208,12 @@ def test_histogram_mapping( mock_pipeline_settings, ): mock_spice_function.side_effect = mock_update_spice_parameters - time_val = 1111111.11 - # A = 2.318 - # B = 69.5454 + time_val = np.double(1111111.11) expected_temp = 100 test_hists = np.zeros((200, 3600)) # For temp - encoded_val = expected_temp * 2.318 + 69.5454 + encoded_val = np.double(expected_temp * 2.3182 + 69.5455) # For now, testing types and number of inputs pipeline_settings = PipelineSettings( @@ -221,7 +226,9 @@ def test_histogram_mapping( dataclasses.asdict( HistogramL1B( test_hists, - "test", + np.array([67], dtype=np.uint32), + np.array(["test_packet_file.pkts"], dtype=object), + np.array(["v999"], dtype=object), 0, 0, 0, @@ -249,10 +256,12 @@ def test_histogram_mapping( ).values() ) - assert output[18] == time_val - # Correctly decoded temperature - assert output[10] - expected_temp < 0.1 + assert np.isclose(output[12], expected_temp, 0.1) + + # Ensure time values are correctly mapped + assert output[20] == time_val + assert output[23] == time_val @patch.object(HistogramL1B, "update_spice_parameters", autospec=True) @@ -265,14 +274,12 @@ def test_process_histogram( ): mock_spice_function.side_effect = mock_update_spice_parameters - time_val = np.single(1111111.11) - # A = 2.318 - # B = 69.5454 + time_val = np.double(1111111.11) expected_temp = 100 test_hists = np.zeros((200,)) # For temp - encoded_val = np.single(expected_temp * 2.318 + 69.5454) + encoded_val = np.double(expected_temp * 2.3182 + 69.5455) pipeline_settings = PipelineSettings( mock_pipeline_settings.sel( @@ -282,7 +289,9 @@ def test_process_histogram( test_l1b = HistogramL1B( test_hists, - "test", + np.array([67], dtype=np.uint32), + np.array(["test_packet_file.pkts"], dtype=object), + np.array(["v999"], dtype=object), 0, 0, 0, @@ -356,16 +365,14 @@ def test_glows_l1b( mock_conversion_table_dict, ) - assert hist_output["histogram"].dims == ("epoch", "bins") - assert hist_output["histogram"].shape == (20, 3600) + assert hist_output["histogram"].dims == ("epoch", "scalar", "bins") + assert hist_output["histogram"].shape == (20, 1, 3600) # This needs to be added eventually, but is skipped for now. expected_de_data = [ "flight_software_version", - "ground_software_version", - "pkts_file_name", "seq_count_in_pkts_file", - "l1a_file_name", + "pkts_file_name", "ancillary_data_files", ] @@ -401,6 +408,9 @@ def test_glows_l1b( "spacecraft_velocity_average", "spacecraft_velocity_std_dev", "flags", + "flight_software_version", + "ground_software_version", + "pkts_file_name", ] for key in expected_hist_data: @@ -490,7 +500,9 @@ def test_hist_spice_output( use_fake_spin_data_for_time(data_start_time) params = { "histogram": np.zeros((1, 3600)), - "flight_software_version": "v0.0.1", + "flight_software_version": np.array([67], dtype=np.uint32), + "pkts_file_name": np.array(["test_packet_file.pkts"], dtype=object), + "ground_software_version": np.array(["v999"], dtype=object), "seq_count_in_pkts_file": 0, "first_spin_id": 0, "last_spin_id": 0,