From 19eb7a3a30722461d60628a4f0b0889bf00c1429 Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Thu, 7 May 2026 12:54:42 -0600 Subject: [PATCH 1/4] CoDICE: remove invalid data products --- imap_processing/_version.py | 5 + .../imap_codice_l1a_variable_attrs.yaml | 2 + imap_processing/codice/codice_l1a.py | 12 - .../codice/codice_l1a_lo_angular.py | 536 ------------------ .../codice/codice_l1a_lo_species.py | 25 +- imap_processing/codice/codice_l1b.py | 3 - imap_processing/codice/codice_l2.py | 209 ------- imap_processing/codice/constants.py | 19 - imap_processing/tests/codice/conftest.py | 57 -- .../tests/codice/test_codice_l1a.py | 198 +------ .../tests/codice/test_codice_l1b.py | 194 ------- .../tests/codice/test_codice_l2.py | 225 -------- .../tests/external_test_data_config.py | 14 - 13 files changed, 27 insertions(+), 1472 deletions(-) create mode 100644 imap_processing/_version.py delete mode 100644 imap_processing/codice/codice_l1a_lo_angular.py diff --git a/imap_processing/_version.py b/imap_processing/_version.py new file mode 100644 index 0000000000..a483898838 --- /dev/null +++ b/imap_processing/_version.py @@ -0,0 +1,5 @@ +"""Version information for the imap_processing package.""" + +# These version placeholders will be replaced later during substitution. +__version__ = "1.0.29.post19.dev0+f2be49d7" +__version_tuple__ = (1, 0, 29, "post19", "dev0", "f2be49d7") diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index df646830e2..260d8a508c 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -902,6 +902,7 @@ de_2d_attrs: DEPEND_0: epoch DEPEND_1: priority DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: spectogram FIELDNAM: Direct Event Data FILLVAL: -9223372036854775808 FORMAT: I5 @@ -919,6 +920,7 @@ de_3d_attrs: DEPEND_1: priority DEPEND_2: event_num DICT_KEY: SPASE>Support>SupportQuantity:Other + DISPLAY_TYPE: spectogram FIELDNAM: Direct Event Data FILLVAL: -9223372036854775808 FORMAT: I{num_digits} diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 7d074104a6..794c09730d 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -19,7 +19,6 @@ from imap_processing.codice.codice_l1a_hi_omni import l1a_hi_omni from imap_processing.codice.codice_l1a_hi_priority import l1a_hi_priority from imap_processing.codice.codice_l1a_hi_sectored import l1a_hi_sectored -from imap_processing.codice.codice_l1a_lo_angular import l1a_lo_angular from imap_processing.codice.codice_l1a_lo_counters_aggregated import ( l1a_lo_counters_aggregated, ) @@ -83,17 +82,6 @@ def process_l1a( # noqa: PLR0912 datasets.append( process_by_table_id(datasets_by_apid[apid], lut_file, l1a_lo_species) ) - elif apid == CODICEAPID.COD_LO_NSW_SPECIES_COUNTS: - logger.info("Processing Lo NSW Species Counts") - datasets.append( - process_by_table_id(datasets_by_apid[apid], lut_file, l1a_lo_species) - ) - elif apid == CODICEAPID.COD_LO_SW_ANGULAR_COUNTS: - logger.info("Processing Lo SW Angular Counts") - datasets.append(l1a_lo_angular(datasets_by_apid[apid], lut_file)) - elif apid == CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS: - logger.info("Processing Lo NSW Angular Counts") - datasets.append(l1a_lo_angular(datasets_by_apid[apid], lut_file)) elif apid == CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS: datasets.append( process_by_table_id(datasets_by_apid[apid], lut_file, l1a_hi_omni) diff --git a/imap_processing/codice/codice_l1a_lo_angular.py b/imap_processing/codice/codice_l1a_lo_angular.py deleted file mode 100644 index 907b83789f..0000000000 --- a/imap_processing/codice/codice_l1a_lo_angular.py +++ /dev/null @@ -1,536 +0,0 @@ -"""CoDICE Lo Angular L1A processing functions.""" - -import logging -from pathlib import Path - -import numpy as np -import xarray as xr - -from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.codice import constants -from imap_processing.codice.constants import ( - HALF_SPIN_FILLVAL, - LO_NSW_ANGULAR_VARIABLE_NAMES, - LO_SW_ANGULAR_VARIABLE_NAMES, -) -from imap_processing.codice.decompress import decompress -from imap_processing.codice.utils import ( - CODICEAPID, - CoDICECompression, - ViewTabInfo, - calculate_acq_time_per_step, - get_codice_epoch_time, - get_collapse_pattern_shape, - get_view_tab_info, - index_to_position, - read_sci_lut, -) -from imap_processing.spice.time import met_to_ttj2000ns - -logger = logging.getLogger(__name__) - - -def _despin_species_data( - species_data: np.ndarray, sci_lut_data: dict, view_tab_obj: ViewTabInfo -) -> np.ndarray: - """ - Apply despinning mapping for angular products. - - Despinned data shape is (num_packets, num_species, 24, inst_az) where - we expand spin_sector to 24 by filling with zeros in 12 to 24 or 0 to 11 - based on pixel orientation. - - Parameters - ---------- - species_data : np.ndarray - The species data array to be despun. - sci_lut_data : dict - The science LUT data used for despinning. - view_tab_obj : ViewTabInfo - The view table information object. - - Returns - ------- - np.ndarray - The despun species data array in - (num_packets, num_species, esa_steps, 24, inst_az). - """ - # species_data shape: (num_packets, num_species, esa_steps, *collapsed_dims) - num_packets, num_species, esa_steps = species_data.shape[:3] - collapsed_dims = species_data.shape[3:] - inst_az_dim = collapsed_dims[-1] - - # Prepare despinning output: (num_packets, num_species, esa_steps, 24, inst_az_dim) - # 24 is derived by multiplying spin sector dim from collapse table by 2 - spin_sector_len = constants.LO_DESPIN_SPIN_SECTORS - despun_shape = (num_packets, num_species, esa_steps, spin_sector_len, inst_az_dim) - despun_data = np.full(despun_shape, 0.0, dtype=np.float64) - # Pixel orientation array and mapping positions - pixel_orientation = np.array( - sci_lut_data["lo_stepping_tab"]["pixel_orientation"]["data"] - ) - # index_to_position gets the position from collapse table. Eg. - # [1, 2, 3, 23, 24] for SW angular - angular_position = index_to_position(sci_lut_data, 0, view_tab_obj.collapse_table) - orientation_a_indices = np.where(pixel_orientation == "A")[0] - orientation_b_indices = np.where(pixel_orientation == "B")[0] - - # Despin data based on orientation and angular position - for pos_idx, position in enumerate(angular_position): - if position <= 12: - # Case 1: position 0-12, orientation A, append to first half - despun_data[:, :, orientation_a_indices, :12, pos_idx] = species_data[ - :, :, orientation_a_indices, :, pos_idx - ] - # Case 2: position 13-24, orientation B, append to second half - despun_data[:, :, orientation_b_indices, 12:, pos_idx] = species_data[ - :, :, orientation_b_indices, :, pos_idx - ] - else: - # Case 3: position 13-24, orientation A, append to second half - despun_data[:, :, orientation_a_indices, 12:, pos_idx] = species_data[ - :, :, orientation_a_indices, :, pos_idx - ] - # Case 4: position 0-12, orientation B, append to first half - despun_data[:, :, orientation_b_indices, :12, pos_idx] = species_data[ - :, :, orientation_b_indices, :, pos_idx - ] - - return despun_data - - -def l1a_lo_angular(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset: # noqa: PLR0912 - """ - L1A processing code. - - Parameters - ---------- - unpacked_dataset : xarray.Dataset - The decompressed and unpacked data from the packet file. - lut_file : pathlib.Path - Path to the LUT (Lookup Table) file used for processing. - - Returns - ------- - xarray.Dataset - The processed L1A dataset for the given species product. - """ - # Get these values from unpacked data. These are used to - # lookup in LUT table. - table_id = unpacked_dataset["table_id"].values[0] - view_id = unpacked_dataset["view_id"].values[0] - apid = unpacked_dataset["pkt_apid"].values[0] - plan_id = unpacked_dataset["plan_id"].values[0] - plan_step = unpacked_dataset["plan_step"].values[0] - - logger.info( - f"Processing angular with - APID: {apid} / 0x{apid:X}, View ID: {view_id}, " - f"Table ID: {table_id}, Plan ID: {plan_id}, Plan Step: {plan_step}" - ) - - # ========== Get LUT Data =========== - # Read information from LUT - sci_lut_data = read_sci_lut(lut_file, table_id) - - view_tab_info = get_view_tab_info(sci_lut_data, view_id, apid) - view_tab_obj = ViewTabInfo( - apid=apid, - view_id=view_id, - sensor=view_tab_info["sensor"], - three_d_collapsed=view_tab_info["3d_collapse"], - collapse_table=view_tab_info["collapse_table"], - compression=view_tab_info["compression"], - ) - - if view_tab_obj.sensor != 0: - raise ValueError("Unsupported sensor ID for Lo angular processing.") - - # ========= Decompress and Reshape Data =========== - # Lookup SW or NSW species based on APID - # We also need to determine if there are any species that should be backfilled - # with fill values - if view_tab_obj.apid == CODICEAPID.COD_LO_SW_ANGULAR_COUNTS: - actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["angular"][ - "sw" - ]["species_names"] - desired_species_names = set( - sci_lut_data["data_product_lo_tab"]["0"]["angular"]["sw"][ - "desired_species_names" - ] - + LO_SW_ANGULAR_VARIABLE_NAMES - ) - logical_source_id = "imap_codice_l1a_lo-sw-angular" - elif view_tab_obj.apid == CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS: - actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["angular"][ - "nsw" - ]["species_names"] - desired_species_names = set( - sci_lut_data["data_product_lo_tab"]["0"]["angular"]["nsw"][ - "desired_species_names" - ] - + LO_NSW_ANGULAR_VARIABLE_NAMES - ) - logical_source_id = "imap_codice_l1a_lo-nsw-angular" - else: - raise ValueError(f"Unknown apid {view_tab_obj.apid} in Lo species processing.") - - compression_algorithm = CoDICECompression(view_tab_obj.compression) - # Decompress data using byte count information from decommed data - binary_data_list = unpacked_dataset["data"].values - byte_count_list = unpacked_dataset["byte_count"].values - - # The decompressed data in the shape of (epoch, n). Then reshape later. - decompressed_data = [ - decompress( - packet_data[:byte_count], - compression_algorithm, - ) - for (packet_data, byte_count) in zip( - binary_data_list, byte_count_list, strict=False - ) - ] - - # Look up collapse pattern using LUT table. This should return collapsed shape. - collapsed_shape = get_collapse_pattern_shape( - sci_lut_data, view_tab_obj.sensor, view_tab_obj.collapse_table - ) - - # Reshape decompressed data to: - # (num_packets, num_species, esa_steps, 12, 5) - # 24 includes despinning spin sector. Then at later steps, - # we handle despinning. - num_packets = len(binary_data_list) - num_esa_steps = constants.NUM_ESA_STEPS - num_species = len(actual_species_names) - num_spin_sectors = collapsed_shape[0] - species_data = np.array(decompressed_data, dtype=np.uint32).reshape( - num_packets, num_species, num_esa_steps, *collapsed_shape - ) - - # ========== Get Voltage Data from LUT =========== - # Use plan id and plan step to get voltage data's table_number in ESA sweep table. - # Voltage data is (128,) - esa_table_number = sci_lut_data["plan_tab"][f"({plan_id}, {plan_step})"][ - "lo_stepping" - ] - voltage_data = sci_lut_data["esa_sweep_tab"][f"{esa_table_number}"] - # If data size is less than 128, pad with fillval to make it 128 - half_spin_per_esa_step = sci_lut_data["lo_stepping_tab"]["row_number"].get("data") - if len(half_spin_per_esa_step) < num_esa_steps: - pad_size = num_esa_steps - len(half_spin_per_esa_step) - half_spin_per_esa_step = np.concatenate( - (np.array(half_spin_per_esa_step), np.full(pad_size, HALF_SPIN_FILLVAL)) - ) - # TODO: Handle epoch dependent acquisition time and half spin per esa step - # For now, just tile the same array for all epochs. - # Eventually we may have data from a day where the LUT changed. If this is the - # case, we need to split the data by epoch and assign different acquisition times - half_spin_per_esa_step = np.tile( - np.asarray(half_spin_per_esa_step).astype(np.uint8), - (len(unpacked_dataset["acq_start_seconds"]), 1), - ) - # Get acquisition time per esa step - acquisition_time_per_step = calculate_acq_time_per_step( - sci_lut_data["lo_stepping_tab"] - ) - acquisition_time_per_step = np.tile( - np.asarray(acquisition_time_per_step), - (len(unpacked_dataset["acq_start_seconds"]), 1), - ) - # ========== Apply NSO/RGFO Masking =========== - # After FSW changes on 20260129, The Lo L1A product contains variables that - # indicate the esa step and spin sector during which the RGFO or NSO limits are - # triggered. The spin sector variable ranges from 0-11 and is the instrument - # reported spin sector. The following algorithm defines when to assign NaN to the - # angular data product due to NSO - # operation: - # 1. For half_spin > nso_half_spin a set all data to NaN - # 2. For half_spin = nso_half_spin - # a. For spin_sector > nso_spin_sector a set all data to NaN - # b. For spin_sector = nso_spin_sector - # i. For esa_step > nso_esa_step a set all data to NaN - # For every energy after nso_half_spin, set data to fill values - # For data before 20260129 ( packet_version <=1 ) set all data to NaN where - # half_spin > nso_half_spin - packet_versions = unpacked_dataset["packet_version"].values - nso_half_spin = unpacked_dataset["nso_half_spin"].values - # TODO handle boundary days where the FSW changed halfway through the dataset. E.g - # Some packet_version = 1 and some = 2 - if packet_versions[0] <= 1: - # For half_spin >= NSO_half_spin, set to NaN - half_spin_mask = (half_spin_per_esa_step >= nso_half_spin[:, np.newaxis]) | ( - half_spin_per_esa_step == HALF_SPIN_FILLVAL - ) - species_mask = half_spin_mask[:, np.newaxis, :, np.newaxis, np.newaxis] - species_mask = np.broadcast_to(species_mask, species_data.shape) - else: - # nso_spin_sector and nso_esa_step for comparison. Shape (epoch, 1, 1) - # to broadcast - nso_spin_sector = unpacked_dataset["nso_spin_sector"].values[ - :, np.newaxis, np.newaxis - ] - nso_esa_step = unpacked_dataset["nso_energy_step"].values[ - :, np.newaxis, np.newaxis - ] - # Create arrays for spin sectors and esa steps to compare with nso values. - # Shape (1, 1, spin_sector) and (1, esa_step, 1) - spin_sectors = np.arange(num_spin_sectors)[np.newaxis, np.newaxis, :] - esa_steps = np.arange(num_esa_steps)[np.newaxis, :, np.newaxis] - # Create a mask for half_spin > nso_half_spin. Shape (epoch, esa_step)) - # This will be used below to set half_spin_per_esa_step to fillval and - # acquisition_time_per_step to NaN for those steps. - half_spin_mask = (half_spin_per_esa_step > nso_half_spin[:, np.newaxis]) | ( - half_spin_per_esa_step == HALF_SPIN_FILLVAL - ) - # Create a mask for the boundary condition where half_spin == nso_half_spin. - at_boundary = ( - half_spin_per_esa_step[:, :, np.newaxis] - == nso_half_spin[:, np.newaxis, np.newaxis] - ) - boundary_half_spin_mask = ( - at_boundary - & - # For spin_sector > nso_spin_sector, set to NaN - ( - (spin_sectors > nso_spin_sector) - | - # For spin_sector = nso_spin_sector and esa_step > nso_esa_step, - # set to NaN - ((spin_sectors == nso_spin_sector) & (esa_steps > nso_esa_step)) - ) - ) - - # Combine masks. Shape (epoch, esa_step, spin_sector). This mask is True - # where data should be set to NaN - nso_mask = half_spin_mask[:, :, np.newaxis] | boundary_half_spin_mask - # Expand nso_mask to (epoch, 1, esa_step, spin_sector, 1) to apply to - # species_data. - species_mask = np.broadcast_to( - nso_mask[:, np.newaxis, :, :, np.newaxis], species_data.shape - ) - - species_data = species_data.astype(np.float64) - species_data[species_mask] = np.nan - # Set half_spin_per_esa_step to (fillval) where half_spin mask is True - half_spin_per_esa_step[half_spin_mask] = HALF_SPIN_FILLVAL - # Set acquisition_time_per_step to nan where half_spin_mask is True - acquisition_time_per_step[half_spin_mask] = np.nan - - # Despinning - # ---------------- - species_data = _despin_species_data(species_data, sci_lut_data, view_tab_obj) - - # ========= Get Epoch Time Data =========== - # Epoch center time and delta - epoch_center, deltas = get_codice_epoch_time( - unpacked_dataset["acq_start_seconds"].values, - unpacked_dataset["acq_start_subseconds"].values, - unpacked_dataset["spin_period"].values, - view_tab_obj, - ) - - # ========== Create CDF Dataset with Metadata =========== - cdf_attrs = ImapCdfAttributes() - cdf_attrs.add_instrument_global_attrs("codice") - cdf_attrs.add_instrument_variable_attrs("codice", "l1a") - - l1a_dataset = xr.Dataset( - coords={ - "epoch": xr.DataArray( - met_to_ttj2000ns(epoch_center), - dims=("epoch",), - attrs=cdf_attrs.get_variable_attributes("epoch", check_schema=False), - ), - "epoch_delta_minus": xr.DataArray( - deltas, - dims=("epoch",), - attrs=cdf_attrs.get_variable_attributes( - "epoch_delta_minus", check_schema=False - ), - ), - "epoch_delta_plus": xr.DataArray( - deltas, - dims=("epoch",), - attrs=cdf_attrs.get_variable_attributes( - "epoch_delta_plus", check_schema=False - ), - ), - "esa_step": xr.DataArray( - np.arange(128), - dims=("esa_step",), - attrs=cdf_attrs.get_variable_attributes("esa_step", check_schema=False), - ), - "half_spin_per_esa_step": xr.DataArray( - half_spin_per_esa_step, - dims=( - "epoch", - "esa_step", - ), - attrs=cdf_attrs.get_variable_attributes( - "half_spin_per_esa_step", check_schema=False - ), - ), - "esa_step_label": xr.DataArray( - np.arange(128).astype(str), - dims=("esa_step",), - attrs=cdf_attrs.get_variable_attributes( - "esa_step_label", check_schema=False - ), - ), - "inst_az": xr.DataArray( - index_to_position(sci_lut_data, 0, view_tab_obj.collapse_table), - dims=("inst_az",), - attrs=cdf_attrs.get_variable_attributes("inst_az", check_schema=False), - ), - "inst_az_label": xr.DataArray( - index_to_position(sci_lut_data, 0, view_tab_obj.collapse_table).astype( - str - ), - dims=("inst_az",), - attrs=cdf_attrs.get_variable_attributes( - "inst_az_label", check_schema=False - ), - ), - "k_factor": xr.DataArray( - np.array([constants.K_FACTOR]), - dims=("k_factor",), - attrs=cdf_attrs.get_variable_attributes("k_factor", check_schema=False), - ), - "spin_sector": xr.DataArray( - np.arange(24, dtype=np.uint8), - dims=("spin_sector",), - attrs=cdf_attrs.get_variable_attributes( - "spin_sector", check_schema=False - ), - ), - "spin_sector_label": xr.DataArray( - np.arange(24).astype(str), - dims=("spin_sector",), - attrs=cdf_attrs.get_variable_attributes( - "spin_sector_label", check_schema=False - ), - ), - }, - attrs=cdf_attrs.get_global_attributes(logical_source_id), - ) - # Add first few unique variables - l1a_dataset["k_factor"] = xr.DataArray( - np.array([constants.K_FACTOR]), - dims=("k_factor",), - attrs=cdf_attrs.get_variable_attributes("k_factor_attrs", check_schema=False), - ) - l1a_dataset["spin_period"] = xr.DataArray( - unpacked_dataset["spin_period"].values * constants.SPIN_PERIOD_CONVERSION, - dims=("epoch",), - attrs=cdf_attrs.get_variable_attributes("spin_period"), - ) - l1a_dataset["voltage_table"] = xr.DataArray( - np.array(voltage_data), - dims=("esa_step",), - attrs=cdf_attrs.get_variable_attributes("voltage_table", check_schema=False), - ) - l1a_dataset["data_quality"] = xr.DataArray( - unpacked_dataset["suspect"].values, - dims=("epoch",), - attrs=cdf_attrs.get_variable_attributes("data_quality"), - ) - l1a_dataset["acquisition_time_per_esa_step"] = xr.DataArray( - acquisition_time_per_step, - dims=("epoch", "esa_step"), - attrs=cdf_attrs.get_variable_attributes( - "acquisition_time_per_esa_step", check_schema=False - ), - ) - # Rename vars - unpacked_dataset = unpacked_dataset.rename( - { - k: v - for k, v in [ - ("rgfo_energy_step", "rgfo_esa_step"), - ("nso_energy_step", "nso_esa_step"), - ] - if k in unpacked_dataset - } - ) - # These variables were added to the packet definition after 20260129, so they only - # exist in the unpacked dataset if packet_version > 1 - # If they don't exist, initialize them with fill val arrays since they won't be - # used in the NSO/RGFO masking logic but should still exist in l1a for SPDF - # compliance/consistency. - l1a_additional_vars = [ - "rgfo_spin_sector", - "rgfo_esa_step", - "nso_spin_sector", - "nso_esa_step", - ] - for var in l1a_additional_vars: - if var not in unpacked_dataset: - unpacked_dataset[var] = np.full( - unpacked_dataset.sizes["epoch"], fill_value=np.nan - ) - - # Carry over these variables from unpacked data to l1a_dataset - l1a_carryover_vars = [ - "sw_bias_gain_mode", - "st_bias_gain_mode", - "rgfo_half_spin", - "nso_half_spin", - "packet_version", - *l1a_additional_vars, - ] - # Loop through them since we need to set their attrs too - for var in l1a_carryover_vars: - l1a_dataset[var] = xr.DataArray( - unpacked_dataset[var].values, - dims=("epoch",), - attrs=cdf_attrs.get_variable_attributes(var), - ) - # Loop through the species we want in the final dataset (desired_species_names) and - # add them if they exist in the actual species names from the LUT. - # This is to handle the bug in which the spacecraft was sending data down "off by - # one" and getting mislabeled. - for species in desired_species_names: - if species not in actual_species_names: - logger.warning( - f"Desired species {species} not found in actual species names from " - f"LUT. This species will be filled with fill values in the final " - f"dataset. Actual species names: {actual_species_names}" - ) - species_data_individual = np.full(species_data[:, 0, :, :, :].shape, np.nan) - else: - species_idx = actual_species_names.index(species) - species_data_individual = species_data[:, species_idx, :, :, :] - - species_attrs = cdf_attrs.get_variable_attributes("lo-angular-attrs") - unc_attrs = cdf_attrs.get_variable_attributes("lo-angular-unc-attrs") - direction = ( - "Sunward" - if view_tab_obj.apid == CODICEAPID.COD_LO_SW_ANGULAR_COUNTS - else "Non-Sunward" - ) - # Replace {species} and {direction} in attrs - species_attrs["CATDESC"] = species_attrs["CATDESC"].format( - species=species, direction=direction - ) - species_attrs["FIELDNAM"] = species_attrs["FIELDNAM"].format( - species=species, direction=direction - ) - l1a_dataset[species] = xr.DataArray( - species_data_individual, - dims=("epoch", "esa_step", "spin_sector", "inst_az"), - attrs=species_attrs, - ) - # Uncertainty data - unc_attrs["CATDESC"] = unc_attrs["CATDESC"].format( - species=species, direction=direction - ) - unc_attrs["FIELDNAM"] = unc_attrs["FIELDNAM"].format( - species=species, direction=direction - ) - l1a_dataset[f"unc_{species}"] = xr.DataArray( - np.sqrt(l1a_dataset[species].values), - dims=("epoch", "esa_step", "spin_sector", "inst_az"), - attrs=unc_attrs, - ) - - return l1a_dataset diff --git a/imap_processing/codice/codice_l1a_lo_species.py b/imap_processing/codice/codice_l1a_lo_species.py index 01eb2fefde..b7aa3f6f23 100644 --- a/imap_processing/codice/codice_l1a_lo_species.py +++ b/imap_processing/codice/codice_l1a_lo_species.py @@ -11,7 +11,6 @@ from imap_processing.codice.constants import ( HALF_SPIN_FILLVAL, LO_IALIRT_VARIABLE_NAMES, - LO_NSW_SPECIES_VARIABLE_NAMES, LO_SW_SPECIES_VARIABLE_NAMES, ) from imap_processing.codice.decompress import decompress @@ -73,7 +72,7 @@ def l1a_lo_species( # noqa: PLR0912 raise ValueError("Unsupported sensor ID for Lo species processing.") # ========= Decompress and Reshape Data =========== - # Lookup SW or NSW species based on APID + # Lookup SW species based on APID if view_tab_obj.apid == CODICEAPID.COD_LO_SW_SPECIES_COUNTS: actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["species"][ "sw" @@ -85,28 +84,6 @@ def l1a_lo_species( # noqa: PLR0912 + LO_SW_SPECIES_VARIABLE_NAMES ) logical_source_id = "imap_codice_l1a_lo-sw-species" - elif view_tab_obj.apid == CODICEAPID.COD_LO_NSW_SPECIES_COUNTS: - actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["species"][ - "nsw" - ]["species_names"] - desired_species_names = set( - sci_lut_data["data_product_lo_tab"]["0"]["species"]["nsw"][ - "desired_species_names" - ] - + LO_NSW_SPECIES_VARIABLE_NAMES - ) - logical_source_id = "imap_codice_l1a_lo-nsw-species" - # Rename "cnoplus" to "junk" if we are processing NSW angular data. Although - # cnoplus is in desired, and actual species name in the LUT, it is referencing - # different "cnoplus" data his is to handle the bug in which the spacecraft was - # sending data down "off by one" and getting mislabeled. The cnoplus data we - # are referencing is actually data that we want to toss out and fill with - # fill vals. This only affects data before the LUT was updated - # (table_id 3978152295). - if int(table_id) <= 3978152295: - actual_species_names = [ - "junk" if name == "cnoplus" else name for name in actual_species_names - ] elif view_tab_obj.apid == CODICEAPID.COD_LO_IAL: actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["ialirt"]["sw"][ "species_names" diff --git a/imap_processing/codice/codice_l1b.py b/imap_processing/codice/codice_l1b.py index e6805e5f7d..644f59a3b9 100644 --- a/imap_processing/codice/codice_l1b.py +++ b/imap_processing/codice/codice_l1b.py @@ -67,8 +67,6 @@ def convert_to_rates(dataset: xr.Dataset, descriptor: str) -> np.ndarray: if descriptor in [ "lo-counters-aggregated", "lo-counters-singles", - "lo-nsw-angular", - "lo-sw-angular", "lo-nsw-priority", "lo-sw-priority", ]: @@ -93,7 +91,6 @@ def convert_to_rates(dataset: xr.Dataset, descriptor: str) -> np.ndarray: ] dataset = dataset.drop_vars(drop_variables) elif descriptor in [ - "lo-nsw-species", "lo-sw-species", "lo-ialirt", ]: diff --git a/imap_processing/codice/codice_l2.py b/imap_processing/codice/codice_l2.py index 6567278073..0a7998ee91 100644 --- a/imap_processing/codice/codice_l2.py +++ b/imap_processing/codice/codice_l2.py @@ -28,18 +28,13 @@ HI_OMNI_VARIABLE_NAMES, HI_SECTORED_VARIABLE_NAMES, L2_HI_SECTORED_ANGLE, - LO_NSW_ANGULAR_VARIABLE_NAMES, - LO_NSW_SPECIES_VARIABLE_NAMES, LO_POSITION_TO_ELEVATION_ANGLE, - LO_SW_ANGULAR_VARIABLE_NAMES, LO_SW_PICKUP_ION_SPECIES_VARIABLE_NAMES, LO_SW_SOLAR_WIND_SPECIES_VARIABLE_NAMES, - NSW_POSITIONS, PUI_POSITIONS, SOLAR_WIND_POSITIONS, SSD_ID_TO_ELEVATION, SSD_ID_TO_SPIN_ANGLE, - SW_POSITIONS, ) from imap_processing.codice.utils import apply_replacements_to_attrs @@ -571,156 +566,6 @@ def process_lo_species_intensity( return dataset -def process_lo_angular_intensity( - dataset: xr.Dataset, - species_list: list, - geometric_factors: xr.DataArray, - efficiency: pd.DataFrame, - positions: list, -) -> xr.Dataset: - """ - Process the lo-species L2 dataset to calculate angular intensities. - - Parameters - ---------- - dataset : xarray.Dataset - The L2 dataset to process. - species_list : list - List of species variable names to calculate intensity. - geometric_factors : xarray.DataArray - The geometric factors array with shape (epoch, esa_steps). - efficiency : pandas.DataFrame - The efficiency lookup table. - positions : list - A list of position indices to select from the geometric factor and - efficiency lookup tables. - - Returns - ------- - xarray.Dataset - The updated L2 dataset with angular intensities calculated. - """ - # Calculate the angular intensities using the provided geometric factors and - # efficiency. - dataset = calculate_intensity( - dataset, - species_list, - geometric_factors, - efficiency, - positions, - average_across_positions=False, - ) - - # transform positions to elevation angles - if positions == SW_POSITIONS: - pos_to_el = LO_POSITION_TO_ELEVATION_ANGLE["sw"] - position_index_to_adjust = 0 - direction = "Sunward" - elif positions == NSW_POSITIONS: - pos_to_el = LO_POSITION_TO_ELEVATION_ANGLE["nsw"] - position_index_to_adjust = 9 - direction = "Non-Sunward" - else: - raise ValueError("Unknown positions for elevation angle mapping.") - - # Create a new coordinate for elevation_angle based on inst_az - dataset = dataset.assign_coords( - elevation_angle=( - "inst_az", - [pos_to_el[pos] for pos in dataset["inst_az"].data], - ) - ) - # add uncertainties to species list - species_list = species_list + [f"unc_{var}" for var in species_list] - - # Take the mean across elevation angles and restore the original dimension order - dataset_converted = ( - dataset[species_list] - .groupby("elevation_angle") - .sum(keep_attrs=True, skipna=False) # One position should always contain zeros - # so sum is safe - # Restore original dimension order because groupby moves the grouped - # dimension to the front - .transpose("epoch", "esa_step", "spin_sector", "elevation_angle", ...) - ) - # Create a new coordinate for spin angle based on spin_sector - # Use equation from section 11.2.2 of algorithm document - dataset = dataset.assign_coords( - spin_angle=("spin_sector", dataset["spin_sector"].data * 15.0 + 7.5) - ) - dataset = dataset.drop_vars(species_list).merge(dataset_converted) - - # Positions 0 and 10 only observe half of the 24 spins for each esa step. - # To account for this, we replicate the counts observed in position 0 and 10 for - # each esa step to either spin angles 0-11 or 12-23, depending on the pixel - # orientation (A/B). See section 11.2.2 of the CoDICE algorithm document - # Use the variable "half_spin_per_esa_step" to determine the pixel orientations. - # When the half spin number is even, the configuration is A and when the half spin - # is odd, the configuration is B. - # TODO handle when half_spin_per_esa_step changes in the middle of the dataset - half_spin_per_esa_step = dataset["half_spin_per_esa_step"].data[0] - # only consider valid half spin values - valid_half_spin = half_spin_per_esa_step != HALF_SPIN_FILLVAL - a_inds = np.nonzero(valid_half_spin & (half_spin_per_esa_step % 2 == 0))[0] - b_inds = np.nonzero(valid_half_spin & (half_spin_per_esa_step % 2 == 1))[0] - - position_index = position_index_to_adjust - for species in species_list: - # Create a copy of the dataset to avoid modifying the original - species_data = dataset[species].data.copy() - # Determine the correct spin indices based on the position - spin_sectors = dataset["spin_sector"].data - spin_inds_1 = np.where(spin_sectors >= 12)[0] - spin_inds_2 = np.where(spin_sectors < 12)[0] - - # if position_index is 9, swap the spin indices - if position_index == 9: - spin_inds_1, spin_inds_2 = spin_inds_2, spin_inds_1 - - # Assign the values to the correct positions and spin sectors - dataset[species].values[ - :, a_inds[:, np.newaxis], spin_inds_1, position_index - ] = species_data[:, a_inds[:, np.newaxis], spin_inds_2, position_index] - - dataset[species].values[ - :, b_inds[:, np.newaxis], spin_inds_2, position_index - ] = species_data[:, b_inds[:, np.newaxis], spin_inds_1, position_index] - cdf_attrs = ImapCdfAttributes() - cdf_attrs.add_instrument_variable_attrs("codice", "l2-lo-angular") - species_attrs = cdf_attrs.get_variable_attributes("lo-angular-attrs") - unc_attrs = cdf_attrs.get_variable_attributes("lo-angular-unc-attrs") - - # update species attrs - for species in species_list: - attrs = unc_attrs if "unc" in species else species_attrs - # Replace {species} and {direction} in attrs - attrs = apply_replacements_to_attrs( - attrs, {"species": species, "direction": direction} - ) - dataset[species].attrs.update(attrs) - - # make sure elevation_angle is a coordinate and has the right attrs - dataset["elevation_angle"].attrs.update( - cdf_attrs.get_variable_attributes("elevation_angle", check_schema=False) - ) - dataset["elevation_angle_label"] = xr.DataArray( - dataset["elevation_angle"].data.astype(str), - dims=("elevation_angle",), - attrs=cdf_attrs.get_variable_attributes( - "elevation_angle_label", check_schema=False - ), - ) - # update spin angle attributes - dataset["spin_angle"].attrs = cdf_attrs.get_variable_attributes( - "spin_angle", check_schema=False - ) - # update spin sector attributes - dataset["spin_sector"].attrs = cdf_attrs.get_variable_attributes( - "spin_sector", check_schema=False - ) - return dataset - - def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: """ Process the hi-omni L1B dataset to calculate omni-directional intensities. @@ -1517,9 +1362,6 @@ def process_codice_l2( # Compute geometric factors needed for intensity calculations if dataset_name in [ "imap_codice_l2_lo-sw-species", - "imap_codice_l2_lo-nsw-species", - "imap_codice_l2_lo-nsw-angular", - "imap_codice_l2_lo-sw-angular", ]: cdf_attrs = ImapCdfAttributes() cdf_attrs.add_instrument_global_attrs("codice") @@ -1556,57 +1398,6 @@ def process_codice_l2( l2_dataset.attrs.update( cdf_attrs.get_global_attributes("imap_codice_l2_lo-sw-species") ) - elif dataset_name == "imap_codice_l2_lo-nsw-species": - geometric_factors = compute_geometric_factors( - l2_dataset, geometric_factor_lookup - ) - # Filter the efficiency lookup table for non-solar wind efficiencies - efficiencies = efficiency_lookup[efficiency_lookup["product"] == "nsw"] - # Calculate the non-sunward species intensities using equation - # described in section 11.2.3 of algorithm document. - l2_dataset = process_lo_species_intensity( - l2_dataset, - LO_NSW_SPECIES_VARIABLE_NAMES, - geometric_factors, - efficiencies, - NSW_POSITIONS, - ) - l2_dataset.attrs.update( - cdf_attrs.get_global_attributes("imap_codice_l2_lo-nsw-species") - ) - elif dataset_name == "imap_codice_l2_lo-sw-angular": - geometric_factors = compute_geometric_factors( - l2_dataset, geometric_factor_lookup, angular_product=True - ) - efficiencies = efficiency_lookup[efficiency_lookup["product"] == "sw"] - # Calculate the sunward solar wind angular intensities using equation - # described in section 11.2.2 of algorithm document. - l2_dataset = process_lo_angular_intensity( - l2_dataset, - LO_SW_ANGULAR_VARIABLE_NAMES, - geometric_factors, - efficiencies, - SW_POSITIONS, - ) - l2_dataset.attrs.update( - cdf_attrs.get_global_attributes("imap_codice_l2_lo-sw-angular") - ) - if dataset_name == "imap_codice_l2_lo-nsw-angular": - geometric_factors = compute_geometric_factors( - l2_dataset, geometric_factor_lookup, angular_product=True - ) - # Calculate the non sunward angular intensities - efficiencies = efficiency_lookup[efficiency_lookup["product"] == "nsw"] - l2_dataset = process_lo_angular_intensity( - l2_dataset, - LO_NSW_ANGULAR_VARIABLE_NAMES, - geometric_factors, - efficiencies, - NSW_POSITIONS, - ) - l2_dataset.attrs.update( - cdf_attrs.get_global_attributes("imap_codice_l2_lo-nsw-angular") - ) if dataset_name in [ "imap_codice_l2_hi-counters-singles", diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 5d61424dd6..c2a53aff9a 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -846,21 +846,12 @@ "lo-counters-singles": { "num_spin_sectors": 2, }, - "lo-nsw-angular": { - "num_spin_sectors": 1, - }, - "lo-sw-angular": { - "num_spin_sectors": 1, - }, "lo-nsw-priority": { "num_spin_sectors": 1, }, "lo-sw-priority": { "num_spin_sectors": 1, }, - "lo-nsw-species": { - "num_spin_sectors": 12, - }, "lo-sw-species": { "num_spin_sectors": 12, }, @@ -890,16 +881,6 @@ "heplus", "cnoplus", ] -LO_NSW_SPECIES_VARIABLE_NAMES = [ - "hplus", - "heplusplus", - "c", - "o", - "ne_si_mg", - "fe", - "heplus", - "cnoplus", -] HI_OMNI_VARIABLE_NAMES = ["h", "he3", "he4", "c", "o", "ne_mg_si", "fe", "uh", "junk"] HI_SECTORED_VARIABLE_NAMES = ["h", "he3he4", "cno", "fe"] HI_PRIORITY_VARIABLE_NAMES = [ diff --git a/imap_processing/tests/codice/conftest.py b/imap_processing/tests/codice/conftest.py index 4ff50f42dd..34d703d47e 100644 --- a/imap_processing/tests/codice/conftest.py +++ b/imap_processing/tests/codice/conftest.py @@ -37,33 +37,6 @@ def _side_effect(descriptor: str = None, data_type: str = None) -> list[Path]: / "l1a_input" / f"imap_codice_l0_lo-sw-species_{VALIDATION_FILE_DATE}_v001.pkts" ] - elif descriptor == "lo-nsw-species" and data_type == "l0": - return [ - imap_module_directory - / "tests" - / "codice" - / "data" - / "l1a_input" - / f"imap_codice_l0_lo-nsw-species_{VALIDATION_FILE_DATE}_v001.pkts" - ] - elif descriptor == "lo-sw-angular" and data_type == "l0": - return [ - imap_module_directory - / "tests" - / "codice" - / "data" - / "l1a_input" - / f"imap_codice_l0_lo-sw-angular_{VALIDATION_FILE_DATE}_v001.pkts" - ] - elif descriptor == "lo-nsw-angular" and data_type == "l0": - return [ - imap_module_directory - / "tests" - / "codice" - / "data" - / "l1a_input" - / f"imap_codice_l0_lo-nsw-angular_{VALIDATION_FILE_DATE}_v001.pkts" - ] elif descriptor == "hi-sectored" and data_type == "l0": return [ imap_module_directory @@ -184,36 +157,6 @@ def _side_effect(descriptor: str = None, data_type: str = None) -> list[Path]: f"_{VALIDATION_FILE_VERSION}.cdf" ) ] - elif descriptor == "lo-sw-species" and data_type == "l1b": - return [ - imap_module_directory - / "tests" - / "codice" - / "data" - / "l1b_validation" - / ( - f"imap_codice_l1b_lo-sw-species_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ] - elif descriptor == "lo-nsw-angular" and data_type == "l1b": - return [ - TEST_DATA_PATH - / "l1b_validation" - / ( - f"imap_codice_l1b_lo-nsw-angular_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ] - elif descriptor == "lo-sw-angular" and data_type == "l1b": - return [ - TEST_DATA_PATH - / "l1b_validation" - / ( - f"imap_codice_l1b_lo-sw-angular_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ] elif descriptor == "hi-sectored" and data_type == "l1b": return [ imap_module_directory diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 105ff30073..d936bbb029 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -40,7 +40,7 @@ def test_updated_packet_version(mock_get_file_paths, codice_lut_path, caplog): ] datasets = process_l1a(dependency=ProcessingInputCollection()) # Assert that we have all of the expected datasets - assert len(datasets) == 17 + assert len(datasets) == 14 for ds in datasets: # Only check lo products. Skip direct-events if ( @@ -60,23 +60,24 @@ def test_updated_packet_version(mock_get_file_paths, codice_lut_path, caplog): f"Expected variable '{var}' not found in dataset" ) - # check that warnings are logged for missing "desired" species - assert ( - "Desired species heplusplus not found in actual species names from LUT" - in caplog.text - ) - assert ( - "Desired species oplus6 not found in actual species names from LUT" - in caplog.text - ) - assert ( - "Desired species heplus not found in actual species names from LUT" - in caplog.text - ) - assert ( - "Desired species cnoplus not found in actual species names from LUT" - in caplog.text - ) + print(caplog.text) + # # check that warnings are logged for missing "desired" species + # assert ( + # "Desired species heplusplus not found in actual species names from LUT" + # in caplog.text + # ) + # assert ( + # "Desired species oplus6 not found in actual species names from LUT" + # in caplog.text + # ) + # assert ( + # "Desired species heplus not found in actual species names from LUT" + # in caplog.text + # ) + # assert ( + # "Desired species cnoplus not found in actual species names from LUT" + # in caplog.text + # ) @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") @@ -335,167 +336,6 @@ def test_lo_sw_species(mock_get_file_paths, codice_lut_path): ) -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_lo_nsw_species(mock_get_file_paths, codice_lut_path): - """Tests lo-nsw-species.""" - - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-nsw-species", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - - # Validation - val_path = ( - imap_module_directory - / "tests/codice/data/l1a_validation/" - / ( - f"imap_codice_l1a_lo-nsw-species_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - - val_data = load_cdf(val_path) - - # Process the input data - processed_data = process_l1a(dependency=ProcessingInputCollection())[0] - # Compare only the common variables - for variable in val_data.data_vars: - # Skip cnopus because this variable should be thrown out for lo nsw species - # for table_ids <= 3978152295 - if "cnoplus" in variable: - continue - np.testing.assert_allclose( - processed_data[variable].values, - val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - - for variable in val_data.coords: - if variable.endswith("_label"): - assert np.array_equal( - processed_data[variable].values, - val_data[variable].values, - ), f"Mismatch in coordinate '{variable}'" - continue - np.testing.assert_allclose( - processed_data[variable].values, - val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in coordinate '{variable}'", - ) - - processed_data.attrs["Data_version"] = "002" - cdf_file = write_cdf(processed_data, terminate_on_warning=True, istp=True) - assert ( - cdf_file.name - == f"imap_codice_l1a_lo-nsw-species_{VALIDATION_FILE_DATE}_v002.cdf" - ) - - -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_lo_sw_angular(mock_get_file_paths, codice_lut_path): - """Tests lo-sw-angular.""" - - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-sw-angular", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - - # Validation - val_path = ( - imap_module_directory - / "tests/codice/data/l1a_validation/" - / ( - f"imap_codice_l1a_lo-sw-angular_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - val_data = load_cdf(val_path) - - # Process the input data - processed_data = process_l1a(dependency=ProcessingInputCollection())[0] - for variable in val_data.data_vars: - np.testing.assert_allclose( - processed_data[variable].values, - val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - - for variable in val_data.coords: - if variable.endswith("_label"): - assert np.array_equal( - processed_data[variable].values, - val_data[variable].values, - ), f"Mismatch in coordinate '{variable}'" - continue - np.testing.assert_allclose( - processed_data[variable].values, - val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in coordinate '{variable}'", - ) - - processed_data.attrs["Data_version"] = "002" - cdf_file = write_cdf(processed_data, terminate_on_warning=True) - assert ( - cdf_file.name - == f"imap_codice_l1a_lo-sw-angular_{VALIDATION_FILE_DATE}_v002.cdf" - ) - - -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_lo_nsw_angular(mock_get_file_paths, codice_lut_path): - """Tests lo-nsw-angular.""" - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-nsw-angular", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - - # Validation - val_path = ( - imap_module_directory - / "tests/codice/data/l1a_validation/" - / ( - f"imap_codice_l1a_lo-nsw-angular_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - val_data = load_cdf(val_path) - - # Process the input data - processed_data = process_l1a(dependency=ProcessingInputCollection())[0] - for variable in val_data.data_vars: - np.testing.assert_allclose( - processed_data[variable].values, - val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - - for variable in val_data.coords: - if variable.endswith("_label"): - assert np.array_equal( - processed_data[variable].values, - val_data[variable].values, - ), f"Mismatch in coordinate '{variable}'" - continue - np.testing.assert_allclose( - processed_data[variable].values, - val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in coordinate '{variable}'", - ) - - processed_data.attrs["Data_version"] = "002" - cdf_file = write_cdf(processed_data, terminate_on_warning=True) - assert ( - cdf_file.name - == f"imap_codice_l1a_lo-nsw-angular_{VALIDATION_FILE_DATE}_v002.cdf" - ) - - @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") def test_hi_counters_aggregated(mock_get_file_paths, codice_lut_path): """Tests hi-counters-aggregated.""" diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py index 77aa35d0cc..3c4532d8a4 100644 --- a/imap_processing/tests/codice/test_codice_l1b.py +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -85,200 +85,6 @@ def test_l1b_lo_sw_species(mock_get_file_paths, codice_lut_path): ) -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_l1b_lo_nsw_species(mock_get_file_paths, codice_lut_path): - """Tests lo-nsw-species.""" - - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-nsw-species", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - - processed_l1a_file = write_cdf(process_l1a(ProcessingInputCollection())[0]) - - l1b_val_data = ( - imap_module_directory - / "tests" - / "codice" - / "data" - / "l1b_validation" - / ( - f"imap_codice_l1b_lo-nsw-species_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - l1b_val_data = load_cdf(l1b_val_data) - processed_data = process_codice_l1b(processed_l1a_file) - - for variable in l1b_val_data.data_vars: - # Skip cnopus because this variable should be thrown out for lo nsw species - # for table_ids <= 3978152295 - if "cnoplus" in variable: - continue - np.testing.assert_allclose( - processed_data[variable].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - - for variable in l1b_val_data.coords: - if variable.endswith("_label"): - assert np.array_equal( - processed_data[variable].values, - l1b_val_data[variable].values, - ), f"Mismatch in coordinate '{variable}'" - continue - elif variable == "energy_table": - np.testing.assert_allclose( - processed_data["energy_per_charge"].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - continue - np.testing.assert_allclose( - processed_data[variable].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in coordinate '{variable}'", - ) - # Write to CDF - processed_data.attrs["Data_version"] = "002" - cdf_file = write_cdf(processed_data, terminate_on_warning=True) - assert ( - cdf_file.name - == f"imap_codice_l1b_lo-nsw-species_{VALIDATION_FILE_DATE}_v002.cdf" - ) - - -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_l1b_lo_sw_angular(mock_get_file_paths, codice_lut_path): - """Tests lo-sw-angular.""" - - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-sw-angular", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - - processed_l1a_file = write_cdf(process_l1a(ProcessingInputCollection())[0]) - - l1b_val_data = ( - imap_module_directory - / "tests" - / "codice" - / "data" - / "l1b_validation" - / ( - f"imap_codice_l1b_lo-sw-angular_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - l1b_val_data = load_cdf(l1b_val_data) - processed_data = process_codice_l1b(processed_l1a_file) - - for variable in l1b_val_data.data_vars: - assert processed_data[variable].shape == l1b_val_data[variable].shape - np.testing.assert_allclose( - processed_data[variable].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - - for variable in l1b_val_data.coords: - if variable.endswith("_label"): - assert np.array_equal( - processed_data[variable].values, - l1b_val_data[variable].values, - ), f"Mismatch in coordinate '{variable}'" - continue - elif variable == "energy_table": - np.testing.assert_allclose( - processed_data["energy_per_charge"].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - continue - np.testing.assert_allclose( - processed_data[variable].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in coordinate '{variable}'", - ) - # Write to CDF - processed_data.attrs["Data_version"] = "002" - cdf_file = write_cdf(processed_data, terminate_on_warning=True) - assert ( - cdf_file.name - == f"imap_codice_l1b_lo-sw-angular_{VALIDATION_FILE_DATE}_v002.cdf" - ) - - -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_l1b_lo_nsw_angular(mock_get_file_paths, codice_lut_path): - """Tests lo-nsw-angular.""" - - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-nsw-angular", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - - processed_l1a_file = write_cdf(process_l1a(ProcessingInputCollection())[0]) - - l1b_val_data = ( - imap_module_directory - / "tests" - / "codice" - / "data" - / "l1b_validation" - / ( - f"imap_codice_l1b_lo-nsw-angular_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - l1b_val_data = load_cdf(l1b_val_data) - processed_data = process_codice_l1b(processed_l1a_file) - - for variable in l1b_val_data.data_vars: - np.testing.assert_allclose( - processed_data[variable].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - - for variable in l1b_val_data.coords: - if variable.endswith("_label"): - assert np.array_equal( - processed_data[variable].values, - l1b_val_data[variable].values, - ), f"Mismatch in coordinate '{variable}'" - continue - elif variable == "energy_table": - np.testing.assert_allclose( - processed_data["energy_per_charge"].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - continue - np.testing.assert_allclose( - processed_data[variable].values, - l1b_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in coordinate '{variable}'", - ) - # Write to CDF - processed_data.attrs["Data_version"] = "002" - cdf_file = write_cdf(processed_data, terminate_on_warning=True) - assert ( - cdf_file.name - == f"imap_codice_l1b_lo-nsw-angular_{VALIDATION_FILE_DATE}_v002.cdf" - ) - - @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") def test_l1b_hi_omni(mock_get_file_paths, codice_lut_path): mock_get_file_paths.side_effect = [ diff --git a/imap_processing/tests/codice/test_codice_l2.py b/imap_processing/tests/codice/test_codice_l2.py index ae738e00b0..86e356e4d3 100644 --- a/imap_processing/tests/codice/test_codice_l2.py +++ b/imap_processing/tests/codice/test_codice_l2.py @@ -23,13 +23,10 @@ get_mpq_calc_energy_conversion_vals, get_mpq_calc_tof_conversion_vals, process_codice_l2, - process_lo_angular_intensity, process_lo_species_intensity, ) from imap_processing.codice.constants import ( - LO_SW_ANGULAR_VARIABLE_NAMES, LO_SW_SOLAR_WIND_SPECIES_VARIABLE_NAMES, - SW_POSITIONS, ) from imap_processing.tests.codice.conftest import ( VALIDATION_FILE_DATE, @@ -343,77 +340,6 @@ def test_process_lo_missing_species_intensity(): ) -def test_process_lo_angular_intensity(mock_get_file_paths, codice_lut_path): - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-sw-angular", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - processed_l1a_file = write_cdf(process_l1a(ProcessingInputCollection())[0]) - l1b_data = process_codice_l1b(processed_l1a_file) - l1b_val_data_processed = l1b_data.copy() - gf = xr.DataArray( - np.ones((len(l1b_data.epoch), 128, 24)) * 2, - dims=("epoch", "esa_step", "inst_az"), - ) - with mock.patch( - "imap_processing.codice.codice_l2.get_species_efficiency", - return_value=xr.DataArray(np.ones((128, 24)) * 2, dims=("esa_step", "inst_az")), - ): - l1b_val_data_processed = process_lo_angular_intensity( - l1b_val_data_processed, - LO_SW_ANGULAR_VARIABLE_NAMES, - gf, - None, - SW_POSITIONS, - ) - - for var in LO_SW_ANGULAR_VARIABLE_NAMES: - # Heplus is not in older CDFs - if var == "heplus" or var not in l1b_val_data_processed: - continue - assert var in l1b_val_data_processed, f"Missing variable {var} after processing" - # Check that values are non-negative - assert np.all( - (l1b_val_data_processed[var].values >= 0) - | np.isnan(l1b_val_data_processed[var].values) - ), f"Variable {var} contains negative values" - # Check shape - expected_shape = ( - len(l1b_data.epoch), - len(l1b_data.energy_per_charge), - len(l1b_data.spin_sector), - 3, # 3 elevation angles map to 5 positions - ) - np.testing.assert_allclose( - expected_shape, l1b_val_data_processed[var].shape, rtol=1e-5 - ) - # Check that values match expected calculation - expected_intensity = ( - l1b_data[var] - / (4 * l1b_data["energy_per_charge"].data)[ - np.newaxis, :, np.newaxis, np.newaxis - ] - ) - # convert pos to el - expected_intensity = ( - expected_intensity.assign_coords(group=("inst_az", [0, 1, 2, 2, 1])) - .groupby("group") - .sum() - ) - # Skip checking the first elevations. Those get reassigned and will be - # validated below. - np.testing.assert_allclose( - l1b_val_data_processed[var].values[:, :, :, 1:], - expected_intensity.values[:, :, :, 1:], - rtol=1e-5, - ) - # Check coords - np.testing.assert_allclose(l1b_val_data_processed["elevation_angle"], [0, 15, 30]) - np.testing.assert_allclose( - l1b_val_data_processed["spin_angle"], np.arange(24) * 15 + 7.5 - ) - - @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") def test_codice_l2_sw_species_intensity(mock_get_file_paths, codice_lut_path): mock_get_file_paths.side_effect = [ @@ -456,157 +382,6 @@ def test_codice_l2_sw_species_intensity(mock_get_file_paths, codice_lut_path): write_cdf(processed_2_ds) -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_codice_l2_nsw_species_intensity(mock_get_file_paths, codice_lut_path): - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-nsw-species", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - processed_l1a_file = write_cdf(process_l1a(ProcessingInputCollection())[0]) - processed_l1b_file = write_cdf(process_codice_l1b(processed_l1a_file)) - # Mock get_files for l2 - mock_get_file_paths.side_effect = [ - [processed_l1b_file.as_posix()], - codice_lut_path(descriptor="l2-lo-gfactor"), - codice_lut_path(descriptor="l2-lo-efficiency"), - ] - processed_2_ds = process_codice_l2("lo-nsw-species", ProcessingInputCollection()) - l2_val_data = ( - imap_module_directory - / "tests" - / "codice" - / "data" - / "l2_validation" - / ( - f"imap_codice_l2_lo-nsw-species_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - l2_val_data = load_cdf(l2_val_data) - for variable in l2_val_data.data_vars: - # Skip cnopus because this variable should be thrown out for lo nsw species - # for table_ids <= 3978152295 - if "cnoplus" in variable: - continue - # NOTE: Replace nan with 0 for comparison as the validation data uses 0 - processed_val = processed_2_ds[variable].values - processed_val[np.isnan(processed_val)] = 0.0 - np.testing.assert_allclose( - processed_val, - l2_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - processed_2_ds.attrs["Data_version"] = "001" - assert processed_2_ds.attrs["Logical_source"] == "imap_codice_l2_lo-nsw-species" - write_cdf(processed_2_ds) - - -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_codice_l2_nsw_angular_intensity(mock_get_file_paths, codice_lut_path): - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-nsw-angular", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - processed_l1a_file = write_cdf(process_l1a(ProcessingInputCollection())[0]) - processed_l1b_file = write_cdf(process_codice_l1b(processed_l1a_file)) - # Mock get_files for l2 - mock_get_file_paths.side_effect = [ - [processed_l1b_file.as_posix()], - codice_lut_path(descriptor="l2-lo-gfactor"), - codice_lut_path(descriptor="l2-lo-efficiency"), - ] - processed_2_ds = process_codice_l2("lo-nsw-angular", ProcessingInputCollection()) - l2_val_data = ( - imap_module_directory - / "tests" - / "codice" - / "data" - / "l2_validation" - / ( - f"imap_codice_l2_lo-nsw-angular_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - l2_val_data = load_cdf(l2_val_data) - for variable in l2_val_data.variables: - np.testing.assert_allclose( - processed_2_ds[variable].values, - l2_val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) - processed_2_ds.attrs["Data_version"] = "001" - assert processed_2_ds.attrs["Logical_source"] == "imap_codice_l2_lo-nsw-angular" - write_cdf(processed_2_ds) - - -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_codice_l2_sw_angular_intensity(mock_get_file_paths, codice_lut_path): - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="lo-sw-angular", data_type="l0"), - codice_lut_path(descriptor="l1a-sci-lut"), - ] - processed_l1a_file = write_cdf(process_l1a(ProcessingInputCollection())[0]) - processed_l1b_file = write_cdf(process_codice_l1b(processed_l1a_file)) - # Mock get_files for l2 - mock_get_file_paths.side_effect = [ - [processed_l1b_file.as_posix()], - codice_lut_path(descriptor="l2-lo-gfactor"), - codice_lut_path(descriptor="l2-lo-efficiency"), - ] - processed_2_ds = process_codice_l2("lo-sw-angular", ProcessingInputCollection()) - l2_val_data = ( - imap_module_directory - / "tests" - / "codice" - / "data" - / "l2_validation" - / ( - f"imap_codice_l2_lo-sw-angular_{VALIDATION_FILE_DATE}" - f"_{VALIDATION_FILE_VERSION}.cdf" - ) - ) - l2_val_data = load_cdf(l2_val_data) - for variable in l2_val_data.variables: - np.testing.assert_allclose( - processed_2_ds[variable].values, - l2_val_data[variable].values, - rtol=1e-4, - err_msg=f"Mismatch in variable '{variable}'", - ) - - processed_2_ds.attrs["Data_version"] = "001" - assert processed_2_ds.attrs["Logical_source"] == "imap_codice_l2_lo-sw-angular" - write_cdf(processed_2_ds) - - -@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") -def test_codice_l2_sw_angular_intensity_rgfo_masking( - mock_get_file_paths, codice_lut_path -): - """Tests RGFO masking after FSW changes (jan 2026).""" - codice_lut_path_jan = codice_lut_path(descriptor="l1a-sci-lut-jan") - mock_get_file_paths.side_effect = [ - codice_lut_path(descriptor="fsw-changes", data_type="l0"), - *([codice_lut_path_jan] * 20), - ] - datasets = process_l1a(dependency=ProcessingInputCollection()) - - ang_dataset = next(ds for ds in datasets if "angular" in ds.attrs["Data_type"]) - # process the first angular dataset - processed_l1a_file = write_cdf(ang_dataset) - processed_l1b_file = write_cdf(process_codice_l1b(processed_l1a_file)) - # Mock get_files for l2 - mock_get_file_paths.side_effect = [ - [processed_l1b_file.as_posix()], - codice_lut_path(descriptor="l2-lo-gfactor"), - codice_lut_path(descriptor="l2-lo-efficiency"), - ] - # TODO verify the results using validation data once we have some - process_codice_l2("lo-nsw-angular", ProcessingInputCollection()) - - @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path): mock_get_file_paths.side_effect = [ diff --git a/imap_processing/tests/external_test_data_config.py b/imap_processing/tests/external_test_data_config.py index c749fe1da2..84c7020588 100644 --- a/imap_processing/tests/external_test_data_config.py +++ b/imap_processing/tests/external_test_data_config.py @@ -17,9 +17,6 @@ # CoDICE # L0 data ("imap_codice_l0_lo-sw-species_20250814_v001.pkts", "codice/data/l1a_input/"), - ("imap_codice_l0_lo-nsw-species_20250814_v001.pkts", "codice/data/l1a_input/"), - ("imap_codice_l0_lo-sw-angular_20250814_v001.pkts", "codice/data/l1a_input/"), - ("imap_codice_l0_lo-nsw-angular_20250814_v001.pkts", "codice/data/l1a_input/"), ("imap_codice_l0_lo-nsw-priority_20250814_v001.pkts", "codice/data/l1a_input/"), ("imap_codice_l0_lo-sw-priority_20250814_v001.pkts", "codice/data/l1a_input/"), ("imap_codice_l0_lo-counters-aggregated_20250814_v001.pkts", "codice/data/l1a_input/"), @@ -55,10 +52,7 @@ (f"imap_codice_l1a_lo-ialirt_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), ("imap_codice_l1a_hi-ialirt_20260331_v0.0.22.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-nsw-priority_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), - (f"imap_codice_l1a_lo-nsw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), - (f"imap_codice_l1a_lo-sw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-sw-priority_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), - (f"imap_codice_l1a_lo-nsw-species_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-sw-species_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), # L1B Input data is same as L1A validation data @@ -72,14 +66,9 @@ (f"imap_codice_l1b_lo-counters-aggregated_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), (f"imap_codice_l1b_lo-counters-singles_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), (f"imap_codice_l1b_lo-ialirt_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), - (f"imap_codice_l1b_lo-nsw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), (f"imap_codice_l1b_lo-nsw-priority_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), - (f"imap_codice_l1b_lo-sw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), (f"imap_codice_l1b_lo-sw-priority_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), - (f"imap_codice_l1b_lo-nsw-species_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), (f"imap_codice_l1b_lo-sw-species_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), - (f"imap_codice_l1b_lo-nsw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), - (f"imap_codice_l1b_lo-sw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1b_validation"), # L2 LUT input data ("imap_codice_l2-hi-omni-efficiency_20251212_v003.csv", "codice/data/l2_lut/"), ("imap_codice_l2-hi-sectored-efficiency_20251212_v003.csv", "codice/data/l2_lut/"), @@ -95,9 +84,6 @@ # L2 Validation data (f"imap_codice_l2_hi-omni_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), (f"imap_codice_l2_hi-sectored_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), - (f"imap_codice_l2_lo-nsw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), - (f"imap_codice_l2_lo-sw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), - (f"imap_codice_l2_lo-nsw-species_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), (f"imap_codice_l2_lo-sw-species_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), (f"imap_codice_l2_lo-direct-events_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), (f"imap_codice_l2_hi-direct-events_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), From 7fdbd763d0e46e66b4bf26a3cdfa789e08ba6cde Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Thu, 7 May 2026 13:44:04 -0600 Subject: [PATCH 2/4] minor test fix --- .../tests/codice/test_codice_l1a.py | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index d936bbb029..af9dec2bb6 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -60,24 +60,11 @@ def test_updated_packet_version(mock_get_file_paths, codice_lut_path, caplog): f"Expected variable '{var}' not found in dataset" ) - print(caplog.text) - # # check that warnings are logged for missing "desired" species - # assert ( - # "Desired species heplusplus not found in actual species names from LUT" - # in caplog.text - # ) - # assert ( - # "Desired species oplus6 not found in actual species names from LUT" - # in caplog.text - # ) - # assert ( - # "Desired species heplus not found in actual species names from LUT" - # in caplog.text - # ) - # assert ( - # "Desired species cnoplus not found in actual species names from LUT" - # in caplog.text - # ) + # check that a warning is logged for the missing "cnoplus" species + assert ( + "Desired species cnoplus not found in actual species names from LUT" + in caplog.text + ) @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") From 51c19608df66d5f465db0d1ae70ca9cd0ac08167 Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Tue, 12 May 2026 12:40:18 -0600 Subject: [PATCH 3/4] feedback changes --- imap_processing/tests/codice/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imap_processing/tests/codice/conftest.py b/imap_processing/tests/codice/conftest.py index 34d703d47e..9d77379545 100644 --- a/imap_processing/tests/codice/conftest.py +++ b/imap_processing/tests/codice/conftest.py @@ -145,7 +145,7 @@ def _side_effect(descriptor: str = None, data_type: str = None) -> list[Path]: / "l1a_input" / "imap_codice_l0_hskp_20250814_v001.pkts" ] - if descriptor == "lo-nsw-species" and data_type == "l1b": + elif descriptor == "lo-sw-species" and data_type == "l1b": return [ imap_module_directory / "tests" @@ -153,7 +153,7 @@ def _side_effect(descriptor: str = None, data_type: str = None) -> list[Path]: / "data" / "l1b_validation" / ( - f"imap_codice_l1b_lo-nsw-species_{VALIDATION_FILE_DATE}" + f"imap_codice_l1b_lo-sw-species_{VALIDATION_FILE_DATE}" f"_{VALIDATION_FILE_VERSION}.cdf" ) ] From a9d3ada469aa045782f241735ce29073e5d68b35 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 May 2026 19:48:29 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- imap_processing/codice/codice_l1a_lo_species.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a_lo_species.py b/imap_processing/codice/codice_l1a_lo_species.py index b7aa3f6f23..d5f9fbeef2 100644 --- a/imap_processing/codice/codice_l1a_lo_species.py +++ b/imap_processing/codice/codice_l1a_lo_species.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) -def l1a_lo_species( # noqa: PLR0912 +def l1a_lo_species( group_ds: xr.Dataset, lut_file: Path, table_id: str,