diff --git a/imap_processing/tests/external_test_data_config.py b/imap_processing/tests/external_test_data_config.py index 3f6b3de60a..b962bb96d9 100644 --- a/imap_processing/tests/external_test_data_config.py +++ b/imap_processing/tests/external_test_data_config.py @@ -160,7 +160,7 @@ ("status_test_data_repoint00047.csv", "ultra/data/l1/"), ("voltage_culling_results_repoint00047.csv", "ultra/data/l1/"), ("validate_high_energy_culling_results_repoint00047_v2.csv", "ultra/data/l1/"), - ("validate_stat_culling_results_repoint00047_v2.csv", "ultra/data/l1/"), + ("validate_stat_culling_results_repoint00047_v3.csv", "ultra/data/l1/"), ("validate_upstream_ion_1_culling_results_repoint00047_v1.csv", "ultra/data/l1/"), ("validate_spectral_culling_results_repoint00047_v1.csv", "ultra/data/l1/"), ("de_test_data_repoint00047.csv", "ultra/data/l1/"), diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index 1df4845720..94852f9b9b 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -839,10 +839,9 @@ def test_flag_statistical_outliers_invalid_events(): energy_range_edges, mask, ) - # check that no flags are set because there were no valid events to calculate - # statistics on. + # check that all flags are set because the mask marks all events as invalid. np.testing.assert_array_equal( - quality_flags, np.zeros_like(quality_flags, dtype=bool) + quality_flags, np.ones_like(quality_flags, dtype=bool) ) # check that all energy bins are marked as converged (no valid events is not a # failure case for convergence since we just can't calculate statistics. @@ -873,11 +872,11 @@ def test_validate_stat_cull(setup_repoint_47_data): """Validate that statistical-outlier quality flags match expected results.""" # read test data from csv files results_df = pd.read_csv( - TEST_PATH / "validate_stat_culling_results_repoint00047_v2.csv" + TEST_PATH / "validate_stat_culling_results_repoint00047_v3.csv" ) de_ds, _, spin_tbin_edges = setup_repoint_47_data # Get the energy ranges - energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) + energy_ranges = get_binned_energy_ranges(build_energy_bins()[0]) # Create a mask of flagged events to test that the stat cull algorithm # properly ignores these. The test data was created using this exact mask as well. diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index a4a0e0eb79..5112204267 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -206,7 +206,7 @@ class UltraConstants: MAX_ENERGY_THRESHOLD = 116.0 # Angle threshold in radians for ULTRA 45 degree culling. # This is only needed for ULTRA 45 since Earth may be in the FOV. - EARTH_ANGLE_45_THRESHOLD = np.radians(20) + EARTH_ANGLE_45_THRESHOLD = np.radians(15) # An array of energy thresholds to use for culling. Each one corresponds to # the number of energy bins used. # n_bins=len(PSET_ENERGY_BIN_EDGES)[BASE_CULL_EBIN:] // N_CULL_EBINS diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index 147b542afc..8fdd394999 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -102,9 +102,10 @@ def calculate_extendedspin( energy_thresholds, instrument_id, ) - # Combine high energy and voltage flags to use for statistical outlier flagging. - mask = ( - voltage_qf[np.newaxis, :] | high_energy_qf + # For the following culls, mask the spins that have already been flagged for + # low voltage + mask = np.repeat( + voltage_qf[np.newaxis, :], len(energy_ranges) - 1, axis=0 ) # Shape (n_energy_bins, n_spins_bins) upstream_ion_qf_1 = flag_upstream_ion( de_dataset, @@ -114,9 +115,6 @@ def calculate_extendedspin( UltraConstants.UPSTREAM_ION_ENERGY_CHANNELS_1, instrument_id, ) - # Update mask to include upstream ion flags from the first set of energy channels - # before flagging with the second set of energy channels - mask = mask | upstream_ion_qf_1 upstream_ion_qf_2 = flag_upstream_ion( de_dataset, spin_tbin_edges, @@ -132,9 +130,9 @@ def calculate_extendedspin( UltraConstants.SPECTRAL_ENERGY_CHANNELS, instrument_id, ) - # Update mask to include upstream ion flags #2 and spectral flags before flagging - # statistical outliers - mask = mask | upstream_ion_qf_2 | spectral_qf + # Update mask to include high energy, upstream ion flags and spectral flags + # before flagging statistical outliers + mask = mask | upstream_ion_qf_1 | upstream_ion_qf_2 | spectral_qf | high_energy_qf stat_outliers_qf, _, _, _ = flag_statistical_outliers( de_dataset, spin_tbin_edges, @@ -178,6 +176,12 @@ def calculate_extendedspin( stop_per_spin[valid] = pulses.stop_per_spin[idx[valid]] coin_per_spin[valid] = pulses.coin_per_spin[idx[valid]] + # To be consistent with the ULTRA IT implementation, apply the low voltage mask + # to the high energy and upstream ion flags + upstream_ion_qf_2 |= voltage_qf + upstream_ion_qf_1 |= voltage_qf + high_energy_qf |= voltage_qf + spectral_qf |= voltage_qf # high energy and statistical outlier flags are energy dependent boolean arrays # with shape (n_energy_bins, n_spin_bins). We want to collapse the energy dimension # using a bitwise OR to get a single boolean flag per spin. diff --git a/imap_processing/ultra/l1b/ultra_l1b_culling.py b/imap_processing/ultra/l1b/ultra_l1b_culling.py index 2261cc9039..3ec5c2a9b0 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_culling.py +++ b/imap_processing/ultra/l1b/ultra_l1b_culling.py @@ -844,7 +844,7 @@ def flag_statistical_outliers( quality_stats: np.ndarray = np.zeros((n_energy_bins, spin_bin_size), dtype=bool) # Initialize a mask to keep track of spin bins that have been flagged across all # energy bins - all_channel_mask: np.ndarray = np.zeros(spin_bin_size, dtype=bool) + all_channel_mask: np.ndarray = curr_mask[0, :].copy() # Initialize convergence array to keep track of poisson stats convergence = np.full(n_energy_bins, False) # Keep track of how many iterations we have done of flagging outliers and