ULTRA l1b: Validation result updates

imap_processing/tests/external_test_data_config.py

@@ -160,7 +160,7 @@
     ("status_test_data_repoint00047.csv", "ultra/data/l1/"),
     ("voltage_culling_results_repoint00047.csv", "ultra/data/l1/"),
     ("validate_high_energy_culling_results_repoint00047_v2.csv", "ultra/data/l1/"),
-    ("validate_stat_culling_results_repoint00047_v2.csv", "ultra/data/l1/"),
+    ("validate_stat_culling_results_repoint00047_v3.csv", "ultra/data/l1/"),
     ("validate_upstream_ion_1_culling_results_repoint00047_v1.csv", "ultra/data/l1/"),
     ("validate_spectral_culling_results_repoint00047_v1.csv", "ultra/data/l1/"),
     ("de_test_data_repoint00047.csv", "ultra/data/l1/"),

imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py

@@ -839,10 +839,9 @@ def test_flag_statistical_outliers_invalid_events():
         energy_range_edges,
         mask,
     )
-    # check that no flags are set because there were no valid events to calculate
-    # statistics on.
+    # check that all flags are set because the mask marks all events as invalid.
     np.testing.assert_array_equal(
-        quality_flags, np.zeros_like(quality_flags, dtype=bool)
+        quality_flags, np.ones_like(quality_flags, dtype=bool)
     )
     # check that all energy bins are marked as converged (no valid events is not a
     # failure case for convergence since we just can't calculate statistics.
@@ -873,11 +872,11 @@ def test_validate_stat_cull(setup_repoint_47_data):
     """Validate that statistical-outlier quality flags match expected results."""
     # read test data from csv files
     results_df = pd.read_csv(
-        TEST_PATH / "validate_stat_culling_results_repoint00047_v2.csv"
+        TEST_PATH / "validate_stat_culling_results_repoint00047_v3.csv"
     )
     de_ds, _, spin_tbin_edges = setup_repoint_47_data
     # Get the energy ranges
-    energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335])
+    energy_ranges = get_binned_energy_ranges(build_energy_bins()[0])
 
     # Create a mask of flagged events to test that the stat cull algorithm
     # properly ignores these. The test data was created using this exact mask as well.

imap_processing/ultra/constants.py

@@ -206,7 +206,7 @@ class UltraConstants:
     MAX_ENERGY_THRESHOLD = 116.0
     # Angle threshold in radians for ULTRA 45 degree culling.
     # This is only needed for ULTRA 45 since Earth may be in the FOV.
-    EARTH_ANGLE_45_THRESHOLD = np.radians(20)
+    EARTH_ANGLE_45_THRESHOLD = np.radians(15)
     # An array of energy thresholds to use for culling. Each one corresponds to
     # the number of energy bins used.
     # n_bins=len(PSET_ENERGY_BIN_EDGES)[BASE_CULL_EBIN:] // N_CULL_EBINS

imap_processing/ultra/l1b/extendedspin.py

@@ -102,9 +102,10 @@ def calculate_extendedspin(
         energy_thresholds,
         instrument_id,
     )
-    # Combine high energy and voltage flags to use for statistical outlier flagging.
-    mask = (
-        voltage_qf[np.newaxis, :] | high_energy_qf
+    # For the following culls, mask the spins that have already been flagged for
+    # low voltage
+    mask = np.repeat(
+        voltage_qf[np.newaxis, :], len(energy_ranges) - 1, axis=0
     )  # Shape (n_energy_bins, n_spins_bins)
     upstream_ion_qf_1 = flag_upstream_ion(
         de_dataset,
@@ -114,9 +115,6 @@ def calculate_extendedspin(
         UltraConstants.UPSTREAM_ION_ENERGY_CHANNELS_1,
         instrument_id,
     )
-    # Update mask to include upstream ion flags from the first set of energy channels
-    # before flagging with the second set of energy channels
-    mask = mask | upstream_ion_qf_1
     upstream_ion_qf_2 = flag_upstream_ion(
         de_dataset,
         spin_tbin_edges,
@@ -132,9 +130,9 @@ def calculate_extendedspin(
         UltraConstants.SPECTRAL_ENERGY_CHANNELS,
         instrument_id,
     )
-    # Update mask to include upstream ion flags #2 and spectral flags before flagging
-    # statistical outliers
-    mask = mask | upstream_ion_qf_2 | spectral_qf
+    # Update mask to include high energy,  upstream ion flags and spectral flags
+    # before flagging statistical outliers
+    mask = mask | upstream_ion_qf_1 | upstream_ion_qf_2 | spectral_qf | high_energy_qf
     stat_outliers_qf, _, _, _ = flag_statistical_outliers(
         de_dataset,
         spin_tbin_edges,
@@ -178,6 +176,12 @@ def calculate_extendedspin(
     stop_per_spin[valid] = pulses.stop_per_spin[idx[valid]]
     coin_per_spin[valid] = pulses.coin_per_spin[idx[valid]]
 
+    # To be consistent with the ULTRA IT implementation, apply the low voltage mask
+    # to the high energy and upstream ion flags
+    upstream_ion_qf_2 |= voltage_qf
+    upstream_ion_qf_1 |= voltage_qf
+    high_energy_qf |= voltage_qf
+    spectral_qf |= voltage_qf
     # high energy and statistical outlier flags are energy dependent boolean arrays
     # with shape (n_energy_bins, n_spin_bins). We want to collapse the energy dimension
     # using a bitwise OR to get a single boolean flag per spin.

imap_processing/ultra/l1b/ultra_l1b_culling.py

@@ -844,7 +844,7 @@ def flag_statistical_outliers(
     quality_stats: np.ndarray = np.zeros((n_energy_bins, spin_bin_size), dtype=bool)
     # Initialize a mask to keep track of spin bins that have been flagged across all
     # energy bins
-    all_channel_mask: np.ndarray = np.zeros(spin_bin_size, dtype=bool)
+    all_channel_mask: np.ndarray = curr_mask[0, :].copy()
     # Initialize convergence array to keep track of poisson stats
     convergence = np.full(n_energy_bins, False)
     # Keep track of how many iterations we have done of flagging outliers and