Add the introduction part to Snippet Tutorial (#397)

* Add the introduction part

* Add textual context and Improve the flow and figures

* [WIP] Revise the intro section and legend of figures

* Add the introduction part

* Add textual context and Improve the flow and figures

* [WIP] Revise the intro section and legend of figures

* [WIP]: save temp changes

* add the slices of indices for each snippet in the output

* use stumpy.mplstyle to refactor the rcparams

* remove blank line after docstring of a function

* add the plot of snippets regime to check out the functionality of snippets_regime added to snippets modules

* correct the block of code for plotting the snippets regimes

* Change snippet_regime from list to numpy array

* update the notebook according to the updated version of snippet module

* resolve issues raised by flake8

* remove snippet tutorial local file from its developing branch tto allow merge from the main

* Copy back Snippet Tutorial notebook to the Snipperts_Tutorial branch

* allow git to track the file (?)

* copy back Snippet Tutorial notebook

* checking out older version of snippet tutorial notebook

* checkout file in a parent

* get back to the ours modified version of the notebook file

* remove snippet tutorial notebook on the branch

* add snippet tutorial notebook from ancestor common file

* handle merge manually: add changes of ours to theirs version of the file

* move the function _get_mask_slices(mask) to core module

* revise the snippet_regime calculation: use list to append new slices since it is faster than appending to numpy array

* slight modification on snippet_regime calculation

* revise the notebook since the structure of snippet_regimes, returned by the snippets and aampdist_snippets modules, is changed

* add dtype=object according to numpy warning. Revise docstring to include snippets_regime

* Run all cells of notebooks to make sure things are okay

* revise docstring (small modification)

* revise the snippet_regime caclulation to get 2D NumPy array

* revise slicing the snippets regimes returned by the snippet module

* create a new variable to reduce the length of lines of code that caused error in flake8

* add unit test for function _get_mask_slices

* resolve indentation. and, move naive implementation to the TOP of the test file

* move the naive implementation of _get_mask_slices function to naive module

* add snippets_regimes in the naive implementation mpdist_snippets and aampdist_snippet

* add snippet_regime test to the unit test

* add snippet_regime to the unit test

* minor modification on the format

* reformat the code to follow Black code formatter style

* add snippet_regimes to the non_normalized decorator unit test

Author: NimaSarajpoor

docs/Tutorial_Time_Series_Snippets.ipynb

@@ -4,7 +4,7 @@
 
 import math
 import numpy as np
-from .core import check_window_size
+from .core import check_window_size, _get_mask_slices
 from .aampdist import _aampdist_vect
 
 
@@ -178,6 +178,9 @@ def aampdist_snippets(
         The area under the curve corresponding to each profile for each of the top `k`
         snippets
 
+    snippets_regimes: ndarray
+        The slices of indices that show the starting and ending indices of snippets
+
     Notes
     -----
     `DOI: 10.1109/ICBK.2018.00058 \
@@ -213,6 +216,7 @@ def aampdist_snippets(
     snippets_areas = np.empty(k)
     Q = np.full(D.shape[-1], np.inf)
     indices = np.arange(0, n_padded - m, m)
+    snippets_regimes_list = []
 
     for i in range(k):
         profile_areas = np.sum(np.minimum(D, Q), axis=1)
@@ -231,11 +235,19 @@ def aampdist_snippets(
         mask = snippets_profiles[i] <= total_min
         snippets_fractions[i] = np.sum(mask) / total_min.shape[0]
         total_min = total_min - mask.astype(np.float64)
+        slices = _get_mask_slices(mask)
+        snippets_regimes_list.append(slices)
+
+    n_slices = [regime.shape[0] for regime in snippets_regimes_list]
+    snippets_regimes = np.empty((sum(n_slices), 3), dtype=object)
+    snippets_regimes[:, 0] = np.repeat(np.arange(len(snippets_regimes_list)), n_slices)
+    snippets_regimes[:, 1:] = np.vstack(snippets_regimes_list)
 
     return (
         snippets,
         snippets_indices,
         snippets_profiles,
         snippets_fractions,
         snippets_areas,
+        snippets_regimes,
     )

stumpy/aampdist_snippets.py

@@ -1806,3 +1806,31 @@ def _jagged_list_to_array(a, fill_value, dtype):
         out[i, : row.size] = row
 
     return out
+
+
+def _get_mask_slices(mask):
+    """
+    For a boolean vector mask, returns the slices of indices at which the mask is True.
+
+    Parameters
+    ----------
+    mask: ndarray
+        A boolean 1D array
+
+    Returns
+    -------
+    slices: ndarray
+        slices of indices where the mask is True. Each slice has a size of two:
+        The first number is the start index (inclusive)
+        The second number is the end index (exclusive)
+
+    """
+    m1 = np.r_[0, mask]
+    m2 = np.r_[mask, 0]
+
+    (starts,) = np.where(~m1 & m2)
+    (ends,) = np.where(m1 & ~m2)
+
+    slices = np.c_[starts, ends]
+
+    return slices

stumpy/core.py

@@ -5,7 +5,7 @@
 import math
 import numpy as np
 from . import core
-from .core import check_window_size
+from .core import check_window_size, _get_mask_slices
 from .mpdist import _mpdist_vect
 from .aampdist_snippets import aampdist_snippets
 
@@ -187,6 +187,9 @@ def snippets(
         The area under the curve corresponding to each profile for each of the top `k`
         snippets
 
+    snippets_regimes: ndarray
+        The slices of indices that show the starting and ending indices of snippets
+
     Notes
     -----
     `DOI: 10.1109/ICBK.2018.00058 \
@@ -222,6 +225,7 @@ def snippets(
     snippets_areas = np.empty(k)
     Q = np.full(D.shape[-1], np.inf)
     indices = np.arange(0, n_padded - m, m)
+    snippets_regimes_list = []
 
     for i in range(k):
         profile_areas = np.sum(np.minimum(D, Q), axis=1)
@@ -240,11 +244,19 @@ def snippets(
         mask = snippets_profiles[i] <= total_min
         snippets_fractions[i] = np.sum(mask) / total_min.shape[0]
         total_min = total_min - mask.astype(np.float64)
+        slices = _get_mask_slices(mask)
+        snippets_regimes_list.append(slices)
+
+    n_slices = [regime.shape[0] for regime in snippets_regimes_list]
+    snippets_regimes = np.empty((sum(n_slices), 3), dtype=object)
+    snippets_regimes[:, 0] = np.repeat(np.arange(len(snippets_regimes_list)), n_slices)
+    snippets_regimes[:, 1:] = np.vstack(snippets_regimes_list)
 
     return (
         snippets,
         snippets_indices,
         snippets_profiles,
         snippets_fractions,
         snippets_areas,
+        snippets_regimes,
     )

stumpy/snippets.py

@@ -1110,6 +1110,7 @@ def mpdist_snippets(
     snippets_areas = np.empty(k)
     Q = np.inf
     indices = np.arange(0, n_padded - m, m)
+    snippets_regimes_list = []
 
     for snippet_idx in range(k):
         min_area = np.inf
@@ -1132,13 +1133,21 @@ def mpdist_snippets(
         mask = snippets_profiles[i] <= total_min
         snippets_fractions[i] = np.sum(mask) / total_min.shape[0]
         total_min = total_min - mask.astype(float)
+        slices = _get_mask_slices(mask)
+        snippets_regimes_list.append(slices)
+
+    n_slices = [regime.shape[0] for regime in snippets_regimes_list]
+    snippets_regimes = np.empty((sum(n_slices), 3), dtype=object)
+    snippets_regimes[:, 0] = np.repeat(np.arange(len(snippets_regimes_list)), n_slices)
+    snippets_regimes[:, 1:] = np.vstack(snippets_regimes_list)
 
     return (
         snippets,
         snippets_indices,
         snippets_profiles,
         snippets_fractions,
         snippets_areas,
+        snippets_regimes,
     )
 
 
@@ -1173,6 +1182,7 @@ def aampdist_snippets(
     snippets_areas = np.empty(k)
     Q = np.inf
     indices = np.arange(0, n_padded - m, m)
+    snippets_regimes_list = []
 
     for snippet_idx in range(k):
         min_area = np.inf
@@ -1195,13 +1205,21 @@ def aampdist_snippets(
         mask = snippets_profiles[i] <= total_min
         snippets_fractions[i] = np.sum(mask) / total_min.shape[0]
         total_min = total_min - mask.astype(float)
+        slices = _get_mask_slices(mask)
+        snippets_regimes_list.append(slices)
+
+    n_slices = [regime.shape[0] for regime in snippets_regimes_list]
+    snippets_regimes = np.empty((sum(n_slices), 3), dtype=object)
+    snippets_regimes[:, 0] = np.repeat(np.arange(len(snippets_regimes_list)), n_slices)
+    snippets_regimes[:, 1:] = np.vstack(snippets_regimes_list)
 
     return (
         snippets,
         snippets_indices,
         snippets_profiles,
         snippets_fractions,
         snippets_areas,
+        snippets_regimes,
     )
 
 
@@ -1351,3 +1369,19 @@ def transform_pan(pan, ms, threshold, bfs_indices, n_processed):
     pan[np.isnan(pan)] = np.nanmax(pan)
 
     return pan
+
+
+def _get_mask_slices(mask):
+    idx = []
+
+    tmp = np.r_[0, mask]
+    for i, val in enumerate(np.diff(tmp)):
+        if val == 1:
+            idx.append(i)
+        if val == -1:
+            idx.append(i)
+
+    if tmp[-1]:
+        idx.append(len(mask))
+
+    return np.array(idx).reshape(len(idx) // 2, 2)

tests/naive.py

@@ -23,13 +23,15 @@ def test_aampdist_snippets(T, m, k):
         ref_profiles,
         ref_fractions,
         ref_areas,
+        ref_regimes,
     ) = naive.aampdist_snippets(T, m, k)
     (
         cmp_snippets,
         cmp_indices,
         cmp_profiles,
         cmp_fractions,
         cmp_areas,
+        cmp_regimes,
     ) = aampdist_snippets(T, m, k)
 
     npt.assert_almost_equal(
@@ -47,6 +49,9 @@ def test_aampdist_snippets(T, m, k):
     # npt.assert_almost_equal(
     #     ref_areas, cmp_areas, decimal=config.STUMPY_TEST_PRECISION
     # )
+    npt.assert_almost_equal(
+        ref_regimes, cmp_regimes, decimal=config.STUMPY_TEST_PRECISION
+    )
 
 
 @pytest.mark.parametrize("T", test_data)
@@ -60,13 +65,15 @@ def test_mpdist_snippets_percentage(T, m, k, percentage):
         ref_profiles,
         ref_fractions,
         ref_areas,
+        ref_regimes,
     ) = naive.aampdist_snippets(T, m, k, percentage=percentage)
     (
         cmp_snippets,
         cmp_indices,
         cmp_profiles,
         cmp_fractions,
         cmp_areas,
+        cmp_regimes,
     ) = aampdist_snippets(T, m, k, percentage=percentage)
 
     npt.assert_almost_equal(
@@ -84,6 +91,9 @@ def test_mpdist_snippets_percentage(T, m, k, percentage):
     # npt.assert_almost_equal(
     #     ref_areas, cmp_areas, decimal=config.STUMPY_TEST_PRECISION
     # )
+    npt.assert_almost_equal(
+        ref_regimes, cmp_regimes, decimal=config.STUMPY_TEST_PRECISION
+    )
 
 
 @pytest.mark.parametrize("T", test_data)
@@ -97,13 +107,15 @@ def test_mpdist_snippets_s(T, m, k, s):
         ref_profiles,
         ref_fractions,
         ref_areas,
+        ref_regimes,
     ) = naive.aampdist_snippets(T, m, k, s=s)
     (
         cmp_snippets,
         cmp_indices,
         cmp_profiles,
         cmp_fractions,
         cmp_areas,
+        cmp_regimes,
     ) = aampdist_snippets(T, m, k, s=s)
 
     npt.assert_almost_equal(
@@ -121,3 +133,6 @@ def test_mpdist_snippets_s(T, m, k, s):
     # npt.assert_almost_equal(
     #     ref_areas, cmp_areas, decimal=config.STUMPY_TEST_PRECISION
     # )
+    npt.assert_almost_equal(
+        ref_regimes, cmp_regimes, decimal=config.STUMPY_TEST_PRECISION
+    )

tests/test_aampdist_snippets.py

@@ -827,3 +827,19 @@ def test_jagged_list_to_array_empty():
     left = np.array([[]], dtype="float64")
     right = core._jagged_list_to_array(arr, fill_value=np.nan, dtype="float64")
     npt.assert_array_equal(left, right)
+
+
+def test_get_mask_slices():
+    bool_lst = [False, True]
+    mask_cases = [
+        [x, y, z, w]
+        for x in bool_lst
+        for y in bool_lst
+        for z in bool_lst
+        for w in bool_lst
+    ]
+
+    for mask in mask_cases:
+        ref_slices = naive._get_mask_slices(mask)
+        comp_slices = core._get_mask_slices(mask)
+        npt.assert_array_equal(ref_slices, comp_slices)

tests/test_core.py

@@ -300,12 +300,14 @@ def test_snippets():
         ref_profiles,
         ref_fractions,
         ref_areas,
+        ref_regimes,
     ) = stumpy.aampdist_snippets(T, m, k)
     (
         cmp_snippets,
         cmp_indices,
         cmp_profiles,
         cmp_fractions,
         cmp_areas,
+        cmp_regimes,
     ) = stumpy.snippets(T, m, k, normalize=False)
     npt.assert_almost_equal(ref_snippets, cmp_snippets)