Merge pull request #7896 from seth-p/ewma_missing_weights_fix1

BUG: fixing ewma() for adjust=False and ignore_na=False

Author: jreback

pandas/algos.pyx

@@ -1018,7 +1018,10 @@ def ewma(ndarray[double_t] input, double_t com, int adjust, int ignore_na):
             if cur == cur:
                 old_wt *= old_wt_factor
                 weighted_avg = ((old_wt * weighted_avg) + (new_wt * cur)) / (old_wt + new_wt)
-                old_wt += new_wt
+                if adjust:
+                    old_wt += new_wt
+                else:
+                    old_wt = 1.
             elif not ignore_na:
                 old_wt *= old_wt_factor
         else:

pandas/stats/moments.py

@@ -107,6 +107,23 @@
 :math:`c = (s - 1) / 2`
 
 So a "20-day EWMA" would have center 9.5.
+
+When adjust is True (default), weighted averages are calculated using weights
+    (1-alpha)**(n-1), (1-alpha)**(n-2), ..., 1-alpha, 1.
+
+When adjust is False, weighted averages are calculated recursively as:
+    weighted_average[0] = arg[0];
+    weighted_average[i] = (1-alpha)*weighted_average[i-1] + alpha*arg[i].
+
+When ignore_na is False (default), weights are based on absolute positions.
+For example, the weights of x and y used in calculating the final weighted
+average of [x, None, y] are (1-alpha)**2 and 1 (if adjust is True), and
+(1-alpha)**2 and alpha (if adjust is False).
+
+When ignore_na is True (reproducing pre-0.15.0 behavior), weights are based on
+relative positions. For example, the weights of x and y used in calculating
+the final weighted average of [x, None, y] are 1-alpha and 1 (if adjust is 
+True), and 1-alpha and alpha (if adjust is False).
 """
 
 _expanding_kw = """min_periods : int, default None

pandas/stats/tests/test_moments.py

@@ -551,25 +551,30 @@ def test_ewma_nan_handling(self):
         s0 = Series([np.nan, 1., 101.])
         s1 = Series([1., np.nan, 101.])
         s2 = Series([np.nan, 1., np.nan, np.nan, 101., np.nan])
+        s3 = Series([1., np.nan, 101., 50.])
         com = 2.
         alpha = 1. / (1. + com)
 
         def simple_wma(s, w):
             return (s.multiply(w).cumsum() / w.cumsum()).fillna(method='ffill')
 
         for (s, adjust, ignore_na, w) in [
-                (s0, True, False, [np.nan, (1.0 - alpha), 1.]),
-                (s0, True, True, [np.nan, (1.0 - alpha), 1.]),
-                (s0, False, False, [np.nan, (1.0 - alpha), alpha]),
-                (s0, False, True, [np.nan, (1.0 - alpha), alpha]),
-                (s1, True, False, [(1.0 - alpha)**2, np.nan, 1.]),
-                (s1, True, True, [(1.0 - alpha), np.nan, 1.]),
-                (s1, False, False, [(1.0 - alpha)**2, np.nan, alpha]),
-                (s1, False, True, [(1.0 - alpha), np.nan, alpha]),
-                (s2, True, False, [np.nan, (1.0 - alpha)**3, np.nan, np.nan, 1., np.nan]),
-                (s2, True, True, [np.nan, (1.0 - alpha), np.nan, np.nan, 1., np.nan]),
-                (s2, False, False, [np.nan, (1.0 - alpha)**3, np.nan, np.nan, alpha, np.nan]),
-                (s2, False, True, [np.nan, (1.0 - alpha), np.nan, np.nan, alpha, np.nan]),
+                (s0, True, False, [np.nan, (1. - alpha), 1.]),
+                (s0, True, True, [np.nan, (1. - alpha), 1.]),
+                (s0, False, False, [np.nan, (1. - alpha), alpha]),
+                (s0, False, True, [np.nan, (1. - alpha), alpha]),
+                (s1, True, False, [(1. - alpha)**2, np.nan, 1.]),
+                (s1, True, True, [(1. - alpha), np.nan, 1.]),
+                (s1, False, False, [(1. - alpha)**2, np.nan, alpha]),
+                (s1, False, True, [(1. - alpha), np.nan, alpha]),
+                (s2, True, False, [np.nan, (1. - alpha)**3, np.nan, np.nan, 1., np.nan]),
+                (s2, True, True, [np.nan, (1. - alpha), np.nan, np.nan, 1., np.nan]),
+                (s2, False, False, [np.nan, (1. - alpha)**3, np.nan, np.nan, alpha, np.nan]),
+                (s2, False, True, [np.nan, (1. - alpha), np.nan, np.nan, alpha, np.nan]),
+                (s3, True, False, [(1. - alpha)**3, np.nan, (1. - alpha), 1.]),
+                (s3, True, True, [(1. - alpha)**2, np.nan, (1. - alpha), 1.]),
+                (s3, False, False, [(1. - alpha)**3, np.nan, (1. - alpha) * alpha, alpha * ((1. - alpha)**2 + alpha)]),
+                (s3, False, True, [(1. - alpha)**2, np.nan, (1. - alpha) * alpha, alpha]),
                 ]:
             expected = simple_wma(s, Series(w))
             result = mom.ewma(s, com=com, adjust=adjust, ignore_na=ignore_na)