COMPAT: capture chain indexing even on single-dtyped

Author: jreback

doc/source/indexing.rst

@@ -1481,7 +1481,8 @@ which can take the values ``['raise','warn',None]``, where showing a warning is
                            'three', 'two', 'one', 'six'],
                     'c' : np.arange(7)})
 
-   # passed via reference (will stay)
+   # This will show the SettingWithCopyWarning
+   # but the frame values will be set
    dfb['c'][dfb.a.str.startswith('o')] = 42
 
 This however is operating on a copy and will not work.

pandas/core/generic.py

@@ -1088,11 +1088,10 @@ def _maybe_cache_changed(self, item, value):
     @property
     def _is_cached(self):
         """ boolean : return if I am cached """
-        cacher = getattr(self, '_cacher', None)
-        return cacher is not None
+        return getattr(self, '_cacher', None) is not None
 
     def _get_cacher(self):
-        """ return my cahcer or None """
+        """ return my cacher or None """
         cacher = getattr(self, '_cacher', None)
         if cacher is not None:
             cacher = cacher[1]()
@@ -1167,13 +1166,18 @@ def _check_is_chained_assignment_possible(self):
         if so, then force a setitem_copy check
 
         should be called just near setting a value
+
+        will return a boolean if it we are a view and are cached, but a single-dtype
+        meaning that the cacher should be updated following setting
         """
         if self._is_view and self._is_cached:
             ref = self._get_cacher()
             if ref is not None and ref._is_mixed_type:
-                self._check_setitem_copy(stacklevel=5, t='referant', force=True)
+                self._check_setitem_copy(stacklevel=4, t='referant', force=True)
+            return True
         elif self.is_copy:
-            self._check_setitem_copy(stacklevel=5, t='referant')
+            self._check_setitem_copy(stacklevel=4, t='referant')
+        return False
 
     def _check_setitem_copy(self, stacklevel=4, t='setting', force=False):
         """

pandas/core/series.py

@@ -587,61 +587,68 @@ def _get_values(self, indexer):
             return self.values[indexer]
 
     def __setitem__(self, key, value):
-        try:
-            self._set_with_engine(key, value)
-            return
-        except (SettingWithCopyError):
-            raise
-        except (KeyError, ValueError):
-            values = self.values
-            if (com.is_integer(key)
-                    and not self.index.inferred_type == 'integer'):
 
-                values[key] = value
+        def setitem(key, value):
+            try:
+                self._set_with_engine(key, value)
                 return
-            elif key is Ellipsis:
-                self[:] = value
+            except (SettingWithCopyError):
+                raise
+            except (KeyError, ValueError):
+                values = self.values
+                if (com.is_integer(key)
+                                and not self.index.inferred_type == 'integer'):
+
+                    values[key] = value
+                    return
+                elif key is Ellipsis:
+                    self[:] = value
+                    return
+                elif _is_bool_indexer(key):
+                    pass
+                elif com.is_timedelta64_dtype(self.dtype):
+                    # reassign a null value to iNaT
+                    if isnull(value):
+                        value = tslib.iNaT
+
+                        try:
+                            self.index._engine.set_value(self.values, key, value)
+                            return
+                        except (TypeError):
+                            pass
+
+                self.loc[key] = value
                 return
-            elif _is_bool_indexer(key):
-                pass
-            elif com.is_timedelta64_dtype(self.dtype):
-                # reassign a null value to iNaT
-                if isnull(value):
-                    value = tslib.iNaT
-
-                    try:
-                        self.index._engine.set_value(self.values, key, value)
-                        return
-                    except (TypeError):
-                        pass
-
-            self.loc[key] = value
-            return
 
-        except TypeError as e:
-            if isinstance(key, tuple) and not isinstance(self.index,
-                                                         MultiIndex):
-                raise ValueError("Can only tuple-index with a MultiIndex")
+            except TypeError as e:
+                if isinstance(key, tuple) and not isinstance(self.index,
+                                                             MultiIndex):
+                    raise ValueError("Can only tuple-index with a MultiIndex")
 
-            # python 3 type errors should be raised
-            if 'unorderable' in str(e):  # pragma: no cover
-                raise IndexError(key)
+                # python 3 type errors should be raised
+                if 'unorderable' in str(e):  # pragma: no cover
+                    raise IndexError(key)
 
-        if _is_bool_indexer(key):
-            key = _check_bool_indexer(self.index, key)
-            try:
-                self.where(~key, value, inplace=True)
-                return
-            except (InvalidIndexError):
-                pass
+            if _is_bool_indexer(key):
+                key = _check_bool_indexer(self.index, key)
+                try:
+                    self.where(~key, value, inplace=True)
+                    return
+                except (InvalidIndexError):
+                    pass
+
+            self._set_with(key, value)
 
-        self._set_with(key, value)
+        # do the setitem
+        cacher_needs_updating = self._check_is_chained_assignment_possible()
+        setitem(key, value)
+        if cacher_needs_updating:
+            self._maybe_update_cacher()
 
     def _set_with_engine(self, key, value):
         values = self.values
         try:
             self.index._engine.set_value(values, key, value)
-            self._check_is_chained_assignment_possible()
             return
         except KeyError:
             values[self.index.get_loc(key)] = value

pandas/io/tests/test_pytables.py

@@ -1278,8 +1278,8 @@ def test_append_with_data_columns(self):
             # data column selection with a string data_column
             df_new = df.copy()
             df_new['string'] = 'foo'
-            df_new['string'][1:4] = np.nan
-            df_new['string'][5:6] = 'bar'
+            df_new.loc[1:4,'string'] = np.nan
+            df_new.loc[5:6,'string'] = 'bar'
             _maybe_remove(store, 'df')
             store.append('df', df_new, data_columns=['string'])
             result = store.select('df', [Term('string=foo')])
@@ -1317,14 +1317,14 @@ def check_col(key,name,size):
         with ensure_clean_store(self.path) as store:
             # multiple data columns
             df_new = df.copy()
-            df_new.loc[:,'A'].iloc[0] = 1.
-            df_new.loc[:,'B'].iloc[0] = -1.
+            df_new.ix[0,'A'] = 1.
+            df_new.ix[0,'B'] = -1.
             df_new['string'] = 'foo'
-            df_new['string'][1:4] = np.nan
-            df_new['string'][5:6] = 'bar'
+            df_new.loc[1:4,'string'] = np.nan
+            df_new.loc[5:6,'string'] = 'bar'
             df_new['string2'] = 'foo'
-            df_new['string2'][2:5] = np.nan
-            df_new['string2'][7:8] = 'bar'
+            df_new.loc[2:5,'string2'] = np.nan
+            df_new.loc[7:8,'string2'] = 'bar'
             _maybe_remove(store, 'df')
             store.append(
                 'df', df_new, data_columns=['A', 'B', 'string', 'string2'])

pandas/tests/test_format.py

@@ -1348,8 +1348,8 @@ def test_to_string(self):
                             'B': tm.makeStringIndex(200)},
                            index=lrange(200))
 
-        biggie['A'][:20] = nan
-        biggie['B'][:20] = nan
+        biggie.loc[:20,'A'] = nan
+        biggie.loc[:20,'B'] = nan
         s = biggie.to_string()
 
         buf = StringIO()
@@ -1597,8 +1597,8 @@ def test_to_html(self):
                             'B': tm.makeStringIndex(200)},
                            index=lrange(200))
 
-        biggie['A'][:20] = nan
-        biggie['B'][:20] = nan
+        biggie.loc[:20,'A'] = nan
+        biggie.loc[:20,'B'] = nan
         s = biggie.to_html()
 
         buf = StringIO()
@@ -1624,8 +1624,8 @@ def test_to_html_filename(self):
                             'B': tm.makeStringIndex(200)},
                            index=lrange(200))
 
-        biggie['A'][:20] = nan
-        biggie['B'][:20] = nan
+        biggie.loc[:20,'A'] = nan
+        biggie.loc[:20,'B'] = nan
         with tm.ensure_clean('test.html') as path:
             biggie.to_html(path)
             with open(path, 'r') as f:

pandas/tests/test_frame.py

@@ -4438,8 +4438,8 @@ def test_repr_mixed_big(self):
         biggie = DataFrame({'A': randn(200),
                             'B': tm.makeStringIndex(200)},
                            index=lrange(200))
-        biggie['A'][:20] = nan
-        biggie['B'][:20] = nan
+        biggie.loc[:20,'A'] = nan
+        biggie.loc[:20,'B'] = nan
 
         foo = repr(biggie)
 
@@ -7412,19 +7412,19 @@ def test_drop(self):
         assert_frame_equal(df,expected)
 
     def test_fillna(self):
-        self.tsframe['A'][:5] = nan
-        self.tsframe['A'][-5:] = nan
+        self.tsframe.ix[:5,'A'] = nan
+        self.tsframe.ix[-5:,'A'] = nan
 
         zero_filled = self.tsframe.fillna(0)
-        self.assertTrue((zero_filled['A'][:5] == 0).all())
+        self.assertTrue((zero_filled.ix[:5,'A'] == 0).all())
 
         padded = self.tsframe.fillna(method='pad')
-        self.assertTrue(np.isnan(padded['A'][:5]).all())
-        self.assertTrue((padded['A'][-5:] == padded['A'][-5]).all())
+        self.assertTrue(np.isnan(padded.ix[:5,'A']).all())
+        self.assertTrue((padded.ix[-5:,'A'] == padded.ix[-5,'A']).all())
 
         # mixed type
-        self.mixed_frame['foo'][5:20] = nan
-        self.mixed_frame['A'][-10:] = nan
+        self.mixed_frame.ix[5:20,'foo'] = nan
+        self.mixed_frame.ix[-10:,'A'] = nan
         result = self.mixed_frame.fillna(value=0)
         result = self.mixed_frame.fillna(method='pad')
 
@@ -7433,7 +7433,7 @@ def test_fillna(self):
 
         # mixed numeric (but no float16)
         mf = self.mixed_float.reindex(columns=['A','B','D'])
-        mf['A'][-10:] = nan
+        mf.ix[-10:,'A'] = nan
         result = mf.fillna(value=0)
         _check_mixed_float(result, dtype = dict(C = None))
 
@@ -7605,8 +7605,8 @@ def test_replace_inplace(self):
         self.assertRaises(TypeError, self.tsframe.replace, nan)
 
         # mixed type
-        self.mixed_frame['foo'][5:20] = nan
-        self.mixed_frame['A'][-10:] = nan
+        self.mixed_frame.ix[5:20,'foo'] = nan
+        self.mixed_frame.ix[-10:,'A'] = nan
 
         result = self.mixed_frame.replace(np.nan, 0)
         expected = self.mixed_frame.fillna(value=0)
@@ -8194,8 +8194,8 @@ def test_replace_convert(self):
         assert_series_equal(expec, res)
 
     def test_replace_mixed(self):
-        self.mixed_frame['foo'][5:20] = nan
-        self.mixed_frame['A'][-10:] = nan
+        self.mixed_frame.ix[5:20,'foo'] = nan
+        self.mixed_frame.ix[-10:,'A'] = nan
 
         result = self.mixed_frame.replace(np.nan, -18)
         expected = self.mixed_frame.fillna(value=-18)
@@ -11717,11 +11717,11 @@ def test_rename_objects(self):
         self.assertNotIn('foo', renamed)
 
     def test_fill_corner(self):
-        self.mixed_frame['foo'][5:20] = nan
-        self.mixed_frame['A'][-10:] = nan
+        self.mixed_frame.ix[5:20,'foo'] = nan
+        self.mixed_frame.ix[-10:,'A'] = nan
 
         filled = self.mixed_frame.fillna(value=0)
-        self.assertTrue((filled['foo'][5:20] == 0).all())
+        self.assertTrue((filled.ix[5:20,'foo'] == 0).all())
         del self.mixed_frame['foo']
 
         empty_float = self.frame.reindex(columns=[])
@@ -12716,6 +12716,7 @@ def __nonzero__(self):
                 self.assertTrue(r1.all())
 
     def test_strange_column_corruption_issue(self):
+
         df = DataFrame(index=[0, 1])
         df[0] = nan
         wasCol = {}

pandas/tests/test_groupby.py

@@ -1664,7 +1664,7 @@ def test_cythonized_aggers(self):
                 'B': ['A', 'B'] * 6,
                 'C': np.random.randn(12)}
         df = DataFrame(data)
-        df['C'][2:10:2] = nan
+        df.loc[2:10:2,'C'] = nan
 
         def _testit(op):
             # single column

pandas/tests/test_indexing.py

@@ -3079,16 +3079,26 @@ def test_setitem_cache_updating(self):
 
         # GH 7084
         # not updating cache on series setting with slices
+        expected = DataFrame({'A': [600, 600, 600]}, index=date_range('5/7/2014', '5/9/2014'))
         out = DataFrame({'A': [0, 0, 0]}, index=date_range('5/7/2014', '5/9/2014'))
         df = DataFrame({'C': ['A', 'A', 'A'], 'D': [100, 200, 300]})
 
         #loop through df to update out
         six = Timestamp('5/7/2014')
         eix = Timestamp('5/9/2014')
         for ix, row in df.iterrows():
-            out[row['C']][six:eix] = out[row['C']][six:eix] + row['D']
+            out.loc[six:eix,row['C']] = out.loc[six:eix,row['C']] + row['D']
+
+        assert_frame_equal(out, expected)
+        assert_series_equal(out['A'], expected['A'])
+
+        # try via a chain indexing
+        # this actually works
+        out = DataFrame({'A': [0, 0, 0]}, index=date_range('5/7/2014', '5/9/2014'))
+        for ix, row in df.iterrows():
+            v = out[row['C']][six:eix] + row['D']
+            out[row['C']][six:eix] = v
 
-        expected = DataFrame({'A': [600, 600, 600]}, index=date_range('5/7/2014', '5/9/2014'))
         assert_frame_equal(out, expected)
         assert_series_equal(out['A'], expected['A'])
 
@@ -3176,8 +3186,6 @@ def f():
             indexer = df.a.str.startswith('o')
             df[indexer]['c'] = 42
         self.assertRaises(com.SettingWithCopyError, f)
-        df['c'][df.a.str.startswith('o')] = 42
-        assert_frame_equal(df,expected)
 
         expected = DataFrame({'A':[111,'bbb','ccc'],'B':[1,2,3]})
         df = DataFrame({'A':['aaa','bbb','ccc'],'B':[1,2,3]})
@@ -3187,6 +3195,8 @@ def f():
         def f():
             df.loc[0]['A'] = 111
         self.assertRaises(com.SettingWithCopyError, f)
+
+        df.loc[0,'A'] = 111
         assert_frame_equal(df,expected)
 
         # make sure that is_copy is picked up reconstruction