Merge pull request #8549 from mtkni/master

PERF: Slowness in multi-level indexes with datetime levels (part 2)

Author: jreback

pandas/core/index.py

@@ -2937,11 +2937,16 @@ def values(self):
 
         values = []
         for lev, lab in zip(self.levels, self.labels):
-            lev_values = lev.values
             # Need to box timestamps, etc.
-            if hasattr(lev, '_box_values'):
-                lev_values = lev._box_values(lev_values)
-            taken = com.take_1d(lev_values, lab)
+            box = hasattr(lev, '_box_values')
+            # Try to minimize boxing.
+            if box and len(lev) > len(lab):
+                taken = lev._box_values(com.take_1d(lev.values, lab))
+            elif box:
+                taken = com.take_1d(lev._box_values(lev.values), lab,
+                                    fill_value=_get_na_value(lev.dtype.type))
+            else:
+                taken = com.take_1d(lev.values, lab)
             values.append(taken)
 
         self._tuples = lib.fast_zip(values)

pandas/tests/test_index.py

@@ -2335,6 +2335,18 @@ def test_from_product_datetimeindex(self):
                                               (2, pd.Timestamp('2000-01-02'))])
         assert_array_equal(mi.values, etalon)
 
+    def test_values_boxed(self):
+        tuples = [(1, pd.Timestamp('2000-01-01')),
+                  (2, pd.NaT),
+                  (3, pd.Timestamp('2000-01-03')),
+                  (1, pd.Timestamp('2000-01-04')),
+                  (2, pd.Timestamp('2000-01-02')),
+                  (3, pd.Timestamp('2000-01-03'))]
+        mi = pd.MultiIndex.from_tuples(tuples)
+        assert_array_equal(mi.values, pd.lib.list_to_object_array(tuples))
+        # Check that code branches for boxed values produce identical results
+        assert_array_equal(mi.values[:4], mi[:4].values)
+
     def test_append(self):
         result = self.index[:3].append(self.index[3:])
         self.assertTrue(result.equals(self.index))

vb_suite/index_object.py

@@ -123,12 +123,19 @@
 
 setup = common_setup + """
 level1 = range(1000)
-level2 = date_range(start='1/1/2012', periods=10)
+level2 = date_range(start='1/1/2012', periods=100)
+mi = MultiIndex.from_product([level1, level2])
 """
 
-multiindex_with_datetime_level = \
-    Benchmark("MultiIndex.from_product([level1, level2]).values", setup,
-              name='multiindex_with_datetime_level',
+multiindex_with_datetime_level_full = \
+    Benchmark("mi.copy().values", setup,
+              name='multiindex_with_datetime_level_full',
+              start_date=datetime(2014, 10, 11))
+
+
+multiindex_with_datetime_level_sliced = \
+    Benchmark("mi[:10].values", setup,
+              name='multiindex_with_datetime_level_sliced',
               start_date=datetime(2014, 10, 11))
 
 #----------------------------------------------------------------------