Merge "Allow profiling of animation performance"

Author: ChrisCraik

tests/TileBenchmark/res/values/strings.xml

@@ -49,8 +49,9 @@
     <!-- Drop down menu entry - automatically scroll to the end of the page
     with scrollBy() [CHAR LIMIT=15] -->
     <string name="movement_auto_scroll">Auto-scroll</string>
-    <!-- Drop down menu entry -  [CHAR LIMIT=15] -->
-    <string name="movement_auto_fling">Auto-fling</string>
+    <!-- Drop down menu entry - automatically record for a set time before
+    stopping [CHAR LIMIT=15] -->
+    <string name="movement_timed">Timed</string>
     <!-- Drop down menu entry - manually navigate the page(s), hit 'capture'
     button [CHAR LIMIT=15] -->
     <string name="movement_manual">Manual</string>
@@ -67,14 +68,21 @@
     <!-- 75th percentile - 75% of frames fall below this value [CHAR LIMIT=12]
     -->
     <string name="percentile_75">75%ile</string>
+    <!-- standard deviation [CHAR LIMIT=12] -->
+    <string name="std_dev">StdDev</string>
+    <!-- mean [CHAR LIMIT=12] -->
+    <string name="mean">mean</string>
+
+
+
     <!-- Frame rate [CHAR LIMIT=15] -->
     <string name="frames_per_second">Frames/sec</string>
     <!-- Portion of viewport covered by good tiles [CHAR LIMIT=15] -->
     <string name="viewport_coverage">Coverage</string>
     <!-- Milliseconds taken to inval, and re-render the page [CHAR LIMIT=15] -->
     <string name="render_millis">RenderMillis</string>
-    <!-- Number of rendering stalls while running the test [CHAR LIMIT=15] -->
-    <string name="render_stalls">Stalls</string>
+    <!-- Animation Framerate [CHAR LIMIT=15] -->
+    <string name="animation_framerate">AnimFramerate</string>
     <!-- Format string for stat value overlay [CHAR LIMIT=15] -->
     <string name="format_stat">%4.4f</string>
 

tests/TileBenchmark/src/com/test/tilebenchmark/PerformanceTest.java

@@ -27,17 +27,57 @@
 import android.os.Environment;
 import android.test.ActivityInstrumentationTestCase2;
 import android.util.Log;
+import android.webkit.WebSettings;
+import android.widget.Spinner;
 
 public class PerformanceTest extends
         ActivityInstrumentationTestCase2<ProfileActivity> {
 
+    public static class AnimStat {
+        double aggVal = 0;
+        double aggSqrVal = 0;
+        double count = 0;
+    }
+
     private class StatAggregator extends PlaybackGraphs {
         private HashMap<String, Double> mDataMap = new HashMap<String, Double>();
+        private HashMap<String, AnimStat> mAnimDataMap = new HashMap<String, AnimStat>();
         private int mCount = 0;
 
+
         public void aggregate() {
+            boolean inAnimTests = mAnimTests != null;
+            Resources resources = mWeb.getResources();
+            String animFramerateString = resources.getString(R.string.animation_framerate);
+            for (Map.Entry<String, Double> e : mSingleStats.entrySet()) {
+                String name = e.getKey();
+                if (inAnimTests) {
+                    if (name.equals(animFramerateString)) {
+                        // in animation testing phase, record animation framerate and aggregate
+                        // stats, differentiating on values of mAnimTestNr and mDoubleBuffering
+                        String fullName = ANIM_TEST_NAMES[mAnimTestNr] + " " + name;
+                        fullName += mDoubleBuffering ? " tiled" : " webkit";
+
+                        if (!mAnimDataMap.containsKey(fullName)) {
+                            mAnimDataMap.put(fullName, new AnimStat());
+                        }
+                        AnimStat statVals = mAnimDataMap.get(fullName);
+                        statVals.aggVal += e.getValue();
+                        statVals.aggSqrVal += e.getValue() * e.getValue();
+                        statVals.count += 1;
+                    }
+                } else {
+                    double aggVal = mDataMap.containsKey(name)
+                            ? mDataMap.get(name) : 0;
+                    mDataMap.put(name, aggVal + e.getValue());
+                }
+            }
+
+            if (inAnimTests) {
+                return;
+            }
+
             mCount++;
-            Resources resources = mView.getResources();
             for (int metricIndex = 0; metricIndex < Metrics.length; metricIndex++) {
                 for (int statIndex = 0; statIndex < Stats.length; statIndex++) {
                     String metricLabel = resources.getString(
@@ -53,34 +93,47 @@ public void aggregate() {
                     mDataMap.put(label, aggVal);
                 }
             }
-            for (Map.Entry<String, Double> e : mSingleStats.entrySet()) {
-                double aggVal = mDataMap.containsKey(e.getKey())
-                        ? mDataMap.get(e.getKey()) : 0;
-                mDataMap.put(e.getKey(), aggVal + e.getValue());
-            }
+
         }
 
+        // build the final bundle of results
         public Bundle getBundle() {
             Bundle b = new Bundle();
-            int count = 0 == mCount ? Integer.MAX_VALUE : mCount;
+            int count = (0 == mCount) ? Integer.MAX_VALUE : mCount;
             for (Map.Entry<String, Double> e : mDataMap.entrySet()) {
                 b.putDouble(e.getKey(), e.getValue() / count);
             }
+
+            for (Map.Entry<String, AnimStat> e : mAnimDataMap.entrySet()) {
+                String statName = e.getKey();
+                AnimStat statVals = e.getValue();
+
+                double avg = statVals.aggVal/statVals.count;
+                double stdDev = Math.sqrt((statVals.aggSqrVal / statVals.count) - avg * avg);
+
+                b.putDouble(statName, avg);
+                b.putDouble(statName + " STD DEV", stdDev);
+            }
+
             return b;
         }
     }
 
     ProfileActivity mActivity;
-    ProfiledWebView mView;
-    StatAggregator mStats = new StatAggregator();
+    ProfiledWebView mWeb;
+    Spinner mMovementSpinner;
+    StatAggregator mStats;
 
     private static final String LOGTAG = "PerformanceTest";
     private static final String TEST_LOCATION = "webkit/page_cycler";
     private static final String URL_PREFIX = "file://";
     private static final String URL_POSTFIX = "/index.html?skip=true";
     private static final int MAX_ITERATIONS = 4;
-    private static final String TEST_DIRS[] = {
-        "alexa25_2011"//, "alexa_us", "android", "dom", "intl2", "moz", "moz2"
+    private static final String SCROLL_TEST_DIRS[] = {
+        "alexa25_2011"
+    };
+    private static final String ANIM_TEST_DIRS[] = {
+        "dhtml"
     };
 
     public PerformanceTest() {
@@ -91,7 +144,22 @@ public PerformanceTest() {
     protected void setUp() throws Exception {
         super.setUp();
         mActivity = getActivity();
-        mView = (ProfiledWebView) mActivity.findViewById(R.id.web);
+        mWeb = (ProfiledWebView) mActivity.findViewById(R.id.web);
+        mMovementSpinner = (Spinner) mActivity.findViewById(R.id.movement);
+        mStats = new StatAggregator();
+
+        // use mStats as a condition variable between the UI thread and
+        // this(the testing) thread
+        mActivity.setCallback(new ProfileCallback() {
+            @Override
+            public void profileCallback(RunData data) {
+                mStats.setData(data);
+                synchronized (mStats) {
+                    mStats.notify();
+                }
+            }
+        });
+
     }
 
     private boolean loadUrl(final String url) {
@@ -100,12 +168,13 @@ private boolean loadUrl(final String url) {
             mActivity.runOnUiThread(new Runnable() {
                 @Override
                 public void run() {
-                    mView.loadUrl(url);
+                    mWeb.loadUrl(url);
                 }
             });
             synchronized (mStats) {
                 mStats.wait();
             }
+
             mStats.aggregate();
         } catch (InterruptedException e) {
             e.printStackTrace();
@@ -114,15 +183,30 @@ public void run() {
         return true;
     }
 
-    private boolean runIteration() {
+    private boolean validTest(String nextTest) {
+        // if testing animations, test must be in mAnimTests
+        if (mAnimTests == null)
+            return true;
+
+        for (String test : mAnimTests) {
+            if (test.equals(nextTest)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean runIteration(String[] testDirs) {
         File sdFile = Environment.getExternalStorageDirectory();
-        for (String testDirName : TEST_DIRS) {
+        for (String testDirName : testDirs) {
             File testDir = new File(sdFile, TEST_LOCATION + "/" + testDirName);
             Log.d(LOGTAG, "Testing dir: '" + testDir.getAbsolutePath()
                     + "', exists=" + testDir.exists());
+
             for (File siteDir : testDir.listFiles()) {
-                if (!siteDir.isDirectory())
+                if (!siteDir.isDirectory() || !validTest(siteDir.getName())) {
                     continue;
+                }
 
                 if (!loadUrl(URL_PREFIX + siteDir.getAbsolutePath()
                         + URL_POSTFIX)) {
@@ -133,35 +217,88 @@ private boolean runIteration() {
         return true;
     }
 
-    public void testMetrics() {
+    private boolean  runTestDirs(String[] testDirs) {
+        for (int i = 0; i < MAX_ITERATIONS; i++)
+            if (!runIteration(testDirs)) {
+                return false;
+            }
+        return true;
+    }
+
+    private void pushDoubleBuffering() {
+        getInstrumentation().runOnMainSync(new Runnable() {
+            public void run() {
+                mWeb.setDoubleBuffering(mDoubleBuffering);
+            }
+        });
+    }
+
+    private void setScrollingTestingMode(final boolean scrolled) {
+        getInstrumentation().runOnMainSync(new Runnable() {
+            public void run() {
+                mMovementSpinner.setSelection(scrolled ? 0 : 2);
+            }
+        });
+    }
+
+
+    private String[] mAnimTests = null;
+    private int mAnimTestNr = -1;
+    private boolean mDoubleBuffering = true;
+    private static final String[] ANIM_TEST_NAMES = {
+        "slow", "fast"
+    };
+    private static final String[][] ANIM_TESTS = {
+        {"scrolling", "replaceimages", "layers5", "layers1"},
+        {"slidingballs", "meter", "slidein", "fadespacing", "colorfade",
+                "mozilla", "movingtext", "diagball", "zoom", "imageslide"},
+    };
+
+    private boolean checkMedia() {
         String state = Environment.getExternalStorageState();
 
         if (!Environment.MEDIA_MOUNTED.equals(state)
                 && !Environment.MEDIA_MOUNTED_READ_ONLY.equals(state)) {
             Log.d(LOGTAG, "ARG Can't access sd card!");
             // Can't read the SD card, fail and die!
             getInstrumentation().sendStatus(1, null);
-            return;
+            return false;
         }
+        return true;
+    }
 
-        // use mGraphs as a condition variable between the UI thread and
-        // this(the testing) thread
-        mActivity.setCallback(new ProfileCallback() {
-            @Override
-            public void profileCallback(RunData data) {
-                Log.d(LOGTAG, "test completion callback");
-                mStats.setData(data);
-                synchronized (mStats) {
-                    mStats.notify();
+    public void testMetrics() {
+        setScrollingTestingMode(true);
+        if (checkMedia() && runTestDirs(SCROLL_TEST_DIRS)) {
+            getInstrumentation().sendStatus(0, mStats.getBundle());
+        } else {
+            getInstrumentation().sendStatus(1, null);
+        }
+    }
+
+    private boolean runAnimationTests() {
+        for (int doubleBuffer = 0; doubleBuffer <= 1; doubleBuffer++) {
+            mDoubleBuffering = doubleBuffer == 1;
+            pushDoubleBuffering();
+            for (mAnimTestNr = 0; mAnimTestNr < ANIM_TESTS.length; mAnimTestNr++) {
+                mAnimTests = ANIM_TESTS[mAnimTestNr];
+                if (!runTestDirs(ANIM_TEST_DIRS)) {
+                    return false;
                 }
             }
-        });
+        }
+        return true;
+    }
 
-        for (int i = 0; i < MAX_ITERATIONS; i++)
-            if (!runIteration()) {
-                getInstrumentation().sendStatus(1, null);
-                return;
-            }
-        getInstrumentation().sendStatus(0, mStats.getBundle());
+    public void testAnimations() {
+        // instead of autoscrolling, load each page until either an timer fires,
+        // or the animation signals complete via javascript
+        setScrollingTestingMode(false);
+
+        if (checkMedia() && runAnimationTests()) {
+            getInstrumentation().sendStatus(0, mStats.getBundle());
+        } else {
+            getInstrumentation().sendStatus(1, null);
+        }
     }
 }