Sample atomic network stats buckets, full poll.

When sampling network stats, always use atomic buckets instead of
interpolating.  Always poll iface and UID together so we distribute
into buckets equally.  Move stale bucket trimming to just before
writing stats.

Bug: 5321340
Change-Id: I78a2226778a79c875f3668336e39ea24a7b4d5c4

Author: jsharkey

core/java/android/net/NetworkStatsHistory.java

@@ -441,10 +441,10 @@ public Entry getValues(long start, long end, long now, Entry recycle) {
             final long curStart = bucketStart[i];
             final long curEnd = curStart + bucketDuration;
 
-            // bucket is older than record; we're finished
-            if (curEnd < start) break;
-            // bucket is newer than record; keep looking
-            if (curStart > end) continue;
+            // bucket is older than request; we're finished
+            if (curEnd <= start) break;
+            // bucket is newer than request; keep looking
+            if (curStart >= end) continue;
 
             // include full value for active buckets, otherwise only fractional
             final boolean activeBucket = curStart < now && curEnd > now;
@@ -466,7 +466,6 @@ public Entry getValues(long start, long end, long now, Entry recycle) {
             if (txPackets != null) entry.txPackets += txPackets[i] * overlap / bucketDuration;
             if (operations != null) entry.operations += operations[i] * overlap / bucketDuration;
         }
-
         return entry;
     }
 

core/java/android/provider/Settings.java

@@ -4031,8 +4031,6 @@ public static final String getBluetoothInputDevicePriorityKey(String address) {
         public static final String NETSTATS_UID_MAX_HISTORY = "netstats_uid_max_history";
         /** {@hide} */
         public static final String NETSTATS_TAG_MAX_HISTORY = "netstats_tag_max_history";
-        /** {@hide} */
-        public static final String NETSTATS_FORCE_COMPLETE_POLL = "netstats_force_complete_poll";
 
         /** Preferred NTP server. {@hide} */
         public static final String NTP_SERVER = "ntp_server";

services/java/com/android/server/net/NetworkStatsService.java

@@ -36,7 +36,6 @@
 import static android.net.NetworkTemplate.buildTemplateWifi;
 import static android.net.TrafficStats.UID_REMOVED;
 import static android.net.TrafficStats.UID_TETHERING;
-import static android.provider.Settings.Secure.NETSTATS_FORCE_COMPLETE_POLL;
 import static android.provider.Settings.Secure.NETSTATS_NETWORK_BUCKET_DURATION;
 import static android.provider.Settings.Secure.NETSTATS_NETWORK_MAX_HISTORY;
 import static android.provider.Settings.Secure.NETSTATS_PERSIST_THRESHOLD;
@@ -136,15 +135,10 @@ public class NetworkStatsService extends INetworkStatsService.Stub {
     private static final int MSG_PERFORM_POLL = 0x1;
 
     /** Flags to control detail level of poll event. */
-    private static final int FLAG_POLL_NETWORK = 0x1;
-    private static final int FLAG_POLL_UID = 0x2;
-    private static final int FLAG_POLL_TETHER = 0x3;
     private static final int FLAG_PERSIST_NETWORK = 0x10;
     private static final int FLAG_PERSIST_UID = 0x20;
-    private static final int FLAG_FORCE_PERSIST = 0x100;
-
-    private static final int FLAG_POLL_ALL = FLAG_POLL_NETWORK | FLAG_POLL_UID | FLAG_POLL_TETHER;
     private static final int FLAG_PERSIST_ALL = FLAG_PERSIST_NETWORK | FLAG_PERSIST_UID;
+    private static final int FLAG_PERSIST_FORCE = 0x100;
 
     private final Context mContext;
     private final INetworkManagementService mNetworkManager;
@@ -182,7 +176,6 @@ public interface NetworkStatsSettings {
         public long getUidMaxHistory();
         public long getTagMaxHistory();
         public long getTimeCacheMaxAge();
-        public boolean getForceCompletePoll();
     }
 
     private final Object mStatsLock = new Object();
@@ -529,7 +522,7 @@ public void setUidForeground(int uid, boolean uidForeground) {
     @Override
     public void forceUpdate() {
         mContext.enforceCallingOrSelfPermission(READ_NETWORK_USAGE_HISTORY, TAG);
-        performPoll(FLAG_POLL_ALL | FLAG_PERSIST_ALL);
+        performPoll(FLAG_PERSIST_ALL);
     }
 
     /**
@@ -561,7 +554,7 @@ public void onReceive(Context context, Intent intent) {
         public void onReceive(Context context, Intent intent) {
             // on background handler thread, and verified CONNECTIVITY_INTERNAL
             // permission above.
-            performPoll(FLAG_POLL_TETHER);
+            performPoll(FLAG_PERSIST_NETWORK);
         }
     };
 
@@ -570,7 +563,7 @@ public void onReceive(Context context, Intent intent) {
         public void onReceive(Context context, Intent intent) {
             // on background handler thread, and verified UPDATE_DEVICE_STATS
             // permission above.
-            performPoll(FLAG_POLL_ALL | FLAG_PERSIST_ALL);
+            performPoll(FLAG_PERSIST_ALL);
 
             // verify that we're watching global alert
             registerGlobalAlert();
@@ -617,7 +610,7 @@ public void limitReached(String limitName, String iface) {
             if (LIMIT_GLOBAL_ALERT.equals(limitName)) {
                 // kick off background poll to collect network stats; UID stats
                 // are handled during normal polling interval.
-                final int flags = FLAG_POLL_NETWORK | FLAG_PERSIST_NETWORK;
+                final int flags = FLAG_PERSIST_NETWORK;
                 mHandler.obtainMessage(MSG_PERFORM_POLL, flags, 0).sendToTarget();
 
                 // re-arm global alert for next update
@@ -639,10 +632,9 @@ private void updateIfacesLocked() {
         // isn't perfect, since the kernel may already be counting traffic from
         // the updated network.
 
-        // poll both network and UID stats, but only persist network stats,
-        // since this codepath should stay fast. UID stats will be persisted
-        // during next alarm poll event.
-        performPollLocked(FLAG_POLL_ALL | FLAG_PERSIST_NETWORK);
+        // poll, but only persist network stats to keep codepath fast. UID stats
+        // will be persisted during next alarm poll event.
+        performPollLocked(FLAG_PERSIST_NETWORK);
 
         final NetworkState[] states;
         try {
@@ -706,21 +698,9 @@ private void performPollLocked(int flags) {
         if (LOGV) Slog.v(TAG, "performPollLocked(flags=0x" + Integer.toHexString(flags) + ")");
         final long startRealtime = SystemClock.elapsedRealtime();
 
-        boolean pollNetwork = (flags & FLAG_POLL_NETWORK) != 0;
-        boolean pollUid = (flags & FLAG_POLL_UID) != 0;
-        boolean pollTether = (flags & FLAG_POLL_TETHER) != 0;
-
-        // when complete poll requested, any partial poll enables everything
-        final boolean forceCompletePoll = mSettings.getForceCompletePoll();
-        if (forceCompletePoll && (pollNetwork || pollUid || pollTether)) {
-            pollNetwork = true;
-            pollUid = true;
-            pollTether = true;
-        }
-
         final boolean persistNetwork = (flags & FLAG_PERSIST_NETWORK) != 0;
         final boolean persistUid = (flags & FLAG_PERSIST_UID) != 0;
-        final boolean forcePersist = (flags & FLAG_FORCE_PERSIST) != 0;
+        final boolean persistForce = (flags & FLAG_PERSIST_FORCE) != 0;
 
         // try refreshing time source when stale
         if (mTime.getCacheAge() > mSettings.getTimeCacheMaxAge()) {
@@ -733,41 +713,36 @@ private void performPollLocked(int flags) {
         final long threshold = mSettings.getPersistThreshold();
 
         try {
-            if (pollNetwork) {
-                final NetworkStats networkSnapshot = mNetworkManager.getNetworkStatsSummary();
-                performNetworkPollLocked(networkSnapshot, currentTime);
-
-                // persist when enough network data has occurred
-                final NetworkStats persistNetworkDelta = computeStatsDelta(
-                        mLastPersistNetworkSnapshot, networkSnapshot, true);
-                final boolean pastThreshold = persistNetworkDelta.getTotalBytes() > threshold;
-                if (forcePersist || (persistNetwork && pastThreshold)) {
-                    writeNetworkStatsLocked();
-                    mLastPersistNetworkSnapshot = networkSnapshot;
-                }
+            // record network stats
+            final NetworkStats networkSnapshot = mNetworkManager.getNetworkStatsSummary();
+            performNetworkPollLocked(networkSnapshot, currentTime);
+
+            // persist when enough network data has occurred
+            final NetworkStats persistNetworkDelta = computeStatsDelta(
+                    mLastPersistNetworkSnapshot, networkSnapshot, true);
+            final boolean networkPastThreshold = persistNetworkDelta.getTotalBytes() > threshold;
+            if (persistForce || (persistNetwork && networkPastThreshold)) {
+                writeNetworkStatsLocked();
+                mLastPersistNetworkSnapshot = networkSnapshot;
             }
 
-            if (pollTether) {
-                final String[] ifacePairs = mConnManager.getTetheredIfacePairs();
-                final NetworkStats tetherSnapshot = mNetworkManager.getNetworkStatsTethering(
-                        ifacePairs);
-                performTetherPollLocked(tetherSnapshot, currentTime);
-
-                // persisted during normal UID cycle below
-            }
-
-            if (pollUid) {
-                final NetworkStats uidSnapshot = mNetworkManager.getNetworkStatsUidDetail(UID_ALL);
-                performUidPollLocked(uidSnapshot, currentTime);
-
-                // persist when enough network data has occurred
-                final NetworkStats persistUidDelta = computeStatsDelta(
-                        mLastPersistUidSnapshot, uidSnapshot, true);
-                final boolean pastThreshold = persistUidDelta.getTotalBytes() > threshold;
-                if (forcePersist || (persistUid && pastThreshold)) {
-                    writeUidStatsLocked();
-                    mLastPersistUidSnapshot = uidSnapshot;
-                }
+            // record tethering stats; persisted during normal UID cycle below
+            final String[] ifacePairs = mConnManager.getTetheredIfacePairs();
+            final NetworkStats tetherSnapshot = mNetworkManager.getNetworkStatsTethering(
+                    ifacePairs);
+            performTetherPollLocked(tetherSnapshot, currentTime);
+
+            // record uid stats
+            final NetworkStats uidSnapshot = mNetworkManager.getNetworkStatsUidDetail(UID_ALL);
+            performUidPollLocked(uidSnapshot, currentTime);
+
+            // persist when enough network data has occurred
+            final NetworkStats persistUidDelta = computeStatsDelta(
+                    mLastPersistUidSnapshot, uidSnapshot, true);
+            final boolean uidPastThreshold = persistUidDelta.getTotalBytes() > threshold;
+            if (persistForce || (persistUid && uidPastThreshold)) {
+                writeUidStatsLocked();
+                mLastPersistUidSnapshot = uidSnapshot;
             }
         } catch (IllegalStateException e) {
             Log.wtf(TAG, "problem reading network stats", e);
@@ -781,9 +756,7 @@ private void performPollLocked(int flags) {
         }
 
         // sample stats after each full poll
-        if (pollNetwork && pollUid) {
-            performSample();
-        }
+        performSample();
 
         // finally, dispatch updated event to any listeners
         final Intent updatedIntent = new Intent(ACTION_NETWORK_STATS_UPDATED);
@@ -813,12 +786,6 @@ private void performNetworkPollLocked(NetworkStats networkSnapshot, long current
             history.recordData(timeStart, currentTime, entry);
         }
 
-        // trim any history beyond max
-        final long maxHistory = mSettings.getNetworkMaxHistory();
-        for (NetworkStatsHistory history : mNetworkStats.values()) {
-            history.removeBucketsBefore(currentTime - maxHistory);
-        }
-
         mLastPollNetworkSnapshot = networkSnapshot;
 
         if (LOGD && unknownIface.size() > 0) {
@@ -862,20 +829,6 @@ private void performUidPollLocked(NetworkStats uidSnapshot, long currentTime) {
             history.recordData(timeStart, currentTime, entry);
         }
 
-        // trim any history beyond max
-        final long maxUidHistory = mSettings.getUidMaxHistory();
-        final long maxTagHistory = mSettings.getTagMaxHistory();
-        for (UidStatsKey key : mUidStats.keySet()) {
-            final NetworkStatsHistory history = mUidStats.get(key);
-
-            // detailed tags are trimmed sooner than summary in TAG_NONE
-            if (key.tag == TAG_NONE) {
-                history.removeBucketsBefore(currentTime - maxUidHistory);
-            } else {
-                history.removeBucketsBefore(currentTime - maxTagHistory);
-            }
-        }
-
         mLastPollUidSnapshot = uidSnapshot;
         mLastPollOperationsSnapshot = mOperations;
         mOperations = new NetworkStats(0L, 10);
@@ -917,9 +870,13 @@ private void performTetherPollLocked(NetworkStats tetherSnapshot, long currentTi
      * Sample recent statistics summary into {@link EventLog}.
      */
     private void performSample() {
-        // take sample as total over last 4 hours
-        final long end = mTime.hasCache() ? mTime.currentTimeMillis() : System.currentTimeMillis();
-        final long start = end - (4 * HOUR_IN_MILLIS);
+        final long largestBucketSize = Math.max(
+                mSettings.getNetworkBucketDuration(), mSettings.getUidBucketDuration());
+
+        // take sample as atomic buckets
+        final long now = mTime.hasCache() ? mTime.currentTimeMillis() : System.currentTimeMillis();
+        final long end = now - (now % largestBucketSize) + largestBucketSize;
+        final long start = end - largestBucketSize;
 
         NetworkTemplate template = null;
         NetworkStats.Entry ifaceTotal = null;
@@ -929,21 +886,17 @@ private void performSample() {
         template = buildTemplateMobileAll(getActiveSubscriberId(mContext));
         ifaceTotal = getSummaryForNetwork(template, start, end).getTotal(ifaceTotal);
         uidTotal = getSummaryForAllUid(template, start, end, false).getTotal(uidTotal);
-        EventLogTags.writeNetstatsMobileSample(
-                ifaceTotal.rxBytes, ifaceTotal.txBytes,
-                ifaceTotal.rxPackets, ifaceTotal.txPackets,
-                uidTotal.rxBytes, uidTotal.txBytes,
-                uidTotal.rxPackets, uidTotal.txPackets);
+        EventLogTags.writeNetstatsMobileSample(ifaceTotal.rxBytes, ifaceTotal.rxPackets,
+                ifaceTotal.txBytes, ifaceTotal.txPackets, uidTotal.rxBytes, uidTotal.rxPackets,
+                uidTotal.txBytes, uidTotal.rxPackets);
 
         // collect wifi sample
         template = buildTemplateWifi();
         ifaceTotal = getSummaryForNetwork(template, start, end).getTotal(ifaceTotal);
         uidTotal = getSummaryForAllUid(template, start, end, false).getTotal(uidTotal);
-        EventLogTags.writeNetstatsWifiSample(
-                ifaceTotal.rxBytes, ifaceTotal.txBytes,
-                ifaceTotal.rxPackets, ifaceTotal.txPackets,
-                uidTotal.rxBytes, uidTotal.txBytes,
-                uidTotal.rxPackets, uidTotal.txPackets);
+        EventLogTags.writeNetstatsWifiSample(ifaceTotal.rxBytes, ifaceTotal.rxPackets,
+                ifaceTotal.txBytes, ifaceTotal.txPackets, uidTotal.rxBytes, uidTotal.rxPackets,
+                uidTotal.txBytes, uidTotal.rxPackets);
     }
 
     /**
@@ -1143,6 +1096,15 @@ private void writeNetworkStatsLocked() {
 
         // TODO: consider duplicating stats and releasing lock while writing
 
+        // trim any history beyond max
+        if (mTime.hasCache()) {
+            final long currentTime = mTime.currentTimeMillis();
+            final long maxHistory = mSettings.getNetworkMaxHistory();
+            for (NetworkStatsHistory history : mNetworkStats.values()) {
+                history.removeBucketsBefore(currentTime - maxHistory);
+            }
+        }
+
         FileOutputStream fos = null;
         try {
             fos = mNetworkFile.startWrite();
@@ -1178,6 +1140,23 @@ private void writeUidStatsLocked() {
 
         // TODO: consider duplicating stats and releasing lock while writing
 
+        // trim any history beyond max
+        if (mTime.hasCache()) {
+            final long currentTime = mTime.currentTimeMillis();
+            final long maxUidHistory = mSettings.getUidMaxHistory();
+            final long maxTagHistory = mSettings.getTagMaxHistory();
+            for (UidStatsKey key : mUidStats.keySet()) {
+                final NetworkStatsHistory history = mUidStats.get(key);
+
+                // detailed tags are trimmed sooner than summary in TAG_NONE
+                if (key.tag == TAG_NONE) {
+                    history.removeBucketsBefore(currentTime - maxUidHistory);
+                } else {
+                    history.removeBucketsBefore(currentTime - maxTagHistory);
+                }
+            }
+        }
+
         // build UidStatsKey lists grouped by ident
         final HashMap<NetworkIdentitySet, ArrayList<UidStatsKey>> keysByIdent = Maps.newHashMap();
         for (UidStatsKey key : mUidStats.keySet()) {
@@ -1242,7 +1221,7 @@ protected void dump(FileDescriptor fd, PrintWriter pw, String[] args) {
             }
 
             if (argSet.contains("poll")) {
-                performPollLocked(FLAG_POLL_ALL | FLAG_PERSIST_ALL | FLAG_FORCE_PERSIST);
+                performPollLocked(FLAG_PERSIST_ALL | FLAG_PERSIST_FORCE);
                 pw.println("Forced poll");
                 return;
             }
@@ -1470,8 +1449,5 @@ public long getTagMaxHistory() {
         public long getTimeCacheMaxAge() {
             return DAY_IN_MILLIS;
         }
-        public boolean getForceCompletePoll() {
-            return getSecureBoolean(NETSTATS_FORCE_COMPLETE_POLL, false);
-        }
     }
 }

services/tests/servicestests/src/com/android/server/NetworkStatsServiceTest.java

@@ -272,7 +272,11 @@ public void testStatsRebootPersist() throws Exception {
 
         // graceful shutdown system, which should trigger persist of stats, and
         // clear any values in memory.
+        expectCurrentTime();
+        expectDefaultSettings();
+        replay();
         mServiceContext.sendBroadcast(new Intent(Intent.ACTION_SHUTDOWN));
+        verifyAndReset();
 
         // talk with zombie service to assert stats have gone; and assert that
         // we persisted them to file.
@@ -487,6 +491,7 @@ public void testUidRemovedIsMoved() throws Exception {
 
         // now pretend two UIDs are uninstalled, which should migrate stats to
         // special "removed" bucket.
+        expectCurrentTime();
         expectDefaultSettings();
         replay();
         final Intent intent = new Intent(ACTION_UID_REMOVED);
@@ -758,7 +763,6 @@ private void expectSettings(long persistThreshold, long bucketDuration, long max
         expect(mSettings.getUidMaxHistory()).andReturn(maxHistory).anyTimes();
         expect(mSettings.getTagMaxHistory()).andReturn(maxHistory).anyTimes();
         expect(mSettings.getTimeCacheMaxAge()).andReturn(DAY_IN_MILLIS).anyTimes();
-        expect(mSettings.getForceCompletePoll()).andReturn(false).anyTimes();
     }
 
     private void expectCurrentTime() throws Exception {