Skip to content

Commit 63abc37

Browse files
jsharkeyjsharkey
committed
Move network stats to FileRotator pattern.
Split existing network stats into two separate classes: a recorder which generates historical data based on periodic counter snapshots, and a collection of historical data with persistance logic. Recorder keeps a pending history in memory until outstanding data crosses a specific threshold. Persisting is handled through a given FileRotator. This pattern significantly reduces disk churn and memory overhead. Separate UID data from UID tag data, enabling a shorter rotation cycle. Migrate existing stats into new structure. Remove "xt" stats until iptables hooks are ready. Avoid consuming Entry values when recording into NetworkStatsHistory. Assign operation counts to default route interface. Introduce "Rewriter" interface in FileRotator with methods to enable rewriteAll(). Introduce IndentingPrintWriter to handle indenting in dump() methods. Bug: 5386531 Change-Id: Ibe086230a17999a197206ca62d45f266225fdff1
1 parent cf5c78f commit 63abc37

File tree

16 files changed

+1640
-1086
lines changed

16 files changed

+1640
-1086
lines changed

‎core/java/android/net/NetworkStats.java‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,15 @@ public Entry(String iface, int uid, int set, int tag, long rxBytes, long rxPacke
102102
this.operations = operations;
103103
}
104104

105+
public boolean isNegative() {
106+
return rxBytes < 0 || rxPackets < 0 || txBytes < 0 || txPackets < 0 || operations < 0;
107+
}
108+
109+
public boolean isEmpty() {
110+
return rxBytes == 0 && rxPackets == 0 && txBytes == 0 && txPackets == 0
111+
&& operations == 0;
112+
}
113+
105114
@Override
106115
public String toString() {
107116
final StringBuilder builder = new StringBuilder();
@@ -343,6 +352,7 @@ public int findIndexHinted(String iface, int uid, int set, int tag, int hintInde
343352
* on matching {@link #uid} and {@link #tag} rows. Ignores {@link #iface},
344353
* since operation counts are at data layer.
345354
*/
355+
@Deprecated
346356
public void spliceOperationsFrom(NetworkStats stats) {
347357
for (int i = 0; i < size; i++) {
348358
final int j = stats.findIndex(IFACE_ALL, uid[i], set[i], tag[i]);
@@ -397,23 +407,27 @@ public long getTotalBytes() {
397407
* Return total of all fields represented by this snapshot object.
398408
*/
399409
public Entry getTotal(Entry recycle) {
400-
return getTotal(recycle, null, UID_ALL);
410+
return getTotal(recycle, null, UID_ALL, false);
401411
}
402412

403413
/**
404414
* Return total of all fields represented by this snapshot object matching
405415
* the requested {@link #uid}.
406416
*/
407417
public Entry getTotal(Entry recycle, int limitUid) {
408-
return getTotal(recycle, null, limitUid);
418+
return getTotal(recycle, null, limitUid, false);
409419
}
410420

411421
/**
412422
* Return total of all fields represented by this snapshot object matching
413423
* the requested {@link #iface}.
414424
*/
415425
public Entry getTotal(Entry recycle, HashSet<String> limitIface) {
416-
return getTotal(recycle, limitIface, UID_ALL);
426+
return getTotal(recycle, limitIface, UID_ALL, false);
427+
}
428+
429+
public Entry getTotalIncludingTags(Entry recycle) {
430+
return getTotal(recycle, null, UID_ALL, true);
417431
}
418432

419433
/**
@@ -423,7 +437,8 @@ public Entry getTotal(Entry recycle, HashSet<String> limitIface) {
423437
* @param limitIface Set of {@link #iface} to include in total; or {@code
424438
* null} to include all ifaces.
425439
*/
426-
private Entry getTotal(Entry recycle, HashSet<String> limitIface, int limitUid) {
440+
private Entry getTotal(
441+
Entry recycle, HashSet<String> limitIface, int limitUid, boolean includeTags) {
427442
final Entry entry = recycle != null ? recycle : new Entry();
428443

429444
entry.iface = IFACE_ALL;
@@ -442,7 +457,7 @@ private Entry getTotal(Entry recycle, HashSet<String> limitIface, int limitUid)
442457

443458
if (matchesUid && matchesIface) {
444459
// skip specific tags, since already counted in TAG_NONE
445-
if (tag[i] != TAG_NONE) continue;
460+
if (tag[i] != TAG_NONE && !includeTags) continue;
446461

447462
entry.rxBytes += rxBytes[i];
448463
entry.rxPackets += rxPackets[i];
@@ -460,7 +475,7 @@ private Entry getTotal(Entry recycle, HashSet<String> limitIface, int limitUid)
460475
* time, and that none of them have disappeared.
461476
*/
462477
public NetworkStats subtract(NetworkStats right) {
463-
return subtract(this, right, null);
478+
return subtract(this, right, null, null);
464479
}
465480

466481
/**
@@ -471,12 +486,12 @@ public NetworkStats subtract(NetworkStats right) {
471486
* If counters have rolled backwards, they are clamped to {@code 0} and
472487
* reported to the given {@link NonMonotonicObserver}.
473488
*/
474-
public static NetworkStats subtract(
475-
NetworkStats left, NetworkStats right, NonMonotonicObserver observer) {
489+
public static <C> NetworkStats subtract(
490+
NetworkStats left, NetworkStats right, NonMonotonicObserver<C> observer, C cookie) {
476491
long deltaRealtime = left.elapsedRealtime - right.elapsedRealtime;
477492
if (deltaRealtime < 0) {
478493
if (observer != null) {
479-
observer.foundNonMonotonic(left, -1, right, -1);
494+
observer.foundNonMonotonic(left, -1, right, -1, cookie);
480495
}
481496
deltaRealtime = 0;
482497
}
@@ -510,7 +525,7 @@ public static NetworkStats subtract(
510525
if (entry.rxBytes < 0 || entry.rxPackets < 0 || entry.txBytes < 0
511526
|| entry.txPackets < 0 || entry.operations < 0) {
512527
if (observer != null) {
513-
observer.foundNonMonotonic(left, i, right, j);
528+
observer.foundNonMonotonic(left, i, right, j, cookie);
514529
}
515530
entry.rxBytes = Math.max(entry.rxBytes, 0);
516531
entry.rxPackets = Math.max(entry.rxPackets, 0);
@@ -663,8 +678,8 @@ public NetworkStats[] newArray(int size) {
663678
}
664679
};
665680

666-
public interface NonMonotonicObserver {
681+
public interface NonMonotonicObserver<C> {
667682
public void foundNonMonotonic(
668-
NetworkStats left, int leftIndex, NetworkStats right, int rightIndex);
683+
NetworkStats left, int leftIndex, NetworkStats right, int rightIndex, C cookie);
669684
}
670685
}

‎core/java/android/net/NetworkStatsHistory.java‎

Lines changed: 55 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,18 @@
2626
import static android.net.NetworkStatsHistory.Entry.UNKNOWN;
2727
import static android.net.NetworkStatsHistory.ParcelUtils.readLongArray;
2828
import static android.net.NetworkStatsHistory.ParcelUtils.writeLongArray;
29+
import static com.android.internal.util.ArrayUtils.total;
2930

3031
import android.os.Parcel;
3132
import android.os.Parcelable;
3233
import android.util.MathUtils;
3334

35+
import com.android.internal.util.IndentingPrintWriter;
36+
3437
import java.io.CharArrayWriter;
3538
import java.io.DataInputStream;
3639
import java.io.DataOutputStream;
3740
import java.io.IOException;
38-
import java.io.PrintWriter;
3941
import java.net.ProtocolException;
4042
import java.util.Arrays;
4143
import java.util.Random;
@@ -74,6 +76,7 @@ public class NetworkStatsHistory implements Parcelable {
7476
private long[] txBytes;
7577
private long[] txPackets;
7678
private long[] operations;
79+
private long totalBytes;
7780

7881
public static class Entry {
7982
public static final long UNKNOWN = -1;
@@ -106,6 +109,12 @@ public NetworkStatsHistory(long bucketDuration, int initialSize, int fields) {
106109
if ((fields & FIELD_TX_PACKETS) != 0) txPackets = new long[initialSize];
107110
if ((fields & FIELD_OPERATIONS) != 0) operations = new long[initialSize];
108111
bucketCount = 0;
112+
totalBytes = 0;
113+
}
114+
115+
public NetworkStatsHistory(NetworkStatsHistory existing, long bucketDuration) {
116+
this(bucketDuration, existing.estimateResizeBuckets(bucketDuration));
117+
recordEntireHistory(existing);
109118
}
110119

111120
public NetworkStatsHistory(Parcel in) {
@@ -118,6 +127,7 @@ public NetworkStatsHistory(Parcel in) {
118127
txPackets = readLongArray(in);
119128
operations = readLongArray(in);
120129
bucketCount = bucketStart.length;
130+
totalBytes = in.readLong();
121131
}
122132

123133
/** {@inheritDoc} */
@@ -130,6 +140,7 @@ public void writeToParcel(Parcel out, int flags) {
130140
writeLongArray(out, txBytes, bucketCount);
131141
writeLongArray(out, txPackets, bucketCount);
132142
writeLongArray(out, operations, bucketCount);
143+
out.writeLong(totalBytes);
133144
}
134145

135146
public NetworkStatsHistory(DataInputStream in) throws IOException {
@@ -144,6 +155,7 @@ public NetworkStatsHistory(DataInputStream in) throws IOException {
144155
txPackets = new long[bucketStart.length];
145156
operations = new long[bucketStart.length];
146157
bucketCount = bucketStart.length;
158+
totalBytes = total(rxBytes) + total(txBytes);
147159
break;
148160
}
149161
case VERSION_ADD_PACKETS:
@@ -158,6 +170,7 @@ public NetworkStatsHistory(DataInputStream in) throws IOException {
158170
txPackets = readVarLongArray(in);
159171
operations = readVarLongArray(in);
160172
bucketCount = bucketStart.length;
173+
totalBytes = total(rxBytes) + total(txBytes);
161174
break;
162175
}
163176
default: {
@@ -207,6 +220,13 @@ public long getEnd() {
207220
}
208221
}
209222

223+
/**
224+
* Return total bytes represented by this history.
225+
*/
226+
public long getTotalBytes() {
227+
return totalBytes;
228+
}
229+
210230
/**
211231
* Return index of bucket that contains or is immediately before the
212232
* requested time.
@@ -266,13 +286,16 @@ public void recordData(long start, long end, long rxBytes, long txBytes) {
266286
* distribute across internal buckets, creating new buckets as needed.
267287
*/
268288
public void recordData(long start, long end, NetworkStats.Entry entry) {
269-
if (entry.rxBytes < 0 || entry.rxPackets < 0 || entry.txBytes < 0 || entry.txPackets < 0
270-
|| entry.operations < 0) {
289+
long rxBytes = entry.rxBytes;
290+
long rxPackets = entry.rxPackets;
291+
long txBytes = entry.txBytes;
292+
long txPackets = entry.txPackets;
293+
long operations = entry.operations;
294+
295+
if (entry.isNegative()) {
271296
throw new IllegalArgumentException("tried recording negative data");
272297
}
273-
if (entry.rxBytes == 0 && entry.rxPackets == 0 && entry.txBytes == 0 && entry.txPackets == 0
274-
&& entry.operations == 0) {
275-
// nothing to record; skip
298+
if (entry.isEmpty()) {
276299
return;
277300
}
278301

@@ -295,21 +318,23 @@ public void recordData(long start, long end, NetworkStats.Entry entry) {
295318
if (overlap <= 0) continue;
296319

297320
// integer math each time is faster than floating point
298-
final long fracRxBytes = entry.rxBytes * overlap / duration;
299-
final long fracRxPackets = entry.rxPackets * overlap / duration;
300-
final long fracTxBytes = entry.txBytes * overlap / duration;
301-
final long fracTxPackets = entry.txPackets * overlap / duration;
302-
final long fracOperations = entry.operations * overlap / duration;
321+
final long fracRxBytes = rxBytes * overlap / duration;
322+
final long fracRxPackets = rxPackets * overlap / duration;
323+
final long fracTxBytes = txBytes * overlap / duration;
324+
final long fracTxPackets = txPackets * overlap / duration;
325+
final long fracOperations = operations * overlap / duration;
303326

304327
addLong(activeTime, i, overlap);
305-
addLong(rxBytes, i, fracRxBytes); entry.rxBytes -= fracRxBytes;
306-
addLong(rxPackets, i, fracRxPackets); entry.rxPackets -= fracRxPackets;
307-
addLong(txBytes, i, fracTxBytes); entry.txBytes -= fracTxBytes;
308-
addLong(txPackets, i, fracTxPackets); entry.txPackets -= fracTxPackets;
309-
addLong(operations, i, fracOperations); entry.operations -= fracOperations;
328+
addLong(this.rxBytes, i, fracRxBytes); rxBytes -= fracRxBytes;
329+
addLong(this.rxPackets, i, fracRxPackets); rxPackets -= fracRxPackets;
330+
addLong(this.txBytes, i, fracTxBytes); txBytes -= fracTxBytes;
331+
addLong(this.txPackets, i, fracTxPackets); txPackets -= fracTxPackets;
332+
addLong(this.operations, i, fracOperations); operations -= fracOperations;
310333

311334
duration -= overlap;
312335
}
336+
337+
totalBytes += entry.rxBytes + entry.txBytes;
313338
}
314339

315340
/**
@@ -394,6 +419,7 @@ private void insertBucket(int index, long start) {
394419
/**
395420
* Remove buckets older than requested cutoff.
396421
*/
422+
@Deprecated
397423
public void removeBucketsBefore(long cutoff) {
398424
int i;
399425
for (i = 0; i < bucketCount; i++) {
@@ -415,6 +441,8 @@ public void removeBucketsBefore(long cutoff) {
415441
if (txPackets != null) txPackets = Arrays.copyOfRange(txPackets, i, length);
416442
if (operations != null) operations = Arrays.copyOfRange(operations, i, length);
417443
bucketCount -= i;
444+
445+
// TODO: subtract removed values from totalBytes
418446
}
419447
}
420448

@@ -527,19 +555,17 @@ public static long randomLong(Random r, long start, long end) {
527555
return (long) (start + (r.nextFloat() * (end - start)));
528556
}
529557

530-
public void dump(String prefix, PrintWriter pw, boolean fullHistory) {
531-
pw.print(prefix);
558+
public void dump(IndentingPrintWriter pw, boolean fullHistory) {
532559
pw.print("NetworkStatsHistory: bucketDuration="); pw.println(bucketDuration);
560+
pw.increaseIndent();
533561

534562
final int start = fullHistory ? 0 : Math.max(0, bucketCount - 32);
535563
if (start > 0) {
536-
pw.print(prefix);
537-
pw.print(" (omitting "); pw.print(start); pw.println(" buckets)");
564+
pw.print("(omitting "); pw.print(start); pw.println(" buckets)");
538565
}
539566

540567
for (int i = start; i < bucketCount; i++) {
541-
pw.print(prefix);
542-
pw.print(" bucketStart="); pw.print(bucketStart[i]);
568+
pw.print("bucketStart="); pw.print(bucketStart[i]);
543569
if (activeTime != null) { pw.print(" activeTime="); pw.print(activeTime[i]); }
544570
if (rxBytes != null) { pw.print(" rxBytes="); pw.print(rxBytes[i]); }
545571
if (rxPackets != null) { pw.print(" rxPackets="); pw.print(rxPackets[i]); }
@@ -548,12 +574,14 @@ public void dump(String prefix, PrintWriter pw, boolean fullHistory) {
548574
if (operations != null) { pw.print(" operations="); pw.print(operations[i]); }
549575
pw.println();
550576
}
577+
578+
pw.decreaseIndent();
551579
}
552580

553581
@Override
554582
public String toString() {
555583
final CharArrayWriter writer = new CharArrayWriter();
556-
dump("", new PrintWriter(writer), false);
584+
dump(new IndentingPrintWriter(writer, " "), false);
557585
return writer.toString();
558586
}
559587

@@ -579,6 +607,10 @@ private static void addLong(long[] array, int i, long value) {
579607
if (array != null) array[i] += value;
580608
}
581609

610+
public int estimateResizeBuckets(long newBucketDuration) {
611+
return (int) (size() * getBucketDuration() / newBucketDuration);
612+
}
613+
582614
/**
583615
* Utility methods for interacting with {@link DataInputStream} and
584616
* {@link DataOutputStream}, mostly dealing with writing partial arrays.

‎core/java/android/net/TrafficStats.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ public static NetworkStats stopDataProfiling(Context context) {
195195
// subtract starting values and return delta
196196
final NetworkStats profilingStop = getDataLayerSnapshotForUid(context);
197197
final NetworkStats profilingDelta = NetworkStats.subtract(
198-
profilingStop, sActiveProfilingStart, null);
198+
profilingStop, sActiveProfilingStart, null, null);
199199
sActiveProfilingStart = null;
200200
return profilingDelta;
201201
}

‎core/java/android/provider/Settings.java‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4104,17 +4104,38 @@ public static final String getBluetoothInputDevicePriorityKey(String address) {
41044104
/** {@hide} */
41054105
public static final String NETSTATS_POLL_INTERVAL = "netstats_poll_interval";
41064106
/** {@hide} */
4107-
public static final String NETSTATS_PERSIST_THRESHOLD = "netstats_persist_threshold";
4107+
public static final String NETSTATS_TIME_CACHE_MAX_AGE = "netstats_time_cache_max_age";
41084108
/** {@hide} */
4109-
public static final String NETSTATS_NETWORK_BUCKET_DURATION = "netstats_network_bucket_duration";
4109+
public static final String NETSTATS_GLOBAL_ALERT_BYTES = "netstats_global_alert_bytes";
41104110
/** {@hide} */
4111-
public static final String NETSTATS_NETWORK_MAX_HISTORY = "netstats_network_max_history";
4111+
public static final String NETSTATS_SAMPLE_ENABLED = "netstats_sample_enabled";
4112+
4113+
/** {@hide} */
4114+
public static final String NETSTATS_DEV_BUCKET_DURATION = "netstats_dev_bucket_duration";
4115+
/** {@hide} */
4116+
public static final String NETSTATS_DEV_PERSIST_BYTES = "netstats_dev_persist_bytes";
4117+
/** {@hide} */
4118+
public static final String NETSTATS_DEV_ROTATE_AGE = "netstats_dev_rotate_age";
4119+
/** {@hide} */
4120+
public static final String NETSTATS_DEV_DELETE_AGE = "netstats_dev_delete_age";
4121+
41124122
/** {@hide} */
41134123
public static final String NETSTATS_UID_BUCKET_DURATION = "netstats_uid_bucket_duration";
41144124
/** {@hide} */
4115-
public static final String NETSTATS_UID_MAX_HISTORY = "netstats_uid_max_history";
4125+
public static final String NETSTATS_UID_PERSIST_BYTES = "netstats_uid_persist_bytes";
4126+
/** {@hide} */
4127+
public static final String NETSTATS_UID_ROTATE_AGE = "netstats_uid_rotate_age";
4128+
/** {@hide} */
4129+
public static final String NETSTATS_UID_DELETE_AGE = "netstats_uid_delete_age";
4130+
4131+
/** {@hide} */
4132+
public static final String NETSTATS_UID_TAG_BUCKET_DURATION = "netstats_uid_tag_bucket_duration";
4133+
/** {@hide} */
4134+
public static final String NETSTATS_UID_TAG_PERSIST_BYTES = "netstats_uid_tag_persist_bytes";
4135+
/** {@hide} */
4136+
public static final String NETSTATS_UID_TAG_ROTATE_AGE = "netstats_uid_tag_rotate_age";
41164137
/** {@hide} */
4117-
public static final String NETSTATS_TAG_MAX_HISTORY = "netstats_tag_max_history";
4138+
public static final String NETSTATS_UID_TAG_DELETE_AGE = "netstats_uid_tag_delete_age";
41184139

41194140
/** Preferred NTP server. {@hide} */
41204141
public static final String NTP_SERVER = "ntp_server";

‎core/java/com/android/internal/util/ArrayUtils.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,14 @@ public static boolean contains(int[] array, int value) {
142142
return false;
143143
}
144144

145+
public static long total(long[] array) {
146+
long total = 0;
147+
for (long value : array) {
148+
total += value;
149+
}
150+
return total;
151+
}
152+
145153
/**
146154
* Appends an element to a copy of the array and returns the copy.
147155
* @param array The original array, or null to represent an empty array.

0 commit comments

Comments
 (0)