Use streaming JSON parser for profile fetch

package.json

@@ -71,6 +71,7 @@
     "@fluent/langneg": "^0.7.0",
     "@fluent/react": "^0.15.2",
     "@lezer/highlight": "^1.2.3",
+    "@streamparser/json": "^0.0.22",
     "@tgwf/co2": "^0.17.0",
     "array-move": "^3.0.1",
     "array-range": "^1.0.1",

src/actions/receive-profile.ts

@@ -2,6 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 import { oneLine } from 'common-tags';
+import { JSONParser } from '@streamparser/json';
 import queryString from 'query-string';
 import type JSZip from 'jszip';
 import {
@@ -1209,6 +1210,151 @@ async function _extractJsonFromArrayBuffer(
   return JSON.parse(textDecoder.decode(profileBytes));
 }
 
+function _concatUint8Chunks(
+  chunks: Array<Uint8Array>,
+  totalBytes: number
+): Uint8Array<ArrayBuffer> {
+  const result = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return result;
+}
+
+function _normalizeStreamJsonParseError(
+  chunks: Array<Uint8Array>,
+  totalBytes: number,
+  fallbackError: unknown
+): Error {
+  const textDecoder = new TextDecoder();
+  try {
+    JSON.parse(textDecoder.decode(_concatUint8Chunks(chunks, totalBytes)));
+  } catch (nativeError) {
+    if (nativeError instanceof Error) {
+      return nativeError;
+    }
+    return new Error(String(nativeError));
+  }
+  if (fallbackError instanceof Error) {
+    return fallbackError;
+  }
+  return new Error(String(fallbackError));
+}
+
+/**
+ * Parse JSON from a ReadableStream incrementally.
+ *
+ * This avoids creating one massive JS string (which can hit engine string-size limits)
+ * before JSON parsing.
+ */
+async function _extractJsonFromReadableStream(
+  stream: ReadableStream<Uint8Array>
+): Promise<unknown> {
+  const reader = stream.getReader();
+  const initialChunks: Array<Uint8Array> = [];
+  let initialChunksLength = 0;
+  const headerBytes = new Uint8Array(3);
+  let headerBytesLength = 0;
+
+  // Read enough bytes to reliably detect gzip headers even if the stream
+  // splits the first three bytes across chunks.
+  while (headerBytesLength < 3) {
+    const { done, value } = await reader.read();
+    if (done || value === undefined) {
+      break;
+    }
+    initialChunks.push(value);
+    initialChunksLength += value.byteLength;
+    for (
+      let headerIndex = 0;
+      headerIndex < value.byteLength && headerBytesLength < 3;
+      headerIndex++
+    ) {
+      headerBytes[headerBytesLength++] = value[headerIndex];
+    }
+  }
+
+  if (initialChunksLength === 0) {
+    throw new SyntaxError('Unexpected end of JSON input');
+  }
+
+  if (headerBytesLength === 3 && isGzip(headerBytes)) {
+    let totalBytes = initialChunksLength;
+    const chunks = [...initialChunks];
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done || value === undefined) {
+        break;
+      }
+      chunks.push(value);
+      totalBytes += value.byteLength;
+    }
+    const compressedBytes = _concatUint8Chunks(chunks, totalBytes);
+    return _extractJsonFromArrayBuffer(compressedBytes.buffer);
+  }
+
+  const parser = new JSONParser();
+  const parserErrorProbeChunks: Array<Uint8Array> = [];
+  let parserErrorProbeLength = 0;
+  const maxErrorProbeBytes = 64 * 1024;
+  let rootValue: unknown;
+  let hasRootValue = false;
+
+  const appendParserErrorProbe = (chunk: Uint8Array) => {
+    if (parserErrorProbeLength >= maxErrorProbeBytes) {
+      return;
+    }
+    const bytesRemaining = maxErrorProbeBytes - parserErrorProbeLength;
+    if (chunk.byteLength <= bytesRemaining) {
+      parserErrorProbeChunks.push(chunk);
+      parserErrorProbeLength += chunk.byteLength;
+      return;
+    }
+    parserErrorProbeChunks.push(chunk.subarray(0, bytesRemaining));
+    parserErrorProbeLength += bytesRemaining;
+  };
+
+  parser.onValue = ({ value, stack }) => {
+    if (stack.length === 0 && value !== undefined) {
+      rootValue = value;
+      hasRootValue = true;
+    }
+  };
+
+  try {
+    for (const chunk of initialChunks) {
+      appendParserErrorProbe(chunk);
+      parser.write(chunk);
+    }
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done || value === undefined) {
+        break;
+      }
+      appendParserErrorProbe(value);
+      parser.write(value);
+    }
+    if (!parser.isEnded) {
+      parser.end();
+    }
+  } catch (error) {
+    throw _normalizeStreamJsonParseError(
+      parserErrorProbeChunks,
+      parserErrorProbeLength,
+      error
+    );
+  }
+
+  if (!hasRootValue) {
+    throw new SyntaxError('Unexpected end of JSON input');
+  }
+
+  return rootValue;
+}
+
 /**
  * Don't trust third party responses, try and handle a variety of responses gracefully.
  */
@@ -1219,7 +1365,11 @@ async function _extractJsonFromResponse(
 ): Promise<unknown> {
   let arrayBuffer: ArrayBuffer | null = null;
   try {
-    // await before returning so that we can catch JSON parse errors.
+    if (fileType === 'application/json' && response.body) {
+      return await _extractJsonFromReadableStream(response.body);
+    }
+
+    // Await before returning so that we can catch JSON parse errors.
     arrayBuffer = await response.arrayBuffer();
     return await _extractJsonFromArrayBuffer(arrayBuffer);
   } catch (error) {

yarn.lock

@@ -2116,6 +2116,11 @@
   dependencies:
     "@sinonjs/commons" "^3.0.1"
 
+"@streamparser/json@^0.0.22":
+  version "0.0.22"
+  resolved "https://registry.yarnpkg.com/@streamparser/json/-/json-0.0.22.tgz#8ddcbcc8c3ca77aeadf80af47f54a64c8739a037"
+  integrity sha512-b6gTSBjJ8G8SuO3Gbbj+zXbVx8NSs1EbpbMKpzGLWMdkR+98McH9bEjSz3+0mPJf68c5nxa3CrJHp5EQNXM6zQ==
+
 "@surma/rollup-plugin-off-main-thread@^2.2.3":
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-2.2.3.tgz#ee34985952ca21558ab0d952f00298ad2190c053"
@@ -6873,7 +6878,7 @@ jsonify@^0.0.1:
 jsonparse@^1.2.0:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"
-  integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=
+  integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==
 
 jsonpointer@^5.0.0:
   version "5.0.1"
@@ -10294,16 +10299,7 @@ string-length@^4.0.2:
     char-regex "^1.0.2"
     strip-ansi "^6.0.0"
 
-"string-width-cjs@npm:string-width@^4.2.0":
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
-string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -10433,7 +10429,7 @@ stringify-object@^3.3.0:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -10447,13 +10443,6 @@ strip-ansi@^0.3.0:
   dependencies:
     ansi-regex "^0.2.1"
 
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.1, strip-ansi@^7.1.0:
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45"
@@ -11829,16 +11818,7 @@ workbox-window@7.4.0, workbox-window@^7.4.0:
     "@types/trusted-types" "^2.0.2"
     workbox-core "7.4.0"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
-wrap-ansi@^7.0.0:
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==