diff --git a/package.json b/package.json index e7ae0fe14a..072753295e 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "@fluent/langneg": "^0.7.0", "@fluent/react": "^0.15.2", "@lezer/highlight": "^1.2.3", + "@streamparser/json": "^0.0.22", "@tgwf/co2": "^0.17.0", "array-move": "^3.0.1", "array-range": "^1.0.1", diff --git a/src/actions/receive-profile.ts b/src/actions/receive-profile.ts index 76909edbdb..c3b598fd89 100644 --- a/src/actions/receive-profile.ts +++ b/src/actions/receive-profile.ts @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { oneLine } from 'common-tags'; +import { JSONParser } from '@streamparser/json'; import queryString from 'query-string'; import type JSZip from 'jszip'; import { @@ -1209,6 +1210,151 @@ async function _extractJsonFromArrayBuffer( return JSON.parse(textDecoder.decode(profileBytes)); } +function _concatUint8Chunks( + chunks: Array, + totalBytes: number +): Uint8Array { + const result = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} + +function _normalizeStreamJsonParseError( + chunks: Array, + totalBytes: number, + fallbackError: unknown +): Error { + const textDecoder = new TextDecoder(); + try { + JSON.parse(textDecoder.decode(_concatUint8Chunks(chunks, totalBytes))); + } catch (nativeError) { + if (nativeError instanceof Error) { + return nativeError; + } + return new Error(String(nativeError)); + } + if (fallbackError instanceof Error) { + return fallbackError; + } + return new Error(String(fallbackError)); +} + +/** + * Parse JSON from a ReadableStream incrementally. + * + * This avoids creating one massive JS string (which can hit engine string-size limits) + * before JSON parsing. + */ +async function _extractJsonFromReadableStream( + stream: ReadableStream +): Promise { + const reader = stream.getReader(); + const initialChunks: Array = []; + let initialChunksLength = 0; + const headerBytes = new Uint8Array(3); + let headerBytesLength = 0; + + // Read enough bytes to reliably detect gzip headers even if the stream + // splits the first three bytes across chunks. + while (headerBytesLength < 3) { + const { done, value } = await reader.read(); + if (done || value === undefined) { + break; + } + initialChunks.push(value); + initialChunksLength += value.byteLength; + for ( + let headerIndex = 0; + headerIndex < value.byteLength && headerBytesLength < 3; + headerIndex++ + ) { + headerBytes[headerBytesLength++] = value[headerIndex]; + } + } + + if (initialChunksLength === 0) { + throw new SyntaxError('Unexpected end of JSON input'); + } + + if (headerBytesLength === 3 && isGzip(headerBytes)) { + let totalBytes = initialChunksLength; + const chunks = [...initialChunks]; + while (true) { + const { done, value } = await reader.read(); + if (done || value === undefined) { + break; + } + chunks.push(value); + totalBytes += value.byteLength; + } + const compressedBytes = _concatUint8Chunks(chunks, totalBytes); + return _extractJsonFromArrayBuffer(compressedBytes.buffer); + } + + const parser = new JSONParser(); + const parserErrorProbeChunks: Array = []; + let parserErrorProbeLength = 0; + const maxErrorProbeBytes = 64 * 1024; + let rootValue: unknown; + let hasRootValue = false; + + const appendParserErrorProbe = (chunk: Uint8Array) => { + if (parserErrorProbeLength >= maxErrorProbeBytes) { + return; + } + const bytesRemaining = maxErrorProbeBytes - parserErrorProbeLength; + if (chunk.byteLength <= bytesRemaining) { + parserErrorProbeChunks.push(chunk); + parserErrorProbeLength += chunk.byteLength; + return; + } + parserErrorProbeChunks.push(chunk.subarray(0, bytesRemaining)); + parserErrorProbeLength += bytesRemaining; + }; + + parser.onValue = ({ value, stack }) => { + if (stack.length === 0 && value !== undefined) { + rootValue = value; + hasRootValue = true; + } + }; + + try { + for (const chunk of initialChunks) { + appendParserErrorProbe(chunk); + parser.write(chunk); + } + + while (true) { + const { done, value } = await reader.read(); + if (done || value === undefined) { + break; + } + appendParserErrorProbe(value); + parser.write(value); + } + if (!parser.isEnded) { + parser.end(); + } + } catch (error) { + throw _normalizeStreamJsonParseError( + parserErrorProbeChunks, + parserErrorProbeLength, + error + ); + } + + if (!hasRootValue) { + throw new SyntaxError('Unexpected end of JSON input'); + } + + return rootValue; +} + /** * Don't trust third party responses, try and handle a variety of responses gracefully. */ @@ -1219,7 +1365,11 @@ async function _extractJsonFromResponse( ): Promise { let arrayBuffer: ArrayBuffer | null = null; try { - // await before returning so that we can catch JSON parse errors. + if (fileType === 'application/json' && response.body) { + return await _extractJsonFromReadableStream(response.body); + } + + // Await before returning so that we can catch JSON parse errors. arrayBuffer = await response.arrayBuffer(); return await _extractJsonFromArrayBuffer(arrayBuffer); } catch (error) { diff --git a/yarn.lock b/yarn.lock index 35af0f4847..f79eeb916c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2116,6 +2116,11 @@ dependencies: "@sinonjs/commons" "^3.0.1" +"@streamparser/json@^0.0.22": + version "0.0.22" + resolved "https://registry.yarnpkg.com/@streamparser/json/-/json-0.0.22.tgz#8ddcbcc8c3ca77aeadf80af47f54a64c8739a037" + integrity sha512-b6gTSBjJ8G8SuO3Gbbj+zXbVx8NSs1EbpbMKpzGLWMdkR+98McH9bEjSz3+0mPJf68c5nxa3CrJHp5EQNXM6zQ== + "@surma/rollup-plugin-off-main-thread@^2.2.3": version "2.2.3" resolved "https://registry.yarnpkg.com/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-2.2.3.tgz#ee34985952ca21558ab0d952f00298ad2190c053" @@ -6873,7 +6878,7 @@ jsonify@^0.0.1: jsonparse@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" - integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= + integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg== jsonpointer@^5.0.0: version "5.0.1" @@ -10294,16 +10299,7 @@ string-length@^4.0.2: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -10433,7 +10429,7 @@ stringify-object@^3.3.0: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -10447,13 +10443,6 @@ strip-ansi@^0.3.0: dependencies: ansi-regex "^0.2.1" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -11829,16 +11818,7 @@ workbox-window@7.4.0, workbox-window@^7.4.0: "@types/trusted-types" "^2.0.2" workbox-core "7.4.0" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==