optimize wasm impl

Author: gfx

benchmark/decode-string.ts

@@ -5,6 +5,11 @@ import { getWasmError, utf8DecodeWasm } from "../src/utils/utf8-wasm.ts";
 // @ts-ignore
 import Benchmark from "benchmark";
 
+// description
+console.log("utf8DecodeJs - pure JS implementation");
+console.log("utf8DecodeTD - TextDecoder implementation");
+console.log("utf8DecodeWasm - WebAssembly implementation");
+
 // Show wasm status
 console.log("=".repeat(60));
 console.log("WebAssembly Status:");
@@ -42,7 +47,7 @@ for (const baseStr of ["A", "あ", "🌏"]) {
       }
     });
 
-    suite.add("utf8DecodeTD (TextDecoder)", () => {
+    suite.add("utf8DecodeTD", () => {
       if (utf8DecodeTD(bytes, 0, byteLength) !== str) {
         throw new Error("wrong result!");
       }

benchmark/encode-string.ts

@@ -5,6 +5,11 @@ import { getWasmError, utf8EncodeWasm } from "../src/utils/utf8-wasm.ts";
 // @ts-ignore
 import Benchmark from "benchmark";
 
+// description
+console.log("utf8EncodeJs - pure JS implementation");
+console.log("utf8EncodeTE - TextEncoder implementation");
+console.log("utf8EncodeWasm - WebAssembly implementation");
+
 // Show wasm status
 console.log("=".repeat(60));
 console.log("WebAssembly Status:");
@@ -39,7 +44,7 @@ for (const baseStr of ["A", "あ", "🌏"]) {
       utf8EncodeJs(str, buffer, 0);
     });
 
-    suite.add("utf8EncodeTE (TextEncoder)", () => {
+    suite.add("utf8EncodeTE", () => {
       utf8EncodeTE(str, buffer, 0);
     });
 

src/utils/utf8-wasm-binary.ts

@@ -1,4 +1,4 @@
 // Auto-generated by wasm/build.sh - do not edit manually
 // Source: wasm/utf8.wat
 
-export const wasmBinary = "AGFzbQEAAAABHgVgAW8Bf2ACb38Bf2ABfwFkb2ACb28BZG9gAX8BfwJrBA53YXNtOmpzLXN0cmluZwZsZW5ndGgAAA53YXNtOmpzLXN0cmluZwpjaGFyQ29kZUF0AAEOd2FzbTpqcy1zdHJpbmcMZnJvbUNoYXJDb2RlAAIOd2FzbTpqcy1zdHJpbmcGY29uY2F0AAMDBAMAAQQFAwEAAQYIAX8AQYCAAgsHOAQGbWVtb3J5AgAJdXRmOENvdW50AAQKdXRmOEVuY29kZQAFEnV0ZjhEZWNvZGVUb01lbW9yeQAGCqoGA3UBBH8gABAAIQICQANAIAEgAk8NASAAIAEQASEEIARBgAFJBEAgA0EBaiEDBSAEQYAQSQRAIANBAmohAwUgBEGAsANPIARB/7cDTXEEQCADQQRqIQMgAUEBaiEBBSADQQNqIQMLCwsgAUEBaiEBDAALAAsgAwu9AgEFfyAAEAAhAyABIQQCQANAIAIgA08NASAAIAIQASEFIAVBgAFJBEAgBCAFOgAAIARBAWohBAUgBUGAEEkEQCAEIAVBBnZBwAFyOgAAIARBAWogBUE/cUGAAXI6AAAgBEECaiEEBSAFQYCwA08gBUH/twNNcQRAIAJBAWohAiAAIAIQASEGIAVBgLADa0EKdCAGQYC4A2tqQYCABGohBSAEIAVBEnZB8AFyOgAAIARBAWogBUEMdkE/cUGAAXI6AAAgBEECaiAFQQZ2QT9xQYABcjoAACAEQQNqIAVBP3FBgAFyOgAAIARBBGohBAUgBCAFQQx2QeABcjoAACAEQQFqIAVBBnZBP3FBgAFyOgAAIARBAmogBUE/cUGAAXI6AAAgBEEDaiEECwsLIAJBAWohAgwACwALIAQgAWsL8gIBCH9BACEBIAAhAiMAIQMCQANAIAEgAk8NASABLQAAIQQgBEGAAXFFBEAgAyAEOwEAIANBAmohAyABQQFqIQEMAQsgBEHgAXFBwAFGBEAgAUEBai0AACEFIARBH3FBBnQgBUE/cXIhCCADIAg7AQAgA0ECaiEDIAFBAmohAQwBCyAEQfABcUHgAUYEQCABQQFqLQAAIQUgAUECai0AACEGIARBD3FBDHQgBUE/cUEGdHIgBkE/cXIhCCADIAg7AQAgA0ECaiEDIAFBA2ohAQwBCyAEQfgBcUHwAUYEQCABQQFqLQAAIQUgAUECai0AACEGIAFBA2otAAAhByAEQQdxQRJ0IAVBP3FBDHRyIAZBP3FBBnRyIAdBP3FyIQggCEGAgARrIQggAyAIQQp2QYCwA3I7AQAgA0ECaiEDIAMgCEH/B3FBgLgDcjsBACADQQJqIQMgAUEEaiEBDAELIAFBAWohAQwACwALIAMjAGtBAXYL";
+export const wasmBinary = "AGFzbQEAAAABNQhedwFgAW8Bf2ADb2QAfwF/YANkAH9/AWRvYAJvfwF/YAJ/ZAABf2ABfwFkAGADZAB/fwFvAl8DDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nEWludG9DaGFyQ29kZUFycmF5AAIOd2FzbTpqcy1zdHJpbmcRZnJvbUNoYXJDb2RlQXJyYXkAAwMGBQEEBQYHBQMBAAEHVAYGbWVtb3J5AgAJdXRmOENvdW50AAMKdXRmOEVuY29kZQAEEXV0ZjhEZWNvZGVUb0FycmF5AAUKYWxsb2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwr7BgWUAQIEfwFkACAAEAAhASABRQRAQQAPC0EAIAH7BgAhBSAAIAVBABABGgJAA0AgAiABTw0BIAUgAvsNACEEIARBgAFJBEAgA0EBaiEDBSAEQYAQSQRAIANBAmohAwUgBEGAsANPIARB/7cDTXEEQCADQQRqIQMgAkEBaiECBSADQQNqIQMLCwsgAkEBaiECDAALAAsgAwvdAgIFfwFkACAAEAAhAiABIQQgAkUEQEEADwtBACAC+wYAIQcgACAHQQAQARoCQANAIAMgAk8NASAHIAP7DQAhBSAFQYABSQRAIAQgBToAACAEQQFqIQQFIAVBgBBJBEAgBCAFQQZ2QcABcjoAACAEQQFqIAVBP3FBgAFyOgAAIARBAmohBAUgBUGAsANPIAVB/7cDTXEEQCADQQFqIQMgByAD+w0AIQYgBUGAsANrQQp0IAZBgLgDa2pBgIAEaiEFIAQgBUESdkHwAXI6AAAgBEEBaiAFQQx2QT9xQYABcjoAACAEQQJqIAVBBnZBP3FBgAFyOgAAIARBA2ogBUE/cUGAAXI6AAAgBEEEaiEEBSAEIAVBDHZB4AFyOgAAIARBAWogBUEGdkE/cUGAAXI6AAAgBEECaiAFQT9xQYABcjoAACAEQQNqIQQLCwsgA0EBaiEDDAALAAsgBCABawvuAgEIfyAAIQMCQANAIAIgA08NASACLQAAIQUgBUGAAXFFBEAgASAEIAX7DgAgBEEBaiEEIAJBAWohAgwBCyAFQeABcUHAAUYEQCACQQFqLQAAIQYgBUEfcUEGdCAGQT9xciEJIAEgBCAJ+w4AIARBAWohBCACQQJqIQIMAQsgBUHwAXFB4AFGBEAgAkEBai0AACEGIAJBAmotAAAhByAFQQ9xQQx0IAZBP3FBBnRyIAdBP3FyIQkgASAEIAn7DgAgBEEBaiEEIAJBA2ohAgwBCyAFQfgBcUHwAUYEQCACQQFqLQAAIQYgAkECai0AACEHIAJBA2otAAAhCCAFQQdxQRJ0IAZBP3FBDHRyIAdBP3FBBnRyIAhBP3FyIQkgCUGAgARrIQkgASAEIAlBCnZBgLADcvsOACAEQQFqIQQgASAEIAlB/wdxQYC4A3L7DgAgBEEBaiEEIAJBBGohAgwBCyACQQFqIQIMAAsACyAECwkAQQAgAPsGAAsKACAAIAEgAhACCw==";

src/utils/utf8-wasm.ts

@@ -1,14 +1,12 @@
 /**
- * WebAssembly-based UTF-8 string processing using js-string-builtins.
+ * WebAssembly-based UTF-8 string processing using js-string-builtins with GC arrays.
  *
  * Environment variables:
  * - MSGPACK_WASM=force: Force wasm mode, throw error if wasm fails to load
  * - MSGPACK_WASM=never: Disable wasm, always use pure JS
  *
- * Three-tier fallback:
- * 1. Native js-string-builtins (Chrome 130+, Firefox 134+)
- * 2. Wasm + polyfill (older browsers with WebAssembly)
- * 3. Pure JS (no WebAssembly support)
+ * This implementation uses WASM GC arrays with intoCharCodeArray/fromCharCodeArray
+ * for efficient bulk string operations instead of character-by-character processing.
  */
 
 import { wasmBinary } from "./utf8-wasm-binary.ts";
@@ -39,19 +37,18 @@ function getWasmMode(): "force" | "never" | "auto" {
 
 const WASM_MODE = getWasmMode();
 
+// GC array type (opaque reference)
+type I16Array = object;
+
 interface WasmExports {
   memory: WebAssembly.Memory;
   utf8Count(str: string): number;
   utf8Encode(str: string, offset: number): number;
-  utf8DecodeToMemory(length: number): number;
+  utf8DecodeToArray(length: number, arr: I16Array): number;
+  allocArray(size: number): I16Array;
+  arrayToString(arr: I16Array, start: number, end: number): string;
 }
 
-// Memory layout constants (must match WAT file)
-const UTF16_OFFSET = 32768; // 32KB offset for UTF-16 output
-
-// Shared TextDecoder for UTF-16LE decoding
-const utf16Decoder = new TextDecoder("utf-16le");
-
 let wasmInstance: WasmExports | null = null;
 let wasmInitError: Error | null = null;
 
@@ -68,17 +65,6 @@ function base64ToBytes(base64: string): Uint8Array {
   return new Uint8Array(Buffer.from(base64, "base64"));
 }
 
-// Polyfill for js-string-builtins (used when native builtins unavailable)
-const jsStringPolyfill = {
-  // eslint-disable-next-line @typescript-eslint/naming-convention
-  "wasm:js-string": {
-    length: (s: string) => s.length,
-    charCodeAt: (s: string, i: number) => s.charCodeAt(i),
-    fromCharCode: (code: number) => String.fromCharCode(code),
-    concat: (a: string, b: string) => a + b,
-  },
-};
-
 function tryInitWasm(): void {
   if (wasmInstance !== null || wasmInitError !== null) {
     return; // Already initialized or failed
@@ -96,14 +82,9 @@ function tryInitWasm(): void {
 
     const bytes = base64ToBytes(wasmBinary);
 
-    // Try with builtins option (native support)
-    // If builtins not supported, option is ignored and polyfill is used
-
-
+    // Requires js-string builtins support (Node.js 24+ / Chrome 130+ / Firefox 134+)
     const module: WebAssembly.Module = new (WebAssembly.Module as any)(bytes, { builtins: ["js-string"] });
-
-
-    const instance = new (WebAssembly.Instance)(module, jsStringPolyfill);
+    const instance = new WebAssembly.Instance(module);
     wasmInstance = instance.exports as unknown as WasmExports;
   } catch (e) {
     wasmInitError = e instanceof Error ? e : new Error(String(e));
@@ -121,7 +102,7 @@ tryInitWasm();
  * Whether wasm is available and initialized.
  */
 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-export const WASM_AVAILABLE = (wasmInstance !== null);
+export const WASM_AVAILABLE = wasmInstance !== null;
 
 /**
  * Get the wasm initialization error, if any.
@@ -156,16 +137,20 @@ export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: nu
     throw new Error("wasm not initialized");
   }
 
+  // Estimate max byte length without a full pass over the string.
+  // Each UTF-16 code unit can produce at most 3 UTF-8 bytes (BMP chars).
+  // Surrogate pairs (2 code units) produce 4 bytes, so 3 bytes/code unit is safe.
+  const maxByteLength = str.length * 3;
+
   // Ensure wasm memory is large enough
-  const byteLength = wasmInstance.utf8Count(str);
-  const requiredPages = Math.ceil((outputOffset + byteLength) / 65536);
+  const requiredPages = Math.ceil(maxByteLength / 65536);
   const currentPages = wasmInstance.memory.buffer.byteLength / 65536;
 
   if (requiredPages > currentPages) {
     wasmInstance.memory.grow(requiredPages - currentPages);
   }
 
-  // Encode to wasm memory
+  // Encode to wasm memory (uses intoCharCodeArray for bulk char extraction)
   const bytesWritten = wasmInstance.utf8Encode(str, 0);
 
   // Copy from wasm memory to output buffer
@@ -177,32 +162,36 @@ export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: nu
 
 /**
  * Decode UTF-8 bytes to string.
+ * Uses GC arrays with fromCharCodeArray for efficient string creation.
  */
 export function utf8DecodeWasm(bytes: Uint8Array, inputOffset: number, byteLength: number): string {
   if (wasmInstance === null) {
     throw new Error("wasm not initialized");
   }
 
-  // Ensure wasm memory is large enough
-  // Need space for UTF-8 input (0 to byteLength) and UTF-16 output (UTF16_OFFSET onwards)
-  // Max UTF-16 output is 2 bytes per code unit, and max expansion is 2x (for ASCII)
-  const utf16MaxBytes = byteLength * 2;
-  const requiredBytes = UTF16_OFFSET + utf16MaxBytes;
-  const requiredPages = Math.ceil(requiredBytes / 65536);
+  // Handle empty input
+  if (byteLength === 0) {
+    return "";
+  }
+
+  // Ensure wasm memory is large enough for UTF-8 input
+  const requiredPages = Math.ceil(byteLength / 65536);
   const currentPages = wasmInstance.memory.buffer.byteLength / 65536;
 
   if (requiredPages > currentPages) {
     wasmInstance.memory.grow(requiredPages - currentPages);
   }
 
-  // Copy bytes to wasm memory at offset 0
+  // Copy UTF-8 bytes to wasm linear memory at offset 0
   const wasmBytes = new Uint8Array(wasmInstance.memory.buffer, 0, byteLength);
   wasmBytes.set(bytes.subarray(inputOffset, inputOffset + byteLength));
 
-  // Decode UTF-8 to UTF-16 in wasm memory, get number of code units
-  const codeUnits = wasmInstance.utf8DecodeToMemory(byteLength);
+  // Allocate GC array for UTF-16 output (max size = byteLength for ASCII)
+  const arr = wasmInstance.allocArray(byteLength);
+
+  // Decode UTF-8 to UTF-16 in GC array
+  const codeUnits = wasmInstance.utf8DecodeToArray(byteLength, arr);
 
-  // Read UTF-16 code units from wasm memory and decode to string
-  const utf16Bytes = new Uint8Array(wasmInstance.memory.buffer, UTF16_OFFSET, codeUnits * 2);
-  return utf16Decoder.decode(utf16Bytes);
+  // Create string directly from GC array using fromCharCodeArray
+  return wasmInstance.arrayToString(arr, 0, codeUnits);
 }

test/utf8-wasm.test.ts

@@ -20,7 +20,9 @@ describe("utf8-wasm", () => {
         assert.ok(exports !== null);
         assert.ok(typeof exports!.utf8Count === "function");
         assert.ok(typeof exports!.utf8Encode === "function");
-        assert.ok(typeof exports!.utf8DecodeToMemory === "function");
+        assert.ok(typeof exports!.utf8DecodeToArray === "function");
+        assert.ok(typeof exports!.allocArray === "function");
+        assert.ok(typeof exports!.arrayToString === "function");
         assert.ok(exports!.memory instanceof WebAssembly.Memory);
       } else {
         assert.strictEqual(exports, null);

wasm/build.sh

@@ -7,7 +7,7 @@ set -e
 cd "$(dirname "$0")"
 
 echo "Compiling utf8.wat -> utf8.wasm..."
-wasm-as utf8.wat -o utf8.wasm --enable-reference-types --enable-gc
+wasm-as utf8.wat -o utf8.wasm --enable-reference-types --enable-gc --enable-strings
 
 echo "Generating base64 TypeScript module..."
 cat > ../src/utils/utf8-wasm-binary.ts << 'HEADER'