feat: Support buffer views serialization

Author: tteles-cf

.changeset/fix-arraybuffer-serialization.md

@@ -0,0 +1,5 @@
+---
+"capnweb": patch
+---
+
+Support exact ArrayBuffer, DataView, and typed array serialization over RPC.

README.md

@@ -199,15 +199,14 @@ The following types can be passed over RPC (in arguments or return values), and
 * Arrays
 * `bigint`
 * `Date`
-* `Uint8Array`
+* `ArrayBuffer`, `DataView`, and typed arrays
 * `Error` and its well-known subclasses
 * `Blob`
 * `ReadableStream` and `WritableStream`, with automatic flow control.
 * `Headers`, `Request`, and `Response` from the Fetch API.
 
 The following types are not supported as of this writing, but may be added in the future:
 * `Map` and `Set`
-* `ArrayBuffer` and typed arrays other than `Uint8Array`
 * `RegExp`
 
 The following are intentionally NOT supported:

__tests__/index.test.ts

@@ -160,6 +160,16 @@ describe("simple serialization", () => {
     expect(new Uint8Array(deserialized)).toStrictEqual(bytes);
   })
 
+  it("can serialize Uint8Array as legacy bytes without a type marker", () => {
+    let bytes = new Uint8Array([72, 101, 108, 108, 111]);
+    let serialized = serialize(bytes);
+    expect(serialized).toBe('["bytes","SGVsbG8"]');
+
+    let deserialized = deserialize(serialized) as Uint8Array;
+    expect(deserialized).toBeInstanceOf(Uint8Array);
+    expect(new Uint8Array(deserialized)).toStrictEqual(bytes);
+  })
+
   it("can serialize Node.js Buffer as bytes", () => {
     if (typeof Buffer === "undefined") return; // skip in browsers
     let buf = Buffer.from("hello!");
@@ -170,6 +180,73 @@ describe("simple serialization", () => {
     expect(new Uint8Array(deserialized)).toStrictEqual(new Uint8Array(buf));
   })
 
+  it("can serialize ArrayBuffer as bytes with an ArrayBuffer marker", () => {
+    let bytes = new Uint8Array([72, 101, 108, 108, 111]);
+    let serialized = serialize(bytes.buffer);
+    expect(serialized).toBe('["bytes","SGVsbG8","ArrayBuffer"]');
+
+    let deserialized = deserialize(serialized);
+    expect(deserialized).toBeInstanceOf(ArrayBuffer);
+    expect(new Uint8Array(deserialized as ArrayBuffer)).toStrictEqual(bytes);
+  })
+
+  it("can serialize typed array views as bytes with type markers", () => {
+    let cases = [
+      {
+        name: "DataView",
+        elementSize: 1,
+        makeView: (buffer: ArrayBuffer, offset: number, byteLength: number) =>
+            new DataView(buffer, offset, byteLength),
+      },
+      ...[
+        Int8Array,
+        Uint8ClampedArray,
+        Int16Array,
+        Uint16Array,
+        Int32Array,
+        Uint32Array,
+        BigInt64Array,
+        BigUint64Array,
+        Float32Array,
+        Float64Array,
+      ].map(Type => ({
+        name: Type.name,
+        elementSize: Type.BYTES_PER_ELEMENT,
+        makeView: (buffer: ArrayBuffer, offset: number, byteLength: number) =>
+            new Type(buffer, offset, byteLength / Type.BYTES_PER_ELEMENT),
+      })),
+    ];
+
+    for (let {name, elementSize, makeView} of cases) {
+      // Use a non-zero offset and extra trailing byte to verify only the view's visible byte range
+      // is serialized, not the whole backing buffer.
+      let byteLength = elementSize * 2;
+      let offset = elementSize;
+      let backing = new ArrayBuffer(offset + byteLength + 1);
+      let bytes = new Uint8Array(byteLength);
+      for (let i = 0; i < bytes.length; i++) bytes[i] = i + 1;
+      new Uint8Array(backing, offset, byteLength).set(bytes);
+
+      let view = makeView(backing, offset, byteLength);
+      let serialized = serialize(view);
+      let parsed = JSON.parse(serialized) as [string, string, string];
+      expect(parsed[0]).toBe("bytes");
+      expect(parsed[2]).toBe(name);
+
+      let deserialized = deserialize(serialized) as ArrayBufferView;
+      expect(Object.getPrototypeOf(deserialized)).toBe(Object.getPrototypeOf(view));
+      expect(new Uint8Array(
+          deserialized.buffer, deserialized.byteOffset, deserialized.byteLength)).toStrictEqual(bytes);
+    }
+  })
+
+  it("throws for unknown bytes type markers", () => {
+    expect(() => deserialize('["bytes","SGVsbG8","invalidUint8Array"]')).toThrowError(
+        "Unknown bytes type marker: invalidUint8Array");
+    expect(() => deserialize('["bytes","SGVsbG8",123]')).toThrowError(
+        "Unknown bytes type marker type: number");
+  })
+
   it("preserves Invalid Date values through serialization", () => {
     let invalidDate = new Date(NaN);
     let serialized = serialize(invalidDate);

protocol.md

@@ -168,9 +168,11 @@ The literal value `undefined`.
 
 The values Infinity, -Infinity, and NaN.
 
-`["bytes", base64]`
+`["bytes", base64]`, `["bytes", base64, type]`
 
-A `Uint8Array`, represented as a base64-encoded string.
+A `Uint8Array`, `ArrayBuffer`, `DataView`, or typed array, represented as a base64-encoded string.
+The optional third element preserves the byte container type across the wire. If it is
+omitted, the receiver should deserialize bytes as its default `Uint8Array` for backwards compatibility.
 
 `["blob", type, readableExpression]`
 

src/core.ts

@@ -95,6 +95,18 @@ export function typeForRpc(value: unknown): TypeForRpc {
 
     case Uint8Array.prototype:
     case BUFFER_PROTOTYPE:
+    case ArrayBuffer.prototype:
+    case DataView.prototype:
+    case Int8Array.prototype:
+    case Uint8ClampedArray.prototype:
+    case Int16Array.prototype:
+    case Uint16Array.prototype:
+    case Int32Array.prototype:
+    case Uint32Array.prototype:
+    case BigInt64Array.prototype:
+    case BigUint64Array.prototype:
+    case Float32Array.prototype:
+    case Float64Array.prototype:
       return "bytes";
 
     case WritableStream.prototype:

src/serialize.ts

@@ -162,12 +162,59 @@ export class Devaluator {
       }
 
       case "bytes": {
-        let bytes = value as Uint8Array;
-        if (bytes.toBase64) {
-          return ["bytes", bytes.toBase64({omitPadding: true})];
+        let alternateTypeName: string | undefined;
+        let bytes: Uint8Array | undefined;
+        switch (Object.getPrototypeOf(value)) {
+          case ArrayBuffer.prototype:
+            alternateTypeName = "ArrayBuffer";
+            bytes = new Uint8Array(value as ArrayBuffer);
+            break;
+          case DataView.prototype:
+            alternateTypeName = "DataView";
+            break;
+          case Int8Array.prototype:
+            alternateTypeName = "Int8Array";
+            break;
+          case Uint8ClampedArray.prototype:
+            alternateTypeName = "Uint8ClampedArray";
+            break;
+          case Int16Array.prototype:
+            alternateTypeName = "Int16Array";
+            break;
+          case Uint16Array.prototype:
+            alternateTypeName = "Uint16Array";
+            break;
+          case Int32Array.prototype:
+            alternateTypeName = "Int32Array";
+            break;
+          case Uint32Array.prototype:
+            alternateTypeName = "Uint32Array";
+            break;
+          case BigInt64Array.prototype:
+            alternateTypeName = "BigInt64Array";
+            break;
+          case BigUint64Array.prototype:
+            alternateTypeName = "BigUint64Array";
+            break;
+          case Float32Array.prototype:
+            alternateTypeName = "Float32Array";
+            break;
+          case Float64Array.prototype:
+            alternateTypeName = "Float64Array";
+            break;
+          default:
+            bytes = value as Uint8Array;
+            break;
+        }
+        if (bytes === undefined) {
+          let view = value as ArrayBufferView;
+          bytes = new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
         }
+
         let b64: string;
-        if (typeof Buffer !== "undefined") {
+        if (bytes.toBase64) {
+          b64 = bytes.toBase64({omitPadding: true});
+        } else if (typeof Buffer !== "undefined") {
           let buf = bytes instanceof Buffer ? bytes
               : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
           b64 = buf.toString("base64");
@@ -178,7 +225,8 @@ export class Devaluator {
           }
           b64 = btoa(binary);
         }
-        return ["bytes", b64.replace(/=+$/, "")];
+        b64 = b64.replace(/=+$/, "");
+        return alternateTypeName === undefined ? ["bytes", b64] : ["bytes", b64, alternateTypeName];
       }
 
       case "headers":
@@ -601,19 +649,45 @@ export class Evaluator {
           break;
         case "bytes": {
           if (typeof value[1] == "string") {
+            let bytes: Uint8Array;
             if (typeof Buffer !== "undefined") {
-              return Buffer.from(value[1], "base64");
+              bytes = Buffer.from(value[1], "base64");
             } else if (Uint8Array.fromBase64) {
-              return Uint8Array.fromBase64(value[1]);
+              bytes = Uint8Array.fromBase64(value[1]);
             } else {
               let bs = atob(value[1]);
               let len = bs.length;
-              let bytes = new Uint8Array(len);
+              bytes = new Uint8Array(len);
               for (let i = 0; i < len; i++) {
                 bytes[i] = bs.charCodeAt(i);
               }
+            }
+            if (value.length === 2) {
               return bytes;
             }
+            if (typeof value[2] !== "string") {
+              throw new TypeError(`Unknown bytes type marker type: ${typeof value[2]}`);
+            }
+
+            let buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+            switch (value[2]) {
+              case "ArrayBuffer": return buffer;
+              case "DataView": return new DataView(buffer);
+              case "Int8Array": return new Int8Array(buffer);
+              case "Uint8ClampedArray": return new Uint8ClampedArray(buffer);
+              case "Int16Array": return new Int16Array(buffer);
+              case "Uint16Array": return new Uint16Array(buffer);
+              case "Int32Array": return new Int32Array(buffer);
+              case "Uint32Array": return new Uint32Array(buffer);
+              case "BigInt64Array": return new BigInt64Array(buffer);
+              case "BigUint64Array": return new BigUint64Array(buffer);
+              case "Float32Array": return new Float32Array(buffer);
+              case "Float64Array": return new Float64Array(buffer);
+              default: {
+                let marker = value[2].slice(0, 64);
+                throw new TypeError(`Unknown bytes type marker: ${marker}`);
+              }
+            }
           }
           break;
         }