fix(session): allow processing of prekeys in omemo:2 wire-format

`processPreKey` fed the bundle's signed prekey and prekey public
keys straight to ECDHE. omemo:2 transfers these as raw 32-byte
curve keys, which the version 2.0.1 DH hardening rejects with
"Invalid public key".

Normalize them to the internal 33-byte 0x05-prefixed form via a new
`normalizeRemotePreKey` protocol-profile hook.

Author: jcbrand

CHANGELOG.md

@@ -1,5 +1,16 @@
 # CHANGES
 
+## 2.0.2 (Unreleased)
+
+### Fix: omemo:2 session establishment from a wire-form PreKey bundle
+
+- `SessionBuilder.processPreKey` now normalises the bundle's signed-pre-key and
+  pre-key public keys to the internal 33-byte `0x05`-prefixed curve form, via a new
+  `normalizeRemotePreKey` protocol-profile hook. omemo:2 transfers these keys in
+  their raw 32-byte curve form, which the 2.0.1 DH hardening rejected with
+  "Invalid public key", breaking the establishment of every new omemo:2
+  session.
+
 ## 2.0.1 (2026-06-19)
 
 ### Security / compatibility: eval-free protobuf (strict CSP support)

package-lock.json

@@ -1,7 +1,7 @@
 {
     "name": "libomemo.js",
     "repository": "github:conversejs/libomemo.js",
-    "version": "2.0.1",
+    "version": "2.0.2",
     "license": "GPL-3.0",
     "type": "module",
     "main": "./dist/libomemo.umd.js",

package.json

@@ -37,6 +37,7 @@ export class SessionBuilder {
             // Normalise the remote identity key: for omemo:2 the wire form is
             // Ed25519 and is converted to its Curve25519 equivalent for DH.
             const remoteId = await this.#profile.normalizeRemoteIdentityKey(device.identityKey);
+
             // Trust is keyed on the form the consumer published in the bundle
             // (Ed25519 for omemo:2, Curve25519 for 0.3.0).
             const trustKey = remoteId.ed ?? remoteId.curve;
@@ -51,21 +52,30 @@ export class SessionBuilder {
                 throw new Error("Identity key changed");
             }
 
+            // Normalise the prekeys, which may be in the raw 32-byte curve
+            // form, to the internal 33-byte form, since the DH path
+            // no longer accepts unprefixed keys.
+            const signedPreKeyPub = this.#profile.normalizeRemotePreKey(
+                device.signedPreKey.publicKey
+            );
+            const devicePreKey = device.preKey?.publicKey
+                ? this.#profile.normalizeRemotePreKey(device.preKey.publicKey)
+                : undefined;
+
             await internalCrypto.Ed25519Verify(
                 remoteId.curve,
-                this.#profile.signedPreKeySignatureData(device.signedPreKey.publicKey),
+                this.#profile.signedPreKeySignatureData(signedPreKeyPub),
                 device.signedPreKey.signature
             );
 
             const baseKey = await internalCrypto.createKeyPair();
-            const devicePreKey = device.preKey ? device.preKey.publicKey : undefined;
             const session = await this.#initSession(
                 true,
                 baseKey,
                 undefined,
                 remoteId.curve,
                 devicePreKey,
-                device.signedPreKey.publicKey,
+                signedPreKeyPub,
                 this.#registrationId(device.registrationId),
                 remoteId.ed
             );

src/session/builder.ts

@@ -161,6 +161,15 @@ export interface ProtocolProfile {
      * libomemo-c.
      */
     signedPreKeySignatureData(publicKey: ArrayBuffer): ArrayBuffer;
+
+    /**
+     * Normalise a (signed-)pre-key public received on the wire (from a PreKey
+     * bundle) into the library's internal 33-byte 0x05-prefixed curve form. 0.3.0
+     * publishes that form already; omemo:2 transfers the raw 32-byte curve form
+     * and restores the prefix here. This mirrors the wire→internal normalisation
+     * the profile already applies to ratchet/base keys on the message paths.
+     */
+    normalizeRemotePreKey(wireKey: ArrayBuffer): ArrayBuffer;
 }
 
 /**
@@ -346,6 +355,12 @@ const OMEMO_0_3_0: ProtocolProfile = {
     signedPreKeySignatureData(publicKey: ArrayBuffer): ArrayBuffer {
         return publicKey;
     },
+
+    normalizeRemotePreKey(wireKey: ArrayBuffer): ArrayBuffer {
+        // 0.3.0 publishes the 33-byte 0x05-prefixed form; a raw 32-byte key is
+        // malformed and is left to fail closed on the DH path.
+        return wireKey;
+    },
 };
 
 /** Profile for OMEMO 2 (urn:xmpp:omemo:2). */
@@ -500,6 +515,12 @@ const OMEMO_2: ProtocolProfile = {
         // omemo:2 signs the raw 32-byte Curve25519 (Montgomery) form.
         return stripKeyType(publicKey);
     },
+
+    normalizeRemotePreKey(wireKey: ArrayBuffer): ArrayBuffer {
+        // omemo:2 transfers the raw 32-byte curve form; restore the 0x05 prefix
+        // to the library's internal form (no-op if already prefixed).
+        return addKeyType(wireKey);
+    },
 };
 
 /** Render an ArrayBuffer/Uint8Array as a binary string (one char per byte). */

src/session/protocol-profile.ts

@@ -199,6 +199,59 @@ describe("OMEMO 2 trust-key enforcement", function () {
     });
 });
 
+describe("OMEMO 2 wire-form pre-key bundle", function () {
+    const ALICE = new OMEMOAddress("alice@example.org", 1);
+    const BOB = new OMEMOAddress("bob@example.org", 1);
+
+    /** Drop the 0x05 type prefix, yielding the raw 32-byte curve key. */
+    function stripPrefix(key: ArrayBuffer): ArrayBuffer {
+        const bytes = new Uint8Array(key);
+        assert.strictEqual(bytes.byteLength, 33);
+        assert.strictEqual(bytes[0], 5);
+        return key.slice(1);
+    }
+
+    // omemo:2 (XEP-0384) publishes the signed-pre-key and pre-keys in their raw
+    // 32-byte curve form, without the 0x05 type prefix. processPreKey must accept
+    // that wire form (regression: a stricter curve check rejected the unprefixed
+    // keys with "Invalid public key", breaking all new omemo:2 sessions).
+    it("builds a session from raw 32-byte (unprefixed) signed-pre-key and pre-key", async function () {
+        const aliceStore = new InMemoryStore();
+        const bobStore = new InMemoryStore();
+        await Promise.all([generateIdentity(aliceStore), generateIdentity(bobStore)]);
+
+        const bundle = await makeBundle(bobStore, "urn:xmpp:omemo:2", 1337, 1);
+        const wireBundle: PreKeyBundle = {
+            ...bundle,
+            preKey: {
+                keyId: bundle.preKey!.keyId,
+                publicKey: stripPrefix(bundle.preKey!.publicKey!),
+            },
+            signedPreKey: {
+                ...bundle.signedPreKey,
+                publicKey: stripPrefix(bundle.signedPreKey.publicKey),
+            },
+        };
+
+        await new SessionBuilder(aliceStore, BOB, "urn:xmpp:omemo:2").processPreKey(wireBundle);
+
+        // The session is established and a full key-exchange round-trip works.
+        const aliceCipher = new SessionCipher(aliceStore, BOB, "urn:xmpp:omemo:2");
+        const bobCipher = new SessionCipher(bobStore, ALICE, "urn:xmpp:omemo:2");
+        const msg = util.toArrayBuffer("wire-form kex") as ArrayBuffer;
+        const ct = await aliceCipher.encrypt(msg);
+        assert.strictEqual(ct.type, 3);
+        const { plaintext } = await bobCipher.decryptPreKeyWhisperMessage(ct.body, "binary");
+        assertEqualArrayBuffers(plaintext, msg);
+
+        // The stored remote ratchet seed is the normalised 33-byte 0x05 form.
+        const record = SessionRecord.deserialize(aliceStore.loadSession(BOB.toString())!);
+        const lastRemote = record.getOpenSession()!.currentRatchet.lastRemoteEphemeralKey;
+        assert.strictEqual(lastRemote.byteLength, 33);
+        assert.strictEqual(new Uint8Array(lastRemote)[0], 5);
+    });
+});
+
 describe("0.3.0 registrationId enforcement", function () {
     it("refuses to send a key-exchange message without a local registrationId", async function () {
         const BOB = new OMEMOAddress("bob@example.org", 1);