refactor: bb.js non-inlined web workers (#13736)

Inlined web workers are completely unsupported in browser extensions,
which created problems for external teams. This PR moves to a more
modern approach, leveraging the integrated support in latest versions of
`webpack` for web workers.

Unfortunately, I've run into quite a few problems implementing this
approach:

* Webpack didn't support proper module workers up until recently,
forcing me to update it (so it wouldn't transpile the imports to the
unsupported `importScripts`). Took the opportunity to equalize versions
accross the board.
* There's a bug in webpack that forbids loading them asynchronously and
threw me for a while: webpack/webpack#17014.
Solution is to carefully handle our dynamic imports so they're only
applied to the wasm files.
* We were using `worker-loader` in `webpack` 5, which is explicitly
deprecated, but we needed it for the inlining. This gets rid of it.
* Unfortunately again, new webpack handles `.d.ts` and `.d.ts.map` files
differently now, which was causing trouble for downstream apps that also
used webpack. Test apps and boxes have been updated to ignore them at
build time (since they're only needed during development). Vite has no
issues.
* Took the opportunity to clean up our API, since we were never
initializing the `SyncApi`with a worker and conversely, the `Async` API
with a plain wasm module.

Author: Thunkar

barretenberg/acir_tests/browser-test-app/package.json

@@ -15,13 +15,14 @@
     "@types/pako": "^2.0.3",
     "debug": "^4.3.4",
     "html-webpack-plugin": "^5.6.0",
+    "ignore-loader": "^0.1.2",
     "pako": "^2.1.0",
     "serve": "^14.2.1",
     "ts-loader": "^9.5.1",
     "typescript": "^5.4.2",
-    "webpack": "^5.90.3",
-    "webpack-cli": "^5.1.4",
-    "webpack-dev-server": "^5.2.0"
+    "webpack": "^5.99.6",
+    "webpack-cli": "^6.0.1",
+    "webpack-dev-server": "^5.2.1"
   },
   "packageManager": "yarn@4.5.2"
 }

barretenberg/acir_tests/browser-test-app/tsconfig.json

@@ -11,6 +11,7 @@
     "inlineSourceMap": true,
     "declarationMap": true,
     "importHelpers": true,
+    "isolatedModules": true,
     "resolveJsonModule": true,
     "composite": true,
     "outDir": "dest",

barretenberg/acir_tests/browser-test-app/webpack.config.js

@@ -13,7 +13,12 @@ export default {
     rules: [
       {
         test: /\.tsx?$/,
-        use: [{ loader: "ts-loader" }],
+        loader: 'ts-loader',
+        exclude: /node_modules|\.d\.ts(.map)?$/
+      },
+      {
+          test: /\.d\.ts(.map)?$/,
+          loader: 'ignore-loader'
       },
     ],
   },

barretenberg/acir_tests/yarn.lock

@@ -75,10 +75,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@discoveryjs/json-ext@npm:^0.5.0":
-  version: 0.5.7
-  resolution: "@discoveryjs/json-ext@npm:0.5.7"
-  checksum: 10c0/e10f1b02b78e4812646ddf289b7d9f2cb567d336c363b266bd50cd223cf3de7c2c74018d91cd2613041568397ef3a4a2b500aba588c6e5bd78c38374ba68f38c
+"@discoveryjs/json-ext@npm:^0.6.1":
+  version: 0.6.3
+  resolution: "@discoveryjs/json-ext@npm:0.6.3"
+  checksum: 10c0/778a9f9d5c3696da3c1f9fa4186613db95a1090abbfb6c2601430645c0d0158cd5e4ba4f32c05904e2dd2747d57710f6aab22bd2f8aa3c4e8feab9b247c65d85
   languageName: node
   linkType: hard
 
@@ -390,7 +390,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/express-serve-static-core@npm:^4.17.33":
+"@types/express-serve-static-core@npm:^4.17.21, @types/express-serve-static-core@npm:^4.17.33":
   version: 4.19.6
   resolution: "@types/express-serve-static-core@npm:4.19.6"
   dependencies:
@@ -733,36 +733,36 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@webpack-cli/configtest@npm:^2.1.1":
-  version: 2.1.1
-  resolution: "@webpack-cli/configtest@npm:2.1.1"
+"@webpack-cli/configtest@npm:^3.0.1":
+  version: 3.0.1
+  resolution: "@webpack-cli/configtest@npm:3.0.1"
   peerDependencies:
-    webpack: 5.x.x
-    webpack-cli: 5.x.x
-  checksum: 10c0/a8da1f15702cb289807da99235ed95326ed7dabeb1a36ca59bd3a5dbe6adcc946a9a2767936050fc4d5ed14efab0e5b5a641dfe8e3d862c36caa5791ac12759d
+    webpack: ^5.82.0
+    webpack-cli: 6.x.x
+  checksum: 10c0/edd24ecfc429298fe86446f7d7daedfe82d72e7f6236c81420605484fdadade5d59c6bcef3d76bd724e11d9727f74e75de183223ae62d3a568b2d54199688cbe
   languageName: node
   linkType: hard
 
-"@webpack-cli/info@npm:^2.0.2":
-  version: 2.0.2
-  resolution: "@webpack-cli/info@npm:2.0.2"
+"@webpack-cli/info@npm:^3.0.1":
+  version: 3.0.1
+  resolution: "@webpack-cli/info@npm:3.0.1"
   peerDependencies:
-    webpack: 5.x.x
-    webpack-cli: 5.x.x
-  checksum: 10c0/ca88a35604dc9aedac7c26e8f6793c5039dc1eea2b12a85fbfd669a5f21ecf9cf169d7fd157ea366a62666e3fa05b776306a96742ac61a9868f44fdce6b40f7d
+    webpack: ^5.82.0
+    webpack-cli: 6.x.x
+  checksum: 10c0/b23b94e7dc8c93e79248f20d5f1bd0fbb7b9ba4b012803e2fdc5440b8f2ee1f3eca7f4933bbca346c8168673bf572b1858169a3cb2c17d9b8bcd833d480c2170
   languageName: node
   linkType: hard
 
-"@webpack-cli/serve@npm:^2.0.5":
-  version: 2.0.5
-  resolution: "@webpack-cli/serve@npm:2.0.5"
+"@webpack-cli/serve@npm:^3.0.1":
+  version: 3.0.1
+  resolution: "@webpack-cli/serve@npm:3.0.1"
   peerDependencies:
-    webpack: 5.x.x
-    webpack-cli: 5.x.x
+    webpack: ^5.82.0
+    webpack-cli: 6.x.x
   peerDependenciesMeta:
     webpack-dev-server:
       optional: true
-  checksum: 10c0/36079d34971ff99a58b66b13f4184dcdd8617853c48cccdbc3f9ab7ea9e5d4fcf504e873c298ea7aa15e0b51ad2c4aee4d7a70bd7d9364e60f57b0eb93ca15fc
+  checksum: 10c0/65245e45bfa35e11a5b30631b99cfed0c1b39b2cc8320fa2d2a4185264535618827d349ec032c58af4201d6236cbc43bec894fcb840fdd06314611537a80e210
   languageName: node
   linkType: hard
 
@@ -1180,13 +1180,14 @@ __metadata:
     "@types/pako": "npm:^2.0.3"
     debug: "npm:^4.3.4"
     html-webpack-plugin: "npm:^5.6.0"
+    ignore-loader: "npm:^0.1.2"
     pako: "npm:^2.1.0"
     serve: "npm:^14.2.1"
     ts-loader: "npm:^9.5.1"
     typescript: "npm:^5.4.2"
-    webpack: "npm:^5.90.3"
-    webpack-cli: "npm:^5.1.4"
-    webpack-dev-server: "npm:^5.2.0"
+    webpack: "npm:^5.99.6"
+    webpack-cli: "npm:^6.0.1"
+    webpack-dev-server: "npm:^5.2.1"
   languageName: unknown
   linkType: soft
 
@@ -1487,13 +1488,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"commander@npm:^10.0.1":
-  version: 10.0.1
-  resolution: "commander@npm:10.0.1"
-  checksum: 10c0/53f33d8927758a911094adadda4b2cbac111a5b377d8706700587650fd8f45b0bbe336de4b5c3fe47fd61f420a3d9bd452b6e0e6e5600a7e74d7bf0174f6efe3
-  languageName: node
-  linkType: hard
-
 "commander@npm:^12.1.0":
   version: 12.1.0
   resolution: "commander@npm:12.1.0"
@@ -1947,7 +1941,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"envinfo@npm:^7.7.3":
+"envinfo@npm:^7.14.0":
   version: 7.14.0
   resolution: "envinfo@npm:7.14.0"
   bin:
@@ -2740,6 +2734,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"ignore-loader@npm:^0.1.2":
+  version: 0.1.2
+  resolution: "ignore-loader@npm:0.1.2"
+  checksum: 10c0/0ba7f0fbcd4d796e68cc05723279b662dea32f229f5912c6ce4fbcc5a50f592d2b6f32baaf047b47b24dccdfdc9faf2db504b48cb6eb886a4969bfc293739cab
+  languageName: node
+  linkType: hard
+
 "import-fresh@npm:^3.3.0":
   version: 3.3.1
   resolution: "import-fresh@npm:3.3.1"
@@ -5138,35 +5139,33 @@ __metadata:
   languageName: node
   linkType: hard
 
-"webpack-cli@npm:^5.1.4":
-  version: 5.1.4
-  resolution: "webpack-cli@npm:5.1.4"
+"webpack-cli@npm:^6.0.1":
+  version: 6.0.1
+  resolution: "webpack-cli@npm:6.0.1"
   dependencies:
-    "@discoveryjs/json-ext": "npm:^0.5.0"
-    "@webpack-cli/configtest": "npm:^2.1.1"
-    "@webpack-cli/info": "npm:^2.0.2"
-    "@webpack-cli/serve": "npm:^2.0.5"
+    "@discoveryjs/json-ext": "npm:^0.6.1"
+    "@webpack-cli/configtest": "npm:^3.0.1"
+    "@webpack-cli/info": "npm:^3.0.1"
+    "@webpack-cli/serve": "npm:^3.0.1"
     colorette: "npm:^2.0.14"
-    commander: "npm:^10.0.1"
+    commander: "npm:^12.1.0"
     cross-spawn: "npm:^7.0.3"
-    envinfo: "npm:^7.7.3"
+    envinfo: "npm:^7.14.0"
     fastest-levenshtein: "npm:^1.0.12"
     import-local: "npm:^3.0.2"
     interpret: "npm:^3.1.1"
     rechoir: "npm:^0.8.0"
-    webpack-merge: "npm:^5.7.3"
+    webpack-merge: "npm:^6.0.1"
   peerDependencies:
-    webpack: 5.x.x
+    webpack: ^5.82.0
   peerDependenciesMeta:
-    "@webpack-cli/generators":
-      optional: true
     webpack-bundle-analyzer:
       optional: true
     webpack-dev-server:
       optional: true
   bin:
-    webpack-cli: bin/cli.js
-  checksum: 10c0/4266909ae5e2e662c8790ac286e965b2c7fd5a4a2f07f48e28576234c9a5f631847ccddc18e1b3281c7b4be04a7ff4717d2636033a322dde13ac995fd0d9de10
+    webpack-cli: ./bin/cli.js
+  checksum: 10c0/2aaca78e277427f03f528602abd707d224696048fb46286ea636c7975592409c4381ca94d68bbbb3900f195ca97f256e619583e8feb34a80da531461323bf3e2
   languageName: node
   linkType: hard
 
@@ -5189,13 +5188,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"webpack-dev-server@npm:^5.2.0":
-  version: 5.2.0
-  resolution: "webpack-dev-server@npm:5.2.0"
+"webpack-dev-server@npm:^5.2.1":
+  version: 5.2.1
+  resolution: "webpack-dev-server@npm:5.2.1"
   dependencies:
     "@types/bonjour": "npm:^3.5.13"
     "@types/connect-history-api-fallback": "npm:^1.5.4"
     "@types/express": "npm:^4.17.21"
+    "@types/express-serve-static-core": "npm:^4.17.21"
     "@types/serve-index": "npm:^1.9.4"
     "@types/serve-static": "npm:^1.15.5"
     "@types/sockjs": "npm:^0.3.36"
@@ -5229,18 +5229,18 @@ __metadata:
       optional: true
   bin:
     webpack-dev-server: bin/webpack-dev-server.js
-  checksum: 10c0/afb2e51945ac54ef3039e11e377241e1cb97a8d3f526f39f13c3fa924c530fb6063200c2c3ae4e33e6bcc110d4abed777c09ce18e2d261012853d81f3c5820ab
+  checksum: 10c0/22bcf2bcc7c72cd2065883ed4368fbcdf20078bc746b07689d10a0546ee99ea00bc50f0474112278ffd8598a5bc237df2bf7bb7f6dcda940a16b1eb91137efea
   languageName: node
   linkType: hard
 
-"webpack-merge@npm:^5.7.3":
-  version: 5.10.0
-  resolution: "webpack-merge@npm:5.10.0"
+"webpack-merge@npm:^6.0.1":
+  version: 6.0.1
+  resolution: "webpack-merge@npm:6.0.1"
   dependencies:
     clone-deep: "npm:^4.0.1"
     flat: "npm:^5.0.2"
-    wildcard: "npm:^2.0.0"
-  checksum: 10c0/b607c84cabaf74689f965420051a55a08722d897bdd6c29cb0b2263b451c090f962d41ecf8c9bf56b0ab3de56e65476ace0a8ecda4f4a4663684243d90e0512b
+    wildcard: "npm:^2.0.1"
+  checksum: 10c0/bf1429567858b353641801b8a2696ca0aac270fc8c55d4de8a7b586fe07d27fdcfc83099a98ab47e6162383db8dd63bb8cc25b1beb2ec82150422eec843b0dc0
   languageName: node
   linkType: hard
 
@@ -5251,9 +5251,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"webpack@npm:^5.90.3":
-  version: 5.98.0
-  resolution: "webpack@npm:5.98.0"
+"webpack@npm:^5.99.6":
+  version: 5.99.6
+  resolution: "webpack@npm:5.99.6"
   dependencies:
     "@types/eslint-scope": "npm:^3.7.7"
     "@types/estree": "npm:^1.0.6"
@@ -5283,7 +5283,7 @@ __metadata:
       optional: true
   bin:
     webpack: bin/webpack.js
-  checksum: 10c0/bee4fa77f444802f0beafb2ff30eb5454a606163ad7d3cc9a5dcc9d24033c62407bed04601b25dea49ea3969b352c1b530a86c753246f42560a4a084eefb094e
+  checksum: 10c0/b8b44b16933a0dd83e185ad42f292bbdfa9c47e245cbe786c48520d681556ece9af6ea7fff33059fafdf3d2cd62674715308d70a6f15eda6c6de7e03ef01842a
   languageName: node
   linkType: hard
 
@@ -5336,7 +5336,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"wildcard@npm:^2.0.0":
+"wildcard@npm:^2.0.1":
   version: 2.0.1
   resolution: "wildcard@npm:2.0.1"
   checksum: 10c0/08f70cd97dd9a20aea280847a1fe8148e17cae7d231640e41eb26d2388697cbe65b67fd9e68715251c39b080c5ae4f76d71a9a69fa101d897273efdfb1b58bf7

barretenberg/ts/package.json

@@ -85,21 +85,18 @@
     "buffer": "^6.0.3",
     "eslint": "^8.35.0",
     "eslint-config-prettier": "^8.8.0",
-    "html-webpack-plugin": "^5.5.1",
+    "html-webpack-plugin": "^5.6.3",
     "idb-keyval": "^6.2.1",
     "jest": "^29.5.0",
     "msgpackr": "^1.11.2",
     "prettier": "^2.8.4",
-    "resolve-typescript-plugin": "^2.0.1",
-    "source-map-support": "^0.5.21",
-    "terser-webpack-plugin": "^5.3.11",
+    "terser-webpack-plugin": "^5.3.14",
     "ts-jest": "^29.1.0",
     "ts-loader": "^9.4.2",
     "ts-node": "^10.9.1",
     "typescript": "5.4.5",
-    "webpack": "^5.82.1",
-    "webpack-cli": "^5.1.1",
-    "webpack-dev-server": "^5.2.0",
-    "worker-loader": "^3.0.8"
+    "webpack": "^5.99.6",
+    "webpack-cli": "^6.0.1",
+    "webpack-dev-server": "^5.2.1"
   }
 }

barretenberg/ts/src/barretenberg/index.ts

@@ -3,10 +3,10 @@ import { BarretenbergApi, BarretenbergApiSync } from '../barretenberg_api/index.
 import { createMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/factory/node/index.js';
 import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/index.js';
 import { getRemoteBarretenbergWasm } from '../barretenberg_wasm/helpers/index.js';
-import { BarretenbergWasmWorker, fetchModuleAndThreads } from '../barretenberg_wasm/index.js';
 import createDebug from 'debug';
 import { Crs, GrumpkinCrs } from '../crs/index.js';
 import { RawBuffer } from '../types/raw_buffer.js';
+import { fetchModuleAndThreads } from '../barretenberg_wasm/index.js';
 
 export { BarretenbergVerifier } from './verifier.js';
 export { UltraPlonkBackend, UltraHonkBackend, AztecClientBackend } from './backend.js';
@@ -40,7 +40,7 @@ export type CircuitOptions = {
 export class Barretenberg extends BarretenbergApi {
   private options: BackendOptions;
 
-  private constructor(private worker: any, wasm: BarretenbergWasmWorker, options: BackendOptions) {
+  private constructor(private worker: any, wasm: BarretenbergWasmMainWorker, options: BackendOptions) {
     super(wasm);
     this.options = options;
   }
@@ -52,7 +52,7 @@ export class Barretenberg extends BarretenbergApi {
    * It threads > 1 (defaults to hardware availability), child threads will be created on their own workers.
    */
   static async new(options: BackendOptions = {}) {
-    const worker = createMainWorker();
+    const worker = await createMainWorker();
     const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
     const { module, threads } = await fetchModuleAndThreads(options.threads, options.wasmPath, options.logger);
     await wasm.init(
@@ -97,6 +97,10 @@ export class Barretenberg extends BarretenbergApi {
     await this.wasm.destroy();
     await this.worker.terminate();
   }
+
+  getWasm() {
+    return this.wasm;
+  }
 }
 
 let barrentenbergSyncSingletonPromise: Promise<BarretenbergSync>;

barretenberg/ts/src/barretenberg_api/index.ts

@@ -1,7 +1,6 @@
 // WARNING: FILE CODE GENERATED BY BINDGEN UTILITY. DO NOT EDIT!
 /* eslint-disable @typescript-eslint/no-unused-vars */
-import { BarretenbergWasmMain } from '../barretenberg_wasm/barretenberg_wasm_main/index.js';
-import { BarretenbergWasmWorker, BarretenbergWasm } from '../barretenberg_wasm/index.js';
+import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/index.js';
 import {
   BufferDeserializer,
   NumberDeserializer,
@@ -11,10 +10,10 @@ import {
   serializeBufferable,
   OutputType,
 } from '../serialize/index.js';
-import { Fr, Fq, Point, Buffer32, Buffer128, Ptr } from '../types/index.js';
+import { Fr, Point, Buffer32, Ptr } from '../types/index.js';
 
 export class BarretenbergApi {
-  constructor(protected wasm: BarretenbergWasmWorker | BarretenbergWasmMain) {}
+  constructor(protected wasm: BarretenbergWasmMainWorker) {}
 
   async pedersenCommit(inputsBuffer: Fr[], ctxIndex: number): Promise<Point> {
     const inArgs = [inputsBuffer, ctxIndex].map(serializeBufferable);
@@ -606,7 +605,7 @@ export class BarretenbergApi {
   }
 }
 export class BarretenbergApiSync {
-  constructor(protected wasm: BarretenbergWasm) {}
+  constructor(protected wasm: BarretenbergWasmMain) {}
 
   pedersenCommit(inputsBuffer: Fr[], ctxIndex: number): Point {
     const inArgs = [inputsBuffer, ctxIndex].map(serializeBufferable);