Merge pull request #1893 from polywrap/kris/polywrap-embed

Add `embed` option to `polywrap codegen`

Author: dOrgJelli

packages/cli/lang/en.json

@@ -82,9 +82,9 @@
   "commands_codegen_description": "Generate Code For Polywrap Projects",
   "commands_codegen_error_domain": "domain",
   "commands_codegen_error_optionMissingArgument": "{option} option missing {argument} argument",
+  "commands_codegen_error_embedAppOnly": "The --embed option is currently only available for app projects",
   "commands_codegen_options_codegen": "Output directory for the generated code (default: {default})",
-  "commands_codegen_options_e": "ENS address to lookup external schemas (default: 0x0000...2e1e)",
-  "commands_codegen_options_e_address": "address",
+  "commands_codegen_options_e": "Locally embed Wasm wraps",
   "commands_codegen_options_h": "Show usage information",
   "commands_codegen_options_s": "Path to a custom generation script (JavaScript | TypeScript)",
   "commands_codegen_options_i": "IPFS node to load external schemas (default: ipfs.io & localhost)",

packages/cli/lang/es.json

@@ -82,9 +82,9 @@
   "commands_codegen_description": "Generate Code For Polywrap Projects",
   "commands_codegen_error_domain": "domain",
   "commands_codegen_error_optionMissingArgument": "{option} option missing {argument} argument",
+  "commands_codegen_error_embedAppOnly": "The --embed option is currently only available for app projects",
   "commands_codegen_options_codegen": "Output directory for the generated code (default: {default})",
-  "commands_codegen_options_e": "ENS address to lookup external schemas (default: 0x0000...2e1e)",
-  "commands_codegen_options_e_address": "address",
+  "commands_codegen_options_e": "Locally embed Wasm wraps",
   "commands_codegen_options_h": "Show usage information",
   "commands_codegen_options_s": "Path to a custom generation script (JavaScript | TypeScript)",
   "commands_codegen_options_i": "IPFS node to load external schemas (default: ipfs.io & localhost)",

packages/cli/src/__tests__/e2e/p2/codegen.spec.ts

@@ -19,6 +19,7 @@ Options:
   -b, --bindgen <URI>                Uri for custom bindgen wrap (must
                                      implement wrap-abi-bindgen interface; see
                                      https://github.com/polywrap/wrap-abi-bindgen)
+  -e, --embed                        Locally embed Wasm wraps
   -s, --script <path>                Path to a custom generation script
                                      (JavaScript | TypeScript)
   -c, --client-config <config-path>  Add custom configuration to the

packages/cli/src/commands/build.ts

@@ -242,11 +242,12 @@ async function run(options: Required<BuildCommandOptions>) {
         project,
         client,
       });
+      const abi = await schemaComposer.getComposedAbis();
 
       if (canRunCodegen && !noCodegen) {
         const codeGenerator = new CodeGenerator({
           project,
-          schemaComposer,
+          abi,
           codegenDirAbs: codegenDir || undefined,
           bindgenUri,
         });
@@ -261,7 +262,7 @@ async function run(options: Required<BuildCommandOptions>) {
       const compiler = new Compiler({
         project: project as PolywrapProject,
         outputDir,
-        schemaComposer,
+        abi,
         buildStrategy,
       });
 

packages/cli/src/commands/codegen.ts

@@ -14,6 +14,8 @@ import {
   defaultPolywrapManifestFiles,
   parseLogFileOption,
   parseWrapperEnvsOption,
+  WasmEmbed,
+  getWasmEmbeds,
 } from "../lib";
 import { ScriptCodegenerator } from "../lib/codegen/ScriptCodeGenerator";
 import { DEFAULT_CODEGEN_DIR } from "../lib/defaults";
@@ -29,6 +31,7 @@ export interface CodegenCommandOptions extends BaseCommandOptions {
   manifestFile: string;
   codegenDir: string | false;
   bindgen: string | false;
+  embed: boolean | false;
   script: string | false;
   clientConfig: string | false;
   wrapperEnvs: string | false;
@@ -54,6 +57,7 @@ export const codegen: Command = {
         })}`
       )
       .option(`-b, --bindgen <URI>`, `${intlMsg.commands_codegen_options_b()}`)
+      .option(`-e, --embed`, `${intlMsg.commands_codegen_options_e()}`)
       .option(
         `-s, --script <${pathStr}>`,
         `${intlMsg.commands_codegen_options_s()}`
@@ -81,6 +85,7 @@ export const codegen: Command = {
           ),
           codegenDir: parseDirOptionNoDefault(options.codegenDir),
           bindgen: options.bindgen || false,
+          embed: options.embed || false,
           script: parseCodegenScriptOption(options.script),
           clientConfig: options.clientConfig || false,
           wrapperEnvs: options.wrapperEnvs || false,
@@ -100,6 +105,7 @@ async function run(options: Required<CodegenCommandOptions>) {
     wrapperEnvs,
     codegenDir,
     bindgen,
+    embed,
     script,
     verbose,
     quiet,
@@ -133,21 +139,34 @@ async function run(options: Required<CodegenCommandOptions>) {
     project,
     client,
   });
+  const abi = await schemaComposer.getComposedAbis();
+
+  const projectLang = await project.getManifestLanguage();
+  if (embed && !projectLang.startsWith("app")) {
+    logger.error(intlMsg.commands_codegen_error_embedAppOnly());
+    process.exit(1);
+  }
+
+  const embeds: WasmEmbed[] | undefined =
+    embed && abi.importedModuleTypes
+      ? await getWasmEmbeds(abi.importedModuleTypes, client, logger)
+      : undefined;
 
   const codeGenerator = script
     ? new ScriptCodegenerator({
         codegenDirAbs: codegenDir || undefined,
         script,
-        schemaComposer,
+        abi,
         project,
         omitHeader: false,
         mustacheView: undefined,
       })
     : new CodeGenerator({
         codegenDirAbs: codegenDir || undefined,
-        schemaComposer,
+        abi,
         project,
         bindgenUri,
+        embeds,
       });
 
   const execute = async (): Promise<boolean> => {

packages/cli/src/lib/Compiler.ts

@@ -8,7 +8,6 @@ import {
   PolywrapProject,
   PluginProject,
   resetDir,
-  SchemaComposer,
   logActivity,
   loadDocsManifest,
 } from "./";
@@ -21,12 +20,13 @@ import fs from "fs";
 import fse from "fs-extra";
 import path from "path";
 import { DocsManifest } from "@polywrap/polywrap-manifest-types-js";
+import { Abi } from "@polywrap/schema-parse";
 
 export interface CompilerConfig {
   outputDir: string;
   project: PolywrapProject | PluginProject;
   buildStrategy?: BuildStrategy;
-  schemaComposer: SchemaComposer;
+  abi: Abi;
 }
 
 export class Compiler {
@@ -111,13 +111,12 @@ export class Compiler {
   }
 
   private async _outputWrapManifest(): Promise<unknown> {
-    const { outputDir, project, schemaComposer } = this._config;
+    const { outputDir, project, abi } = this._config;
     const manifestPath = `${outputDir}/wrap.info`;
     const run = async () => {
       const manifest = await project.getManifest();
 
       const type = manifest.project.type.split("/")[0];
-      const abi = await schemaComposer.getComposedAbis();
       await generateWrapFile(
         abi,
         manifest.project.name,

packages/cli/src/lib/codegen/CodeGenerator.ts

@@ -15,19 +15,21 @@ import {
   Project,
 } from "../project";
 import { resetDir } from "../system";
-import { SchemaComposer } from "../SchemaComposer";
 import { CodegenOverrides, tryGetCodegenOverrides } from "./CodegenOverrides";
+import { WasmEmbed } from "./WasmEmbed";
 
 import path from "path";
 import { BindLanguage } from "@polywrap/schema-bind";
 import { writeDirectorySync } from "@polywrap/os-js";
 import { Uri } from "@polywrap/core-js";
+import { Abi } from "@polywrap/schema-parse";
 
 export interface CodeGeneratorConfig {
   project: Project<AnyProjectManifest>;
-  schemaComposer: SchemaComposer;
+  abi: Abi;
   codegenDirAbs?: string;
   bindgenUri?: Uri;
+  embeds?: WasmEmbed[];
 }
 
 export class CodeGenerator {
@@ -88,13 +90,19 @@ export class CodeGenerator {
         )
       : undefined;
 
-    const bindConfig = overrides
+    let bindConfig = overrides
       ? await overrides.getSchemaBindConfig(this._config.project)
       : {};
 
-    const abi = await this._config.schemaComposer.getComposedAbis();
+    if (this._config.embeds) {
+      bindConfig = {
+        ...bindConfig,
+        embeds: this._config.embeds,
+      };
+    }
+
     const binding = await this._config.project.generateSchemaBindings(
-      abi,
+      this._config.abi,
       codegenDir,
       this._config.bindgenUri?.toString(),
       bindConfig

packages/cli/src/lib/codegen/ScriptCodeGenerator.ts

@@ -1,7 +1,6 @@
 import { intlMsg } from "../intl";
 import { AnyProjectManifest, Project } from "../project";
 import { isTypescriptFile, importTypescriptModule, resetDir } from "../system";
-import { SchemaComposer } from "../SchemaComposer";
 import { CodeGenerator } from "./CodeGenerator";
 
 import { writeDirectorySync } from "@polywrap/os-js";
@@ -14,6 +13,7 @@ import { readFileSync } from "fs-extra";
 import Mustache from "mustache";
 import path from "path";
 import { latestWrapManifestVersion } from "@polywrap/wrap-manifest-types-js";
+import { Abi } from "@polywrap/schema-parse";
 
 export class ScriptCodegenerator extends CodeGenerator {
   private readonly _script: string;
@@ -23,7 +23,7 @@ export class ScriptCodegenerator extends CodeGenerator {
 
   constructor(config: {
     project: Project<AnyProjectManifest>;
-    schemaComposer: SchemaComposer;
+    abi: Abi;
     codegenDirAbs?: string;
     script: string;
     mustacheView: Record<string, unknown> | undefined;
@@ -32,7 +32,7 @@ export class ScriptCodegenerator extends CodeGenerator {
   }) {
     super({
       project: config.project,
-      schemaComposer: config.schemaComposer,
+      abi: config.abi,
       codegenDirAbs: config.codegenDirAbs,
     });
 
@@ -72,7 +72,7 @@ export class ScriptCodegenerator extends CodeGenerator {
         version: latestWrapManifestVersion,
         name: await this._config.project.getName(),
         type: bindLanguageToWrapInfoType(bindLanguage),
-        abi: await this._config.schemaComposer.getComposedAbis(),
+        abi: this._config.abi,
       },
       config: this._mustacheView,
       outputDirAbs,

packages/cli/src/lib/codegen/WasmEmbed.ts

@@ -0,0 +1,47 @@
+import { Logger } from "../logging";
+
+import { ImportedModuleDefinition } from "@polywrap/wrap-manifest-types-js";
+import { PolywrapClient } from "@polywrap/client-js";
+
+export interface WasmEmbed {
+  uri: string;
+  namespace: string;
+  wrapInfo: Array<number>;
+  wrapWasm: Array<number>;
+}
+
+export async function getWasmEmbeds(
+  importedModules: ImportedModuleDefinition[],
+  client: PolywrapClient,
+  logger: Logger
+): Promise<WasmEmbed[]> {
+  const embeds: WasmEmbed[] = [];
+
+  for (const importedModule of importedModules ?? []) {
+    if (importedModule.isInterface) {
+      continue;
+    }
+    const uri = importedModule.uri;
+
+    const manifest = await client.getFile(uri, { path: "wrap.info" });
+    if (!manifest.ok) {
+      logger.error(JSON.stringify(manifest.error, null, 2));
+      process.exit(1);
+    }
+
+    const module = await client.getFile(uri, { path: "wrap.wasm" });
+    if (!module.ok) {
+      // The error is ignored because getFile is expected to fail for plugins and interfaces
+      continue;
+    }
+
+    embeds.push({
+      uri,
+      namespace: importedModule.namespace,
+      wrapInfo: Array.from(manifest.value as Uint8Array),
+      wrapWasm: Array.from(module.value as Uint8Array),
+    });
+  }
+
+  return embeds;
+}

packages/cli/src/lib/codegen/index.ts

@@ -1,3 +1,4 @@
 export * from "./CodeGenerator";
 export * from "./CodegenOverrides";
 export * from "./ScriptCodeGenerator";
+export * from "./WasmEmbed";