feat(apps): introduce alternative node-runtime

.changeset/fruity-items-fix.md

@@ -0,0 +1,6 @@
+---
+'@rocket.chat/apps': minor
+'@rocket.chat/meteor': minor
+---
+
+Adds an alternative runtime runner for apps. It can be enabled via environment variable `APPS_ENGINE_RUNTIME_BACKEND='node'`

.github/workflows/ci-test-e2e.yml

@@ -167,7 +167,7 @@ jobs:
         run: echo "DEBUG_LOG_LEVEL=2" >> "$GITHUB_ENV"
 
       - name: Start httpbin container and wait for it to be ready
-        if: inputs.type == 'api' || inputs.type == 'api-livechat'
+        if: startsWith(inputs.type, 'api')
         run: |
           docker compose -f docker-compose-ci.yml up -d httpbin
 
@@ -186,7 +186,7 @@ jobs:
           # behavior on (rate-limiter bypass, short cache TTLs) while letting
           # the deprecation logger log without throwing. Other suites use the
           # docker-compose default of TEST_MODE='true'.
-          TEST_MODE: ${{ (inputs.type == 'api' || inputs.type == 'api-livechat') && 'api' || 'true' }}
+          TEST_MODE: ${{ startsWith(inputs.type, 'api') && 'api' || 'true' }}
         run: |
           # when we are testing CE, we only need to start the rocketchat container
           DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d rocketchat --wait
@@ -197,7 +197,8 @@ jobs:
           ENTERPRISE_LICENSE: ${{ inputs.enterprise-license }}
           TRANSPORTER: ${{ inputs.transporter }}
           COMPOSE_PROFILES: ${{ inputs.type == 'api' && 'api' || '' }}
-          TEST_MODE: ${{ (inputs.type == 'api' || inputs.type == 'api-livechat') && 'api' || 'true' }}
+          TEST_MODE: ${{ startsWith(inputs.type, 'api') && 'api' || 'true' }}
+          APPS_ENGINE_RUNTIME_BACKEND: ${{ inputs.type == 'api-apps-node' && 'node' || '' }}
         run: |
           DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d --wait
 
@@ -234,6 +235,23 @@ jobs:
           ls -la "$COVERAGE_DIR"
           exit "${s:-0}"
 
+      # This step should be temporary, only here until we remove the deno-runtime
+      - name: E2E Test API (apps + node-runtime)
+        if: (inputs.type == 'api-apps-node' && inputs.release == 'ee')
+        working-directory: ./apps/meteor
+        env:
+          WEBHOOK_TEST_URL: 'http://httpbin'
+          IS_EE: 'true'
+        run: |
+          set -o xtrace
+
+          npm run testapi:apps || s=$?
+
+          docker compose -f ../../docker-compose-ci.yml stop
+
+          ls -la "$COVERAGE_DIR"
+          exit "${s:-0}"
+
       - name: E2E Test API (Livechat)
         if: inputs.type == 'api-livechat'
         working-directory: ./apps/meteor

.github/workflows/ci.yml

@@ -657,6 +657,27 @@ jobs:
       CR_PAT: ${{ secrets.CR_PAT }}
       NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
+  test-api-apps-node-ee:
+    name: 🔨 Test API Apps (node-runtime - EE)
+    needs: [checks, build-gh-docker-publish, release-versions]
+
+    uses: ./.github/workflows/ci-test-e2e.yml
+    with:
+      type: api-apps-node
+      release: ee
+      transporter: 'nats://nats:4222'
+      enterprise-license: ${{ needs.release-versions.outputs.enterprise-license }}
+      mongodb-version: "['8.0']"
+      coverage: '8.0'
+      node-version: ${{ needs.release-versions.outputs.node-version }}
+      deno-version: ${{ needs.release-versions.outputs.deno-version }}
+      lowercase-repo: ${{ needs.release-versions.outputs.lowercase-repo }}
+      gh-docker-tag: ${{ needs.release-versions.outputs.gh-docker-tag }}
+    secrets:
+      CR_USER: ${{ secrets.CR_USER }}
+      CR_PAT: ${{ secrets.CR_PAT }}
+      NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
   test-ui-ee:
     name: 🔨 Test UI (EE)
     needs: [checks, build-gh-docker-publish, release-versions]

apps/meteor/.mocharc.api.apps.js

@@ -0,0 +1,15 @@
+'use strict';
+
+/*
+ * Mocha configuration for REST API integration tests.
+ */
+
+module.exports = /** @satisfies {import('mocha').MochaOptions} */ ({
+	...require('./.mocharc.base.json'), // see https://github.com/mochajs/mocha/issues/3916
+	timeout: 10000,
+	bail: false,
+	retries: 0,
+	file: 'tests/end-to-end/teardown.ts',
+	reporter: 'tests/end-to-end/reporter.ts',
+	spec: ['tests/end-to-end/apps/*'],
+});

apps/meteor/package.json

@@ -56,6 +56,7 @@
 		"test:e2e:nyc": "nyc report --reporter=lcovonly",
 		"testapi": "TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\"}' mocha --config ./.mocharc.api.js",
 		"testapi:livechat": "TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\"}' mocha --config ./.mocharc.api.livechat.js",
+		"testapi:apps": "TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\"}' mocha --config ./.mocharc.api.apps.js",
 		"testunit": "yarn .testunit:definition && yarn .testunit:jest && yarn .testunit:server:cov",
 		"testunit-watch": "TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\"}' mocha --watch --config ./.mocharc.js",
 		"typecheck": "meteor lint && cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsc --noEmit --skipLibCheck",

docker-compose-ci.yml

@@ -14,6 +14,7 @@ services:
     image: ghcr.io/${LOWERCASE_REPOSITORY}/rocket.chat:${DOCKER_TAG}${DOCKER_TAG_SUFFIX_ROCKETCHAT:-}
     environment:
       - 'TEST_MODE=${TEST_MODE:-true}'
+      - APPS_ENGINE_RUNTIME_BACKEND=${APPS_ENGINE_RUNTIME_BACKEND:-}
       - DEBUG=${DEBUG:-}
       - EXIT_UNHANDLEDPROMISEREJECTION=true
       - MONGO_URL=mongodb://mongo:27017/rocketchat?replicaSet=rs0

packages/apps/deno-runtime/main.ts

@@ -17,7 +17,7 @@ import { stdoutTransport } from './lib/transports/stdoutTransport';
 
 if (!process.argv.includes('--subprocess')) {
 	console.error(`
-            This is a Deno wrapper for Rocket.Chat Apps. It is not meant to be executed stand-alone;
+            This is the Deno wrapper for the Rocket.Chat Apps runtime. It is not meant to be executed stand-alone;
             It is instead meant to be executed as a subprocess by the Apps-Engine framework.
 	`);
 

packages/apps/node-runtime/src/error-handlers.ts

@@ -0,0 +1,10 @@
+import * as Messenger from '@rocket.chat/apps/base-runtime/dist/lib/messenger';
+
+export default function registerErrorListeners() {
+	process.on('uncaughtException', (error: Error, origin: 'uncaughtException' | 'unhandledRejection') => {
+		Messenger.sendNotification({
+			method: origin,
+			params: [error.stack || error],
+		});
+	});
+}

packages/apps/node-runtime/src/lib/loader-hook.ts

@@ -0,0 +1,19 @@
+import { registerHooks } from 'node:module';
+import path from 'node:path';
+
+// This file compiles to dist/lib/loader-hook.js.
+// Three levels up from dist/lib/ lands on packages/apps/ — the @rocket.chat/apps package root.
+const appsPackageDir = path.resolve(__dirname, '../../..');
+
+const PACKAGE_PREFIX = '@rocket.chat/apps';
+
+registerHooks({
+	resolve(specifier, context, nextResolve) {
+		if (specifier === PACKAGE_PREFIX || specifier.startsWith(`${PACKAGE_PREFIX}/`)) {
+			const subpath = specifier.slice(PACKAGE_PREFIX.length).replace(/^\//, '');
+			const localPath = subpath ? path.join(appsPackageDir, subpath) : appsPackageDir;
+			return nextResolve(localPath, context);
+		}
+		return nextResolve(specifier, context);
+	},
+});

packages/apps/node-runtime/src/lib/require.ts

@@ -0,0 +1,37 @@
+const ALLOWED_MODULES = [
+	'path',
+	'url',
+	'crypto',
+	'buffer',
+	'stream',
+	'net',
+	'http',
+	'https',
+	'zlib',
+	'util',
+	'punycode',
+	'os',
+	'querystring',
+	'fs',
+	// External libraries
+	'uuid',
+	'@rocket.chat/apps-engine',
+];
+
+// As the apps are bundled, the only times they will call require are
+// 1. To require native modules
+// 2. To require external npm packages we may provide
+// 3. To require apps-engine files
+export const sandboxRequire = (module: string) => {
+	// Normalize Node built-in specifiers: accept both 'crypto' and 'node:crypto'
+	const normalized = module.replace('node:', '');
+
+	// We allow variants like 'fs', 'node:fs' or 'node:fs/promises', or even '@rocket.chat/apps-engine/**'
+	if (!ALLOWED_MODULES.some((mod) => normalized.startsWith(mod))) {
+		throw new Error(`Module ${module} is not allowed`);
+	}
+
+	// This is THE purpose of this function, we can't escape a dinamyc require call
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-return, import/no-dynamic-require, @typescript-eslint/no-require-imports
+	return require(normalized);
+};

packages/apps/node-runtime/src/lib/transports/stdoutTransport.ts

@@ -0,0 +1,16 @@
+import type { Transport } from '@rocket.chat/apps/base-runtime/dist/lib/messenger';
+
+/**
+ * Transport that writes messages to the process' standard output.
+ *
+ * This is the transport used when the runtime is executed as a subprocess by
+ * the Apps-Engine framework, and it is specific to platforms that expose a
+ * Node-compatible `process.stdout`.
+ */
+export const stdoutTransport: Transport = {
+	async send(message: Uint8Array): Promise<void> {
+		await new Promise<void>((resolve, reject) => {
+			process.stdout.write(message, (err) => (err ? reject(err) : resolve()));
+		});
+	},
+};

packages/apps/node-runtime/src/main.ts

@@ -0,0 +1,30 @@
+import './lib/loader-hook';
+
+import { setSandboxGlobals, setSandboxRequire } from '@rocket.chat/apps/base-runtime/dist/handlers/app/construct';
+import { setTransport } from '@rocket.chat/apps/base-runtime/dist/lib/messenger';
+import { startMainLoop } from '@rocket.chat/apps/base-runtime/dist/mainLoop';
+
+import registerErrorListeners from './error-handlers';
+import { sandboxRequire } from './lib/require';
+import { stdoutTransport } from './lib/transports/stdoutTransport';
+
+if (!process.argv.includes('--subprocess')) {
+	console.error(`
+            This is the Node wrapper for the Rocket.Chat Apps runtime. It is not meant to be executed stand-alone;
+            It is instead meant to be executed as a subprocess by the Apps-Engine framework.
+       `);
+
+	process.exit(1);
+}
+
+// This runtime communicates with the Apps-Engine host through stdout
+setTransport(stdoutTransport);
+
+// The sandbox `require` handed to the app is Node's own global `require`; it
+// needs no extra globals beyond the common ones the base eval shell binds.
+setSandboxRequire(sandboxRequire);
+setSandboxGlobals({});
+
+registerErrorListeners();
+
+void startMainLoop();

packages/apps/node-runtime/tsconfig.json

@@ -0,0 +1,18 @@
+{
+	"extends": "../tsconfig.json",
+	"compilerOptions": {
+		"rootDir": "./src",
+		"outDir": "./dist",
+		"baseUrl": ".",
+		"types": ["node"],
+		"lib": ["es2023"],
+		"module": "nodenext",
+		"moduleResolution": "nodenext",
+		"target": "es2023",
+		"declaration": true,
+		"paths": {
+			"@rocket.chat/apps/*": ["../*"]
+		}
+	},
+	"include": ["./src/**/*"]
+}

packages/apps/package.json

@@ -7,14 +7,16 @@
 	"files": [
 		"dist/",
 		"base-runtime/",
+		"node-runtime/",
 		"deno-runtime/",
 		".deno-cache/"
 	],
 	"scripts": {
-		"build": "run-s build:clean build:default build:base-runtime build:deno-cache",
-		"build:clean": "rimraf dist base-runtime/dist",
+		"build": "run-s build:clean build:default build:base-runtime build:node-runtime build:deno-cache",
+		"build:clean": "rimraf dist base-runtime/dist node-runtime/dist",
 		"build:default": "tsc -p tsconfig.json",
 		"build:base-runtime": "tsc -p base-runtime/tsconfig.json",
+		"build:node-runtime": "tsc -p node-runtime/tsconfig.json",
 		"build:deno-cache": "node scripts/deno-cache.js",
 		"dev": "tsc -p tsconfig.json --watch --preserveWatchOutput",
 		"lint": "eslint .",
@@ -24,7 +26,8 @@
 		"testunit": "run-s test:node test:deno test:base-runtime",
 		"typecheck:default": "tsc -p tsconfig.json --noEmit",
 		"typecheck:base-runtime": "tsc -p base-runtime/tsconfig.json --noEmit",
-		"typecheck": "run-s typecheck:default typecheck:base-runtime"
+		"typecheck:node-runtime": "tsc -p node-runtime/tsconfig.json --noEmit",
+		"typecheck": "run-s typecheck:default typecheck:base-runtime typecheck:node-runtime"
 	},
 	"dependencies": {
 		"@msgpack/msgpack": "3.0.0-beta2",

packages/apps/src/server/managers/AppRuntimeManager.ts

@@ -2,6 +2,7 @@ import type { AppManager } from '../AppManager';
 import type { IParseAppPackageResult } from '../compiler';
 import type { IRuntimeController } from '../runtime/IRuntimeController';
 import { DenoRuntimeSubprocessController } from '../runtime/deno/AppsEngineDenoRuntime';
+import { NodeRuntimeSubprocessController } from '../runtime/node/AppsEngineNodeRuntime';
 import type { IAppStorageItem } from '../storage';
 
 export type AppRuntimeParams = {
@@ -18,9 +19,16 @@ export type ExecRequestOptions = {
 	timeout?: number;
 };
 
-const defaultRuntimeFactory = (manager: AppManager, appPackage: IParseAppPackageResult, storageItem: IAppStorageItem) =>
+const { APPS_ENGINE_RUNTIME_BACKEND = 'deno' } = process.env;
+
+export const nodeRuntimeFactory = (manager: AppManager, appPackage: IParseAppPackageResult, storageItem: IAppStorageItem) =>
+	new NodeRuntimeSubprocessController(manager, appPackage, storageItem);
+
+export const denoRuntimeFactory = (manager: AppManager, appPackage: IParseAppPackageResult, storageItem: IAppStorageItem) =>
 	new DenoRuntimeSubprocessController(manager, appPackage, storageItem);
 
+const defaultRuntimeFactory = APPS_ENGINE_RUNTIME_BACKEND === 'node' ? nodeRuntimeFactory : denoRuntimeFactory;
+
 export class AppRuntimeManager {
 	private readonly subprocesses: Record<string, IRuntimeController> = {};