diff --git a/.changeset/fruity-items-fix.md b/.changeset/fruity-items-fix.md new file mode 100644 index 0000000000000..dd9b9d290b81a --- /dev/null +++ b/.changeset/fruity-items-fix.md @@ -0,0 +1,6 @@ +--- +'@rocket.chat/apps': minor +'@rocket.chat/meteor': minor +--- + +Adds an alternative runtime runner for apps. It can be enabled via environment variable `APPS_ENGINE_RUNTIME_BACKEND='node'` diff --git a/.github/workflows/ci-test-e2e.yml b/.github/workflows/ci-test-e2e.yml index 2c4ff87cb30bc..f89fa0ac3ee9e 100644 --- a/.github/workflows/ci-test-e2e.yml +++ b/.github/workflows/ci-test-e2e.yml @@ -167,7 +167,7 @@ jobs: run: echo "DEBUG_LOG_LEVEL=2" >> "$GITHUB_ENV" - name: Start httpbin container and wait for it to be ready - if: inputs.type == 'api' || inputs.type == 'api-livechat' + if: startsWith(inputs.type, 'api') run: | docker compose -f docker-compose-ci.yml up -d httpbin @@ -186,7 +186,7 @@ jobs: # behavior on (rate-limiter bypass, short cache TTLs) while letting # the deprecation logger log without throwing. Other suites use the # docker-compose default of TEST_MODE='true'. - TEST_MODE: ${{ (inputs.type == 'api' || inputs.type == 'api-livechat') && 'api' || 'true' }} + TEST_MODE: ${{ startsWith(inputs.type, 'api') && 'api' || 'true' }} run: | # when we are testing CE, we only need to start the rocketchat container DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d rocketchat --wait @@ -197,7 +197,8 @@ jobs: ENTERPRISE_LICENSE: ${{ inputs.enterprise-license }} TRANSPORTER: ${{ inputs.transporter }} COMPOSE_PROFILES: ${{ inputs.type == 'api' && 'api' || '' }} - TEST_MODE: ${{ (inputs.type == 'api' || inputs.type == 'api-livechat') && 'api' || 'true' }} + TEST_MODE: ${{ startsWith(inputs.type, 'api') && 'api' || 'true' }} + APPS_ENGINE_RUNTIME_BACKEND: ${{ inputs.type == 'api-apps-node' && 'node' || '' }} run: | DEBUG_LOG_LEVEL=${DEBUG_LOG_LEVEL:-0} docker compose -f docker-compose-ci.yml up -d --wait @@ -234,6 +235,23 @@ jobs: ls -la "$COVERAGE_DIR" exit "${s:-0}" + # This step should be temporary, only here until we remove the deno-runtime + - name: E2E Test API (apps + node-runtime) + if: (inputs.type == 'api-apps-node' && inputs.release == 'ee') + working-directory: ./apps/meteor + env: + WEBHOOK_TEST_URL: 'http://httpbin' + IS_EE: 'true' + run: | + set -o xtrace + + npm run testapi:apps || s=$? + + docker compose -f ../../docker-compose-ci.yml stop + + ls -la "$COVERAGE_DIR" + exit "${s:-0}" + - name: E2E Test API (Livechat) if: inputs.type == 'api-livechat' working-directory: ./apps/meteor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a170493f02c37..80b315f1a92d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -657,6 +657,27 @@ jobs: CR_PAT: ${{ secrets.CR_PAT }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + test-api-apps-node-ee: + name: 🔨 Test API Apps (node-runtime - EE) + needs: [checks, build-gh-docker-publish, release-versions] + + uses: ./.github/workflows/ci-test-e2e.yml + with: + type: api-apps-node + release: ee + transporter: 'nats://nats:4222' + enterprise-license: ${{ needs.release-versions.outputs.enterprise-license }} + mongodb-version: "['8.0']" + coverage: '8.0' + node-version: ${{ needs.release-versions.outputs.node-version }} + deno-version: ${{ needs.release-versions.outputs.deno-version }} + lowercase-repo: ${{ needs.release-versions.outputs.lowercase-repo }} + gh-docker-tag: ${{ needs.release-versions.outputs.gh-docker-tag }} + secrets: + CR_USER: ${{ secrets.CR_USER }} + CR_PAT: ${{ secrets.CR_PAT }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + test-ui-ee: name: 🔨 Test UI (EE) needs: [checks, build-gh-docker-publish, release-versions] diff --git a/apps/meteor/.mocharc.api.apps.js b/apps/meteor/.mocharc.api.apps.js new file mode 100644 index 0000000000000..7eea51b389f01 --- /dev/null +++ b/apps/meteor/.mocharc.api.apps.js @@ -0,0 +1,15 @@ +'use strict'; + +/* + * Mocha configuration for REST API integration tests. + */ + +module.exports = /** @satisfies {import('mocha').MochaOptions} */ ({ + ...require('./.mocharc.base.json'), // see https://github.com/mochajs/mocha/issues/3916 + timeout: 10000, + bail: false, + retries: 0, + file: 'tests/end-to-end/teardown.ts', + reporter: 'tests/end-to-end/reporter.ts', + spec: ['tests/end-to-end/apps/*'], +}); diff --git a/apps/meteor/package.json b/apps/meteor/package.json index 876e44865fa00..58634287ed865 100644 --- a/apps/meteor/package.json +++ b/apps/meteor/package.json @@ -56,6 +56,7 @@ "test:e2e:nyc": "nyc report --reporter=lcovonly", "testapi": "TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\"}' mocha --config ./.mocharc.api.js", "testapi:livechat": "TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\"}' mocha --config ./.mocharc.api.livechat.js", + "testapi:apps": "TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\"}' mocha --config ./.mocharc.api.apps.js", "testunit": "yarn .testunit:definition && yarn .testunit:jest && yarn .testunit:server:cov", "testunit-watch": "TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\"}' mocha --watch --config ./.mocharc.js", "typecheck": "meteor lint && cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsc --noEmit --skipLibCheck", diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index ffa7125b5f727..c2c23488913f6 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -14,6 +14,7 @@ services: image: ghcr.io/${LOWERCASE_REPOSITORY}/rocket.chat:${DOCKER_TAG}${DOCKER_TAG_SUFFIX_ROCKETCHAT:-} environment: - 'TEST_MODE=${TEST_MODE:-true}' + - APPS_ENGINE_RUNTIME_BACKEND=${APPS_ENGINE_RUNTIME_BACKEND:-} - DEBUG=${DEBUG:-} - EXIT_UNHANDLEDPROMISEREJECTION=true - MONGO_URL=mongodb://mongo:27017/rocketchat?replicaSet=rs0 diff --git a/packages/apps/deno-runtime/main.ts b/packages/apps/deno-runtime/main.ts index 4e68cc54f475d..d559ad084e214 100644 --- a/packages/apps/deno-runtime/main.ts +++ b/packages/apps/deno-runtime/main.ts @@ -17,7 +17,7 @@ import { stdoutTransport } from './lib/transports/stdoutTransport'; if (!process.argv.includes('--subprocess')) { console.error(` - This is a Deno wrapper for Rocket.Chat Apps. It is not meant to be executed stand-alone; + This is the Deno wrapper for the Rocket.Chat Apps runtime. It is not meant to be executed stand-alone; It is instead meant to be executed as a subprocess by the Apps-Engine framework. `); diff --git a/packages/apps/node-runtime/src/error-handlers.ts b/packages/apps/node-runtime/src/error-handlers.ts new file mode 100644 index 0000000000000..226cc189fa8b9 --- /dev/null +++ b/packages/apps/node-runtime/src/error-handlers.ts @@ -0,0 +1,10 @@ +import * as Messenger from '@rocket.chat/apps/base-runtime/dist/lib/messenger'; + +export default function registerErrorListeners() { + process.on('uncaughtException', (error: Error, origin: 'uncaughtException' | 'unhandledRejection') => { + Messenger.sendNotification({ + method: origin, + params: [error.stack || error], + }); + }); +} diff --git a/packages/apps/node-runtime/src/lib/loader-hook.ts b/packages/apps/node-runtime/src/lib/loader-hook.ts new file mode 100644 index 0000000000000..80411e78d2155 --- /dev/null +++ b/packages/apps/node-runtime/src/lib/loader-hook.ts @@ -0,0 +1,19 @@ +import { registerHooks } from 'node:module'; +import path from 'node:path'; + +// This file compiles to dist/lib/loader-hook.js. +// Three levels up from dist/lib/ lands on packages/apps/ — the @rocket.chat/apps package root. +const appsPackageDir = path.resolve(__dirname, '../../..'); + +const PACKAGE_PREFIX = '@rocket.chat/apps'; + +registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier === PACKAGE_PREFIX || specifier.startsWith(`${PACKAGE_PREFIX}/`)) { + const subpath = specifier.slice(PACKAGE_PREFIX.length).replace(/^\//, ''); + const localPath = subpath ? path.join(appsPackageDir, subpath) : appsPackageDir; + return nextResolve(localPath, context); + } + return nextResolve(specifier, context); + }, +}); diff --git a/packages/apps/node-runtime/src/lib/require.ts b/packages/apps/node-runtime/src/lib/require.ts new file mode 100644 index 0000000000000..5b1f118e38a08 --- /dev/null +++ b/packages/apps/node-runtime/src/lib/require.ts @@ -0,0 +1,37 @@ +const ALLOWED_MODULES = [ + 'path', + 'url', + 'crypto', + 'buffer', + 'stream', + 'net', + 'http', + 'https', + 'zlib', + 'util', + 'punycode', + 'os', + 'querystring', + 'fs', + // External libraries + 'uuid', + '@rocket.chat/apps-engine', +]; + +// As the apps are bundled, the only times they will call require are +// 1. To require native modules +// 2. To require external npm packages we may provide +// 3. To require apps-engine files +export const sandboxRequire = (module: string) => { + // Normalize Node built-in specifiers: accept both 'crypto' and 'node:crypto' + const normalized = module.replace('node:', ''); + + // We allow variants like 'fs', 'node:fs' or 'node:fs/promises', or even '@rocket.chat/apps-engine/**' + if (!ALLOWED_MODULES.some((mod) => normalized.startsWith(mod))) { + throw new Error(`Module ${module} is not allowed`); + } + + // This is THE purpose of this function, we can't escape a dinamyc require call + // eslint-disable-next-line @typescript-eslint/no-unsafe-return, import/no-dynamic-require, @typescript-eslint/no-require-imports + return require(normalized); +}; diff --git a/packages/apps/node-runtime/src/lib/transports/stdoutTransport.ts b/packages/apps/node-runtime/src/lib/transports/stdoutTransport.ts new file mode 100644 index 0000000000000..d835646c388fa --- /dev/null +++ b/packages/apps/node-runtime/src/lib/transports/stdoutTransport.ts @@ -0,0 +1,16 @@ +import type { Transport } from '@rocket.chat/apps/base-runtime/dist/lib/messenger'; + +/** + * Transport that writes messages to the process' standard output. + * + * This is the transport used when the runtime is executed as a subprocess by + * the Apps-Engine framework, and it is specific to platforms that expose a + * Node-compatible `process.stdout`. + */ +export const stdoutTransport: Transport = { + async send(message: Uint8Array): Promise { + await new Promise((resolve, reject) => { + process.stdout.write(message, (err) => (err ? reject(err) : resolve())); + }); + }, +}; diff --git a/packages/apps/node-runtime/src/main.ts b/packages/apps/node-runtime/src/main.ts new file mode 100644 index 0000000000000..7452f34a1f591 --- /dev/null +++ b/packages/apps/node-runtime/src/main.ts @@ -0,0 +1,30 @@ +import './lib/loader-hook'; + +import { setSandboxGlobals, setSandboxRequire } from '@rocket.chat/apps/base-runtime/dist/handlers/app/construct'; +import { setTransport } from '@rocket.chat/apps/base-runtime/dist/lib/messenger'; +import { startMainLoop } from '@rocket.chat/apps/base-runtime/dist/mainLoop'; + +import registerErrorListeners from './error-handlers'; +import { sandboxRequire } from './lib/require'; +import { stdoutTransport } from './lib/transports/stdoutTransport'; + +if (!process.argv.includes('--subprocess')) { + console.error(` + This is the Node wrapper for the Rocket.Chat Apps runtime. It is not meant to be executed stand-alone; + It is instead meant to be executed as a subprocess by the Apps-Engine framework. + `); + + process.exit(1); +} + +// This runtime communicates with the Apps-Engine host through stdout +setTransport(stdoutTransport); + +// The sandbox `require` handed to the app is Node's own global `require`; it +// needs no extra globals beyond the common ones the base eval shell binds. +setSandboxRequire(sandboxRequire); +setSandboxGlobals({}); + +registerErrorListeners(); + +void startMainLoop(); diff --git a/packages/apps/node-runtime/tsconfig.json b/packages/apps/node-runtime/tsconfig.json new file mode 100644 index 0000000000000..3b75a52883b11 --- /dev/null +++ b/packages/apps/node-runtime/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "baseUrl": ".", + "types": ["node"], + "lib": ["es2023"], + "module": "nodenext", + "moduleResolution": "nodenext", + "target": "es2023", + "declaration": true, + "paths": { + "@rocket.chat/apps/*": ["../*"] + } + }, + "include": ["./src/**/*"] +} diff --git a/packages/apps/package.json b/packages/apps/package.json index 1d02722b51bec..29d0a91c8e393 100644 --- a/packages/apps/package.json +++ b/packages/apps/package.json @@ -7,14 +7,16 @@ "files": [ "dist/", "base-runtime/", + "node-runtime/", "deno-runtime/", ".deno-cache/" ], "scripts": { - "build": "run-s build:clean build:default build:base-runtime build:deno-cache", - "build:clean": "rimraf dist base-runtime/dist", + "build": "run-s build:clean build:default build:base-runtime build:node-runtime build:deno-cache", + "build:clean": "rimraf dist base-runtime/dist node-runtime/dist", "build:default": "tsc -p tsconfig.json", "build:base-runtime": "tsc -p base-runtime/tsconfig.json", + "build:node-runtime": "tsc -p node-runtime/tsconfig.json", "build:deno-cache": "node scripts/deno-cache.js", "dev": "tsc -p tsconfig.json --watch --preserveWatchOutput", "lint": "eslint .", @@ -24,7 +26,8 @@ "testunit": "run-s test:node test:deno test:base-runtime", "typecheck:default": "tsc -p tsconfig.json --noEmit", "typecheck:base-runtime": "tsc -p base-runtime/tsconfig.json --noEmit", - "typecheck": "run-s typecheck:default typecheck:base-runtime" + "typecheck:node-runtime": "tsc -p node-runtime/tsconfig.json --noEmit", + "typecheck": "run-s typecheck:default typecheck:base-runtime typecheck:node-runtime" }, "dependencies": { "@msgpack/msgpack": "3.0.0-beta2", diff --git a/packages/apps/src/server/managers/AppRuntimeManager.ts b/packages/apps/src/server/managers/AppRuntimeManager.ts index dd631bc8eebd0..3539f80148121 100644 --- a/packages/apps/src/server/managers/AppRuntimeManager.ts +++ b/packages/apps/src/server/managers/AppRuntimeManager.ts @@ -2,6 +2,7 @@ import type { AppManager } from '../AppManager'; import type { IParseAppPackageResult } from '../compiler'; import type { IRuntimeController } from '../runtime/IRuntimeController'; import { DenoRuntimeSubprocessController } from '../runtime/deno/AppsEngineDenoRuntime'; +import { NodeRuntimeSubprocessController } from '../runtime/node/AppsEngineNodeRuntime'; import type { IAppStorageItem } from '../storage'; export type AppRuntimeParams = { @@ -18,9 +19,16 @@ export type ExecRequestOptions = { timeout?: number; }; -const defaultRuntimeFactory = (manager: AppManager, appPackage: IParseAppPackageResult, storageItem: IAppStorageItem) => +const { APPS_ENGINE_RUNTIME_BACKEND = 'deno' } = process.env; + +export const nodeRuntimeFactory = (manager: AppManager, appPackage: IParseAppPackageResult, storageItem: IAppStorageItem) => + new NodeRuntimeSubprocessController(manager, appPackage, storageItem); + +export const denoRuntimeFactory = (manager: AppManager, appPackage: IParseAppPackageResult, storageItem: IAppStorageItem) => new DenoRuntimeSubprocessController(manager, appPackage, storageItem); +const defaultRuntimeFactory = APPS_ENGINE_RUNTIME_BACKEND === 'node' ? nodeRuntimeFactory : denoRuntimeFactory; + export class AppRuntimeManager { private readonly subprocesses: Record = {}; diff --git a/packages/apps/src/server/runtime/AppsEngineNodeRuntime.ts b/packages/apps/src/server/runtime/AppsEngineNodeRuntime.ts deleted file mode 100644 index 4dadf1588e87e..0000000000000 --- a/packages/apps/src/server/runtime/AppsEngineNodeRuntime.ts +++ /dev/null @@ -1,75 +0,0 @@ -import * as timers from 'node:timers'; -import * as vm from 'node:vm'; - -import type { App } from '@rocket.chat/apps-engine/definition/App'; - -import type { IAppsEngineRuntimeOptions } from './AppsEngineRuntime'; -import { APPS_ENGINE_RUNTIME_DEFAULT_TIMEOUT, AppsEngineRuntime, getFilenameForApp } from './AppsEngineRuntime'; - -export class AppsEngineNodeRuntime extends AppsEngineRuntime { - public static defaultRuntimeOptions = { - timeout: APPS_ENGINE_RUNTIME_DEFAULT_TIMEOUT, - }; - - public static defaultContext = { - ...timers, - Buffer, - console, - process: {}, - exports: {}, - }; - - public static async runCode(code: string, sandbox?: Record, options?: IAppsEngineRuntimeOptions): Promise { - return new Promise((resolve, reject) => { - process.nextTick(() => { - try { - resolve(this.runCodeSync(code, sandbox, options)); - } catch (e) { - reject(e); - } - }); - }); - } - - public static runCodeSync(code: string, sandbox?: Record, options?: IAppsEngineRuntimeOptions): any { - return vm.runInNewContext( - code, - { ...AppsEngineNodeRuntime.defaultContext, ...sandbox }, - { ...AppsEngineNodeRuntime.defaultRuntimeOptions, ...(options || {}) }, - ); - } - - constructor( - private readonly app: App, - private readonly customRequire: (mod: string) => any, - ) { - super(app, customRequire); - } - - public async runInSandbox(code: string, sandbox?: Record, options?: IAppsEngineRuntimeOptions): Promise { - return new Promise((resolve, reject) => { - process.nextTick(async () => { - try { - sandbox ??= {}; - - const result = await vm.runInNewContext( - code, - { - ...AppsEngineNodeRuntime.defaultContext, - ...sandbox, - require: this.customRequire, - }, - { - ...AppsEngineNodeRuntime.defaultRuntimeOptions, - filename: getFilenameForApp(options?.filename || this.app.getName()), - }, - ); - - resolve(result); - } catch (e) { - reject(e); - } - }); - }); - } -} diff --git a/packages/apps/src/server/runtime/node/AppsEngineNodeRuntime.ts b/packages/apps/src/server/runtime/node/AppsEngineNodeRuntime.ts new file mode 100644 index 0000000000000..08b9550d74831 --- /dev/null +++ b/packages/apps/src/server/runtime/node/AppsEngineNodeRuntime.ts @@ -0,0 +1,43 @@ +import * as path from 'node:path'; + +import type { AppManager } from '../../AppManager'; +import type { IParseAppPackageResult } from '../../compiler'; +import type { IAppStorageItem } from '../../storage'; +import { BaseRuntimeSubprocessController, type ProcessConfiguration } from '../base/BaseRuntimeSubprocessController'; + +export class NodeRuntimeSubprocessController extends BaseRuntimeSubprocessController { + private readonly nodeBin = 'node'; + + private readonly scriptRuntimePath: string; + + constructor(manager: AppManager, appPackage: IParseAppPackageResult, storageItem: IAppStorageItem) { + super('node', manager, appPackage, storageItem); + + this.scriptRuntimePath = require.resolve('../../../../node-runtime/dist/main.js'); + } + + protected buildProcessConfiguration(): ProcessConfiguration { + const allowedDirs = [this.tempFilePath, path.resolve(path.dirname(this.scriptRuntimePath), '..', '..'), this.appsEnginePath]; + + const args = [ + '--permission', + ...allowedDirs.map((dir) => `--allow-fs-read=${dir}`), + this.scriptRuntimePath, + '--subprocess', + this.appPackage.info.id, + '--spawnId', + String(this.spawnId++), + ]; + + // SECURITY: We control the command, the arguments and the script that will be executed. + return { + command: this.nodeBin, + args, + options: { + env: { + PATH: process.env.PATH, + }, + }, + }; + } +} diff --git a/packages/apps/turbo.json b/packages/apps/turbo.json index ad6d1f980e819..b58250f0bd902 100644 --- a/packages/apps/turbo.json +++ b/packages/apps/turbo.json @@ -4,7 +4,7 @@ "build": { "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", "$TURBO_ROOT$/.tool-versions"], - "outputs": ["base-runtime/**", "deno-runtime/**", "scripts/**", ".deno-cache/**", "dist/**"] + "outputs": ["base-runtime/**", "node-runtime/**", "deno-runtime/**", "scripts/**", ".deno-cache/**", "dist/**"] } } }