Skip to content

Commit 911e026

Browse files
adrians5jclaude
andauthored
feat(self-hosted): background tasks, scheduler & file manager for the server flavour (#5408)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent a8d7290 commit 911e026

31 files changed

Lines changed: 595 additions & 131 deletions

File tree

packages/api-event-handler-aws/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"dependencies": {
2121
"@webiny/api-core": "0.0.0",
2222
"@webiny/api-event-handler-core": "0.0.0",
23+
"@webiny/api-file-manager-s3": "0.0.0",
2324
"@webiny/api-scheduler-aws": "0.0.0",
2425
"@webiny/api-websockets": "0.0.0",
2526
"@webiny/api-websockets-aws": "0.0.0",

packages/api-event-handler-aws/src/createWebinyApiHandler.ts

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import { DynamoDBCoreFeature } from "@webiny/db-dynamodb";
2424
import { registerApiRequestStack } from "@webiny/api-event-handler-core";
2525
import { WebsocketsAwsFeature } from "@webiny/api-websockets-aws";
2626
import { registerSchedulerAwsExtension } from "@webiny/api-scheduler-aws";
27+
import { FileManagerS3Feature } from "@webiny/api-file-manager-s3";
2728
import { WebSocketLambdaHandler } from "@webiny/api-websockets";
2829
// CognitoIdpFeature must be in the root container so the request auth step
2930
// (ApiGatewayIdentityLoaderDecorator → RequestIdentityLoader) sees CognitoIdentityProvider
@@ -107,21 +108,27 @@ export function createWebinyApiHandler(config: CreateWebinyApiHandlerConfig) {
107108
},
108109

109110
request: async container => {
110-
// The per-request feature stack is transport-agnostic (shared with the future server
111-
// transport). The two AWS-specific interleave points are supplied as hooks.
111+
// The per-request feature stack is transport-agnostic (shared with the server transport).
112+
// The AWS-specific interleave points are supplied as the `transports` adapters.
112113
await registerApiRequestStack(container, {
113114
extensions: config.extensions,
114115
registerRequestStorage: config.registerRequestStorage,
115-
// Real AWS WebSocket transport (API Gateway Management API), registered right after
116-
// WebsocketsFeature so it overrides the NullWebsocketsTransport.
117-
registerRealtimeTransport: c => {
118-
WebsocketsAwsFeature.register(c);
119-
},
120-
// Scheduler transport: the scheduler-aws extension (EventBridge Scheduler).
121-
registerSchedulerTransport: c => {
122-
registerSchedulerAwsExtension(c, {
123-
getClient: schedulerConfig => createSchedulerClient(schedulerConfig)
124-
});
116+
transports: {
117+
// Real AWS WebSocket transport (API Gateway Management API), registered right after
118+
// WebsocketsFeature so it overrides the NullWebsocketsTransport.
119+
realtime: c => {
120+
WebsocketsAwsFeature.register(c);
121+
},
122+
// Scheduler transport: the scheduler-aws extension (EventBridge Scheduler).
123+
scheduler: c => {
124+
registerSchedulerAwsExtension(c, {
125+
getClient: schedulerConfig => createSchedulerClient(schedulerConfig)
126+
});
127+
},
128+
// File-manager storage transport: S3 (asset delivery + S3 file operations + schema).
129+
fileManager: c => {
130+
FileManagerS3Feature.register(c, {});
131+
}
125132
}
126133
});
127134
}

packages/api-event-handler-aws/tsconfig.build.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"references": [
55
{ "path": "../api-core/tsconfig.build.json" },
66
{ "path": "../api-event-handler-core/tsconfig.build.json" },
7+
{ "path": "../api-file-manager-s3/tsconfig.build.json" },
78
{ "path": "../api-scheduler-aws/tsconfig.build.json" },
89
{ "path": "../api-websockets/tsconfig.build.json" },
910
{ "path": "../api-websockets-aws/tsconfig.build.json" },
@@ -26,6 +27,8 @@
2627
"@webiny/api-core": ["../api-core/src"],
2728
"@webiny/api-event-handler-core/*": ["../api-event-handler-core/src/*"],
2829
"@webiny/api-event-handler-core": ["../api-event-handler-core/src"],
30+
"@webiny/api-file-manager-s3/*": ["../api-file-manager-s3/src/*"],
31+
"@webiny/api-file-manager-s3": ["../api-file-manager-s3/src"],
2932
"@webiny/api-scheduler-aws/*": ["../api-scheduler-aws/src/*"],
3033
"@webiny/api-scheduler-aws": ["../api-scheduler-aws/src"],
3134
"@webiny/api-websockets/*": ["../api-websockets/src/*"],

packages/api-event-handler-aws/tsconfig.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"references": [
55
{ "path": "../api-core" },
66
{ "path": "../api-event-handler-core" },
7+
{ "path": "../api-file-manager-s3" },
78
{ "path": "../api-scheduler-aws" },
89
{ "path": "../api-websockets" },
910
{ "path": "../api-websockets-aws" },
@@ -26,6 +27,8 @@
2627
"@webiny/api-core": ["../api-core/src"],
2728
"@webiny/api-event-handler-core/*": ["../api-event-handler-core/src/*"],
2829
"@webiny/api-event-handler-core": ["../api-event-handler-core/src"],
30+
"@webiny/api-file-manager-s3/*": ["../api-file-manager-s3/src/*"],
31+
"@webiny/api-file-manager-s3": ["../api-file-manager-s3/src"],
2932
"@webiny/api-scheduler-aws/*": ["../api-scheduler-aws/src/*"],
3033
"@webiny/api-scheduler-aws": ["../api-scheduler-aws/src"],
3134
"@webiny/api-websockets/*": ["../api-websockets/src/*"],

packages/api-event-handler-core/package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
"@webiny/api-core": "0.0.0",
2424
"@webiny/api-file-manager": "0.0.0",
2525
"@webiny/api-file-manager-aco": "0.0.0",
26-
"@webiny/api-file-manager-s3": "0.0.0",
2726
"@webiny/api-headless-cms": "0.0.0",
2827
"@webiny/api-headless-cms-aco": "0.0.0",
2928
"@webiny/api-headless-cms-scheduler": "0.0.0",

packages/api-event-handler-core/src/registerApiRequestStack.ts

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,16 @@ import { AcoFeature } from "@webiny/api-aco";
1616
import { BackgroundTasksFeature } from "@webiny/background-tasks/api";
1717
import { FileManagerAppFeature } from "@webiny/api-file-manager";
1818
import { FileManagerAcoFeature } from "@webiny/api-file-manager-aco";
19-
import { FileManagerS3Feature } from "@webiny/api-file-manager-s3";
2019
import { WebsiteBuilderFeature, setupWebsiteBuilderModels } from "@webiny/api-website-builder";
2120
import { WebsiteBuilderWorkflowsFeature } from "@webiny/api-website-builder-workflows";
2221
import { WebsiteBuilderSchedulerFeature } from "@webiny/api-website-builder-scheduler";
2322
import { WebsocketsFeature } from "@webiny/api-websockets";
2423
import { WorkflowsFeature } from "@webiny/api-workflows";
2524
import { SchedulerFeature } from "@webiny/api-scheduler";
2625

26+
/** Installs a flavour-specific transport adapter into the per-request container at its interleave point. */
27+
export type TransportRegistrar = (container: Container) => void | Promise<void>;
28+
2729
export interface RegisterApiRequestStackConfig {
2830
/**
2931
* Project-defined extensions, applied at register() time (so extension features — including
@@ -36,18 +38,30 @@ export interface RegisterApiRequestStackConfig {
3638
*/
3739
registerRequestStorage?: (container: Container) => void | Promise<void>;
3840
/**
39-
* Register the real-time (websockets) transport, run immediately AFTER the transport-agnostic
40-
* `WebsocketsFeature`. On AWS this is `WebsocketsAwsFeature` (API Gateway Management API); it
41-
* MUST register after WebsocketsFeature so it overrides the NullWebsocketsTransport
42-
* (nearest-container-last-wins), otherwise every server→client send() is a silent no-op.
43-
* Optional — omit for transports with no real-time push.
44-
*/
45-
registerRealtimeTransport?: (container: Container) => void | Promise<void>;
46-
/**
47-
* Register the scheduler transport, run AFTER `SchedulerFeature` and BEFORE `CmsSchedulerFeature`.
48-
* On AWS this bridges the scheduler-aws extension (EventBridge Scheduler). Optional.
41+
* Flavour-specific transport adapters, each installed at its exact interleave point in the stack.
42+
* Every entry follows the same shape: it runs immediately AFTER the transport-agnostic domain
43+
* Feature has registered its NULL default, and overrides that default (nearest-container-last-wins)
44+
* with the real adapter. Each is optional — omit one for a deployment/transport that lacks that
45+
* capability. AWS supplies AWS adapters (API Gateway Management API / EventBridge / S3); the
46+
* self-hosted server supplies in-process adapters (server WebSockets / Bree / local disk).
4947
*/
50-
registerSchedulerTransport?: (container: Container) => void | Promise<void>;
48+
transports?: {
49+
/**
50+
* Real-time (WebSockets) transport, run right after `WebsocketsFeature`. Overrides the
51+
* NullWebsocketsTransport, otherwise every server→client send() is a silent no-op.
52+
*/
53+
realtime?: TransportRegistrar;
54+
/**
55+
* Scheduler transport, run AFTER `SchedulerFeature` and BEFORE `CmsSchedulerFeature`.
56+
*/
57+
scheduler?: TransportRegistrar;
58+
/**
59+
* File-manager storage transport, run AFTER `FileManagerAppFeature` (which registers the
60+
* AssetDeliveryRoute + NULL asset-delivery impls). Overrides those nulls with real impls and
61+
* adds the file-operation features.
62+
*/
63+
fileManager?: TransportRegistrar;
64+
};
5165
}
5266

5367
/**
@@ -86,10 +100,10 @@ export async function registerApiRequestStack(
86100
AcoHcmsFeature.register(container);
87101
HcmsTasksFeature.register(container);
88102

89-
// ── File Manager ───────────────────────────────────────────
103+
// ── File Manager (domain) + storage transport ──────────────
90104
FileManagerAppFeature.register(container);
91105
FileManagerAcoFeature.register(container);
92-
FileManagerS3Feature.register(container, {});
106+
await config.transports?.fileManager?.(container);
93107

94108
// ── Website Builder ────────────────────────────────────────
95109
WebsiteBuilderFeature.register(container);
@@ -99,7 +113,7 @@ export async function registerApiRequestStack(
99113

100114
// ── Websockets (domain) + real-time transport ──────────────
101115
WebsocketsFeature.register(container);
102-
await config.registerRealtimeTransport?.(container);
116+
await config.transports?.realtime?.(container);
103117

104118
// ── Supporting services ────────────────────────────────────
105119
MailerFeature.register(container);
@@ -114,7 +128,7 @@ export async function registerApiRequestStack(
114128

115129
// ── Scheduler + scheduler transport ────────────────────────
116130
SchedulerFeature.register(container);
117-
await config.registerSchedulerTransport?.(container);
131+
await config.transports?.scheduler?.(container);
118132
CmsSchedulerFeature.register(container);
119133

120134
// ── Extensions ─────────────────────────────────────────────

packages/api-event-handler-core/tsconfig.build.json

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
{ "path": "../api-core/tsconfig.build.json" },
88
{ "path": "../api-file-manager/tsconfig.build.json" },
99
{ "path": "../api-file-manager-aco/tsconfig.build.json" },
10-
{ "path": "../api-file-manager-s3/tsconfig.build.json" },
1110
{ "path": "../api-headless-cms/tsconfig.build.json" },
1211
{ "path": "../api-headless-cms-aco/tsconfig.build.json" },
1312
{ "path": "../api-headless-cms-scheduler/tsconfig.build.json" },
@@ -44,8 +43,6 @@
4443
"@webiny/api-file-manager": ["../api-file-manager/src"],
4544
"@webiny/api-file-manager-aco/*": ["../api-file-manager-aco/src/*"],
4645
"@webiny/api-file-manager-aco": ["../api-file-manager-aco/src"],
47-
"@webiny/api-file-manager-s3/*": ["../api-file-manager-s3/src/*"],
48-
"@webiny/api-file-manager-s3": ["../api-file-manager-s3/src"],
4946
"@webiny/api-headless-cms/*": ["../api-headless-cms/src/*"],
5047
"@webiny/api-headless-cms": ["../api-headless-cms/src"],
5148
"@webiny/api-headless-cms-aco/*": ["../api-headless-cms-aco/src/*"],

packages/api-event-handler-core/tsconfig.json

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
{ "path": "../api-core" },
88
{ "path": "../api-file-manager" },
99
{ "path": "../api-file-manager-aco" },
10-
{ "path": "../api-file-manager-s3" },
1110
{ "path": "../api-headless-cms" },
1211
{ "path": "../api-headless-cms-aco" },
1312
{ "path": "../api-headless-cms-scheduler" },
@@ -44,8 +43,6 @@
4443
"@webiny/api-file-manager": ["../api-file-manager/src"],
4544
"@webiny/api-file-manager-aco/*": ["../api-file-manager-aco/src/*"],
4645
"@webiny/api-file-manager-aco": ["../api-file-manager-aco/src"],
47-
"@webiny/api-file-manager-s3/*": ["../api-file-manager-s3/src/*"],
48-
"@webiny/api-file-manager-s3": ["../api-file-manager-s3/src"],
4946
"@webiny/api-headless-cms/*": ["../api-headless-cms/src/*"],
5047
"@webiny/api-headless-cms": ["../api-headless-cms/src"],
5148
"@webiny/api-headless-cms-aco/*": ["../api-headless-cms-aco/src/*"],

packages/api-event-handler-server/package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020
"dependencies": {
2121
"@webiny/api-core": "0.0.0",
2222
"@webiny/api-event-handler-core": "0.0.0",
23+
"@webiny/api-file-manager-server": "0.0.0",
24+
"@webiny/api-scheduler-server": "0.0.0",
2325
"@webiny/api-websockets-server": "0.0.0",
26+
"@webiny/background-tasks-server": "0.0.0",
2427
"@webiny/event-handler-core": "0.0.0",
2528
"@webiny/event-handler-server": "0.0.0",
2629
"@webiny/handler": "0.0.0"

packages/api-event-handler-server/src/createWebinyApiHandler.ts

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
* plus the auth/tenant loader decorators (extract token / x-tenant from the IncomingMessage → shared
66
* RequestIdentityLoader / RequestTenantLoader). The per-request feature stack is the transport-agnostic
77
* `registerApiRequestStack` from `@webiny/api-event-handler-core` — the SAME stack the AWS handler uses.
8-
* It is called with NEITHER the realtime nor the scheduler hook: those are AWS-specific (API Gateway
9-
* WebSockets / EventBridge Scheduler) and this transport has no equivalent — the hooks being optional
10-
* is the point. The storage variant (and its identity provider) is injected via `registerRootStorage`.
8+
* Both interleave hooks are supplied with SINGLE-PROCESS, in-process equivalents of the AWS transports:
9+
* the realtime hook installs the server WebSockets transport (vs AWS's API Gateway Management API), and
10+
* the scheduler hook installs the Bree/in-process scheduler (vs AWS's EventBridge Scheduler). Background
11+
* tasks are wired in the ROOT container (below), mirroring how the AWS handler registers its background-
12+
* task transport at root. The storage variant (and its identity provider) is injected via `registerRootStorage`.
1113
*
1214
* The identity provider (e.g. `@webiny/self-hosted-auth`'s JWT IdP) must be registered by the variant
1315
* in `registerRootStorage`, so the RequestIdentityLoader driven by the identity decorator can resolve it.
@@ -23,6 +25,9 @@ import {
2325
WebsocketsConnectionManager,
2426
attachWebsocketsServer
2527
} from "@webiny/api-websockets-server";
28+
import { BackgroundTasksServerFeature } from "@webiny/background-tasks-server";
29+
import { registerSchedulerServerExtension } from "@webiny/api-scheduler-server";
30+
import { FileManagerServerFeature } from "@webiny/api-file-manager-server";
2631
import { NodeHttpIdentityLoaderDecorator } from "~/handlers/NodeHttpIdentityLoaderDecorator.js";
2732
import { NodeHttpTenantLoaderDecorator } from "~/handlers/NodeHttpTenantLoaderDecorator.js";
2833

@@ -63,25 +68,55 @@ export function createWebinyApiHandler(config: CreateWebinyApiHandlerConfig) {
6368
// connection registry (ConnectionRegistry) is the storage variant's job (e.g. sql).
6469
container.register(ServerConnectionManager).inSingletonScope();
6570
container.register(NodeWsAdapter).inSingletonScope();
71+
72+
// ── Background tasks (root) ────────────────────────────────
73+
// Mirrors the AWS handler registering its background-task transport at root. There is no
74+
// Step Functions / Lambda re-invocation in a single process, so the transport is in-process:
75+
// WorkerService (the BackgroundTasks/TaskService dispatch abstraction) runs each triggered
76+
// task in a Node worker_thread that POSTs back to this server's `/background-task` HTTP route
77+
// (BackgroundTaskRoute), which runs the task loop (continue/timeout/abort) in-process. Root,
78+
// not per-request, is load-bearing: the shared InternalToken (registered here as a singleton)
79+
// gates the route against the worker's callback, so dispatcher and route MUST see the SAME
80+
// token — a per-request registration would mint a fresh token per request and always 403.
81+
// The route is an HttpRoute; the per-request HttpRouter collects it via the parent chain.
82+
BackgroundTasksServerFeature.register(container);
6683
},
6784

6885
request: async container => {
6986
// The transport-agnostic per-request stack. The realtime hook installs the server
7087
// WebSockets transport (overriding the domain's NullWebsocketsTransport); it resolves the
71-
// shared connection manager + adapter from the root. No scheduler hook — that's AWS-only.
88+
// shared connection manager + adapter from the root. The scheduler hook installs the
89+
// Bree/in-process scheduler transport (the single-process equivalent of EventBridge).
7290
await registerApiRequestStack(container, {
7391
extensions: config.extensions,
74-
// Why a hook (and not just registering the transport ourselves): `.register()` calls
92+
// Why hooks (and not just registering the transports ourselves): `.register()` calls
7593
// are otherwise order-independent — you can register Features in any order, because
7694
// they only REGISTER, they don't RESOLVE during registration (resolution happens
7795
// later, in Initializers / SchemaFactories). The one thing that makes order matter is
78-
// a DEFAULT registration: `WebsocketsFeature` registers `NullWebsocketsTransport` so
79-
// the abstraction is always resolvable. Overriding it (last-registration-wins) means
80-
// our transport MUST be registered AFTER `WebsocketsFeature`. This hook is the seam
81-
// `registerApiRequestStack` provides for exactly that — it runs right after the Null
96+
// a DEFAULT registration: e.g. `WebsocketsFeature` registers `NullWebsocketsTransport`
97+
// so the abstraction is always resolvable. Overriding it (last-registration-wins) means
98+
// our transport MUST be registered AFTER that Feature. These hooks are the seams
99+
// `registerApiRequestStack` provides for exactly that — each runs right after its Null
82100
// default, so the override is guaranteed without the caller knowing the internal order.
83-
registerRealtimeTransport: requestContainer => {
84-
requestContainer.register(ServerWebsocketsTransport);
101+
transports: {
102+
// Server WebSockets transport; resolves the shared connection manager + adapter
103+
// from the root (registered as singletons above).
104+
realtime: requestContainer => {
105+
requestContainer.register(ServerWebsocketsTransport);
106+
},
107+
// Scheduler transport: the Bree/in-process extension. Where AWS bridges EventBridge
108+
// Scheduler, the single-process server drives delayed/scheduled action triggers with
109+
// in-process timers (Bree).
110+
scheduler: requestContainer => {
111+
registerSchedulerServerExtension(requestContainer);
112+
},
113+
// File-manager storage transport: local disk. Where AWS uses S3 (+ a separate asset-
114+
// delivery Lambda), the single-process server stores files on disk and serves them
115+
// in-process — FileManagerServerFeature registers local asset delivery (overriding
116+
// the domain's null impls), the upload/multipart HTTP routes, and disk file ops.
117+
fileManager: requestContainer => {
118+
FileManagerServerFeature.register(requestContainer);
119+
}
85120
}
86121
});
87122
},

0 commit comments

Comments
 (0)