|
| 1 | +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. |
| 2 | + |
| 3 | +/** |
| 4 | + * End-to-end async import-job integration: the REAL create / progress / results |
| 5 | + * / list / cancel routes driven by a REAL {@link ObjectQL} engine + |
| 6 | + * {@link ObjectStackProtocolImplementation}, an in-memory driver, and real |
| 7 | + * registered objects (including a `sys_import_job` mirror) — no protocol mocks. |
| 8 | + * |
| 9 | + * Proves the P1 async pipeline: a create request persists a job row and returns |
| 10 | + * immediately; the background worker streams the batch through the SAME shared |
| 11 | + * runner the sync route uses, updating progress on the row; and readers can poll |
| 12 | + * progress, fetch a capped results report, and list history. |
| 13 | + */ |
| 14 | + |
| 15 | +import { describe, it, expect, beforeEach } from 'vitest'; |
| 16 | +import { ObjectQL, ObjectStackProtocolImplementation } from '@objectstack/objectql'; |
| 17 | +import { RestServer } from './rest-server'; |
| 18 | + |
| 19 | +// In-memory driver — equality + `$in`, with skip/limit (mirrors import-integration). |
| 20 | +function makeMemoryDriver() { |
| 21 | + const stores = new Map<string, Map<string, Record<string, unknown>>>(); |
| 22 | + const storeFor = (o: string) => { |
| 23 | + let s = stores.get(o); |
| 24 | + if (!s) { s = new Map(); stores.set(o, s); } |
| 25 | + return s; |
| 26 | + }; |
| 27 | + let nextId = 0; |
| 28 | + const matchOne = (cell: unknown, cond: unknown): boolean => { |
| 29 | + if (cond && typeof cond === 'object' && !Array.isArray(cond)) { |
| 30 | + const c = cond as Record<string, unknown>; |
| 31 | + if ('$in' in c) return Array.isArray(c.$in) && c.$in.some((x) => (cell ?? null) === (x ?? null)); |
| 32 | + if ('$eq' in c) return (cell ?? null) === ((c.$eq as unknown) ?? null); |
| 33 | + if ('$ne' in c) return (cell ?? null) !== ((c.$ne as unknown) ?? null); |
| 34 | + } |
| 35 | + return (cell ?? null) === ((cond as unknown) ?? null); |
| 36 | + }; |
| 37 | + const matches = (row: Record<string, unknown>, where: any): boolean => { |
| 38 | + if (!where || typeof where !== 'object') return true; |
| 39 | + for (const [k, v] of Object.entries(where)) { |
| 40 | + if (k.startsWith('$')) continue; |
| 41 | + if (!matchOne(row[k], v)) return false; |
| 42 | + } |
| 43 | + return true; |
| 44 | + }; |
| 45 | + const driver: any = { |
| 46 | + name: 'memory', version: '0.0.0', supports: {}, |
| 47 | + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, |
| 48 | + async find(o: string, ast: any) { |
| 49 | + const rows = Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)); |
| 50 | + const skip = Number(ast?.skip ?? ast?.offset ?? 0) || 0; |
| 51 | + const limit = ast?.limit ?? ast?.top; |
| 52 | + return limit != null ? rows.slice(skip, skip + Number(limit)) : rows.slice(skip); |
| 53 | + }, |
| 54 | + findStream() { throw new Error('ns'); }, |
| 55 | + async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, |
| 56 | + async create(o: string, data: Record<string, unknown>) { |
| 57 | + nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; |
| 58 | + }, |
| 59 | + async update(o: string, id: string, data: Record<string, unknown>) { |
| 60 | + const s = storeFor(o); const cur = s.get(id); if (!cur) throw new Error(`nf ${o}/${id}`); |
| 61 | + const up = { ...cur, ...data, id }; s.set(id, up); return up; |
| 62 | + }, |
| 63 | + async upsert(o: string, data: Record<string, unknown>) { const id = data.id as string | undefined; return id && storeFor(o).has(id) ? this.update(o, id, data) : this.create(o, data); }, |
| 64 | + async delete(o: string, id: string) { return storeFor(o).delete(id); }, |
| 65 | + async count(o: string, ast: any) { return (await this.find(o, ast)).length; }, |
| 66 | + async bulkCreate(o: string, rows: Record<string, unknown>[]) { return Promise.all(rows.map((r) => this.create(o, r))); }, |
| 67 | + async bulkUpdate() { return []; }, async bulkDelete() {}, |
| 68 | + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, async commit() {}, async rollback() {}, |
| 69 | + }; |
| 70 | + return { driver, stores }; |
| 71 | +} |
| 72 | + |
| 73 | +const TASK = { |
| 74 | + name: 'task', label: 'Task', systemFields: false, |
| 75 | + fields: { |
| 76 | + id: { name: 'id', type: 'text' as const, primaryKey: true, label: 'ID' }, |
| 77 | + title: { name: 'title', type: 'text' as const, label: '标题' }, |
| 78 | + done: { name: 'done', type: 'boolean' as const, label: '完成' }, |
| 79 | + score: { name: 'score', type: 'number' as const, label: '分数' }, |
| 80 | + }, |
| 81 | +}; |
| 82 | + |
| 83 | +// Minimal sys_import_job mirror the routes read/write through the protocol. |
| 84 | +const SYS_IMPORT_JOB = { |
| 85 | + name: 'sys_import_job', label: 'Import Job', systemFields: false, |
| 86 | + fields: { |
| 87 | + id: { name: 'id', type: 'text' as const, primaryKey: true }, |
| 88 | + object_name: { name: 'object_name', type: 'text' as const }, |
| 89 | + status: { name: 'status', type: 'text' as const }, |
| 90 | + total_rows: { name: 'total_rows', type: 'number' as const }, |
| 91 | + processed_rows: { name: 'processed_rows', type: 'number' as const }, |
| 92 | + created_count: { name: 'created_count', type: 'number' as const }, |
| 93 | + updated_count: { name: 'updated_count', type: 'number' as const }, |
| 94 | + skipped_count: { name: 'skipped_count', type: 'number' as const }, |
| 95 | + error_count: { name: 'error_count', type: 'number' as const }, |
| 96 | + write_mode: { name: 'write_mode', type: 'text' as const }, |
| 97 | + dry_run: { name: 'dry_run', type: 'boolean' as const }, |
| 98 | + run_automations: { name: 'run_automations', type: 'boolean' as const }, |
| 99 | + error: { name: 'error', type: 'textarea' as const }, |
| 100 | + results: { name: 'results', type: 'json' as const }, |
| 101 | + started_at: { name: 'started_at', type: 'text' as const }, |
| 102 | + completed_at: { name: 'completed_at', type: 'text' as const }, |
| 103 | + created_by: { name: 'created_by', type: 'text' as const }, |
| 104 | + created_at: { name: 'created_at', type: 'text' as const }, |
| 105 | + }, |
| 106 | +}; |
| 107 | + |
| 108 | +function createMockServer() { |
| 109 | + const noop = () => {}; |
| 110 | + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; |
| 111 | +} |
| 112 | + |
| 113 | +function makeRes() { |
| 114 | + const res: any = { |
| 115 | + write: () => true, end: () => {}, |
| 116 | + header: () => res, |
| 117 | + status: (code: number) => { res._status = code; return res; }, |
| 118 | + json: (body: any) => { res._json = body; return res; }, |
| 119 | + }; |
| 120 | + return res; |
| 121 | +} |
| 122 | + |
| 123 | +async function boot() { |
| 124 | + const { driver } = makeMemoryDriver(); |
| 125 | + const engine = new ObjectQL(); |
| 126 | + engine.registerDriver(driver, true); |
| 127 | + await engine.init(); |
| 128 | + engine.registry.registerObject(TASK as any); |
| 129 | + engine.registry.registerObject(SYS_IMPORT_JOB as any); |
| 130 | + |
| 131 | + const protocol = new ObjectStackProtocolImplementation(engine as any); |
| 132 | + const rest = new RestServer(createMockServer() as any, protocol as any); |
| 133 | + rest.registerRoutes(); |
| 134 | + const routes = rest.getRoutes(); |
| 135 | + const find = (method: string, path: string) => routes.find((r: any) => r.method === method && r.path === path); |
| 136 | + return { |
| 137 | + engine, protocol, |
| 138 | + create: find('POST', '/api/v1/data/:object/import/jobs'), |
| 139 | + progress: find('GET', '/api/v1/data/import/jobs/:jobId'), |
| 140 | + results: find('GET', '/api/v1/data/import/jobs/:jobId/results'), |
| 141 | + list: find('GET', '/api/v1/data/import/jobs'), |
| 142 | + cancel: find('POST', '/api/v1/data/import/jobs/:jobId/cancel'), |
| 143 | + }; |
| 144 | +} |
| 145 | + |
| 146 | +const callCreate = (route: any, body: any) => { |
| 147 | + const res = makeRes(); |
| 148 | + return route.handler({ params: { object: 'task' }, body } as any, res).then(() => res); |
| 149 | +}; |
| 150 | +const callJob = (route: any, jobId: string, query: any = {}) => { |
| 151 | + const res = makeRes(); |
| 152 | + return route.handler({ params: { jobId }, query } as any, res).then(() => res); |
| 153 | +}; |
| 154 | + |
| 155 | +/** Poll the progress route until the job reaches a terminal state. */ |
| 156 | +async function waitForTerminal(progress: any, jobId: string, tries = 100): Promise<any> { |
| 157 | + for (let i = 0; i < tries; i++) { |
| 158 | + const res = await callJob(progress, jobId); |
| 159 | + const status = res._json?.status; |
| 160 | + if (status === 'succeeded' || status === 'failed' || status === 'cancelled') return res._json; |
| 161 | + await new Promise((r) => setTimeout(r, 5)); |
| 162 | + } |
| 163 | + throw new Error(`import job ${jobId} did not finish`); |
| 164 | +} |
| 165 | + |
| 166 | +describe('async import job — real engine + protocol integration', () => { |
| 167 | + let ctx: Awaited<ReturnType<typeof boot>>; |
| 168 | + beforeEach(async () => { |
| 169 | + ctx = await boot(); |
| 170 | + expect(ctx.create).toBeDefined(); |
| 171 | + expect(ctx.progress && ctx.results && ctx.list && ctx.cancel).toBeTruthy(); |
| 172 | + }); |
| 173 | + |
| 174 | + it('creates a job, processes it in the background, and persists rows', async () => { |
| 175 | + const res = await callCreate(ctx.create, { |
| 176 | + format: 'json', |
| 177 | + rows: [ |
| 178 | + { id: 'a', title: 'one', done: '是', score: '10' }, |
| 179 | + { id: 'b', title: 'two', done: '否', score: '20' }, |
| 180 | + { id: 'c', title: 'three', score: 'not-a-number' }, // one failure |
| 181 | + ], |
| 182 | + }); |
| 183 | + expect(res._status).toBe(201); |
| 184 | + expect(res._json).toMatchObject({ object: 'task', status: 'pending', total: 3 }); |
| 185 | + const jobId = res._json.jobId; |
| 186 | + expect(jobId).toMatch(/^imp_/); |
| 187 | + |
| 188 | + const done = await waitForTerminal(ctx.progress, jobId); |
| 189 | + expect(done).toMatchObject({ status: 'succeeded', total: 3, processed: 3, created: 2, errors: 1 }); |
| 190 | + expect(done.percentComplete).toBe(100); |
| 191 | + |
| 192 | + // Records really landed (coerced: 是→true, "10"→10). |
| 193 | + const a = await ctx.engine.findOne('task', { where: { id: 'a' } }); |
| 194 | + expect(a).toMatchObject({ title: 'one', done: true, score: 10 }); |
| 195 | + |
| 196 | + // Results route returns the capped per-row report (failure present). |
| 197 | + const results = await callJob(ctx.results, jobId); |
| 198 | + expect(results._json.resultsTruncated).toBe(false); |
| 199 | + expect(results._json.results.find((r: any) => !r.ok)).toMatchObject({ field: 'score', code: 'invalid_number' }); |
| 200 | + }); |
| 201 | + |
| 202 | + it('rejects a payload above the 50k async ceiling with 413', async () => { |
| 203 | + const rows = Array.from({ length: 50_001 }, (_, i) => ({ id: `x${i}`, title: 't' })); |
| 204 | + const res = await callCreate(ctx.create, { format: 'json', rows }); |
| 205 | + expect(res._status).toBe(413); |
| 206 | + expect(res._json.code).toBe('PAYLOAD_TOO_LARGE'); |
| 207 | + expect(String(res._json.error)).toMatch(/50000/); |
| 208 | + }); |
| 209 | + |
| 210 | + it('lists jobs in history and filters by status', async () => { |
| 211 | + const r1 = await callCreate(ctx.create, { format: 'json', rows: [{ id: 'l1', title: 'a' }] }); |
| 212 | + await waitForTerminal(ctx.progress, r1._json.jobId); |
| 213 | + const r2 = await callCreate(ctx.create, { format: 'json', rows: [{ id: 'l2', title: 'b' }] }); |
| 214 | + await waitForTerminal(ctx.progress, r2._json.jobId); |
| 215 | + |
| 216 | + const all = await callJob(ctx.list, '', {}); |
| 217 | + expect(all._json.jobs.length).toBe(2); |
| 218 | + expect(all._json.jobs[0]).toHaveProperty('jobId'); |
| 219 | + expect(all._json.jobs[0]).toHaveProperty('createdAt'); |
| 220 | + |
| 221 | + const succeeded = await callJob(ctx.list, '', { status: 'succeeded' }); |
| 222 | + expect(succeeded._json.jobs.length).toBe(2); |
| 223 | + const failedOnly = await callJob(ctx.list, '', { status: 'failed' }); |
| 224 | + expect(failedOnly._json.jobs.length).toBe(0); |
| 225 | + }); |
| 226 | + |
| 227 | + it('404s progress/results/cancel for an unknown job id', async () => { |
| 228 | + const p = await callJob(ctx.progress, 'imp_nope'); |
| 229 | + expect(p._status).toBe(404); |
| 230 | + const r = await callJob(ctx.results, 'imp_nope'); |
| 231 | + expect(r._status).toBe(404); |
| 232 | + const c = await callJob(ctx.cancel, 'imp_nope'); |
| 233 | + expect(c._status).toBe(404); |
| 234 | + }); |
| 235 | + |
| 236 | + it('cancel on an already-finished job is a no-op success', async () => { |
| 237 | + const created = await callCreate(ctx.create, { format: 'json', rows: [{ id: 'k', title: 'x' }] }); |
| 238 | + const jobId = created._json.jobId; |
| 239 | + await waitForTerminal(ctx.progress, jobId); |
| 240 | + const c = await callJob(ctx.cancel, jobId); |
| 241 | + expect(c._json).toMatchObject({ success: true }); |
| 242 | + // Terminal state preserved. |
| 243 | + const after = await callJob(ctx.progress, jobId); |
| 244 | + expect(after._json.status).toBe('succeeded'); |
| 245 | + }); |
| 246 | +}); |
0 commit comments