Skip to content

Commit b7f58de

Browse files
kibertoadkibertoad
authored
Add fauxqs AWS engine (#400)
1 parent da9c9f3 commit b7f58de

51 files changed

Lines changed: 812 additions & 420 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.common.yml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ on:
88
node_version:
99
required: true
1010
type: string
11+
queue_backend:
12+
required: false
13+
type: string
14+
default: 'fauxqs'
1115

1216
jobs:
1317
build:
@@ -32,9 +36,17 @@ jobs:
3236

3337
- name: Docker start
3438
run: npm run docker:start:ci -- --filter=${{ inputs.package_name }}
39+
env:
40+
QUEUE_BACKEND: ${{ inputs.queue_backend }}
3541

3642
- name: Run Tests
37-
run: npm run test:ci -- --filter=${{ inputs.package_name }}
43+
run: |
44+
echo "::notice::Running ${{ inputs.package_name }} with QUEUE_BACKEND=${{ inputs.queue_backend }}"
45+
npm run test:ci -- --filter=${{ inputs.package_name }}
46+
env:
47+
QUEUE_BACKEND: ${{ inputs.queue_backend }}
3848

3949
- name: Docker stop
4050
run: npm run docker:stop:ci -- --filter=${{ inputs.package_name }}
51+
env:
52+
QUEUE_BACKEND: ${{ inputs.queue_backend }}

‎.github/workflows/ci.yml‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ jobs:
1212
runs-on: ubuntu-latest
1313
outputs:
1414
packages: ${{ steps.detect.outputs.packages }}
15+
aws_packages: ${{ steps.detect.outputs.aws_packages }}
1516
steps:
1617
- name: Get changed files
1718
id: changed-files
@@ -39,22 +40,45 @@ jobs:
3940
["packages/sqs"]="@message-queue-toolkit/sqs"
4041
)
4142
43+
AWS_PACKAGE_PATHS=("packages/sqs" "packages/sns")
44+
4245
PACKAGES=()
46+
AWS_PACKAGES=()
4347
for path in "${!PATH_TO_NAME[@]}"; do
4448
if echo "$ALL_CHANGED_FILES" | grep -q "$path/"; then
45-
PACKAGES+=("\"${PATH_TO_NAME[$path]}\"")
49+
is_aws=false
50+
for aws_path in "${AWS_PACKAGE_PATHS[@]}"; do
51+
if [ "$path" = "$aws_path" ]; then
52+
is_aws=true
53+
break
54+
fi
55+
done
56+
if [ "$is_aws" = true ]; then
57+
AWS_PACKAGES+=("\"${PATH_TO_NAME[$path]}\"")
58+
else
59+
PACKAGES+=("\"${PATH_TO_NAME[$path]}\"")
60+
fi
4661
fi
4762
done
4863
4964
if [ ${#PACKAGES[@]} -eq 0 ]; then
5065
echo 'packages=[]' >> $GITHUB_OUTPUT
51-
echo "No packages changed"
66+
echo "No non-AWS packages changed"
5267
else
5368
JSON="[$(IFS=,; echo "${PACKAGES[*]}")]"
5469
echo "packages=$JSON" >> $GITHUB_OUTPUT
5570
echo "Changed packages: $JSON"
5671
fi
5772
73+
if [ ${#AWS_PACKAGES[@]} -eq 0 ]; then
74+
echo 'aws_packages=[]' >> $GITHUB_OUTPUT
75+
echo "No AWS packages changed"
76+
else
77+
AWS_JSON="[$(IFS=,; echo "${AWS_PACKAGES[*]}")]"
78+
echo "aws_packages=$AWS_JSON" >> $GITHUB_OUTPUT
79+
echo "Changed AWS packages: $AWS_JSON"
80+
fi
81+
5882
general:
5983
needs: [changed-files-job]
6084
if: needs.changed-files-job.outputs.packages != '[]'
@@ -67,9 +91,23 @@ jobs:
6791
node_version: ${{ matrix.node-version }}
6892
package_name: ${{ matrix.package-name }}
6993

94+
aws-packages:
95+
needs: [changed-files-job]
96+
if: needs.changed-files-job.outputs.aws_packages != '[]'
97+
strategy:
98+
matrix:
99+
node-version: [22.x, 24.x]
100+
package-name: ${{ fromJson(needs.changed-files-job.outputs.aws_packages) }}
101+
queue-backend: [fauxqs, localstack]
102+
uses: ./.github/workflows/ci.common.yml
103+
with:
104+
node_version: ${{ matrix.node-version }}
105+
package_name: ${{ matrix.package-name }}
106+
queue_backend: ${{ matrix.queue-backend }}
107+
70108
automerge:
71-
needs: [general]
72-
if: always() && (needs.general.result == 'success' || needs.general.result == 'skipped')
109+
needs: [general, aws-packages]
110+
if: always() && (needs.general.result == 'success' || needs.general.result == 'skipped') && (needs.aws-packages.result == 'success' || needs.aws-packages.result == 'skipped')
73111
runs-on: ubuntu-latest
74112
permissions:
75113
pull-requests: write

‎docker-compose.yml‎

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,25 +9,6 @@ services:
99
- rabbit_data:/var/lib/rabbitmq
1010
restart: on-failure
1111

12-
localstack:
13-
image: localstack/localstack:4.13.1
14-
network_mode: bridge
15-
hostname: localstack
16-
ports:
17-
- '127.0.0.1:4566:4566' # LocalStack Gateway
18-
- '127.0.0.1:4510-4559:4510-4559' # external services port range
19-
environment:
20-
- SERVICES=sns,sqs,s3,sts
21-
- DEBUG=0
22-
- DATA_DIR=${DATA_DIR-}
23-
- DOCKER_HOST=unix:///var/run/docker.sock
24-
- LOCALSTACK_HOST=localstack
25-
# - LOCALSTACK_API_KEY=someDummyKey
26-
volumes:
27-
- '${TMPDIR:-/tmp}/localstack:/var/log/localstack'
28-
- '/var/run/docker.sock:/var/run/docker.sock'
29-
restart: on-failure
30-
3112
redis:
3213
image: redis:6.2.7-alpine
3314
command: redis-server --requirepass sOmE_sEcUrE_pAsS
@@ -95,6 +76,22 @@ services:
9576
retries: 10
9677
start_period: 10s
9778

79+
localstack:
80+
image: localstack/localstack:4.13.1
81+
network_mode: bridge
82+
hostname: localstack
83+
ports:
84+
- '127.0.0.1:4566:4566'
85+
- '127.0.0.1:4510-4559:4510-4559'
86+
environment:
87+
- SERVICES=sns,sqs,s3,sts
88+
- DEBUG=0
89+
- LOCALSTACK_HOST=localstack
90+
volumes:
91+
- '${TMPDIR:-/tmp}/localstack:/var/log/localstack'
92+
- '/var/run/docker.sock:/var/run/docker.sock'
93+
restart: on-failure
94+
9895
volumes:
9996
rabbit_data:
10097
driver: local

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
},
1616
"dependencies": {},
1717
"devDependencies": {
18-
"turbo": "^2.6.2"
18+
"turbo": "^2.8.10"
1919
},
2020
"packageManager": "npm@10.7.0"
2121
}

‎packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.payloadOffloading.spec.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { PUBSUB_MESSAGE_MAX_SIZE } from '../../lib/pubsub/AbstractPubSubService.
99
import { OFFLOADED_PAYLOAD_SIZE_ATTRIBUTE } from '../../lib/utils/messageUtils.ts'
1010
import { PubSubPermissionPublisher } from '../publishers/PubSubPermissionPublisher.ts'
1111
import { deletePubSubTopicAndSubscription } from '../utils/cleanupPubSub.ts'
12-
import { assertBucket, emptyBucket } from '../utils/gcsUtils.ts'
12+
import { assertBucket, emptyBuckets } from '../utils/gcsUtils.ts'
1313
import type { Dependencies } from '../utils/testContext.ts'
1414
import { registerDependencies } from '../utils/testContext.ts'
1515
import { PubSubPermissionConsumer } from './PubSubPermissionConsumer.ts'
@@ -75,7 +75,7 @@ describe('PubSubPermissionConsumer - Payload Offloading', () => {
7575
})
7676

7777
afterAll(async () => {
78-
await emptyBucket(gcsStorage, gcsBucketName)
78+
await emptyBuckets(gcsStorage, gcsBucketName)
7979

8080
const { awilixManager } = diContainer.cradle
8181
await awilixManager.executeDispose()
@@ -259,7 +259,7 @@ describe('PubSubPermissionConsumer - Payload Offloading', () => {
259259
})
260260

261261
afterAll(async () => {
262-
await emptyBucket(gcsStorage, gcsBucketName)
262+
await emptyBuckets(gcsStorage, gcsBucketName)
263263

264264
const { awilixManager } = diContainer.cradle
265265
await awilixManager.executeDispose()

‎packages/gcp-pubsub/test/utils/gcsUtils.ts‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,16 @@ export async function assertBucket(storage: Storage, bucketName: string) {
1111
return bucket
1212
}
1313

14-
export async function emptyBucket(storage: Storage, bucketName: string) {
15-
const bucket = storage.bucket(bucketName)
16-
const [exists] = await bucket.exists()
14+
export async function emptyBuckets(storage: Storage, ...bucketNames: string[]) {
15+
for (const bucketName of bucketNames) {
16+
const bucket = storage.bucket(bucketName)
17+
const [exists] = await bucket.exists()
1718

18-
if (!exists) {
19-
return
20-
}
19+
if (!exists) {
20+
continue
21+
}
2122

22-
const [files] = await bucket.getFiles()
23-
await Promise.all(files.map((file) => file.delete({ ignoreNotFound: true })))
23+
const [files] = await bucket.getFiles()
24+
await Promise.all(files.map((file) => file.delete({ ignoreNotFound: true })))
25+
}
2426
}

‎packages/s3-payload-store/package.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@
2727
"test:coverage": "npm run test -- --coverage",
2828
"lint": "biome check . && tsc",
2929
"lint:fix": "biome check --write .",
30-
"docker:start": "docker compose up -d --quiet-pull localstack",
31-
"docker:stop": "docker compose down",
3230
"prepublishOnly": "npm run lint && npm run build"
3331
},
3432
"dependencies": {},
@@ -43,8 +41,9 @@
4341
"@lokalise/tsconfig": "^3.0.0",
4442
"@types/node": "^25.0.2",
4543
"@vitest/coverage-v8": "^4.0.15",
44+
"fauxqs": "^1.9.3",
4645
"rimraf": "^6.0.1",
47-
"typescript": "^5.9.2",
46+
"typescript": "^5.9.3",
4847
"vitest": "^4.0.15"
4948
},
5049
"homepage": "https://github.com/kibertoad/message-queue-toolkit",

‎packages/s3-payload-store/test/store/S3PayloadStore.spec.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { beforeAll, beforeEach, describe, expect, it } from 'vitest'
66
import { resolvePayloadStoreConfig, S3PayloadStore } from '../../lib/S3PayloadStore.ts'
77
import { assertEmptyBucket, getObjectContent, objectExists } from '../utils/s3Utils.ts'
88
import { streamToString } from '../utils/streamUtils.ts'
9-
import { TEST_AWS_CONFIG } from '../utils/testS3Config.ts'
9+
import { getTestS3Config } from '../utils/testS3Config.ts'
1010

1111
const TEST_BUCKET = 'test-bucket'
1212

@@ -15,7 +15,7 @@ describe('S3PayloadStore', () => {
1515
let store: S3PayloadStore
1616

1717
beforeAll(() => {
18-
s3 = new S3(TEST_AWS_CONFIG)
18+
s3 = new S3(getTestS3Config())
1919
store = new S3PayloadStore({ s3 }, { bucketName: TEST_BUCKET })
2020
})
2121
beforeEach(async () => {

‎packages/s3-payload-store/test/utils/fauxqsInstance.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/** biome-ignore-all lint/suspicious/noConsole: test **/
2+
import { type FauxqsServer, startFauxqs } from 'fauxqs'
3+
4+
let server: FauxqsServer | undefined
5+
6+
export async function ensureFauxqsServer(): Promise<void> {
7+
if (server) return
8+
server = await startFauxqs({ port: 0, logger: false, host: 'localstack' })
9+
console.log(`[fauxqs] server started on port ${server.port}`)
10+
}
11+
12+
export function getFauxqsServer(): FauxqsServer | undefined {
13+
return server
14+
}
15+
16+
export function getFauxqsPort(): number {
17+
if (!server) throw new Error('fauxqs server not started — call ensureFauxqsServer() first')
18+
return server.port
19+
}

‎packages/s3-payload-store/test/utils/testS3Config.ts‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,17 @@
11
import type { S3ClientConfig } from '@aws-sdk/client-s3'
2+
import { createLocalhostHandler } from 'fauxqs'
23

3-
export const TEST_AWS_CONFIG: S3ClientConfig = {
4-
endpoint: 'http://s3.localhost.localstack.cloud:4566',
5-
region: 'eu-west-1',
6-
credentials: {
7-
accessKeyId: 'access',
8-
secretAccessKey: 'secret',
9-
},
4+
import { getFauxqsPort } from './fauxqsInstance.ts'
5+
6+
export function getTestS3Config(): S3ClientConfig {
7+
const port = getFauxqsPort()
8+
return {
9+
endpoint: `http://s3.localhost:${port}`,
10+
region: 'eu-west-1',
11+
credentials: {
12+
accessKeyId: 'access',
13+
secretAccessKey: 'secret',
14+
},
15+
requestHandler: createLocalhostHandler(),
16+
}
1017
}

0 commit comments

Comments
 (0)