Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .changeset/fix-ci-command-injection.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
---
---
14 changes: 10 additions & 4 deletions .github/actions/prepare-workbench-path/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,24 @@ runs:
steps:
- id: prepare
shell: bash
env:
APP_NAME: ${{ inputs.app-name }}
run: |
if [[ "${{ inputs.app-name }}" == "nextjs-turbopack" || "${{ inputs.app-name }}" == "nextjs-webpack" ]]; then
if [[ ! "$APP_NAME" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]]; then
echo "Invalid app-name: $APP_NAME" >&2
exit 1
fi
if [[ "$APP_NAME" == "nextjs-turbopack" || "$APP_NAME" == "nextjs-webpack" ]]; then
STAGE_LOG="$(mktemp)"
node scripts/stage-workbench-with-tarballs.mjs "workbench/${{ inputs.app-name }}" | tee "$STAGE_LOG"
node scripts/stage-workbench-with-tarballs.mjs "workbench/$APP_NAME" | tee "$STAGE_LOG"
WORKBENCH_APP_PATH="$(sed -n 's/^Staged workbench: //p' "$STAGE_LOG" | tail -n 1)"
if [ -z "$WORKBENCH_APP_PATH" ]; then
echo "Failed to parse staged workbench path from stage-workbench-with-tarballs output"
exit 1
fi
else
./scripts/resolve-symlinks.sh "workbench/${{ inputs.app-name }}"
WORKBENCH_APP_PATH="$GITHUB_WORKSPACE/workbench/${{ inputs.app-name }}"
./scripts/resolve-symlinks.sh "workbench/$APP_NAME"
WORKBENCH_APP_PATH="$GITHUB_WORKSPACE/workbench/$APP_NAME"
fi

echo "workbench_app_path=$WORKBENCH_APP_PATH" >> "$GITHUB_OUTPUT"
51 changes: 42 additions & 9 deletions .github/workflows/benchmark-community-world.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,37 +91,70 @@ jobs:
with:
build-packages: 'true'

- name: Validate inputs
env:
APP_NAME: ${{ inputs.app-name }}
WORLD_ID: ${{ inputs.world-id }}
WORLD_PACKAGE: ${{ inputs.world-package }}
run: |
if [[ ! "$APP_NAME" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]]; then
echo "Invalid app-name: $APP_NAME" >&2
exit 1
fi
if [[ ! "$WORLD_ID" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]]; then
echo "Invalid world-id: $WORLD_ID" >&2
exit 1
fi
if [[ ! "$WORLD_PACKAGE" =~ ^(@[a-zA-Z0-9][a-zA-Z0-9_.-]*/)?[a-zA-Z0-9][a-zA-Z0-9_.-]*(@[a-zA-Z0-9][a-zA-Z0-9_.+-]*)?$ ]]; then
echo "Invalid world-package: $WORLD_PACKAGE" >&2
exit 1
fi

- name: Install ${{ inputs.world-name }} World
run: pnpm --filter ${{ inputs.app-name }} add ${{ inputs.world-package }}
env:
APP_NAME: ${{ inputs.app-name }}
WORLD_PACKAGE: ${{ inputs.world-package }}
run: pnpm --filter "$APP_NAME" add "$WORLD_PACKAGE"

- name: Run setup command
if: ${{ inputs.setup-command != '' }}
run: ${{ inputs.setup-command }}
working-directory: workbench/${{ inputs.app-name }}
env:
APP_NAME: ${{ inputs.app-name }}
SETUP_COMMAND: ${{ inputs.setup-command }}
run: |
cd "workbench/$APP_NAME"
eval "$SETUP_COMMAND"

- name: Resolve symlinks
run: ./scripts/resolve-symlinks.sh workbench/${{ inputs.app-name }}
env:
APP_NAME: ${{ inputs.app-name }}
run: ./scripts/resolve-symlinks.sh "workbench/$APP_NAME"

- name: Set environment variables
env:
ENV_VARS_JSON: ${{ inputs.env-vars }}
run: |
echo '${{ inputs.env-vars }}' | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> $GITHUB_ENV
printf '%s' "$ENV_VARS_JSON" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV"

- name: Build workbench
run: pnpm turbo run build --filter='./workbench/${{ inputs.app-name }}'
env:
APP_NAME: ${{ inputs.app-name }}
run: pnpm turbo run build --filter="./workbench/$APP_NAME"

- name: Run benchmarks
env:
DEPLOYMENT_URL: "http://localhost:3000"
APP_NAME: ${{ inputs.app-name }}
WORLD_ID: ${{ inputs.world-id }}
WORKFLOW_BENCH_BACKEND: ${{ inputs.world-id }}
BENCHMARK_FULL_SUITE: ${{ inputs.full-suite }}
run: |
cd workbench/${{ inputs.app-name }}
cd "workbench/$APP_NAME"
pnpm start &
echo "Waiting for server to start..."
sleep 15
cd ../..
pnpm vitest bench packages/core/e2e/bench.bench.ts --run --outputJson=bench-results-${{ inputs.app-name }}-${{ inputs.world-id }}.json
cd "$GITHUB_WORKSPACE"
pnpm vitest bench packages/core/e2e/bench.bench.ts --run "--outputJson=bench-results-$APP_NAME-$WORLD_ID.json"

- name: Render benchmark results
uses: ./.github/actions/render-benchmarks
Expand Down
26 changes: 16 additions & 10 deletions .github/workflows/benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,9 @@ jobs:
if_no_artifact_found: warn

- name: Build workbench
run: pnpm turbo run build --filter='./workbench/${{ matrix.app }}'
env:
APP_NAME: ${{ matrix.app }}
run: pnpm turbo run build --filter="./workbench/$APP_NAME"

- name: Run benchmarks
env:
Expand All @@ -203,12 +205,12 @@ jobs:
# Run full suite on nextjs-turbopack only (stress tests are too slow to run on all apps)
BENCHMARK_FULL_SUITE: ${{ matrix.app == 'nextjs-turbopack' && ((github.event_name == 'workflow_dispatch' && inputs.full_suite) || contains(github.event.pull_request.labels.*.name, 'stress-test')) }}
run: |
cd workbench/${{ matrix.app }}
cd "workbench/$APP_NAME"
pnpm start &
echo "Waiting for server to start..."
sleep 15
cd ../..
pnpm vitest bench packages/core/e2e/bench.bench.ts --run --outputJson=bench-results-${{ matrix.app }}-local.json
cd "$GITHUB_WORKSPACE"
pnpm vitest bench packages/core/e2e/bench.bench.ts --run "--outputJson=bench-results-$APP_NAME-local.json"

- name: Render benchmark results
uses: ./.github/actions/render-benchmarks
Expand Down Expand Up @@ -296,7 +298,9 @@ jobs:

# Build workbench with postgres world (build output differs based on target world)
- name: Build workbench for postgres
run: pnpm turbo run build --filter='./workbench/${{ matrix.app }}'
env:
APP_NAME: ${{ matrix.app }}
run: pnpm turbo run build --filter="./workbench/$APP_NAME"

- name: Run benchmarks
env:
Expand All @@ -305,12 +309,12 @@ jobs:
# Run full suite on nextjs-turbopack only (stress tests are too slow to run on all apps)
BENCHMARK_FULL_SUITE: ${{ matrix.app == 'nextjs-turbopack' && ((github.event_name == 'workflow_dispatch' && inputs.full_suite) || contains(github.event.pull_request.labels.*.name, 'stress-test')) }}
run: |
cd workbench/${{ matrix.app }}
cd "workbench/$APP_NAME"
pnpm start &
echo "Waiting for server to start..."
sleep 15
cd ../..
pnpm vitest bench packages/core/e2e/bench.bench.ts --run --outputJson=bench-results-${{ matrix.app }}-postgres.json
cd "$GITHUB_WORKSPACE"
pnpm vitest bench packages/core/e2e/bench.bench.ts --run "--outputJson=bench-results-$APP_NAME-postgres.json"

- name: Render benchmark results
uses: ./.github/actions/render-benchmarks
Expand Down Expand Up @@ -404,7 +408,7 @@ jobs:
# Run full suite on nextjs-turbopack only (stress tests are too slow to run on all apps)
BENCHMARK_FULL_SUITE: ${{ matrix.app.name == 'nextjs-turbopack' && ((github.event_name == 'workflow_dispatch' && inputs.full_suite) || contains(github.event.pull_request.labels.*.name, 'stress-test')) }}
run: |
pnpm vitest bench packages/core/e2e/bench.bench.ts --run --outputJson=bench-results-${{ matrix.app.name }}-vercel.json
pnpm vitest bench packages/core/e2e/bench.bench.ts --run "--outputJson=bench-results-$APP_NAME-vercel.json"

- name: Render benchmark results
uses: ./.github/actions/render-benchmarks
Expand Down Expand Up @@ -439,7 +443,9 @@ jobs:
build-packages: 'false'

- id: set-matrix
run: echo "matrix=$(node ./scripts/create-community-worlds-matrix.mjs)" >> $GITHUB_OUTPUT
run: |
MATRIX_JSON="$(node ./scripts/create-community-worlds-matrix.mjs | jq -c .)"
echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT"

benchmark-community:
name: Benchmark Community World (${{ matrix.world.name }})
Expand Down
61 changes: 48 additions & 13 deletions .github/workflows/e2e-community-world.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,37 +87,72 @@ jobs:
with:
build-packages: 'true'

- name: Validate inputs
env:
APP_NAME: ${{ inputs.app-name }}
WORLD_ID: ${{ inputs.world-id }}
WORLD_PACKAGE: ${{ inputs.world-package }}
run: |
if [[ ! "$APP_NAME" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]]; then
echo "Invalid app-name: $APP_NAME" >&2
exit 1
fi
if [[ ! "$WORLD_ID" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]]; then
echo "Invalid world-id: $WORLD_ID" >&2
exit 1
fi
if [[ ! "$WORLD_PACKAGE" =~ ^(@[a-zA-Z0-9][a-zA-Z0-9_.-]*/)?[a-zA-Z0-9][a-zA-Z0-9_.-]*(@[a-zA-Z0-9][a-zA-Z0-9_.+-]*)?$ ]]; then
echo "Invalid world-package: $WORLD_PACKAGE" >&2
exit 1
fi

- name: Install ${{ inputs.world-name }} World
run: pnpm --filter ${{ inputs.app-name }} add ${{ inputs.world-package }}
env:
APP_NAME: ${{ inputs.app-name }}
WORLD_PACKAGE: ${{ inputs.world-package }}
run: pnpm --filter "$APP_NAME" add "$WORLD_PACKAGE"

- name: Run setup command
if: ${{ inputs.setup-command != '' }}
run: ${{ inputs.setup-command }}
working-directory: workbench/${{ inputs.app-name }}
env:
APP_NAME: ${{ inputs.app-name }}
SETUP_COMMAND: ${{ inputs.setup-command }}
run: |
cd "workbench/$APP_NAME"
eval "$SETUP_COMMAND"

- name: Resolve symlinks
run: ./scripts/resolve-symlinks.sh workbench/${{ inputs.app-name }}
env:
APP_NAME: ${{ inputs.app-name }}
run: ./scripts/resolve-symlinks.sh "workbench/$APP_NAME"

- name: Set environment variables
env:
ENV_VARS_JSON: ${{ inputs.env-vars }}
run: |
echo '${{ inputs.env-vars }}' | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> $GITHUB_ENV
printf '%s' "$ENV_VARS_JSON" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV"

- name: Run E2E Tests
run: |
cd workbench/${{ inputs.app-name }} && pnpm dev &
echo "Waiting for dev server to start..." && sleep 15
pnpm vitest run packages/core/e2e/dev.test.ts --reporter=default --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-community-${{ inputs.world-id }}-dev.json || true
sleep 10
pnpm vitest run packages/core/e2e/e2e.test.ts --reporter=default --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-community-${{ inputs.world-id }}.json || true
env:
NODE_OPTIONS: "--enable-source-maps"
APP_NAME: ${{ inputs.app-name }}
WORLD_ID: ${{ inputs.world-id }}
DEPLOYMENT_URL: "http://localhost:3000"
DEV_TEST_CONFIG: '{"name":"${{ inputs.app-name }}","project":"workbench-${{ inputs.app-name }}-workflow","generatedStepPath":"app/.well-known/workflow/v1/step/route.js","generatedWorkflowPath":"app/.well-known/workflow/v1/flow/route.js","apiFilePath":"app/api/chat/route.ts","apiFileImportPath":"../../.."}'
run: |
DEV_TEST_CONFIG="$(jq -nc --arg name "$APP_NAME" '{name:$name,project:"workbench-\($name)-workflow",generatedStepPath:"app/.well-known/workflow/v1/step/route.js",generatedWorkflowPath:"app/.well-known/workflow/v1/flow/route.js",apiFilePath:"app/api/chat/route.ts",apiFileImportPath:"../../.."}')"
export DEV_TEST_CONFIG
cd "workbench/$APP_NAME" && pnpm dev &
cd "$GITHUB_WORKSPACE"
echo "Waiting for dev server to start..." && sleep 15
pnpm vitest run packages/core/e2e/dev.test.ts --reporter=default --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-community-$WORLD_ID-dev.json" || true
sleep 10
pnpm vitest run packages/core/e2e/e2e.test.ts --reporter=default --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-community-$WORLD_ID.json" || true

- name: Generate E2E summary
if: always()
run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Tests (${{ inputs.world-name }})" >> $GITHUB_STEP_SUMMARY || true
env:
WORLD_NAME: ${{ inputs.world-name }}
run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Tests ($WORLD_NAME)" >> $GITHUB_STEP_SUMMARY || true

- name: Upload E2E results
uses: actions/upload-artifact@v4
Expand Down
Loading
Loading