Merge pull request #6996 from Shopify/delete-cucumber

Remove Cucumber test infrastructure

Author: ryancbahan

.github/workflows/tests-main.yml

@@ -59,12 +59,6 @@ jobs:
         env:
           VITEST_MIN_THREADS: "1"
           VITEST_MAX_THREADS: "4"
-      - name: Acceptance tests
-        if: ${{ matrix.node == '24.1.0' }}
-        env:
-          SHOPIFY_CLI_PARTNERS_TOKEN: ${{ secrets.SHOPIFY_CLI_PARTNERS_TOKEN }}
-          SHOPIFY_FLAG_CLIENT_ID: ${{ secrets.SHOPIFY_FLAG_CLIENT_ID }}
-        run: pnpm nx run features:test
       - name: Send Slack notification on failure
         uses: slackapi/slack-github-action@007b2c3c751a190b6f0f040e47ed024deaa72844 # v1.23.0
         if: ${{ failure() && !cancelled() }}

.github/workflows/tests-manual.yml

@@ -60,10 +60,6 @@ jobs:
           node-version: ${{ inputs.node-version }}
       - name: Unit tests
         run: pnpm test:unit --output-style=stream
-      - name: Acceptance tests
-        env:
-          SHOPIFY_CLI_PARTNERS_TOKEN: ${{ secrets.SHOPIFY_CLI_PARTNERS_TOKEN }}
-        run: pnpm test:features --output-style=stream
       - name: Setup tmate session
         if: ${{ failure() && inputs.debug-enabled }}
         uses: mxschmitt/action-tmate@c0afd6f790e3a5564914980036ebf83216678101 # v3

.github/workflows/tests-pr.yml

@@ -170,30 +170,6 @@ jobs:
           VITEST_MIN_THREADS: "1"
           VITEST_MAX_THREADS: "4"
 
-  acceptance-tests:
-    name: 'Acceptance tests with Node ${{ matrix.node }} in ${{ matrix.os }}'
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ]
-        node: [ '24.1.0' ]
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: ${{ github.event.pull_request.head.repo.full_name || github.event.repository.full_name }}
-          ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }}
-          fetch-depth: 1
-      - name: Setup deps
-        uses: ./.github/actions/setup-cli-deps
-        with:
-          node-version: ${{ matrix.node }}
-      - name: Acceptance tests
-        env:
-          SHOPIFY_CLI_PARTNERS_TOKEN: ${{ secrets.SHOPIFY_CLI_PARTNERS_TOKEN }}
-          SHOPIFY_FLAG_CLIENT_ID: ${{ secrets.SHOPIFY_FLAG_CLIENT_ID }}
-        run: pnpm test:features --output-style=stream
-
   test-coverage:
     if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }}
     name: 'Test Coverage'
@@ -282,7 +258,7 @@ jobs:
         with:
           node-version: ${{ env.DEFAULT_NODE_VERSION }}
       - name: Build
-        run: pnpm nx run-many --all --skip-nx-cache --target=build --exclude=features --output-style=stream
+        run: pnpm nx run-many --all --skip-nx-cache --target=build --output-style=stream
       - name: Type-diff
         working-directory: workspace
         id: type-diff

docs/cli/testing-strategy.md

@@ -30,7 +30,7 @@ test("loads the app", async () => {
 })
 ```
 
-Tests can be run with `pnpm test` or `pnpm test:watch` for the entire workspace or individual packages. `test:watch` keeps the process alive and runs tests as files are changed. If you want to run a single test, pass the path to the file as argument:
+Tests can be run with `pnpm test` for the Vitest suite, `pnpm test:watch` for watch mode, or `pnpm test:e2e` for the Playwright end-to-end suite. If you want to run a single unit test, pass the path to the file as argument:
 
 ```
 pnpm test path/to/my.test.ts
@@ -73,32 +73,9 @@ test("writes", async () => {
 - [Vitest API](https://vitest.dev/api/)
 - [Examples](https://vitest.dev/guide/#examples)
 
-## Acceptance tests 🥒
+## E2E tests
 
-[Acceptance tests](https://en.wikipedia.org/wiki/Acceptance_testing) play an essential role in ensuring all the small pieces fit together to create user experiences.
-They are defined as user stories which are a set of steps a user would take and the expectations they'd have while navigating through those steps.
-
-```feature
-Scenario: I create a new app with pnpm
-  Given I have a working directory
-  When I create an app named MyApp with pnpm as dependency manager
-  Then I have an app named MyApp with pnpm as dependency manager
-```
-Acceptance tests live under `packages/features` and implemented using [Cucumber](https://cucumber.io/). We create a working directory for every test that isolates the test from the rest. Moreover, the CLIs are invoked, configuring them to store global states in those temporary directories. That way, we prevent the global state from leaking into other tests and making them fail.
-
-Each `.feature` file under `features/` group of scenarios has something in common (e.g. all scenarios related to app development). Steps are declared in Typescript files under `steps`. All the files in that directory or sub-directories are automatically loaded. We recommend keeping parity between those files and the features to quickly locate the steps.
-
-### How to implement an acceptance test
-
-1. Describe the scenario in a `.feature` file. Create a new one if you can't find a feature file your scenario fits into.
-2. Implement the steps of your scenario.
-3. Run the scenario with `FEATURE=path/to/scenario.feature:line yarn test`.
-
-**Note** that we don't need to test all the user scenarios. Unlike unit tests, acceptance tests are slow. Focus on the user journeys that are most common and prefer larger but fewer scenarios over smaller but more.
-
-> :bulb: Try to make them as generic as possible by using regular expressions when defining steps. That way, your steps can easily be reused by other scenarios.
-
-> :bulb: If your scenario relies on a global state, for example, storing a file in the user's environment, adjust the implementation to control the state's location from the acceptance tests. This is extremely important to prevent flakiness.
+End-to-end tests live under `packages/e2e` and are implemented using [Playwright](https://playwright.dev/). They test full user journeys by invoking the CLI and verifying outputs. Run them with `pnpm test:e2e`.
 
 ## Github Actions
 Before being able to marge a PR, it must pass all CI checks executed in Github Actions.

package.json

@@ -29,10 +29,8 @@
     "shopify:run": "node packages/cli/bin/dev.js",
     "shopify": "nx build cli && node packages/cli/bin/dev.js",
     "test:e2e": "nx run-many --target=build --projects=cli,create-app --skip-nx-cache && pnpm --filter e2e exec playwright test",
-    "test:features": "pnpm nx run features:test",
-    "test:regenerate-snapshots": "nx build cli && packages/features/snapshots/regenerate.sh",
     "test:unit": "pnpm vitest run",
-    "test": "pnpm vitest run && pnpm nx run features:test",
+    "test": "pnpm vitest run",
     "type-check:affected": "nx affected --target=type-check",
     "type-check": "nx run-many --target=type-check --all --skip-nx-cache",
     "update-bugsnag": "bin/update-bugsnag.js"
@@ -173,7 +171,6 @@
           "graphql.config.ts"
         ],
         "ignoreBinaries": [
-          "packages/features/snapshots/regenerate.sh",
           "bin/*",
           "shopify",
           "shopify-nightly",
@@ -318,15 +315,6 @@
           "**/src/index.ts"
         ],
         "project": "**/*.{ts,tsx}"
-      },
-      "packages/features": {
-        "entry": [
-          "**/cucumber.js",
-          "**/world/index.ts",
-          "**/steps/*.ts",
-          "**/lib/*.ts"
-        ],
-        "project": "**/*.ts"
       }
     }
   }

packages/eslint-plugin-cli/config.js

@@ -297,14 +297,6 @@ const config = [
     rules: testFileRules,
   },
 
-  // Features package - allow console for test utilities
-  {
-    files: ['**/packages/features/**/*.ts'],
-    rules: {
-      'no-console': 'off',
-    },
-  },
-
   // JS bin files - different rules apply
   {
     files: ['**/bin/*.js'],