Upgrade to Allure v3 (#1603)

* Start of Allure v3

* CI setup for new Allure v3

* correct actions

* lint

* fix test naming

* Update test snapshots

* Update knip config, and remove legacy allure command line

* lint again

* remove now unused action

* fix annoying spacing

* cleanup testing definitions to ignore packages without tests

Author: Adammatthiesen

.codecov.yml

@@ -164,10 +164,6 @@ component_management:
       name: "@withstudiocms/kysely"
       paths:
         - packages/@withstudiocms/kysely/**
-    - component_id: withstudiocms_sdk
-      name: "@withstudiocms/sdk"
-      paths:
-        - packages/@withstudiocms/sdk/**
     - component_id: withstudiocms_cli_kit
       name: "@withstudiocms/cli-kit"
       paths:

.devcontainer/devcontainer.json

@@ -17,7 +17,6 @@
     },
     "vscode": {
       "settings": {
-        "js/ts.tsdk.path": "node_modules/typescript/lib",
         "editor.codeActionsOnSave": {
           "source.fixAll.biome": "explicit"
         },

.github/actions/allure-report/action.yml

@@ -71,6 +71,13 @@ jobs:
         with:
           persist-credentials: true
 
+      - name: Checkout Allure Test history
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: allure/history
+          path: test-history
+          persist-credentials: false
+
       - name: Install Tools & Dependencies
         uses: withstudiocms/automations/.github/actions/install@f32588d80ce35d57905264f7fa473b0f6e297c0a # main
 
@@ -80,17 +87,31 @@ jobs:
       - name: Run Tests & Code Coverage
         run: pnpm ci:test
 
-      # Allure Reports Require to be ran from within our repository only
-      # So we check if the PR is from a fork or from the same repository
-      - name: Generate Allure Report and Deploy
-        if: always() && (github.event.pull_request.head.repo.full_name == github.repository || github.ref == 'refs/heads/main')
-        uses: ./.github/actions/allure-report
+      # Only run on PRs
+      - name: Report Allure results
+        if: github.event_name == 'pull_request'
+        uses: allure-framework/allure-action@bd0373251ed3f1c5ac9aa44ac86ed40d839b3094 # v0.8.0
+        with:
+          report-directory: "./allure-report"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # Only upload Allure report results to Github if we are a push to main
+      - name: Upload Allure Report to Github Branch
+        if: github.event_name == 'push'
+        uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: allure/report
+          publish_dir: ./allure-report
+
+      # Only upload Allure report results to Github if we are a push to main
+      - name: Upload Allure History to Github Branch
+        if: github.event_name == 'push'
+        uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0
         with:
-          gh_pages_branch: testing/allure
-          gh_page_history: testing/allure
-          allure_report_id: studiocms-tests
-          allure_report_dir: allure-results
-          cname: tests.studiocms.dev
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: allure/history
+          publish_dir: ./test-history
 
       - name: Upload coverage report to Codecov
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0

.github/workflows/ci-testing.yml

@@ -18,7 +18,8 @@ junit-report.xml
 dist
 
 # allure results
-allure-results/*
 allure-results
+allure-report
+history.jsonl
 
-.env
+.env

.gitignore

@@ -1,5 +1,4 @@
 {
-  "js/ts.tsdk.path": "node_modules/typescript/lib",
   "editor.fontFamily": "Fira Code Retina, Menlo, Monaco, 'Courier New', monospace",
   "editor.fontLigatures": true,
   "editor.fontSize": 14.2,

.vscode/settings.json

@@ -0,0 +1,75 @@
+import { defineConfig } from 'allure';
+
+const packagesWithTests = [
+	'studiocms',
+	'effectify',
+
+	'@studiocms/auth0',
+	'@studiocms/discord',
+	'@studiocms/github',
+	'@studiocms/google',
+	'@studiocms/devapps',
+	'@studiocms/html',
+	'@studiocms/cloudinary-image-service',
+	'@studiocms/md',
+	'@studiocms/mdx',
+	'@studiocms/blog',
+	'@studiocms/markdoc',
+	'@studiocms/markdown-remark',
+	'@studiocms/wysiwyg',
+
+	'@withstudiocms/auth-kit',
+	'@withstudiocms/buildkit',
+	'@withstudiocms/cli-kit',
+	'@withstudiocms/component-registry',
+	'@withstudiocms/config-utils',
+	'@withstudiocms/effect',
+	'@withstudiocms/internal_helpers',
+	'@withstudiocms/template-lang',
+	'@withstudiocms/kysely',
+];
+
+// Create a plugins configuration object for each package with tests, using the @allurereport/plugin-awesome plugin. The report will be named `${pkg} Tests`, and will be published to the Allure server.
+const pluginsConfig = Object.fromEntries(
+	packagesWithTests.map((pkg) => [
+		pkg,
+		{
+			import: '@allurereport/plugin-awesome',
+			options: {
+				reportName: `${pkg} Tests`,
+				singleFile: false,
+				reportLanguage: 'en',
+				open: false,
+				publish: true,
+				groupBy: ['parentSuite', 'suite', 'subSuite'],
+				filter: ({ labels }) =>
+					labels.find(({ name, value }) => name === 'parentSuite' && value === `${pkg} Tests`),
+			},
+		},
+	])
+);
+
+// Export the Allure configuration object, which includes the plugins configuration for each package with tests, as well as some general settings for the Allure report.
+export default defineConfig({
+	name: 'Allure Report',
+	output: './allure-report',
+	historyPath: './test-history/history.jsonl',
+	appendHistory: true,
+	historyLimit: 20,
+	qualityGate: {
+		rules: [
+			{
+				maxFailures: 0,
+				fastFail: true,
+			},
+		],
+	},
+	plugins: {
+		...pluginsConfig,
+		log: {
+			options: {
+				groupBy: 'none',
+			},
+		},
+	},
+});

allurerc.mjs

@@ -257,6 +257,7 @@
     "YYYYMMDDTHHMMSS",
     "zipx",
     "zizmor",
-    "zizmorcore"
+    "zizmorcore",
+    "allurereport"
   ]
 }

cspell.json

@@ -181,7 +181,13 @@ const config: KnipConfig = {
 	exclude: ['duplicates', 'optionalPeerDependencies'],
 	workspaces: {
 		'.': {
-			ignoreDependencies: ['@changesets/config', 'studiocms', 'vite', 'allure-js-commons'],
+			ignoreDependencies: [
+				'@changesets/config',
+				'studiocms',
+				'vite',
+				'allure-js-commons',
+				'@allurereport/plugin-awesome',
+			],
 			ignoreBinaries: [/ci:.*$/, /bundle-test:.*$/, 'du'],
 			entry: ['.github/workflows/*.yml', '.github/scripts/**/*.mjs', 'scripts/**/*.mjs'],
 			project: ['.github/scripts/**/*.mjs', 'scripts/**/*.mjs'],

knip.config.ts

@@ -66,9 +66,13 @@
     "ci:version": "pnpm changeset version && pnpm ci:update-templates && pnpm install --no-frozen-lockfile && pnpm run lint:fix",
     "ci:publish": "pnpm changeset publish",
     "ci:snapshot": "pnpx pkg-pr-new publish --pnpm './packages/*'",
-    "ci:test": "vitest run --coverage",
+
+    "ci:vitest": "vitest run --coverage",
+    "ci:test": "allure run -- pnpm ci:vitest",
     "ci:build": "pnpm build:packages",
 
+    "allure:view": "allure open allure-report",
+
     "ci:typecheck:studiocms": "pnpm --filter studiocms typecheck",
     "ci:typecheck:studiocms-packages": "pnpm --filter @studiocms/* --filter create-studiocms typecheck",
     "ci:typecheck:withstudiocms": "pnpm --filter @withstudiocms/* --filter effectify typecheck",
@@ -83,6 +87,7 @@
   },
   "devDependencies": {
     "@actions/core": "^3.0.1",
+    "@allurereport/plugin-awesome": "^3.14.2",
     "@biomejs/biome": "^2.5.3",
     "@changesets/changelog-github": "^0.7.0",
     "@changesets/cli": "^2.31.0",
@@ -95,9 +100,9 @@
     "@types/node": "catalog:",
     "@vitest/coverage-v8": "^3.2.4",
     "@withstudiocms/buildkit": "workspace:^",
+    "allure": "^3.14.2",
     "allure-js-commons": "^3.10.2",
     "allure-vitest": "^3.10.2",
-    "allure-commandline": "^2.43.0",
     "execa": "^9.6.1",
     "knip": "^6.26.0",
     "pkg-pr-new": "^0.0.75",