feat(plugin-e2e): fetch e2e-selectors from Grafana at runtime behind a flag

Behind PLUGIN_E2E_RUNTIME_SELECTORS, the selectors fixture fetches /public/e2e-selectors.js from the Grafana under test and resolves from it, falling back to the bundled @grafana/e2e-selectors (quiet on 404, warns on other failures). Off by default; enabled in CI to trial it.

Author: sunker

.github/workflows/playwright-nightly.yml

@@ -37,6 +37,10 @@ jobs:
   playwright-tests:
     needs: resolve-versions
     timeout-minutes: 60
+    # trial: fetch e2e-selectors from the running Grafana at runtime instead of the bundled dep.
+    # grafana-dev serves the file (exercises the fetch); latest stable 404s (exercises the fallback)
+    env:
+      PLUGIN_E2E_RUNTIME_SELECTORS: 'true'
     strategy:
       fail-fast: false
       matrix:

.github/workflows/playwright.yml

@@ -33,6 +33,9 @@ jobs:
   playwright-tests:
     needs: resolve-versions
     timeout-minutes: 60
+    # trial: fetch e2e-selectors from the running Grafana at runtime instead of the bundled dep
+    env:
+      PLUGIN_E2E_RUNTIME_SELECTORS: 'true'
     strategy:
       fail-fast: false
       matrix:

packages/plugin-e2e/src/fixtures/selectors.test.ts

@@ -0,0 +1,130 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// identity resolveSelectors so we can assert which versioned object flowed through;
+// tagged fallback data so we can tell the bundled dep apart from the fetched bundle
+vi.mock('@grafana/e2e-selectors', () => ({
+  resolveSelectors: vi.fn((versioned: unknown) => versioned),
+  versionedComponents: { __source: 'dep-components' },
+  versionedPages: { __source: 'dep-pages' },
+}));
+vi.mock('../selectors/versionedConstants', () => ({ versionedConstants: { __source: 'local-constants' } }));
+vi.mock('../selectors/versionedAPIs', () => ({ versionedAPIs: { __source: 'local-apis' } }));
+
+import { selectors } from './selectors';
+
+const VALID_BUNDLE = `
+  exports.versionedComponents = { __source: 'fetched-components' };
+  exports.versionedPages = { __source: 'fetched-pages' };
+`;
+
+// a fake APIRequestContext.get() result
+function mockResponse({ status = 200, body = '' }: { status?: number; body?: string }) {
+  return { status: () => status, ok: () => status >= 200 && status < 300, text: async () => body };
+}
+
+function mockRequest(get: ReturnType<typeof vi.fn>) {
+  // only `get` is exercised by the fixture
+  return { get } as never;
+}
+
+// drives the fixture and returns whatever it passes to `use`
+async function runFixture(args: { grafanaVersion: string; request: never }) {
+  let captured: unknown;
+  await (selectors as unknown as (args: unknown, use: (value: unknown) => Promise<void>) => Promise<void>)(
+    args,
+    async (value: unknown) => {
+      captured = value;
+    }
+  );
+  return captured as Record<string, unknown>;
+}
+
+describe('selectors fixture', () => {
+  const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+  beforeEach(() => {
+    warnSpy.mockClear();
+  });
+
+  afterEach(() => {
+    delete process.env.PLUGIN_E2E_RUNTIME_SELECTORS;
+  });
+
+  it('flag off: uses the bundled dependency and never fetches', async () => {
+    const get = vi.fn();
+    const result = await runFixture({ grafanaVersion: '11.0.0-off', request: mockRequest(get) });
+
+    expect(get).not.toHaveBeenCalled();
+    expect(result.components).toEqual({ __source: 'dep-components' });
+    expect(result.constants).toEqual({ __source: 'local-constants' });
+  });
+
+  it('flag on + 200: uses fetched components/pages and local constants/apis', async () => {
+    process.env.PLUGIN_E2E_RUNTIME_SELECTORS = 'true';
+    const get = vi.fn().mockResolvedValue(mockResponse({ status: 200, body: VALID_BUNDLE }));
+
+    const result = await runFixture({ grafanaVersion: '11.0.0-200', request: mockRequest(get) });
+
+    expect(get).toHaveBeenCalledWith('/public/e2e-selectors.js');
+    expect(result.components).toEqual({ __source: 'fetched-components' });
+    expect(result.pages).toEqual({ __source: 'fetched-pages' });
+    expect(result.constants).toEqual({ __source: 'local-constants' });
+    expect(result.apis).toEqual({ __source: 'local-apis' });
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('flag on + 404: falls back to the dependency quietly', async () => {
+    process.env.PLUGIN_E2E_RUNTIME_SELECTORS = 'true';
+    const get = vi.fn().mockResolvedValue(mockResponse({ status: 404 }));
+
+    const result = await runFixture({ grafanaVersion: '11.0.0-404', request: mockRequest(get) });
+
+    expect(result.components).toEqual({ __source: 'dep-components' });
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('flag on + 5xx: falls back to the dependency with a warning', async () => {
+    process.env.PLUGIN_E2E_RUNTIME_SELECTORS = 'true';
+    const get = vi.fn().mockResolvedValue(mockResponse({ status: 503 }));
+
+    const result = await runFixture({ grafanaVersion: '11.0.0-503', request: mockRequest(get) });
+
+    expect(result.components).toEqual({ __source: 'dep-components' });
+    expect(warnSpy).toHaveBeenCalled();
+  });
+
+  it('flag on + network error: falls back to the dependency with a warning', async () => {
+    process.env.PLUGIN_E2E_RUNTIME_SELECTORS = 'true';
+    const get = vi.fn().mockRejectedValue(new Error('ECONNREFUSED'));
+
+    const result = await runFixture({ grafanaVersion: '11.0.0-net', request: mockRequest(get) });
+
+    expect(result.components).toEqual({ __source: 'dep-components' });
+    expect(warnSpy).toHaveBeenCalled();
+  });
+
+  it('flag on + malformed bundle (missing exports): falls back to the dependency with a warning', async () => {
+    process.env.PLUGIN_E2E_RUNTIME_SELECTORS = 'true';
+    const get = vi.fn().mockResolvedValue(mockResponse({ status: 200, body: 'exports.nope = 1;' }));
+
+    const result = await runFixture({ grafanaVersion: '11.0.0-bad', request: mockRequest(get) });
+
+    expect(result.components).toEqual({ __source: 'dep-components' });
+    expect(warnSpy).toHaveBeenCalled();
+  });
+
+  it('flag on: concurrent fixtures for the same version share a single fetch', async () => {
+    process.env.PLUGIN_E2E_RUNTIME_SELECTORS = 'true';
+    const get = vi.fn().mockResolvedValue(mockResponse({ status: 200, body: VALID_BUNDLE }));
+    const request = mockRequest(get);
+
+    const [a, b] = await Promise.all([
+      runFixture({ grafanaVersion: '11.0.0-cache', request }),
+      runFixture({ grafanaVersion: '11.0.0-cache', request }),
+    ]);
+
+    expect(get).toHaveBeenCalledTimes(1);
+    expect(a.components).toEqual({ __source: 'fetched-components' });
+    expect(b.components).toEqual({ __source: 'fetched-components' });
+  });
+});

packages/plugin-e2e/src/fixtures/selectors.ts

@@ -1,16 +1,110 @@
-import { TestFixture } from '@playwright/test';
+import { APIRequestContext, TestFixture } from '@playwright/test';
+import {
+  resolveSelectors,
+  versionedComponents as bundledVersionedComponents,
+  versionedPages as bundledVersionedPages,
+} from '@grafana/e2e-selectors';
 import { E2ESelectorGroups, PlaywrightArgs } from '../types';
-import { resolveSelectors, versionedComponents, versionedPages } from '@grafana/e2e-selectors';
 import { versionedConstants } from '../selectors/versionedConstants';
 import { versionedAPIs } from '../selectors/versionedAPIs';
 
 type SelectorFixture = TestFixture<E2ESelectorGroups, PlaywrightArgs>;
 
-export const selectors: SelectorFixture = async ({ grafanaVersion }, use) => {
-  await use({
-    components: resolveSelectors(versionedComponents, grafanaVersion),
-    pages: resolveSelectors(versionedPages, grafanaVersion),
+// served by grafana/grafana from the @grafana/e2e-selectors build (see PLUGIN_E2E_RUNTIME_SELECTORS)
+const SELECTORS_URL = '/public/e2e-selectors.js';
+
+// only the components/pages data comes from Grafana; constants/apis are plugin-e2e's own
+type RuntimeSelectors = {
+  versionedComponents: typeof bundledVersionedComponents;
+  versionedPages: typeof bundledVersionedPages;
+};
+
+// per-worker cache keyed by grafanaVersion so concurrent fixtures share one in-flight fetch
+const selectorsCache = new Map<string, Promise<E2ESelectorGroups>>();
+
+// evaluate the fetched CJS bundle with a require shim that allows only 'semver'
+function evaluateBundle(src: string): RuntimeSelectors {
+  const shimRequire = (id: string) => {
+    if (id !== 'semver') {
+      throw new Error(`@grafana/plugin-e2e: disallowed require('${id}') in e2e-selectors bundle`);
+    }
+    return require('semver');
+  };
+  const module: { exports: Partial<RuntimeSelectors> } = { exports: {} };
+  new Function('require', 'module', 'exports', src)(shimRequire, module, module.exports);
+
+  const { versionedComponents, versionedPages } = module.exports;
+  if (!versionedComponents || !versionedPages) {
+    throw new Error('@grafana/plugin-e2e: e2e-selectors bundle is missing expected exports');
+  }
+  return { versionedComponents, versionedPages };
+}
+
+function buildGroups(runtime: RuntimeSelectors, grafanaVersion: string): E2ESelectorGroups {
+  return {
+    components: resolveSelectors(runtime.versionedComponents, grafanaVersion),
+    pages: resolveSelectors(runtime.versionedPages, grafanaVersion),
     constants: resolveSelectors(versionedConstants, grafanaVersion),
     apis: resolveSelectors(versionedAPIs, grafanaVersion),
-  });
+  };
+}
+
+// fall back to the selectors bundled with the installed @grafana/plugin-e2e release
+function bundledGroups(grafanaVersion: string): E2ESelectorGroups {
+  return buildGroups(
+    { versionedComponents: bundledVersionedComponents, versionedPages: bundledVersionedPages },
+    grafanaVersion
+  );
+}
+
+async function fetchRuntimeSelectors(request: APIRequestContext, grafanaVersion: string): Promise<E2ESelectorGroups> {
+  let response;
+  try {
+    response = await request.get(SELECTORS_URL);
+  } catch (error) {
+    console.warn(
+      `@grafana/plugin-e2e: failed to fetch ${SELECTORS_URL}, falling back to bundled @grafana/e2e-selectors.`,
+      error
+    );
+    return bundledGroups(grafanaVersion);
+  }
+
+  // 404 -> Grafana predates the feature; expected on older images, fall back quietly
+  if (response.status() === 404) {
+    return bundledGroups(grafanaVersion);
+  }
+
+  // any other non-OK status (5xx etc.) -> loud fallback
+  if (!response.ok()) {
+    console.warn(
+      `@grafana/plugin-e2e: ${SELECTORS_URL} returned ${response.status()}, falling back to bundled @grafana/e2e-selectors.`
+    );
+    return bundledGroups(grafanaVersion);
+  }
+
+  try {
+    const src = await response.text();
+    return buildGroups(evaluateBundle(src), grafanaVersion);
+  } catch (error) {
+    console.warn(
+      `@grafana/plugin-e2e: failed to evaluate ${SELECTORS_URL}, falling back to bundled @grafana/e2e-selectors.`,
+      error
+    );
+    return bundledGroups(grafanaVersion);
+  }
+}
+
+export const selectors: SelectorFixture = async ({ grafanaVersion, request }, use) => {
+  // opt-in during the trial period; when off, behave exactly as before (bundled dependency)
+  if (process.env.PLUGIN_E2E_RUNTIME_SELECTORS !== 'true') {
+    await use(bundledGroups(grafanaVersion));
+    return;
+  }
+
+  let groups = selectorsCache.get(grafanaVersion);
+  if (!groups) {
+    groups = fetchRuntimeSelectors(request, grafanaVersion);
+    selectorsCache.set(grafanaVersion, groups);
+  }
+  await use(await groups);
 };