Run benchmarks in test mode by default instead of excluding them (#56418)

Summary:
Pull Request resolved: #56418

Changelog: [internal]

D92150163 excluded benchmark tests by default when running `yarn fantom` without `--benchmarks`. This was incorrect because it means benchmarks could silently break without being caught.

This changes the behavior so:
1. By default (without `--benchmarks`), benchmarks run in test mode (single iteration for correctness only), ensuring they do not break.
2. With `--benchmarks`, benchmarks run in full benchmark mode (multiple iterations for performance measurement).

Also renames `FANTOM_FORCE_TEST_MODE` to `FANTOM_RUN_BENCHMARKS` and `forceTestModeForBenchmarks` to `runBenchmarks` to better reflect the intent (opt-in to full benchmarks rather than opt-in to test mode).

Differential Revision: D100464314

Author: rubennorte

private/react-native-fantom/config/jest.config.js

@@ -29,12 +29,7 @@ module.exports = {
   ] /*:: as ReadonlyArray<string> */,
   // This allows running Meta-internal tests with the `-test.fb.js` suffix.
   testRegex: '/__tests__/.*-itest(\\.fb)?\\.js$',
-  testPathIgnorePatterns: [
-    ...baseConfig.testPathIgnorePatterns,
-    ...(process.env.FANTOM_INCLUDE_BENCHMARKS != null
-      ? []
-      : ['benchmark-itest']),
-  ] /*:: as ReadonlyArray<string> */,
+  testPathIgnorePatterns: baseConfig.testPathIgnorePatterns,
   transformIgnorePatterns: ['.*'],
   testRunner: '<rootDir>/private/react-native-fantom/runner/index.js',
   watchPathIgnorePatterns: ['<rootDir>/private/react-native-fantom/build/'],

private/react-native-fantom/runner/EnvironmentOptions.js

@@ -15,8 +15,7 @@ const VALID_ENVIRONMENT_VARIABLES = [
   'FANTOM_ENABLE_CPP_DEBUGGING',
   'FANTOM_FORCE_CI_MODE',
   'FANTOM_FORCE_OSS_BUILD',
-  'FANTOM_FORCE_TEST_MODE',
-  'FANTOM_INCLUDE_BENCHMARKS',
+  'FANTOM_RUN_BENCHMARKS',
   'FANTOM_LOG_COMMANDS',
   'FANTOM_PRINT_OUTPUT',
   'FANTOM_DEBUG_JS',
@@ -60,11 +59,12 @@ export const isCI: boolean =
   Boolean(process.env.GITHUB_ACTIONS);
 
 /**
- * Forces benchmarks to run in test mode (running a single time to ensure
- * correctness instead of multiples times to measure performance).
+ * Runs benchmarks in benchmark mode (measuring performance with multiple
+ * iterations). When false, benchmarks run in test mode (single iteration
+ * for correctness only).
  */
-export const forceTestModeForBenchmarks: boolean = Boolean(
-  process.env.FANTOM_FORCE_TEST_MODE,
+export const runBenchmarks: boolean = Boolean(
+  process.env.FANTOM_RUN_BENCHMARKS,
 );
 
 export const debugJS: boolean = Boolean(process.env.FANTOM_DEBUG_JS);

private/react-native-fantom/runner/entrypoint-template.js

@@ -37,7 +37,7 @@ module.exports = function entrypointTemplate({
   const constants: FantomRuntimeConstants = {
     isOSS: EnvironmentOptions.isOSS,
     isRunningFromCI: EnvironmentOptions.isCI,
-    forceTestModeForBenchmarks: EnvironmentOptions.forceTestModeForBenchmarks,
+    runBenchmarks: EnvironmentOptions.runBenchmarks,
     fantomConfigSummary: formatFantomConfig(testConfig),
     jsTraceOutputPath,
     jsHeapSnapshotOutputPathTemplate,

private/react-native-fantom/src/Benchmark.js

@@ -113,15 +113,15 @@ export function suite(
       throw new Error('No benchmark tests defined');
     }
 
-    const {isRunningFromCI, forceTestModeForBenchmarks} = getConstants();
+    const {isRunningFromCI, runBenchmarks} = getConstants();
 
     // If we're running from CI and there's no verification function, there's
     // no point in running the benchmark.
     // We still run a single iteration of each test just to make sure that the
     // logic in the benchmark doesn't break.
     const isTestOnly =
       suiteOptions.testOnly === true ||
-      forceTestModeForBenchmarks ||
+      !runBenchmarks ||
       (isRunningFromCI && verifyFns.length === 0);
 
     const benchOptions: BenchOptions = isTestOnly
@@ -180,7 +180,11 @@ export function suite(
       bench.add(task.name, task.fn, options);
     }
 
-    if (!isTestOnly) {
+    if (isTestOnly) {
+      console.log(
+        `Running benchmark in test mode: ${suiteName}. Use --benchmarks to run in benchmark mode.`,
+      );
+    } else {
       console.log(`Running benchmark: ${suiteName}. Please wait.`);
     }
 

private/react-native-fantom/src/Constants.js

@@ -11,7 +11,7 @@
 export type FantomRuntimeConstants = Readonly<{
   isOSS: boolean,
   isRunningFromCI: boolean,
-  forceTestModeForBenchmarks: boolean,
+  runBenchmarks: boolean,
   fantomConfigSummary: string,
   jsHeapSnapshotOutputPathTemplate: string,
   jsHeapSnapshotOutputPathTemplateToken: string,
@@ -21,7 +21,7 @@ export type FantomRuntimeConstants = Readonly<{
 let constants: FantomRuntimeConstants = {
   isOSS: false,
   isRunningFromCI: false,
-  forceTestModeForBenchmarks: false,
+  runBenchmarks: false,
   fantomConfigSummary: '',
   jsHeapSnapshotOutputPathTemplate: '',
   jsHeapSnapshotOutputPathTemplateToken: '',

scripts/fantom.sh

@@ -18,20 +18,13 @@ fi
 
 export NODE_OPTIONS='--max-old-space-size=8192'
 
-# Parse arguments to check for --benchmarks flag
-INCLUDE_BENCHMARKS=false
+# Parse arguments to extract custom flags
 ARGS=()
 for arg in "$@"; do
-  if [[ "$arg" == "--benchmarks" ]]; then
-    INCLUDE_BENCHMARKS=true
-  else
-    ARGS+=("$arg")
-  fi
+  case "$arg" in
+    --benchmarks) export FANTOM_RUN_BENCHMARKS=1 ;;
+    *) ARGS+=("$arg") ;;
+  esac
 done
 
-# When --benchmarks is passed, set env var so jest.config.js includes benchmark tests
-if [[ "$INCLUDE_BENCHMARKS" == true ]]; then
-  FANTOM_INCLUDE_BENCHMARKS=1 yarn jest --config private/react-native-fantom/config/jest.config.js "${ARGS[@]}"
-else
-  yarn jest --config private/react-native-fantom/config/jest.config.js "${ARGS[@]}"
-fi
+yarn jest --config private/react-native-fantom/config/jest.config.js "${ARGS[@]}"