feature-flags.test.ts

import * as fs from "fs";
import * as path from "path";

import test, { ExecutionContext } from "ava";

import * as defaults from "./defaults.json";
import { EnvVar } from "./environment";
import {
  Feature,
  featureConfig,
  FeatureEnablement,
  Features,
  FEATURE_FLAGS_FILE_NAME,
  FeatureConfig,
  FeatureWithoutCLI,
} from "./feature-flags";
import { getRunnerLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import {
  getRecordingLogger,
  initializeFeatures,
  LoggedMessage,
  mockCodeQLVersion,
  mockFeatureFlagApiEndpoint,
  setupActionsVars,
  setupTests,
  stubFeatureFlagApiEndpoint,
} from "./testing-utils";
import { ToolsFeature } from "./tools-features";
import * as util from "./util";
import { GitHubVariant, initializeEnvironment, withTmpDir } from "./util";

setupTests(test);

test.beforeEach(() => {
  initializeEnvironment("1.2.3");
});

const testRepositoryNwo = parseRepositoryNwo("github/example");

test(`All features use default values if running against GHES`, async (t) => {
  await withTmpDir(async (tmpDir) => {
    const loggedMessages: LoggedMessage[] = [];
    const features = setUpFeatureFlagTests(
      tmpDir,
      getRecordingLogger(loggedMessages),
      { type: GitHubVariant.GHES, version: "3.0.0" },
    );

    await assertAllFeaturesHaveDefaultValues(t, features);
    assertLoggedMessage(
      t,
      loggedMessages,
      "Not running against github.com. Using default values for all features.",
    );
  });
});

test(`All features use default values if running in CCR`, async (t) => {
  await withTmpDir(async (tmpDir) => {
    const loggedMessages: LoggedMessage[] = [];
    const features = setUpFeatureFlagTests(
      tmpDir,
      getRecordingLogger(loggedMessages),
    );

    process.env[EnvVar.ANALYSIS_KEY] = "dynamic/copilot-pull-request-reviewer";

    await assertAllFeaturesHaveDefaultValues(t, features);
    assertLoggedMessage(
      t,
      loggedMessages,
      "Feature flags are not supported in Copilot Code Review. Using default values for all features.",
    );
  });
});

test(`Feature flags are requested in GHEC-DR`, async (t) => {
  await withTmpDir(async (tmpDir) => {
    const loggedMessages = [];
    const features = setUpFeatureFlagTests(
      tmpDir,
      getRecordingLogger(loggedMessages),
      { type: GitHubVariant.GHEC_DR },
    );

    mockFeatureFlagApiEndpoint(200, initializeFeatures(true));

    for (const feature of Object.values(Feature)) {
      // Ensure we have gotten a response value back from the Mock API
      t.assert(await getFeatureIncludingCodeQlIfRequired(features, feature));
    }

    // And that we haven't bailed preemptively.
    t.assert(
      loggedMessages.find(
        (v: LoggedMessage) =>
          v.type === "debug" &&
          v.message ===
            "Not running against github.com. Disabling all toggleable features.",
      ) === undefined,
    );
  });
});

test("API response missing and features use default value", async (t) => {
  await withTmpDir(async (tmpDir) => {
    const loggedMessages: LoggedMessage[] = [];
    const features = setUpFeatureFlagTests(
      tmpDir,
      getRecordingLogger(loggedMessages),
    );

    mockFeatureFlagApiEndpoint(403, {});

    for (const feature of Object.values(Feature)) {
      t.assert(
        (await getFeatureIncludingCodeQlIfRequired(features, feature)) ===
          featureConfig[feature].defaultValue,
      );
    }
    assertAllFeaturesUndefinedInApi(t, loggedMessages);
  });
});

test("Features use default value if they're not returned in API response", async (t) => {
  await withTmpDir(async (tmpDir) => {
    const loggedMessages: LoggedMessage[] = [];
    const features = setUpFeatureFlagTests(
      tmpDir,
      getRecordingLogger(loggedMessages),
    );

    mockFeatureFlagApiEndpoint(200, {});

    for (const feature of Object.values(Feature)) {
      t.assert(
        (await getFeatureIncludingCodeQlIfRequired(features, feature)) ===
          featureConfig[feature].defaultValue,
      );
    }

    assertAllFeaturesUndefinedInApi(t, loggedMessages);
  });
});

test("Include no more than 25 features in each API request", async (t) => {
  await withTmpDir(async (tmpDir) => {
    const features = setUpFeatureFlagTests(tmpDir);

    stubFeatureFlagApiEndpoint((request) => {
      const requestedFeatures = (request.features as string).split(",");
      return {
        status: requestedFeatures.length <= 25 ? 200 : 400,
        messageIfError: "Can request a maximum of 25 features.",
        data: {},
      };
    });

    // We only need to call getValue once, and it does not matter which feature
    // we ask for. Under the hood, the features library will request all features
    // from the API.
    const feature = Object.values(Feature)[0];
    await t.notThrowsAsync(async () =>
      getFeatureIncludingCodeQlIfRequired(features, feature),
    );
  });
});

test("Feature flags exception is propagated if the API request errors", async (t) => {
  await withTmpDir(async (tmpDir) => {
    const features = setUpFeatureFlagTests(tmpDir);

    mockFeatureFlagApiEndpoint(500, {});

    const someFeature = Object.values(Feature)[0];

    await t.throwsAsync(
      async () => getFeatureIncludingCodeQlIfRequired(features, someFeature),
      {
        message:
          "Encountered an error while trying to determine feature enablement: Error: some error message",
      },
    );
  });
});

for (const feature of Object.keys(featureConfig)) {
  test(`Only feature '${feature}' is enabled if enabled in the API response. Other features disabled`, async (t) => {
    await withTmpDir(async (tmpDir) => {
      const features = setUpFeatureFlagTests(tmpDir);

      // set all features to false except the one we're testing
      const expectedFeatureEnablement: { [feature: string]: boolean } = {};
      for (const f of Object.keys(featureConfig)) {
        expectedFeatureEnablement[f] = f === feature;
      }
      mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

      // retrieve the values of the actual features
      const actualFeatureEnablement: { [feature: string]: boolean } = {};
      for (const f of Object.keys(featureConfig)) {
        actualFeatureEnablement[f] = await getFeatureIncludingCodeQlIfRequired(
          features,
          f as Feature,
        );
      }

      // All features should be false except the one we're testing
      t.deepEqual(actualFeatureEnablement, expectedFeatureEnablement);
    });
  });

  test(`Only feature '${feature}' is enabled if the associated environment variable is true. Others disabled.`, async (t) => {
    await withTmpDir(async (tmpDir) => {
      const features = setUpFeatureFlagTests(tmpDir);

      const expectedFeatureEnablement = initializeFeatures(false);
      mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

      // feature should be disabled initially
      t.assert(
        !(await getFeatureIncludingCodeQlIfRequired(
          features,
          feature as Feature,
        )),
      );

      // set env var to true and check that the feature is now enabled
      process.env[featureConfig[feature].envVar] = "true";
      t.assert(
        await getFeatureIncludingCodeQlIfRequired(features, feature as Feature),
      );
    });
  });

  test(`Feature '${feature}' is disabled if the associated environment variable is false, even if enabled in API`, async (t) => {
    await withTmpDir(async (tmpDir) => {
      const features = setUpFeatureFlagTests(tmpDir);

      const expectedFeatureEnablement = initializeFeatures(true);
      mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

      // feature should be enabled initially
      t.assert(
        await getFeatureIncludingCodeQlIfRequired(features, feature as Feature),
      );

      // set env var to false and check that the feature is now disabled
      process.env[featureConfig[feature].envVar] = "false";
      t.assert(
        !(await getFeatureIncludingCodeQlIfRequired(
          features,
          feature as Feature,
        )),
      );
    });
  });

  if (
    featureConfig[feature].minimumVersion !== undefined ||
    featureConfig[feature].toolsFeature !== undefined
  ) {
    test(`Getting feature '${feature} should throw if no codeql is provided`, async (t) => {
      await withTmpDir(async (tmpDir) => {
        const features = setUpFeatureFlagTests(tmpDir);

        const expectedFeatureEnablement = initializeFeatures(true);
        mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

        // The type system should prevent this happening, but test that if we
        // bypass it we get the expected error.
        await t.throwsAsync(
          // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
          async () => features.getValue(feature as any),
          {
            message: `Internal error: A ${
              featureConfig[feature].minimumVersion !== undefined
                ? "minimum version"
                : "required tools feature"
            } is specified for feature ${feature}, but no instance of CodeQL was provided.`,
          },
        );
      });
    });
  }

  if (featureConfig[feature].minimumVersion !== undefined) {
    test(`Feature '${feature}' is disabled if the minimum CLI version is below ${featureConfig[feature].minimumVersion}`, async (t) => {
      await withTmpDir(async (tmpDir) => {
        const features = setUpFeatureFlagTests(tmpDir);

        const expectedFeatureEnablement = initializeFeatures(true);
        mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

        // feature should be disabled when an old CLI version is set
        let codeql = mockCodeQLVersion("2.0.0");
        t.assert(!(await features.getValue(feature as Feature, codeql)));

        // even setting the env var to true should not enable the feature if
        // the minimum CLI version is not met
        process.env[featureConfig[feature].envVar] = "true";
        t.assert(!(await features.getValue(feature as Feature, codeql)));

        // feature should be enabled when a new CLI version is set
        // and env var is not set
        process.env[featureConfig[feature].envVar] = "";
        codeql = mockCodeQLVersion(
          featureConfig[feature].minimumVersion as string,
        );
        t.assert(await features.getValue(feature as Feature, codeql));

        // set env var to false and check that the feature is now disabled
        process.env[featureConfig[feature].envVar] = "false";
        t.assert(!(await features.getValue(feature as Feature, codeql)));
      });
    });
  }

  if (featureConfig[feature].toolsFeature !== undefined) {
    test(`Feature '${feature}' is disabled if the required tools feature is not enabled`, async (t) => {
      await withTmpDir(async (tmpDir) => {
        const features = setUpFeatureFlagTests(tmpDir);

        const expectedFeatureEnablement = initializeFeatures(true);
        mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

        // feature should be disabled when the required tools feature is not enabled
        let codeql = mockCodeQLVersion("2.0.0");
        t.assert(!(await features.getValue(feature as Feature, codeql)));

        // even setting the env var to true should not enable the feature if
        // the required tools feature is not enabled
        process.env[featureConfig[feature].envVar] = "true";
        t.assert(!(await features.getValue(feature as Feature, codeql)));

        // feature should be enabled when the required tools feature is enabled
        // and env var is not set
        process.env[featureConfig[feature].envVar] = "";
        codeql = mockCodeQLVersion("2.0.0", {
          [featureConfig[feature].toolsFeature]: true,
        });
        t.assert(await features.getValue(feature as Feature, codeql));

        // set env var to false and check that the feature is now disabled
        process.env[featureConfig[feature].envVar] = "false";
        t.assert(!(await features.getValue(feature as Feature, codeql)));
      });
    });
  }
}

test("Feature flags are saved to disk", async (t) => {
  await withTmpDir(async (tmpDir) => {
    const features = setUpFeatureFlagTests(tmpDir);
    const expectedFeatureEnablement = initializeFeatures(true);
    mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

    const cachedFeatureFlags = path.join(tmpDir, FEATURE_FLAGS_FILE_NAME);

    t.false(
      fs.existsSync(cachedFeatureFlags),
      "Feature flag cached file should not exist before getting feature flags",
    );

    t.true(
      await getFeatureIncludingCodeQlIfRequired(
        features,
        Feature.QaTelemetryEnabled,
      ),
      "Feature flag should be enabled initially",
    );

    t.true(
      fs.existsSync(cachedFeatureFlags),
      "Feature flag cached file should exist after getting feature flags",
    );

    const actualFeatureEnablement = JSON.parse(
      fs.readFileSync(cachedFeatureFlags, "utf8"),
    );
    t.deepEqual(actualFeatureEnablement, expectedFeatureEnablement);

    // now test that we actually use the feature flag cache instead of the server
    actualFeatureEnablement[Feature.QaTelemetryEnabled] = false;
    fs.writeFileSync(
      cachedFeatureFlags,
      JSON.stringify(actualFeatureEnablement),
    );

    // delete the in memory cache so that we are forced to use the cached file
    (features as any).gitHubFeatureFlags.cachedApiResponse = undefined;

    t.false(
      await getFeatureIncludingCodeQlIfRequired(
        features,
        Feature.QaTelemetryEnabled,
      ),
      "Feature flag should be enabled after reading from cached file",
    );
  });
});

test("Environment variable can override feature flag cache", async (t) => {
  await withTmpDir(async (tmpDir) => {
    const features = setUpFeatureFlagTests(tmpDir);
    const expectedFeatureEnablement = initializeFeatures(true);
    mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

    const cachedFeatureFlags = path.join(tmpDir, FEATURE_FLAGS_FILE_NAME);
    t.true(
      await getFeatureIncludingCodeQlIfRequired(
        features,
        Feature.QaTelemetryEnabled,
      ),
      "Feature flag should be enabled initially",
    );

    t.true(
      fs.existsSync(cachedFeatureFlags),
      "Feature flag cached file should exist after getting feature flags",
    );
    process.env.CODEQL_ACTION_QA_TELEMETRY = "false";

    t.false(
      await getFeatureIncludingCodeQlIfRequired(
        features,
        Feature.QaTelemetryEnabled,
      ),
      "Feature flag should be disabled after setting env var",
    );
  });
});

test(`selects CLI from defaults.json on GHES`, async (t) => {
  await withTmpDir(async (tmpDir) => {
    const features = setUpFeatureFlagTests(tmpDir);

    const defaultCliVersion = await features.getDefaultCliVersion(
      GitHubVariant.GHES,
    );
    t.deepEqual(defaultCliVersion, {
      cliVersion: defaults.cliVersion,
      tagName: defaults.bundleVersion,
    });
  });
});

for (const variant of [GitHubVariant.DOTCOM, GitHubVariant.GHEC_DR]) {
  test(`selects CLI v2.20.1 on ${variant} when feature flags enable v2.20.0 and v2.20.1`, async (t) => {
    await withTmpDir(async (tmpDir) => {
      const features = setUpFeatureFlagTests(tmpDir);
      const expectedFeatureEnablement = initializeFeatures(true);
      expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
      expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
      expectedFeatureEnablement["default_codeql_version_2_20_2_enabled"] =
        false;
      expectedFeatureEnablement["default_codeql_version_2_20_3_enabled"] =
        false;
      expectedFeatureEnablement["default_codeql_version_2_20_4_enabled"] =
        false;
      expectedFeatureEnablement["default_codeql_version_2_20_5_enabled"] =
        false;
      mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

      const defaultCliVersion = await features.getDefaultCliVersion(variant);
      t.deepEqual(defaultCliVersion, {
        cliVersion: "2.20.1",
        tagName: "codeql-bundle-v2.20.1",
        toolsFeatureFlagsValid: true,
      });
    });
  });

  test(`selects CLI from defaults.json on ${variant} when no default version feature flags are enabled`, async (t) => {
    await withTmpDir(async (tmpDir) => {
      const features = setUpFeatureFlagTests(tmpDir);
      const expectedFeatureEnablement = initializeFeatures(true);
      mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

      const defaultCliVersion = await features.getDefaultCliVersion(variant);
      t.deepEqual(defaultCliVersion, {
        cliVersion: defaults.cliVersion,
        tagName: defaults.bundleVersion,
        toolsFeatureFlagsValid: false,
      });
    });
  });

  test(`ignores invalid version numbers in default version feature flags on ${variant}`, async (t) => {
    await withTmpDir(async (tmpDir) => {
      const loggedMessages = [];
      const features = setUpFeatureFlagTests(
        tmpDir,
        getRecordingLogger(loggedMessages),
      );
      const expectedFeatureEnablement = initializeFeatures(true);
      expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
      expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
      expectedFeatureEnablement["default_codeql_version_2_20_invalid_enabled"] =
        true;
      mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);

      const defaultCliVersion = await features.getDefaultCliVersion(variant);
      t.deepEqual(defaultCliVersion, {
        cliVersion: "2.20.1",
        tagName: "codeql-bundle-v2.20.1",
        toolsFeatureFlagsValid: true,
      });

      t.assert(
        loggedMessages.find(
          (v: LoggedMessage) =>
            v.type === "warning" &&
            v.message ===
              "Ignoring feature flag default_codeql_version_2_20_invalid_enabled as it does not specify a valid CodeQL version.",
        ) !== undefined,
      );
    });
  });
}

test("legacy feature flags should end with _enabled", async (t) => {
  for (const [feature, config] of Object.entries(featureConfig)) {
    if ((config satisfies FeatureConfig as FeatureConfig).legacyApi) {
      t.assert(
        feature.endsWith("_enabled"),
        `legacy feature ${feature} should end with '_enabled'`,
      );
    }
  }
});

test("non-legacy feature flags should not end with _enabled", async (t) => {
  for (const [feature, config] of Object.entries(featureConfig)) {
    if (!(config satisfies FeatureConfig as FeatureConfig).legacyApi) {
      t.false(
        feature.endsWith("_enabled"),
        `non-legacy feature ${feature} should not end with '_enabled'`,
      );
    }
  }
});

test("non-legacy feature flags should not start with codeql_action_", async (t) => {
  for (const [feature, config] of Object.entries(featureConfig)) {
    if (!(config satisfies FeatureConfig as FeatureConfig).legacyApi) {
      t.false(
        feature.startsWith("codeql_action_"),
        `non-legacy feature ${feature} should not start with 'codeql_action_'`,
      );
    }
  }
});

async function assertAllFeaturesHaveDefaultValues(
  t: ExecutionContext<unknown>,
  features: FeatureEnablement,
) {
  for (const feature of Object.values(Feature)) {
    t.deepEqual(
      await getFeatureIncludingCodeQlIfRequired(features, feature),
      featureConfig[feature].defaultValue,
    );
  }
}

function assertLoggedMessage(
  t: ExecutionContext<unknown>,
  loggedMessages: LoggedMessage[],
  expectedMessage: string,
) {
  t.assert(
    loggedMessages.find(
      (v: LoggedMessage) => v.type === "debug" && v.message === expectedMessage,
    ) !== undefined,
  );
}

function assertAllFeaturesUndefinedInApi(
  t: ExecutionContext<unknown>,
  loggedMessages: LoggedMessage[],
) {
  for (const feature of Object.keys(featureConfig)) {
    t.assert(
      loggedMessages.find(
        (v) =>
          v.type === "debug" &&
          (v.message as string).includes(feature) &&
          (v.message as string).includes("undefined in API response"),
      ) !== undefined,
    );
  }
}

function setUpFeatureFlagTests(
  tmpDir: string,
  logger = getRunnerLogger(true),
  gitHubVersion = { type: GitHubVariant.DOTCOM } as util.GitHubVersion,
): FeatureEnablement {
  setupActionsVars(tmpDir, tmpDir);

  return new Features(gitHubVersion, testRepositoryNwo, tmpDir, logger);
}

/**
 * Returns an argument to pass to `getValue` that if required includes a CodeQL object meeting the
 * minimum version or tool feature requirements specified by the feature.
 */
function getFeatureIncludingCodeQlIfRequired(
  features: FeatureEnablement,
  feature: Feature,
) {
  const config = featureConfig[
    feature
  ] satisfies FeatureConfig as FeatureConfig;
  if (
    config.minimumVersion === undefined &&
    config.toolsFeature === undefined
  ) {
    return features.getValue(feature as FeatureWithoutCLI);
  }

  return features.getValue(
    feature,
    mockCodeQLVersion(
      "9.9.9",
      Object.fromEntries(Object.values(ToolsFeature).map((v) => [v, true])),
    ),
  );
}