Merge pull request #3484 from github/mbg/cli/force-nightly

Add feature for forcing the `nightly` bundle in `dynamic` workflows

src/diagnostics.ts

@@ -66,6 +66,12 @@ interface UnwrittenDiagnostic {
 /** A list of diagnostics which have not yet been written to disk. */
 let unwrittenDiagnostics: UnwrittenDiagnostic[] = [];
 
+/**
+ * A list of diagnostics which have not yet been written to disk,
+ * and where the language does not matter.
+ */
+let unwrittenDefaultLanguageDiagnostics: DiagnosticMessage[] = [];
+
 /**
  * Constructs a new diagnostic message with the specified id and name, as well as optional additional data.
  *
@@ -119,16 +125,20 @@ export function addDiagnostic(
 
 /** Adds a diagnostic that is not specific to any language. */
 export function addNoLanguageDiagnostic(
-  config: Config,
+  config: Config | undefined,
   diagnostic: DiagnosticMessage,
 ) {
-  addDiagnostic(
-    config,
-    // Arbitrarily choose the first language. We could also choose all languages, but that
-    // increases the risk of misinterpreting the data.
-    config.languages[0],
-    diagnostic,
-  );
+  if (config !== undefined) {
+    addDiagnostic(
+      config,
+      // Arbitrarily choose the first language. We could also choose all languages, but that
+      // increases the risk of misinterpreting the data.
+      config.languages[0],
+      diagnostic,
+    );
+  } else {
+    unwrittenDefaultLanguageDiagnostics.push(diagnostic);
+  }
 }
 
 /**
@@ -188,16 +198,21 @@ export function logUnwrittenDiagnostics() {
 /** Writes all unwritten diagnostics to disk. */
 export function flushDiagnostics(config: Config) {
   const logger = getActionsLogger();
-  logger.debug(
-    `Writing ${unwrittenDiagnostics.length} diagnostic(s) to database.`,
-  );
+
+  const diagnosticsCount =
+    unwrittenDiagnostics.length + unwrittenDefaultLanguageDiagnostics.length;
+  logger.debug(`Writing ${diagnosticsCount} diagnostic(s) to database.`);
 
   for (const unwritten of unwrittenDiagnostics) {
     writeDiagnostic(config, unwritten.language, unwritten.diagnostic);
   }
+  for (const unwritten of unwrittenDefaultLanguageDiagnostics) {
+    addNoLanguageDiagnostic(config, unwritten);
+  }
 
-  // Reset the unwritten diagnostics array.
+  // Reset the unwritten diagnostics arrays.
   unwrittenDiagnostics = [];
+  unwrittenDefaultLanguageDiagnostics = [];
 }
 
 /**

src/feature-flags.ts

@@ -46,6 +46,7 @@ export enum Feature {
   DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
   DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
   ExportDiagnosticsEnabled = "export_diagnostics_enabled",
+  ForceNightly = "force_nightly",
   IgnoreGeneratedFiles = "ignore_generated_files",
   ImprovedProxyCertificates = "improved_proxy_certificates",
   OverlayAnalysis = "overlay_analysis",
@@ -164,6 +165,11 @@ export const featureConfig = {
     legacyApi: true,
     minimumVersion: undefined,
   },
+  [Feature.ForceNightly]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: undefined,
+  },
   [Feature.IgnoreGeneratedFiles]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",

src/setup-codeql.test.ts

@@ -1,18 +1,22 @@
 import * as path from "path";
 
+import * as github from "@actions/github";
 import * as toolcache from "@actions/tool-cache";
 import test, { ExecutionContext } from "ava";
 import * as sinon from "sinon";
 
 import * as actionsUtil from "./actions-util";
+import * as api from "./api-client";
 import { Feature, FeatureEnablement } from "./feature-flags";
 import { getRunnerLogger } from "./logging";
 import * as setupCodeql from "./setup-codeql";
+import * as tar from "./tar";
 import {
   LINKED_CLI_VERSION,
   LoggedMessage,
   SAMPLE_DEFAULT_CLI_VERSION,
   SAMPLE_DOTCOM_API_DETAILS,
+  checkExpectedLogMessages,
   createFeatures,
   getRecordingLogger,
   initializeFeatures,
@@ -268,13 +272,127 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
   });
 });
 
+test("getCodeQLSource correctly returns nightly CLI version when tools == nightly", async (t) => {
+  const loggedMessages: LoggedMessage[] = [];
+  const logger = getRecordingLogger(loggedMessages);
+  const features = createFeatures([]);
+
+  const expectedDate = "30260213";
+  const expectedTag = `codeql-bundle-${expectedDate}`;
+
+  // Ensure that we consistently select "zstd" for the test.
+  sinon.stub(process, "platform").value("linux");
+  sinon.stub(tar, "isZstdAvailable").resolves({
+    available: true,
+    foundZstdBinary: true,
+  });
+
+  const client = github.getOctokit("123");
+  const listReleases = sinon.stub(client.rest.repos, "listReleases");
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+  listReleases.resolves({
+    data: [{ tag_name: expectedTag }],
+  } as any);
+  sinon.stub(api, "getApiClient").value(() => client);
+
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    const source = await setupCodeql.getCodeQLSource(
+      "nightly",
+      SAMPLE_DEFAULT_CLI_VERSION,
+      SAMPLE_DOTCOM_API_DETAILS,
+      GitHubVariant.DOTCOM,
+      false,
+      features,
+      logger,
+    );
+
+    // Check that the `CodeQLToolsSource` object matches our expectations.
+    const expectedVersion = `0.0.0-${expectedDate}`;
+    const expectedURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/${setupCodeql.getCodeQLBundleName("zstd")}`;
+    t.deepEqual(source, {
+      bundleVersion: expectedDate,
+      cliVersion: undefined,
+      codeqlURL: expectedURL,
+      compressionMethod: "zstd",
+      sourceType: "download",
+      toolsVersion: expectedVersion,
+    } satisfies setupCodeql.CodeQLToolsSource);
+
+    // Afterwards, ensure that we see the expected messages in the log.
+    checkExpectedLogMessages(t, loggedMessages, [
+      "Using the latest CodeQL CLI nightly, as requested by 'tools: nightly'.",
+      `Bundle version ${expectedDate} is not in SemVer format. Will treat it as pre-release ${expectedVersion}.`,
+      `Attempting to obtain CodeQL tools. CLI version: unknown, bundle tag name: ${expectedTag}`,
+      `Using CodeQL CLI sourced from ${expectedURL}`,
+    ]);
+  });
+});
+
+test("getCodeQLSource correctly returns nightly CLI version when forced by FF", async (t) => {
+  const loggedMessages: LoggedMessage[] = [];
+  const logger = getRecordingLogger(loggedMessages);
+  const features = createFeatures([Feature.ForceNightly]);
+
+  const expectedDate = "30260213";
+  const expectedTag = `codeql-bundle-${expectedDate}`;
+
+  // Ensure that we consistently select "zstd" for the test.
+  sinon.stub(process, "platform").value("linux");
+  sinon.stub(tar, "isZstdAvailable").resolves({
+    available: true,
+    foundZstdBinary: true,
+  });
+
+  const client = github.getOctokit("123");
+  const listReleases = sinon.stub(client.rest.repos, "listReleases");
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+  listReleases.resolves({
+    data: [{ tag_name: expectedTag }],
+  } as any);
+  sinon.stub(api, "getApiClient").value(() => client);
+
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    process.env["GITHUB_EVENT_NAME"] = "dynamic";
+
+    const source = await setupCodeql.getCodeQLSource(
+      undefined,
+      SAMPLE_DEFAULT_CLI_VERSION,
+      SAMPLE_DOTCOM_API_DETAILS,
+      GitHubVariant.DOTCOM,
+      false,
+      features,
+      logger,
+    );
+
+    // Check that the `CodeQLToolsSource` object matches our expectations.
+    const expectedVersion = `0.0.0-${expectedDate}`;
+    const expectedURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/${setupCodeql.getCodeQLBundleName("zstd")}`;
+    t.deepEqual(source, {
+      bundleVersion: expectedDate,
+      cliVersion: undefined,
+      codeqlURL: expectedURL,
+      compressionMethod: "zstd",
+      sourceType: "download",
+      toolsVersion: expectedVersion,
+    } satisfies setupCodeql.CodeQLToolsSource);
+
+    // Afterwards, ensure that we see the expected messages in the log.
+    checkExpectedLogMessages(t, loggedMessages, [
+      `Using the latest CodeQL CLI nightly, as forced by the ${Feature.ForceNightly} feature flag.`,
+      `Bundle version ${expectedDate} is not in SemVer format. Will treat it as pre-release ${expectedVersion}.`,
+      `Attempting to obtain CodeQL tools. CLI version: unknown, bundle tag name: ${expectedTag}`,
+      `Using CodeQL CLI sourced from ${expectedURL}`,
+    ]);
+  });
+});
+
 test("getCodeQLSource correctly returns latest version from toolcache when tools == toolcache", async (t) => {
   const loggedMessages: LoggedMessage[] = [];
   const logger = getRecordingLogger(loggedMessages);
   const features = createFeatures([Feature.AllowToolcacheInput]);
 
-  process.env["GITHUB_EVENT_NAME"] = "dynamic";
-
   const latestToolcacheVersion = "3.2.1";
   const latestVersionPath = "/path/to/latest";
   const testVersions = ["2.3.1", latestToolcacheVersion, "1.2.3"];
@@ -288,6 +406,8 @@ test("getCodeQLSource correctly returns latest version from toolcache when tools
 
   await withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
+    process.env["GITHUB_EVENT_NAME"] = "dynamic";
+
     const source = await setupCodeql.getCodeQLSource(
       "toolcache",
       SAMPLE_DEFAULT_CLI_VERSION,
@@ -343,16 +463,17 @@ const toolcacheInputFallbackMacro = test.macro({
     const logger = getRecordingLogger(loggedMessages);
     const features = createFeatures(featureList);
 
-    for (const [k, v] of Object.entries(environment)) {
-      process.env[k] = v;
-    }
-
     const findAllVersionsStub = sinon
       .stub(toolcache, "findAllVersions")
       .returns(testVersions);
 
     await withTmpDir(async (tmpDir) => {
       setupActionsVars(tmpDir, tmpDir);
+
+      for (const [k, v] of Object.entries(environment)) {
+        process.env[k] = v;
+      }
+
       const source = await setupCodeql.getCodeQLSource(
         "toolcache",
         SAMPLE_DEFAULT_CLI_VERSION,

src/setup-codeql.ts

@@ -10,6 +10,7 @@ import { v4 as uuidV4 } from "uuid";
 import { isDynamicWorkflow, isRunningLocalAction } from "./actions-util";
 import * as api from "./api-client";
 import * as defaults from "./defaults.json";
+import { addNoLanguageDiagnostic, makeDiagnostic } from "./diagnostics";
 import {
   CODEQL_VERSION_ZSTD_BUNDLE,
   CodeQLDefaultVersionInfo,
@@ -55,7 +56,9 @@ function getCodeQLBundleExtension(
   }
 }
 
-function getCodeQLBundleName(compressionMethod: tar.CompressionMethod): string {
+export function getCodeQLBundleName(
+  compressionMethod: tar.CompressionMethod,
+): string {
   const extension = getCodeQLBundleExtension(compressionMethod);
 
   let platform: string;
@@ -196,7 +199,7 @@ export function convertToSemVer(version: string, logger: Logger): string {
   return s;
 }
 
-type CodeQLToolsSource =
+export type CodeQLToolsSource =
   | {
       codeqlTarPath: string;
       compressionMethod: tar.CompressionMethod;
@@ -261,6 +264,20 @@ async function findOverridingToolsInCache(
   return undefined;
 }
 
+/**
+ * Determines where the CodeQL CLI we want to use comes from. This can be from a local file,
+ * the Actions toolcache, or a download.
+ *
+ * @param toolsInput The argument provided for the `tools` input, if any.
+ * @param defaultCliVersion The default CLI version that's linked to the CodeQL Action.
+ * @param apiDetails Information about the GitHub API.
+ * @param variant The GitHub variant we are running on.
+ * @param tarSupportsZstd Whether zstd is supported by `tar`.
+ * @param features Information about enabled features.
+ * @param logger The logger to use.
+ *
+ * @returns Information about where the CodeQL CLI we want to use comes from.
+ */
 export async function getCodeQLSource(
   toolsInput: string | undefined,
   defaultCliVersion: CodeQLDefaultVersionInfo,
@@ -270,6 +287,9 @@ export async function getCodeQLSource(
   features: FeatureEnablement,
   logger: Logger,
 ): Promise<CodeQLToolsSource> {
+  // If there is an explicit `tools` input, it's not one of the reserved values, and it doesn't appear
+  // to point to a URL, then we assume it is a local path and use the CLI from there.
+  // TODO: This appears to misclassify filenames that happen to start with `http` as URLs.
   if (
     toolsInput &&
     !isReservedToolsValue(toolsInput) &&
@@ -302,13 +322,47 @@ export async function getCodeQLSource(
    */
   let url: string | undefined;
 
-  if (
+  // We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the
+  // `tools` input cannot be adjusted to explicitly request it.
+  const canForceNightlyWithFF = isDynamicWorkflow() || util.isInTestMode();
+  const forceNightlyValueFF = await features.getValue(Feature.ForceNightly);
+  const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
+
+  // For advanced workflows, a value from `CODEQL_NIGHTLY_TOOLS_INPUTS` can be specified explicitly
+  // for the `tools` input in the workflow file.
+  const nightlyRequestedByToolsInput =
     toolsInput !== undefined &&
-    CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput)
-  ) {
-    logger.info(
-      `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`,
-    );
+    CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
+
+  if (forceNightly || nightlyRequestedByToolsInput) {
+    if (forceNightly) {
+      logger.info(
+        `Using the latest CodeQL CLI nightly, as forced by the ${Feature.ForceNightly} feature flag.`,
+      );
+      addNoLanguageDiagnostic(
+        undefined,
+        makeDiagnostic(
+          "codeql-action/forced-nightly-cli",
+          "A nightly release of CodeQL was used",
+          {
+            markdownMessage:
+              "GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release.\n\n" +
+              "Nightly releases do not undergo the same validation as regular releases and may lead to analysis instability.\n\n" +
+              "If use of a nightly CodeQL release for this analysis is unexpected, please contact GitHub support.",
+            visibility: {
+              cliSummaryTable: true,
+              statusPage: true,
+              telemetry: true,
+            },
+            severity: "note",
+          },
+        ),
+      );
+    } else {
+      logger.info(
+        `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`,
+      );
+    }
     toolsInput = await getNightlyToolsUrl(logger);
   }
 

src/testing-utils.ts

@@ -145,6 +145,7 @@ export function setupActionsVars(tempDir: string, toolsDir: string) {
   process.env["RUNNER_TEMP"] = tempDir;
   process.env["RUNNER_TOOL_CACHE"] = toolsDir;
   process.env["GITHUB_WORKSPACE"] = tempDir;
+  process.env["GITHUB_EVENT_NAME"] = "push";
 }
 
 export interface LoggedMessage {