Scaffold validation job

Add limited test coverage

.github/workflows/__diagnostics-without-db.yml

@@ -0,0 +1,119 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pr-checks/sync.sh
+# to regenerate this file.
+
+name: PR Check - Diagnostic export without database
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+      - main
+      - releases/v*
+  pull_request: {}
+  merge_group:
+    types:
+      - checks_requested
+  schedule:
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+    inputs: {}
+  workflow_call:
+    inputs: {}
+defaults:
+  run:
+    shell: bash
+concurrency:
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || false }}
+  group: diagnostics-without-db-${{github.ref}}
+jobs:
+  diagnostics-without-db:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            version: linked
+    name: Diagnostic export without database
+    if: github.triggering_actor != 'dependabot[bot]'
+    permissions:
+      contents: read
+      security-events: read
+    timeout-minutes: 45
+    runs-on: ${{ matrix.os }}
+    steps:
+      # We deliberately need to make this step fail before the database is initialised, but after
+      # some diagnostic has been produced. The simplest way to do that right now is to force the
+      # use of a nightly release and then fail because we are attempting to analyse Swift on
+      # a non-macOS runner.
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 20.x
+          cache: npm
+      - name: Install newer npm
+        run: npm install -g npm@11.19.1
+      - name: Install dependencies
+        run: npm ci
+      - name: Prepare test
+        id: prepare-test
+        uses: ./.github/actions/prepare-test
+        with:
+          version: ${{ matrix.version }}
+          use-all-platform-bundle: 'false'
+          setup-kotlin: 'true'
+      - name: Initialise failing analysis
+        id: init
+        uses: ./../action/init
+        continue-on-error: true
+        env:
+          CODEQL_ACTION_FORCE_NIGHTLY: true
+        with:
+          languages: swift
+          tools: ${{ steps.prepare-test.outputs.tools-url }}
+
+      - name: Check diagnostics appear on disk
+        env:
+          SARIF_PATH: '${{ runner.temp }}/codeql-action-temp-diagnostics/'
+        run: |
+          cd ../action
+          npx tsx ./pr-checks/diagnostics-without-db.ts
+    env:
+      CODEQL_ACTION_TEST_MODE: false
+      CODEQL_ACTION_SKIP_SARIF_UPLOAD: true
+      CODEQL_ACTION_TESTING_ENVIRONMENT: codeql-action-pr-checks
+  diagnostics-without-db-validation:
+    if: github.triggering_actor != 'dependabot[bot]'
+    needs:
+      - diagnostics-without-db
+    permissions:
+      contents: read
+      security-events: read
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 20.x
+          cache: npm
+      - name: Install newer npm
+        run: npm install -g npm@11.19.1
+      - name: Install dependencies
+        run: npm ci
+      - name: Download workflow artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: test-mode-sarif
+          path: sarif
+
+      - name: List artifact contents
+        run: ls -laR ./sarif
+    env:
+      CODEQL_ACTION_TEST_MODE: true

pr-checks/checks/diagnostics-without-db.yml

@@ -0,0 +1,49 @@
+name: "Diagnostic export without database"
+description: "Tests that temporary diagnostics are correctly included in the failed SARIF."
+versions:
+  # Overridden by the FF below.
+  - linked
+installNode: true
+
+env:
+  # Allow the failed SARIF file to be generated.
+  CODEQL_ACTION_TEST_MODE: false
+  # But don't actually upload it.
+  CODEQL_ACTION_SKIP_SARIF_UPLOAD: true
+  # Mark telemetry for this workflow so it can be treated separately.
+  CODEQL_ACTION_TESTING_ENVIRONMENT: codeql-action-pr-checks
+
+steps:
+  # We deliberately need to make this step fail before the database is initialised, but after
+  # some diagnostic has been produced. The simplest way to do that right now is to force the
+  # use of a nightly release and then fail because we are attempting to analyse Swift on
+  # a non-macOS runner.
+  - name: Initialise failing analysis
+    id: init
+    uses: ./../action/init
+    continue-on-error: true
+    env:
+      CODEQL_ACTION_FORCE_NIGHTLY: true
+    with:
+      languages: swift
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+
+  - name: Check diagnostics appear on disk
+    env:
+      SARIF_PATH: "${{ runner.temp }}/codeql-action-temp-diagnostics/"
+    run: |
+      cd ../action
+      npx tsx ./pr-checks/diagnostics-without-db.ts
+validationJobs:
+  diagnostics-without-db-validation:
+    checkout: true
+    installNode: true
+    steps:
+      - name: Download workflow artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: test-mode-sarif
+          path: sarif
+
+      - name: List artifact contents
+        run: ls -laR ./sarif

pr-checks/diagnostics-without-db.ts

@@ -0,0 +1,48 @@
+import * as fs from "fs";
+import * as path from "path";
+
+import * as core from "@actions/core";
+
+import type * as diagnostics from "../src/diagnostics";
+
+const DIAGNOSTIC_ID = "codeql-action/forced-nightly-cli";
+const sarifDirectory = process.env["SARIF_PATH"];
+
+if (sarifDirectory === undefined) {
+  throw new Error("SARIF_PATH is undefined.");
+}
+
+const paths = fs
+  .readdirSync(sarifDirectory, {
+    recursive: true,
+    encoding: "utf-8",
+  })
+  .filter((p) => p.endsWith(".json"));
+
+core.info(`Found ${paths.length} paths in '${sarifDirectory}'.`);
+core.info(`Searching for a diagnostic with id '${DIAGNOSTIC_ID}'...`);
+
+let found = false;
+for (const diagnosticPath of paths) {
+  if (path.extname(diagnosticPath) !== ".json") {
+    core.info(`Skipping ${diagnosticPath}.`);
+    continue;
+  }
+  core.info(`Found ${diagnosticPath}.`);
+
+  const contents = JSON.parse(
+    fs.readFileSync(path.resolve(sarifDirectory, diagnosticPath), "utf-8"),
+  ) as diagnostics.DiagnosticMessage;
+
+  if (contents.source.id === DIAGNOSTIC_ID) {
+    core.info(
+      `Found '${DIAGNOSTIC_ID}' in '${diagnosticPath}'. Skipping remaining files.`,
+    );
+    found = true;
+    break;
+  }
+}
+
+if (!found) {
+  throw new Error(`Didn't find '${DIAGNOSTIC_ID}' diagnostic.`);
+}

pr-checks/sync.ts

@@ -90,7 +90,7 @@ interface Specification extends JobSpecification {
   services?: any;
 
   /** Additional jobs to run after the main PR check job. */
-  validationJobs?: Record<string, JobSpecification>;
+  validationJobs?: Record<string, ValidationJobSpecification>;
 
   /** If set, this check is part of a named collection that gets its own caller workflow. */
   collection?: string;
@@ -122,6 +122,10 @@ interface JobSpecification {
   installYq?: boolean;
 }
 
+type ValidationJobSpecification = JobSpecification & {
+  checkout?: boolean;
+};
+
 /** Describes language/framework-specific steps and inputs. */
 interface LanguageSetup {
   specProperty: keyof JobSpecification;
@@ -462,6 +466,20 @@ function generateJobMatrix(
   return matrix;
 }
 
+/**
+ * Gets the `actions/checkout` step.
+ */
+function getCheckoutStep(): Step {
+  return {
+    name: "Check out repository",
+    uses: pinnedUses(
+      "actions/checkout",
+      "3d3c42e5aac5ba805825da76410c181273ba90b1",
+      "v7.0.1",
+    ),
+  };
+}
+
 /**
  * Retrieves setup steps and additional input definitions based on specific languages or frameworks
  * that are requested by the `checkSpecification`.
@@ -531,14 +549,7 @@ function generateJob(
 
   // Construct the workflow steps needed for this check.
   const steps: Step[] = [
-    {
-      name: "Check out repository",
-      uses: pinnedUses(
-        "actions/checkout",
-        "3d3c42e5aac5ba805825da76410c181273ba90b1",
-        "v7.0.1",
-      ),
-    },
+    getCheckoutStep(),
     ...setupInfo.steps,
     {
       name: "Prepare test",
@@ -609,13 +620,16 @@ function generateJob(
 /** Generates a validation job. */
 function generateValidationJob(
   specDocument: yaml.Document,
-  jobSpecification: JobSpecification,
+  jobSpecification: ValidationJobSpecification,
   checkName: string,
   name: string,
 ) {
   // Determine which languages or frameworks have to be installed.
   const { inputs, steps } = getSetupSteps(jobSpecification);
 
+  // Construct a checkout step if requested.
+  const checkoutStep = jobSpecification.checkout ? [getCheckoutStep()] : [];
+
   // Extract the sequence of steps from the YAML document to persist as much formatting as possible.
   const specSteps = specDocument.getIn([
     "validationJobs",
@@ -624,7 +638,7 @@ function generateValidationJob(
   ]) as yaml.YAMLSeq;
 
   // Add the generated steps in front of the ones from the specification.
-  specSteps.items.unshift(...steps);
+  specSteps.items.unshift(...checkoutStep, ...steps);
 
   const validationJob: Record<string, any> = {
     name: jobSpecification.name,
@@ -635,7 +649,7 @@ function generateValidationJob(
       "security-events": "read",
     },
     "timeout-minutes": 5,
-    "runs-on": "ubuntu-slim",
+    "runs-on": "ubuntu-latest",
     steps: specSteps,
   };
 

src/codeql.ts

@@ -199,6 +199,7 @@ export interface CodeQL {
     sarifFile: string,
     automationDetailsId: string | undefined,
     config: ActionConfig<"tempDir">,
+    diagnosticDir: string | undefined,
   ): Promise<void>;
   /** Get the location of an extractor for the specified language. */
   resolveExtractor(language: Language): Promise<string>;
@@ -915,6 +916,7 @@ async function getCodeQLForCmd(
       sarifFile: string,
       automationDetailsId: string | undefined,
       config: ActionConfig<"tempDir">,
+      diagnosticDir: string | undefined,
     ): Promise<void> {
       const args = [
         "diagnostics",
@@ -924,6 +926,9 @@ async function getCodeQLForCmd(
         `--sarif-codescanning-config=${getGeneratedCodeScanningConfigPath(
           config,
         )}`,
+        ...(diagnosticDir !== undefined
+          ? [`--diagnostic-dir=${diagnosticDir}`]
+          : []),
         ...getExtraOptionsFromEnv(["diagnostics", "export"]),
       ];
       if (automationDetailsId !== undefined) {

src/config/db-config.test.ts

@@ -9,8 +9,9 @@ import {
   LoggedMessage,
   makeMacro,
   RecordingLogger,
+  setupActionsVars,
 } from "../testing-utils";
-import { ConfigurationError, prettyPrintPack } from "../util";
+import { ConfigurationError, prettyPrintPack, withTmpDir } from "../util";
 
 import * as dbConfig from "./db-config";
 
@@ -562,69 +563,84 @@ test("mergeDefaultSetupAndUserConfigs - keeps other properties from user-supplie
   t.deepEqual(result, configFile);
 });
 
-test("mergeDefaultSetupAndUserConfigs - ignores, but warns about, unknown keys from Default Setup", async (t) => {
-  const logger = new RecordingLogger();
-  const configFile: dbConfig.UserConfig = {
-    "query-filters": [{ exclude: { a: "b" } }],
-    "paths-ignore": ["path"],
-  };
+test.serial(
+  "mergeDefaultSetupAndUserConfigs - ignores, but warns about, unknown keys from Default Setup",
+  async (t) => {
+    const logger = new RecordingLogger();
+    const configFile: dbConfig.UserConfig = {
+      "query-filters": [{ exclude: { a: "b" } }],
+      "paths-ignore": ["path"],
+    };
 
-  const result = dbConfig.mergeDefaultSetupAndUserConfigs(
-    logger,
-    {
-      "default-setup": {
-        borg: [],
-        org: {
-          unknown: "foo",
-          "model-packs": [],
+    await withTmpDir(async (tmpDir) => {
+      setupActionsVars(tmpDir, tmpDir);
+      const result = dbConfig.mergeDefaultSetupAndUserConfigs(
+        logger,
+        {
+          "default-setup": {
+            borg: [],
+            org: {
+              unknown: "foo",
+              "model-packs": [],
+            },
+          } as unknown as dbConfig.DefaultSetupConfig,
+          "paths-ignore": ["other-path"],
         },
-      } as unknown as dbConfig.DefaultSetupConfig,
-      "paths-ignore": ["other-path"],
-    },
-    configFile,
-  );
+        configFile,
+      );
 
-  t.deepEqual(result, {
-    ...configFile,
-    "default-setup": { org: { "model-packs": [] } },
-  });
+      t.deepEqual(result, {
+        ...configFile,
+        "default-setup": { org: { "model-packs": [] } },
+      });
 
-  const expectedUnrecognisedKeys = [
-    ".default-setup.org.unknown",
-    ".default-setup.borg",
-    ".paths-ignore",
-  ].join(", ");
-  checkExpectedLogMessages(t, logger.messages, [
-    `Unrecognised keys in Default Setup configuration: ${expectedUnrecognisedKeys}`,
-  ]);
-});
+      const expectedUnrecognisedKeys = [
+        ".default-setup.org.unknown",
+        ".default-setup.borg",
+        ".paths-ignore",
+      ].join(", ");
+      checkExpectedLogMessages(t, logger.messages, [
+        `Unrecognised keys in Default Setup configuration: ${expectedUnrecognisedKeys}`,
+      ]);
+    });
+  },
+);
 
-test("mergeDefaultSetupAndUserConfigs - warns about invalid keys from Default Setup", async (t) => {
-  const logger = new RecordingLogger();
-  const configFile: dbConfig.UserConfig = {};
+test.serial(
+  "mergeDefaultSetupAndUserConfigs - warns about invalid keys from Default Setup",
+  async (t) => {
+    const logger = new RecordingLogger();
+    const configFile: dbConfig.UserConfig = {};
 
-  const result = dbConfig.mergeDefaultSetupAndUserConfigs(
-    logger,
-    {
-      "default-setup": {
-        org: {
-          "model-packs": [123],
+    await withTmpDir(async (tmpDir) => {
+      setupActionsVars(tmpDir, tmpDir);
+
+      const result = dbConfig.mergeDefaultSetupAndUserConfigs(
+        logger,
+        {
+          "default-setup": {
+            org: {
+              "model-packs": [123],
+            },
+          } as unknown as dbConfig.DefaultSetupConfig,
         },
-      } as unknown as dbConfig.DefaultSetupConfig,
-    },
-    configFile,
-  );
+        configFile,
+      );
 
-  t.deepEqual(result, {
-    ...configFile,
-    "default-setup": { org: { "model-packs": [123] } },
-  });
+      t.deepEqual(result, {
+        ...configFile,
+        "default-setup": { org: { "model-packs": [123] } },
+      });
 
-  const expectedInvalidKeys = [".default-setup.org.model-packs[0]"].join(", ");
-  checkExpectedLogMessages(t, logger.messages, [
-    `Invalid keys in Default Setup configuration: ${expectedInvalidKeys}`,
-  ]);
-});
+      const expectedInvalidKeys = [".default-setup.org.model-packs[0]"].join(
+        ", ",
+      );
+      checkExpectedLogMessages(t, logger.messages, [
+        `Invalid keys in Default Setup configuration: ${expectedInvalidKeys}`,
+      ]);
+    });
+  },
+);
 
 /** Parses `contents` as a configuration without validating it. */
 function parseUnvalidatedConfig(contents: string): dbConfig.UserConfig {

src/diagnostics.test.ts

@@ -0,0 +1,111 @@
+import * as fs from "fs";
+import path from "path";
+
+import test, { type ExecutionContext } from "ava";
+
+import * as diagnostics from "./diagnostics";
+import { BuiltInLanguage } from "./languages";
+import {
+  checkExpectedLogMessages,
+  createTestConfig,
+  initAllState,
+  RecordingLogger,
+  setupTests,
+} from "./testing-utils";
+import { withTmpDir } from "./util";
+
+setupTests(test);
+
+const id = "codeql-action/test-diagnostic";
+const name = "Test title";
+
+test("makeDiagnostic - adds expected properties", (t) => {
+  const diagnostic = diagnostics.makeDiagnostic(id, name, {});
+
+  t.truthy(diagnostic.timestamp);
+  t.is(diagnostic.source.id, id);
+  t.is(diagnostic.source.name, name);
+});
+
+/**
+ * Diagnostic files that are written to disk don't have deterministic names.
+ * This function recursively finds all `.json` files at {@link rootPath},
+ * asserts that there is exactly one, and that it matches {@link expected}.
+ *
+ * @param rootPath The path to start searching from.
+ */
+function assertSingleDiagnosticIn(
+  t: ExecutionContext<unknown>,
+  rootPath: string,
+  expected: diagnostics.DiagnosticMessage,
+) {
+  // Find all `.json` files in `rootPath` and check that there is exactly one.
+  const filePaths = fs
+    .readdirSync(rootPath, {
+      recursive: true,
+      encoding: "utf-8",
+    })
+    .filter((p) => p.endsWith(".json"));
+
+  t.is(filePaths.length, 1);
+
+  // Check that the written diagnostic matches the one we added.
+  const contents = fs.readFileSync(
+    path.resolve(rootPath, filePaths[0]),
+    "utf-8",
+  );
+
+  const readDiagnostic = JSON.parse(contents) as diagnostics.DiagnosticMessage;
+  t.deepEqual(readDiagnostic, expected);
+}
+
+test("addDiagnostic writes temporary diagnostics and flushDiagnostics moves them to the database", async (t) =>
+  withTmpDir(async (tmpDir) => {
+    const logger = new RecordingLogger();
+
+    const tempDiagDir = path.resolve(tmpDir, "temp");
+    const testDbDir = path.resolve(tmpDir, "test", "database");
+    const diagnostic = diagnostics.makeDiagnostic(id, name, {});
+
+    // Write the diagnostic. The path provided for `dbLocation` doesn't exist,
+    // so the diagnostic will be written to a temporary location.
+    t.notThrows(() => {
+      diagnostics.addDiagnostic(
+        createTestConfig({ tempDir: tempDiagDir, dbLocation: testDbDir }),
+        BuiltInLanguage.actions,
+        diagnostic,
+        logger,
+      );
+    });
+
+    // Check that `tempDiagDir` was created.
+    t.true(fs.existsSync(tempDiagDir));
+
+    // Check that the diagnostic exists under `tempDiagDir`.
+    assertSingleDiagnosticIn(t, tempDiagDir, diagnostic);
+
+    // Flush the diagnostics. This will cause the diagnostic we wrote previously to
+    // be moved to a location under `dbLocation`.
+    t.notThrows(() => {
+      diagnostics.flushDiagnostics(
+        initAllState({ logger }),
+        createTestConfig({
+          tempDir: tempDiagDir,
+          dbLocation: testDbDir,
+        }),
+      );
+    });
+
+    // Check that `testDbDir` was created.
+    t.true(fs.existsSync(testDbDir));
+
+    // Check that the diagnostic exists under `testDbDir`.
+    assertSingleDiagnosticIn(t, testDbDir, diagnostic);
+
+    // Check that we logged that the database directory doesn't exist yet
+    // and that we moved one diagnostic.
+    checkExpectedLogMessages(t, logger.messages, [
+      `but the database at ${path.join(testDbDir, "actions")} does not exist yet`,
+      "Moving 1 diagnostic(s) to their respective databases.",
+    ]);
+  }));

src/diagnostics.ts

@@ -1,10 +1,16 @@
-import { existsSync, mkdirSync, writeFileSync } from "fs";
+import * as fs from "fs";
 import path from "path";
 
+import type { ActionState } from "./action-common";
+import { getTemporaryDirectory } from "./actions-util";
 import type { Config } from "./config-utils";
 import { Language } from "./languages";
-import { getActionsLogger } from "./logging";
-import { getCodeQLDatabasePath } from "./util";
+import { getActionsLogger, type Logger } from "./logging";
+import {
+  getCodeQLDatabasePath,
+  getErrorMessage,
+  isErrnoException,
+} from "./util";
 
 /**
  * Known tags for diagnostics. There is currently only "internal-error",
@@ -84,22 +90,16 @@ export type DiagnosticMessage = DiagnosticMessageOptions & {
   source: DiagnosticSource;
 };
 
-/** Represents a diagnostic message that has not yet been written to the database. */
-interface UnwrittenDiagnostic {
-  /** The diagnostic message that has not yet been written. */
-  diagnostic: DiagnosticMessage;
-  /** The language the diagnostic is for. */
-  language: Language;
+/** Represents a diagnostic message that has not yet been saved to the database. */
+interface TemporaryDiagnostic {
+  /** The path to which the diagnostic has temporarily been written to. */
+  path: string;
+  /** The language the diagnostic is for, if any. */
+  language?: Language;
 }
 
-/** A list of diagnostics which have not yet been written to disk. */
-let unwrittenDiagnostics: UnwrittenDiagnostic[] = [];
-
-/**
- * A list of diagnostics which have not yet been written to disk,
- * and where the language does not matter.
- */
-let unwrittenDefaultLanguageDiagnostics: DiagnosticMessage[] = [];
+/** A list of diagnostics which have not yet been saved to the database. */
+let temporaryDiagnostics: TemporaryDiagnostic[] = [];
 
 /**
  * Counter used to generate a unique suffix for each diagnostic filename, so that
@@ -128,6 +128,29 @@ export function makeDiagnostic(
   };
 }
 
+/**
+ * Gets a path relative to {@link tmpDir} where diagnostics for {@link language}
+ * can temporarily be stored at before the database is initialised. If {@link language}
+ * is `undefined`, then the common base path for all languages relative to {@link tmpDir}
+ * is returned.
+ */
+export function getTempDiagnosticPath(tmpDir: string, language?: Language) {
+  const root = path.join(tmpDir, "codeql-action-temp-diagnostics");
+  return getDiagnosticsPath(root, language);
+}
+
+/**
+ * Gets a path relative to {@link basePath} where diagnostics for {@link language}
+ * should be stored at. If {@link language} is `undefined`, then {@link basePath}
+ * is returned instead.
+ */
+export function getDiagnosticsPath(basePath: string, language?: Language) {
+  if (language !== undefined) {
+    return path.resolve(basePath, language, "diagnostic", "codeql-action");
+  }
+  return basePath;
+}
+
 /**
  * Adds the given diagnostic to the database. If the database does not yet exist,
  * the diagnostic will be written to it once it has been created.
@@ -140,22 +163,23 @@ export function addDiagnostic(
   config: Config,
   language: Language,
   diagnostic: DiagnosticMessage,
+  logger: Logger = getActionsLogger(),
 ) {
-  const logger = getActionsLogger();
   const databasePath = language
     ? getCodeQLDatabasePath(config, language)
     : config.dbLocation;
 
   // Check that the database exists before writing to it. If the database does not yet exist,
   // store the diagnostic in memory and write it later.
-  if (existsSync(databasePath)) {
-    writeDiagnostic(config, language, diagnostic);
+  if (fs.existsSync(databasePath)) {
+    writeDiagnostic({ logger }, config, language, diagnostic);
   } else {
     logger.debug(
       `Writing a diagnostic for ${language}, but the database at ${databasePath} does not exist yet.`,
     );
 
-    unwrittenDiagnostics.push({ diagnostic, language });
+    // Write the diagnostic to a temporary location.
+    writeTemporaryDiagnosticFile(logger, config.tempDir, language, diagnostic);
   }
 }
 
@@ -163,6 +187,7 @@ export function addDiagnostic(
 export function addNoLanguageDiagnostic(
   config: Config | undefined,
   diagnostic: DiagnosticMessage,
+  logger: Logger = getActionsLogger(),
 ) {
   if (config !== undefined) {
     addDiagnostic(
@@ -173,10 +198,82 @@ export function addNoLanguageDiagnostic(
       diagnostic,
     );
   } else {
-    unwrittenDefaultLanguageDiagnostics.push(diagnostic);
+    logger.debug(
+      `Writing a diagnostic not associated with any language, but we have no configuration to tell us where the database is yet.`,
+    );
+
+    // Write the diagnostic to a temporary location.
+    writeTemporaryDiagnosticFile(
+      logger,
+      getTemporaryDirectory(),
+      undefined,
+      diagnostic,
+    );
   }
 }
 
+/**
+ * Writes {@link diagnostic} for an optional {@link language} to a subdirectory of {@link tempDir}
+ * and tracks it as a diagnostic that was written to a temporary location.
+ */
+function writeTemporaryDiagnosticFile(
+  logger: Logger,
+  tempDir: string,
+  language: Language | undefined,
+  diagnostic: DiagnosticMessage,
+) {
+  const tempDiagnosticsPath = getTempDiagnosticPath(tempDir, language);
+
+  try {
+    // Write the diagnostic to a temporary location.
+    const diagnosticPath = writeDiagnosticFile(tempDiagnosticsPath, diagnostic);
+
+    // Track the temporary file.
+    temporaryDiagnostics.push({ path: diagnosticPath, language });
+  } catch (err) {
+    logger.warning(
+      `Unable to write temporary diagnostic to '${tempDiagnosticsPath}': ${getErrorMessage(err)}\n\n${JSON.stringify(diagnostic)}`,
+    );
+  }
+}
+
+/**
+ * Writes {@link diagnostic} to a file in {@link diagnosticsPath}.
+ */
+function writeDiagnosticFile(
+  diagnosticsPath: string,
+  diagnostic: DiagnosticMessage,
+): string {
+  // Create the directory if it doesn't exist yet.
+  fs.mkdirSync(diagnosticsPath, { recursive: true });
+
+  // Include a monotonically increasing suffix to avoid filename collisions
+  // between diagnostics produced within the same millisecond.
+  const uniqueSuffix = (diagnosticCounter++).toString();
+  // We should only need to remove colons, but to be defensive, only allow a restricted set of
+  // characters.
+  const sanitizedTimestamp = diagnostic.timestamp.replace(
+    /[^a-zA-Z0-9.-]/g,
+    "",
+  );
+  const jsonPath = path.resolve(
+    diagnosticsPath,
+    `codeql-action-${sanitizedTimestamp}-${uniqueSuffix}.json`,
+  );
+
+  fs.writeFileSync(jsonPath, JSON.stringify(diagnostic));
+
+  return jsonPath;
+}
+
+/** Gets the path where diagnostics for {@link language} should be stored in the database. */
+export function getDatabaseDiagnosticsPath(
+  config: Config,
+  language: Language | undefined,
+) {
+  return getDiagnosticsPath(config.dbLocation, language);
+}
+
 /**
  * Writes the given diagnostic to the database.
  *
@@ -185,78 +282,85 @@ export function addNoLanguageDiagnostic(
  * @param diagnostic The diagnostic message to add to the database.
  */
 function writeDiagnostic(
+  action: ActionState<["Logger"]>,
   config: Config,
   language: Language | undefined,
   diagnostic: DiagnosticMessage,
 ) {
-  const logger = getActionsLogger();
-  const databasePath = language
-    ? getCodeQLDatabasePath(config, language)
-    : config.dbLocation;
-  const diagnosticsPath = path.resolve(
-    databasePath,
-    "diagnostic",
-    "codeql-action",
-  );
+  const diagnosticsPath = getDatabaseDiagnosticsPath(config, language);
 
   try {
-    // Create the directory if it doesn't exist yet.
-    mkdirSync(diagnosticsPath, { recursive: true });
-
-    // Include a monotonically increasing suffix to avoid filename collisions
-    // between diagnostics produced within the same millisecond.
-    const uniqueSuffix = (diagnosticCounter++).toString();
-    // We should only need to remove colons, but to be defensive, only allow a restricted set of
-    // characters.
-    const sanitizedTimestamp = diagnostic.timestamp.replace(
-      /[^a-zA-Z0-9.-]/g,
-      "",
-    );
-    const jsonPath = path.resolve(
-      diagnosticsPath,
-      `codeql-action-${sanitizedTimestamp}-${uniqueSuffix}.json`,
-    );
-
-    writeFileSync(jsonPath, JSON.stringify(diagnostic));
+    writeDiagnosticFile(diagnosticsPath, diagnostic);
   } catch (err) {
-    logger.warning(`Unable to write diagnostic message to database: ${err}`);
-    logger.debug(JSON.stringify(diagnostic));
+    action.logger.warning(
+      `Unable to write diagnostic message to database: ${err}`,
+    );
+    action.logger.debug(JSON.stringify(diagnostic));
   }
 }
 
-/** Report if there are unwritten diagnostics and write them to the log. */
-export function logUnwrittenDiagnostics() {
-  const logger = getActionsLogger();
-  const num = unwrittenDiagnostics.length;
+/** Report if there are temporary diagnostics and write them to the log. */
+export function logTemporaryDiagnostics(action: ActionState<["Logger"]>) {
+  const num = temporaryDiagnostics.length;
+
   if (num > 0) {
-    logger.warning(
+    action.logger.warning(
       `${num} diagnostic(s) could not be written to the database and will not appear on the Tool Status Page.`,
     );
 
-    for (const unwritten of unwrittenDiagnostics) {
-      logger.debug(JSON.stringify(unwritten.diagnostic));
+    for (const temporary of temporaryDiagnostics) {
+      action.logger.debug(fs.readFileSync(temporary.path, "utf-8"));
     }
   }
 }
 
-/** Writes all unwritten diagnostics to disk. */
-export function flushDiagnostics(config: Config) {
-  const logger = getActionsLogger();
+/** Relocates all temporary diagnostics to the respective databases. */
+export function flushDiagnostics(
+  action: ActionState<["Logger"]>,
+  config: Config,
+) {
+  const diagnosticsCount = temporaryDiagnostics.length;
+  action.logger.debug(
+    `Moving ${diagnosticsCount} diagnostic(s) to their respective databases.`,
+  );
 
-  const diagnosticsCount =
-    unwrittenDiagnostics.length + unwrittenDefaultLanguageDiagnostics.length;
-  logger.debug(`Writing ${diagnosticsCount} diagnostic(s) to database.`);
+  const failedFlushDiagnostics: TemporaryDiagnostic[] = [];
+  for (const temporary of temporaryDiagnostics) {
+    // If `temporary.language` is `undefined`, then we didn't have a `config` at the time that
+    // `addNoLanguageDiagnostic` was called. In that case, we arbitrarily choose the first
+    // configured language here like we would have done in `addNoLanguageDiagnostic`.
+    const directory = getDatabaseDiagnosticsPath(
+      config,
+      temporary.language ?? config.languages[0],
+    );
+    const filename = path.basename(temporary.path);
+    const destination = path.join(directory, filename);
 
-  for (const unwritten of unwrittenDiagnostics) {
-    writeDiagnostic(config, unwritten.language, unwritten.diagnostic);
-  }
-  for (const unwritten of unwrittenDefaultLanguageDiagnostics) {
-    addNoLanguageDiagnostic(config, unwritten);
+    try {
+      fs.mkdirSync(directory, { recursive: true });
+      try {
+        fs.renameSync(temporary.path, destination);
+      } catch (renameErr: unknown) {
+        // Rethrow the error if it isn't an EXDEV one.
+        if (!isErrnoException(renameErr) || renameErr.code !== "EXDEV") {
+          throw renameErr;
+        }
+
+        // The error was an EXDEV one: try to copy the file and delete the temporary one.
+        fs.copyFileSync(temporary.path, destination);
+        fs.unlinkSync(temporary.path);
+      }
+    } catch (err) {
+      action.logger.warning(
+        `Failed to relocate '${temporary.path}' to '${destination}': ${getErrorMessage(err)}`,
+      );
+
+      failedFlushDiagnostics.push(temporary);
+    }
   }
 
-  // Reset the unwritten diagnostics arrays.
-  unwrittenDiagnostics = [];
-  unwrittenDefaultLanguageDiagnostics = [];
+  // Reset the temporary diagnostics arrays.
+  temporaryDiagnostics = failedFlushDiagnostics;
 }
 
 /**

src/init-action-post-helper.test.ts

@@ -1,3 +1,6 @@
+import * as fs from "fs";
+import * as path from "path";
+
 import * as core from "@actions/core";
 import test, { ExecutionContext } from "ava";
 import * as sinon from "sinon";
@@ -8,6 +11,7 @@ import * as apiClient from "./api-client";
 import * as codeql from "./codeql";
 import * as configUtils from "./config-utils";
 import * as debugArtifacts from "./debug-artifacts";
+import { getTempDiagnosticPath } from "./diagnostics";
 import { EnvVar } from "./environment";
 import { Feature } from "./feature-flags";
 import * as initActionPostHelper from "./init-action-post-helper";
@@ -35,6 +39,47 @@ const NUM_BYTES_PER_GIB = 1024 * 1024 * 1024;
 
 setupTests(test);
 
+test.serial("generateFailedSarif picks up unwritten diagnostics", async (t) =>
+  util.withTmpDir(async (tmpDir) => {
+    const logger = new RecordingLogger();
+
+    const tempDiagDir = getTempDiagnosticPath(tmpDir);
+    fs.mkdirSync(tempDiagDir);
+
+    const codeqlObject = await codeql.getCodeQLForTesting();
+    const diagnosticsExport = sinon
+      .stub(codeqlObject, "diagnosticsExport")
+      .resolves();
+
+    sinon.stub(codeql, "getCodeQL").resolves(codeqlObject);
+
+    const sarifFile = path.join(tmpDir, "failed.sarif");
+    const result = await initActionPostHelper.generateFailedSarif(
+      logger,
+      createFeatures([]),
+      createTestConfig({
+        tempDir: tmpDir,
+        // Explicitly undefined to ensure we get the "no database" path.
+        dbLocation: undefined,
+      }),
+      "category",
+      "",
+      sarifFile,
+    );
+
+    t.truthy(result);
+    t.is(diagnosticsExport.callCount, 1);
+    t.true(
+      diagnosticsExport.calledOnceWith(
+        sarifFile,
+        "category",
+        sinon.match.any,
+        tempDiagDir,
+      ),
+    );
+  }),
+);
+
 test.serial("init-post action with debug mode off", async (t) => {
   return await util.withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);

src/init-action-post-helper.ts

@@ -24,7 +24,8 @@ import {
   type UploadDebugArtifactsConfig,
 } from "./debug-artifacts";
 import * as dependencyCaching from "./dependency-caching";
-import { EnvVar, ReadOnlyEnv } from "./environment";
+import { getTempDiagnosticPath } from "./diagnostics";
+import { EnvVar, isInTestMode, ReadOnlyEnv } from "./environment";
 import { Feature, FeatureEnablement } from "./feature-flags";
 import { Logger } from "./logging";
 import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
@@ -168,7 +169,16 @@ async function prepareFailedSarif(
   }
 }
 
-async function generateFailedSarif(
+/**
+ * Generates a SARIF file for a failed run, i.e. an analysis where no SARIF was produced as a result of the analysis.
+ *
+ * Not intended to be called directly, other than by `prepareFailedSarif`.
+ *
+ * @param category The analysis category to include in the SARIF.
+ * @param checkoutPath The path where the repository was checked out at.
+ * @param sarifFile The SARIF file to produce.
+ */
+export async function generateFailedSarif(
   logger: Logger,
   features: FeatureEnablement,
   config: ActionConfig<"codeQLCmd" | "tempDir">,
@@ -189,7 +199,19 @@ async function generateFailedSarif(
     databasePath === undefined ||
     !(await features.getValue(Feature.ExportDiagnosticsEnabled, codeql))
   ) {
-    await codeql.diagnosticsExport(sarifFile, category, config);
+    // If we don't have a database path, then we may have diagnostics that were written to a
+    // temporary location. Get the path that they would have been written to and check whether
+    // it exists. If so, pass it on to the CLI. The CLI will not complain if we give it a
+    // path that doesn't exist, so we are just being extra defensive here.
+    let diagnosticDir: string | undefined = getTempDiagnosticPath(
+      config.tempDir,
+    );
+
+    if (!fs.existsSync(diagnosticDir)) {
+      diagnosticDir = undefined;
+    }
+
+    await codeql.diagnosticsExport(sarifFile, category, config, diagnosticDir);
   } else {
     // We call 'database export-diagnostics' to find any per-database diagnostics.
     await codeql.databaseExportDiagnostics(databasePath, sarifFile, category);
@@ -231,6 +253,19 @@ async function maybeUploadFailedSarif(
     logger,
     { isUnsuccessfulExecution: true },
   );
+
+  // If we are in test mode, `uploadFiles` will not have uploaded the failed SARIF file
+  // and instead will have written it to disk. Upload it as a workflow artifact so that
+  // we can inspect it in validation jobs.
+  if (uploadResult.sarifID && isInTestMode()) {
+    await uploadSarifArtifact(logger, "test-mode-sarif", {
+      // We defaulted to CodeScanning in the `uploadFiles` call above.
+      sarifFile: uploadLib.getSavedPayloadPath(CodeScanning.kind),
+      category: failedSarif.category,
+      checkoutPath: failedSarif.checkoutPath,
+    });
+  }
+
   return uploadResult
     ? { ...uploadResult.statusReport, sarifID: uploadResult.sarifID }
     : {};
@@ -249,7 +284,15 @@ async function maybeUploadFailedSarifArtifact(
   }
 
   const failedSarif = failedSarifResult.value;
+  return uploadSarifArtifact(logger, "sarif-artifact", failedSarif);
+}
 
+/** Uploads a failed SARIF file as workflow artifact. */
+async function uploadSarifArtifact(
+  logger: Logger,
+  prefix: string,
+  failedSarif: FailedSarifInfo,
+): Promise<UploadFailedSarifResult> {
   logger.info(
     `Uploading failed SARIF file ${failedSarif.sarifFile} as artifact`,
   );
@@ -258,7 +301,7 @@ async function maybeUploadFailedSarifArtifact(
   const client = await getArtifactUploaderClient(logger, gitHubVersion.type);
 
   const suffix = getArtifactSuffix(actionsUtil.getOptionalInput("matrix"));
-  const name = sanitizeArtifactName(`sarif-artifact-${suffix}`);
+  const name = sanitizeArtifactName(`${prefix}-${suffix}`);
   await client.uploadArtifact(
     name,
     [path.normalize(failedSarif.sarifFile)],

src/init-action-post.ts

@@ -76,6 +76,9 @@ async function run(action: ActionState<["Base", "Logger", "Env"]>) {
     // Try to load the "failed" configuration file instead which may have been written by the `init` step
     // with a minimal configuration for this post-action.
     if (config === undefined) {
+      logger.debug(
+        "Will try to load the configuration file from an aborted 'init' step, since there is no regular one.",
+      );
       config = getAbortedInitConfig(action, tmpDir);
     }
 

src/init-action.ts

@@ -40,7 +40,7 @@ import {
   addDiagnostic,
   addNoLanguageDiagnostic,
   flushDiagnostics,
-  logUnwrittenDiagnostics,
+  logTemporaryDiagnostics,
   makeDiagnostic,
   makeTelemetryDiagnostic,
 } from "./diagnostics";
@@ -484,6 +484,7 @@ async function run(
     if (statusReportBase !== undefined) {
       await sendStatusReport(statusReportBase);
     }
+    logTemporaryDiagnostics(actionState);
     return;
   }
 
@@ -766,9 +767,9 @@ async function run(
       );
     }
 
-    // Write diagnostics to the database that we previously stored in memory because the database
-    // did not exist until now.
-    flushDiagnostics(config);
+    // Move diagnostics to the database that we previously kept in a temporary location
+    // because the database did not exist until now
+    flushDiagnostics(actionState, config);
 
     // We save the config here instead of at the end of `initConfig` because we
     // may have updated the config returned from `initConfig`, e.g. to revert to
@@ -797,7 +798,7 @@ async function run(
     );
     return;
   } finally {
-    logUnwrittenDiagnostics();
+    logTemporaryDiagnostics(actionState);
   }
   await sendCompletedStatusReport(
     startedAt,

src/setup-codeql.test.ts

@@ -1647,16 +1647,20 @@ test.serial(
       },
     ]);
 
-    const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
-      overlayMatchEnabledVersions,
-      ["javascript"],
-      createFeatures([Feature.OverlayAnalysisMatchCodeqlVersion]),
-      getRunnerLogger(true),
-    );
-    t.deepEqual(result, [
-      { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" },
-      { cliVersion: "2.20.0", tagName: "codeql-bundle-v2.20.0" },
-    ]);
+    await withTmpDir(async (tmpDir) => {
+      setupActionsVars(tmpDir, tmpDir);
+      const result =
+        await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
+          overlayMatchEnabledVersions,
+          ["javascript"],
+          createFeatures([Feature.OverlayAnalysisMatchCodeqlVersion]),
+          getRunnerLogger(true),
+        );
+      t.deepEqual(result, [
+        { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" },
+        { cliVersion: "2.20.0", tagName: "codeql-bundle-v2.20.0" },
+      ]);
+    });
   },
 );
 
@@ -1773,21 +1777,25 @@ test.serial(
       },
     ]);
 
-    const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
-      overlayMatchEnabledVersions,
-      ["javascript"],
-      createFeatures([Feature.OverlayAnalysisMatchCodeqlVersionDryRun]),
-      getRunnerLogger(true),
-    );
-    t.deepEqual(
-      result,
-      [],
-      "Dry-run should return an empty list so the caller falls back.",
-    );
-    t.assert(
-      listStub.calledOnce,
-      "Dry-run should still list Actions caches to populate the diagnostic.",
-    );
+    await withTmpDir(async (tmpDir) => {
+      setupActionsVars(tmpDir, tmpDir);
+      const result =
+        await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
+          overlayMatchEnabledVersions,
+          ["javascript"],
+          createFeatures([Feature.OverlayAnalysisMatchCodeqlVersionDryRun]),
+          getRunnerLogger(true),
+        );
+      t.deepEqual(
+        result,
+        [],
+        "Dry-run should return an empty list so the caller falls back.",
+      );
+      t.assert(
+        listStub.calledOnce,
+        "Dry-run should still list Actions caches to populate the diagnostic.",
+      );
+    });
   },
 );
 
@@ -1801,18 +1809,22 @@ test.serial(
       },
     ]);
 
-    const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
-      overlayMatchEnabledVersions,
-      ["javascript"],
-      createFeatures([
-        Feature.OverlayAnalysisMatchCodeqlVersion,
-        Feature.OverlayAnalysisMatchCodeqlVersionDryRun,
-      ]),
-      getRunnerLogger(true),
-    );
-    t.deepEqual(result, [
-      { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" },
-    ]);
+    await withTmpDir(async (tmpDir) => {
+      setupActionsVars(tmpDir, tmpDir);
+      const result =
+        await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
+          overlayMatchEnabledVersions,
+          ["javascript"],
+          createFeatures([
+            Feature.OverlayAnalysisMatchCodeqlVersion,
+            Feature.OverlayAnalysisMatchCodeqlVersionDryRun,
+          ]),
+          getRunnerLogger(true),
+        );
+      t.deepEqual(result, [
+        { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" },
+      ]);
+    });
   },
 );
 

src/upload-lib.ts

@@ -227,6 +227,14 @@ function getAutomationID(
   return api.computeAutomationID(analysis_key, environment);
 }
 
+/** Gets the path to a saved payload file for {@link analysisKind}. */
+export function getSavedPayloadPath(analysisKind: analyses.AnalysisKind) {
+  return path.join(
+    actionsUtil.getTemporaryDirectory(),
+    `payload-${analysisKind}.json`,
+  );
+}
+
 /**
  * Upload the given payload.
  * If the request fails then this will retry a small number of times.
@@ -241,10 +249,7 @@ export async function uploadPayload(
   logger.info("Uploading results");
 
   if (util.shouldSkipSarifUpload()) {
-    const payloadSaveFile = path.join(
-      actionsUtil.getTemporaryDirectory(),
-      `payload-${analysis.kind}.json`,
-    );
+    const payloadSaveFile = getSavedPayloadPath(analysis.kind);
     logger.info(
       `SARIF upload disabled by an environment variable. Saving to ${payloadSaveFile}`,
     );

src/util.ts

@@ -924,6 +924,21 @@ export function getErrorMessage(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }
 
+/**
+ * Approximates whether {@link error} is like a {@link NodeJS.ErrnoException}.
+ * We only typically care about the `code` property, so that's what we check for here.
+ */
+export function isErrnoException(
+  error: unknown,
+): error is NodeJS.ErrnoException {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    typeof error.code === "string"
+  );
+}
+
 export function prettyPrintPack(pack: Pack) {
   return `${pack.name}${pack.version ? `@${pack.version}` : ""}${
     pack.path ? `:${pack.path}` : ""