Add e2e test

.github/workflows/__diagnostics-without-db.yml

@@ -0,0 +1,138 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pr-checks/sync.sh
+# to regenerate this file.
+
+name: PR Check - Diagnostic export without database
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+      - main
+      - releases/v*
+  pull_request: {}
+  merge_group:
+    types:
+      - checks_requested
+  schedule:
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+    inputs: {}
+  workflow_call:
+    inputs: {}
+defaults:
+  run:
+    shell: bash
+concurrency:
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || false }}
+  group: diagnostics-without-db-${{github.ref}}
+jobs:
+  diagnostics-without-db:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            version: linked
+    name: Diagnostic export without database
+    if: github.triggering_actor != 'dependabot[bot]'
+    permissions:
+      contents: read
+      security-events: read
+    timeout-minutes: 45
+    runs-on: ${{ matrix.os }}
+    steps:
+      # We deliberately need to make this step fail before the database is initialised, but after
+      # some diagnostic has been produced. The simplest way to do that right now is to force the
+      # use of a nightly release and then fail because we are attempting to analyse Swift on
+      # a non-macOS runner.
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install Node.js
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version: 20.x
+          cache: npm
+      - name: Install newer npm
+        run: npm install -g npm@11.19.1
+      - name: Install dependencies
+        run: npm ci
+      - name: Prepare test
+        id: prepare-test
+        uses: ./.github/actions/prepare-test
+        with:
+          version: ${{ matrix.version }}
+          use-all-platform-bundle: 'false'
+          setup-kotlin: 'true'
+      - name: Initialise failing analysis
+        id: init
+        uses: ./../action/init
+        continue-on-error: true
+        env:
+          CODEQL_ACTION_FORCE_NIGHTLY: true
+        with:
+          languages: swift
+          tools: ${{ steps.prepare-test.outputs.tools-url }}
+
+      - name: Analyze
+        uses: ./analyze
+        # In a real workflow, this step wouldn't run. Since we used `continue-on-error`
+        # above, we manually disable it with an `if` condition.
+        if: false
+        with:
+          category: '/test-codeql-version:${{ matrix.version }}'
+
+      - name: Check diagnostics appear on disk
+        env:
+          SARIF_PATH: '${{ runner.temp }}/codeql-action-temp-diagnostics/'
+        run: |
+          cd ../action
+          npx tsx ./pr-checks/diagnostics-without-db.ts
+    env:
+      CODEQL_ACTION_SKIP_GENERATE_FAILED_SARIF: false
+      CODEQL_ACTION_SKIP_SARIF_UPLOAD: true
+      CODEQL_ACTION_TESTING_ENVIRONMENT: codeql-action-pr-checks
+      CODEQL_ACTION_TEST_MODE: true
+  diagnostics-without-db-validation:
+    if: github.triggering_actor != 'dependabot[bot]'
+    needs:
+      - diagnostics-without-db
+    permissions:
+      contents: read
+      security-events: read
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install Node.js
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version: 20.x
+          cache: npm
+      - name: Install newer npm
+        run: npm install -g npm@11.19.1
+      - name: Install dependencies
+        run: npm ci
+      - name: Download workflow artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: test-mode-sarif-*
+          path: sarif
+
+      - name: List artifact contents
+        run: ls -laR ./sarif
+
+      - name: Check diagnostics appear in SARIF
+        env:
+          SARIF_PATH: "./sarif/codeql-failed-run.sarif"
+          DIAGNOSTIC_ID: "codeql-action/forced-nightly-cli"
+          DIAGNOSTIC_NAME: "A nightly release of CodeQL was used"
+          EXPECTED_LOCATIONS: "0"
+          EXPECTED_MESSAGE: |
+            GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release.
+        run: npx tsx ./pr-checks/check-diagnostics.ts
+    env:
+      CODEQL_ACTION_TEST_MODE: true

pr-checks/check-diagnostics.ts

@@ -4,16 +4,32 @@ import * as core from "@actions/core";
 
 import * as sarif from "../src/sarif";
 
-function checkStatusPageNotification(n) {
-  const expectedMessage = "Plaintext message";
-  if (n.message.text !== expectedMessage) {
-    core.setFailed(
-      `Expected the status page diagnostic to have the message '${expectedMessage}', but found '${n.message.text}'.`,
-    );
+const DIAGNOSTIC_ID =
+  process.env["DIAGNOSTIC_ID"] ?? "lang/diagnostics/example";
+const DIAGNOSTIC_NAME = process.env["DIAGNOSTIC_NAME"] ?? "Diagnostic name";
+const EXPECTED_MESSAGE = process.env["EXPECTED_MESSAGE"] ?? "Plaintext message";
+const EXPECTED_LOCATIONS = Number.parseInt(
+  process.env["EXPECTED_LOCATIONS"] ?? "1",
+);
+
+function checkStatusPageNotification(n: sarif.Notification) {
+  const expectedMessage = EXPECTED_MESSAGE.trim();
+  if (!n.message.text?.includes(expectedMessage)) {
+    if (n.message.text !== undefined) {
+      core.setFailed(
+        `Expected the status page diagnostic to contain the message '${expectedMessage}', but found '${n.message.text.replaceAll("\n", "\\n")}'.`,
+      );
+    } else {
+      core.setFailed(
+        `Expected the status page diagnostic to contain the message '${expectedMessage}', but found nothing.`,
+      );
+    }
   }
-  if (n.locations.length !== 1) {
+
+  const locationCount = n.locations?.length ?? 0;
+  if (locationCount !== EXPECTED_LOCATIONS) {
     core.setFailed(
-      `Expected the status page diagnostic to have exactly 1 location, but found ${n.locations.length}.`,
+      `Expected the status page diagnostic to have exactly ${EXPECTED_LOCATIONS} location(s), but found ${locationCount}.`,
     );
   }
 }
@@ -40,8 +56,7 @@ if (toolExecutionNotifications === undefined) {
 
 const statusPageNotifications = toolExecutionNotifications.filter(
   (n) =>
-    n.descriptor?.id === "lang/diagnostics/example" &&
-    n.properties?.visibility?.statusPage,
+    n.descriptor?.id === DIAGNOSTIC_ID && n.properties?.visibility?.statusPage,
 );
 if (statusPageNotifications.length !== 1) {
   core.setFailed(
@@ -61,9 +76,9 @@ if (notifications === undefined) {
 
 const diagnosticNotification = notifications.filter(
   (n) =>
-    n.id === "lang/diagnostics/example" &&
-    n.name === "lang/diagnostics/example" &&
-    n.fullDescription?.text === "Diagnostic name",
+    n.id === DIAGNOSTIC_ID &&
+    n.name === DIAGNOSTIC_ID &&
+    n.fullDescription?.text === DIAGNOSTIC_NAME,
 );
 if (diagnosticNotification.length !== 1) {
   core.setFailed(

pr-checks/checks/diagnostics-without-db.yml

@@ -0,0 +1,67 @@
+name: "Diagnostic export without database"
+description: "Tests that temporary diagnostics are correctly included in the failed SARIF."
+versions:
+  # Overridden by the FF below.
+  - linked
+installNode: true
+
+env:
+  # Allow the failed SARIF file to be generated.
+  CODEQL_ACTION_SKIP_GENERATE_FAILED_SARIF: false
+  # But don't actually upload it.
+  CODEQL_ACTION_SKIP_SARIF_UPLOAD: true
+  # Mark telemetry for this workflow so it can be treated separately.
+  CODEQL_ACTION_TESTING_ENVIRONMENT: codeql-action-pr-checks
+
+steps:
+  # We deliberately need to make this step fail before the database is initialised, but after
+  # some diagnostic has been produced. The simplest way to do that right now is to force the
+  # use of a nightly release and then fail because we are attempting to analyse Swift on
+  # a non-macOS runner.
+  - name: Initialise failing analysis
+    id: init
+    uses: ./../action/init
+    continue-on-error: true
+    env:
+      CODEQL_ACTION_FORCE_NIGHTLY: true
+    with:
+      languages: swift
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+
+  - name: Analyze
+    uses: ./analyze
+    # In a real workflow, this step wouldn't run. Since we used `continue-on-error`
+    # above, we manually disable it with an `if` condition.
+    if: false
+    with:
+      category: "/test-codeql-version:${{ matrix.version }}"
+
+  - name: Check diagnostics appear on disk
+    env:
+      SARIF_PATH: "${{ runner.temp }}/codeql-action-temp-diagnostics/"
+    run: |
+      cd ../action
+      npx tsx ./pr-checks/diagnostics-without-db.ts
+validationJobs:
+  diagnostics-without-db-validation:
+    checkout: true
+    installNode: true
+    steps:
+      - name: Download workflow artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: test-mode-sarif-*
+          path: sarif
+
+      - name: List artifact contents
+        run: ls -laR ./sarif
+
+      - name: Check diagnostics appear in SARIF
+        env:
+          SARIF_PATH: "./sarif/codeql-failed-run.sarif"
+          DIAGNOSTIC_ID: "codeql-action/forced-nightly-cli"
+          DIAGNOSTIC_NAME: "A nightly release of CodeQL was used"
+          EXPECTED_LOCATIONS: "0"
+          EXPECTED_MESSAGE: |
+            GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release.
+        run: npx tsx ./pr-checks/check-diagnostics.ts

pr-checks/diagnostics-without-db.ts

@@ -0,0 +1,48 @@
+import * as fs from "fs";
+import * as path from "path";
+
+import * as core from "@actions/core";
+
+import type * as diagnostics from "../src/diagnostics";
+
+const DIAGNOSTIC_ID = "codeql-action/forced-nightly-cli";
+const sarifDirectory = process.env["SARIF_PATH"];
+
+if (sarifDirectory === undefined) {
+  throw new Error("SARIF_PATH is undefined.");
+}
+
+const paths = fs
+  .readdirSync(sarifDirectory, {
+    recursive: true,
+    encoding: "utf-8",
+  })
+  .filter((p) => p.endsWith(".json"));
+
+core.info(`Found ${paths.length} paths in '${sarifDirectory}'.`);
+core.info(`Searching for a diagnostic with id '${DIAGNOSTIC_ID}'...`);
+
+let found = false;
+for (const diagnosticPath of paths) {
+  if (path.extname(diagnosticPath) !== ".json") {
+    core.info(`Skipping ${diagnosticPath}.`);
+    continue;
+  }
+  core.info(`Found ${diagnosticPath}.`);
+
+  const contents = JSON.parse(
+    fs.readFileSync(path.resolve(sarifDirectory, diagnosticPath), "utf-8"),
+  ) as diagnostics.DiagnosticMessage;
+
+  if (contents.source.id === DIAGNOSTIC_ID) {
+    core.info(
+      `Found '${DIAGNOSTIC_ID}' in '${diagnosticPath}'. Skipping remaining files.`,
+    );
+    found = true;
+    break;
+  }
+}
+
+if (!found) {
+  throw new Error(`Didn't find '${DIAGNOSTIC_ID}' diagnostic.`);
+}

pr-checks/sync.ts

@@ -90,7 +90,7 @@ interface Specification extends JobSpecification {
   services?: any;
 
   /** Additional jobs to run after the main PR check job. */
-  validationJobs?: Record<string, JobSpecification>;
+  validationJobs?: Record<string, ValidationJobSpecification>;
 
   /** If set, this check is part of a named collection that gets its own caller workflow. */
   collection?: string;
@@ -122,6 +122,10 @@ interface JobSpecification {
   installYq?: boolean;
 }
 
+type ValidationJobSpecification = JobSpecification & {
+  checkout?: boolean;
+};
+
 /** Describes language/framework-specific steps and inputs. */
 interface LanguageSetup {
   specProperty: keyof JobSpecification;
@@ -462,6 +466,20 @@ function generateJobMatrix(
   return matrix;
 }
 
+/**
+ * Gets the `actions/checkout` step.
+ */
+function getCheckoutStep(): Step {
+  return {
+    name: "Check out repository",
+    uses: pinnedUses(
+      "actions/checkout",
+      "3d3c42e5aac5ba805825da76410c181273ba90b1",
+      "v7.0.1",
+    ),
+  };
+}
+
 /**
  * Retrieves setup steps and additional input definitions based on specific languages or frameworks
  * that are requested by the `checkSpecification`.
@@ -531,14 +549,7 @@ function generateJob(
 
   // Construct the workflow steps needed for this check.
   const steps: Step[] = [
-    {
-      name: "Check out repository",
-      uses: pinnedUses(
-        "actions/checkout",
-        "3d3c42e5aac5ba805825da76410c181273ba90b1",
-        "v7.0.1",
-      ),
-    },
+    getCheckoutStep(),
     ...setupInfo.steps,
     {
       name: "Prepare test",
@@ -609,13 +620,16 @@ function generateJob(
 /** Generates a validation job. */
 function generateValidationJob(
   specDocument: yaml.Document,
-  jobSpecification: JobSpecification,
+  jobSpecification: ValidationJobSpecification,
   checkName: string,
   name: string,
 ) {
   // Determine which languages or frameworks have to be installed.
   const { inputs, steps } = getSetupSteps(jobSpecification);
 
+  // Construct a checkout step if requested.
+  const checkoutStep = jobSpecification.checkout ? [getCheckoutStep()] : [];
+
   // Extract the sequence of steps from the YAML document to persist as much formatting as possible.
   const specSteps = specDocument.getIn([
     "validationJobs",
@@ -624,7 +638,7 @@ function generateValidationJob(
   ]) as yaml.YAMLSeq;
 
   // Add the generated steps in front of the ones from the specification.
-  specSteps.items.unshift(...steps);
+  specSteps.items.unshift(...checkoutStep, ...steps);
 
   const validationJob: Record<string, any> = {
     name: jobSpecification.name,
@@ -635,7 +649,7 @@ function generateValidationJob(
       "security-events": "read",
     },
     "timeout-minutes": 5,
-    "runs-on": "ubuntu-slim",
+    "runs-on": "ubuntu-latest",
     steps: specSteps,
   };