diff --git a/.github/workflows/__diagnostics-without-db.yml b/.github/workflows/__diagnostics-without-db.yml new file mode 100644 index 000000000..27200f181 --- /dev/null +++ b/.github/workflows/__diagnostics-without-db.yml @@ -0,0 +1,138 @@ +# Warning: This file is generated automatically, and should not be modified. +# Instead, please modify the template in the pr-checks directory and run: +# pr-checks/sync.sh +# to regenerate this file. + +name: PR Check - Diagnostic export without database +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GO111MODULE: auto +on: + push: + branches: + - main + - releases/v* + pull_request: {} + merge_group: + types: + - checks_requested + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + inputs: {} + workflow_call: + inputs: {} +defaults: + run: + shell: bash +concurrency: + cancel-in-progress: ${{ github.event_name == 'pull_request' || false }} + group: diagnostics-without-db-${{github.ref}} +jobs: + diagnostics-without-db: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + version: linked + name: Diagnostic export without database + if: github.triggering_actor != 'dependabot[bot]' + permissions: + contents: read + security-events: read + timeout-minutes: 45 + runs-on: ${{ matrix.os }} + steps: + # We deliberately need to make this step fail before the database is initialised, but after + # some diagnostic has been produced. The simplest way to do that right now is to force the + # use of a nightly release and then fail because we are attempting to analyse Swift on + # a non-macOS runner. + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 20.x + cache: npm + - name: Install newer npm + run: npm install -g npm@11.19.1 + - name: Install dependencies + run: npm ci + - name: Prepare test + id: prepare-test + uses: ./.github/actions/prepare-test + with: + version: ${{ matrix.version }} + use-all-platform-bundle: 'false' + setup-kotlin: 'true' + - name: Initialise failing analysis + id: init + uses: ./../action/init + continue-on-error: true + env: + CODEQL_ACTION_FORCE_NIGHTLY: true + with: + languages: swift + tools: ${{ steps.prepare-test.outputs.tools-url }} + + - name: Analyze + uses: ./analyze + # In a real workflow, this step wouldn't run. Since we used `continue-on-error` + # above, we manually disable it with an `if` condition. + if: false + with: + category: '/test-codeql-version:${{ matrix.version }}' + + - name: Check diagnostics appear on disk + env: + SARIF_PATH: '${{ runner.temp }}/codeql-action-temp-diagnostics/' + run: | + cd ../action + npx tsx ./pr-checks/diagnostics-without-db.ts + env: + CODEQL_ACTION_SKIP_GENERATE_FAILED_SARIF: false + CODEQL_ACTION_SKIP_SARIF_UPLOAD: true + CODEQL_ACTION_TESTING_ENVIRONMENT: codeql-action-pr-checks + CODEQL_ACTION_TEST_MODE: true + diagnostics-without-db-validation: + if: github.triggering_actor != 'dependabot[bot]' + needs: + - diagnostics-without-db + permissions: + contents: read + security-events: read + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 20.x + cache: npm + - name: Install newer npm + run: npm install -g npm@11.19.1 + - name: Install dependencies + run: npm ci + - name: Download workflow artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: test-mode-sarif-* + path: sarif + + - name: List artifact contents + run: ls -laR ./sarif + + - name: Check diagnostics appear in SARIF + env: + SARIF_PATH: "./sarif/codeql-failed-run.sarif" + DIAGNOSTIC_ID: "codeql-action/forced-nightly-cli" + DIAGNOSTIC_NAME: "A nightly release of CodeQL was used" + EXPECTED_LOCATIONS: "0" + EXPECTED_MESSAGE: | + GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release. + run: npx tsx ./pr-checks/check-diagnostics.ts + env: + CODEQL_ACTION_TEST_MODE: true diff --git a/pr-checks/check-diagnostics.ts b/pr-checks/check-diagnostics.ts index 0a35ac28b..6e9d0ba82 100644 --- a/pr-checks/check-diagnostics.ts +++ b/pr-checks/check-diagnostics.ts @@ -4,16 +4,32 @@ import * as core from "@actions/core"; import * as sarif from "../src/sarif"; -function checkStatusPageNotification(n) { - const expectedMessage = "Plaintext message"; - if (n.message.text !== expectedMessage) { - core.setFailed( - `Expected the status page diagnostic to have the message '${expectedMessage}', but found '${n.message.text}'.`, - ); +const DIAGNOSTIC_ID = + process.env["DIAGNOSTIC_ID"] ?? "lang/diagnostics/example"; +const DIAGNOSTIC_NAME = process.env["DIAGNOSTIC_NAME"] ?? "Diagnostic name"; +const EXPECTED_MESSAGE = process.env["EXPECTED_MESSAGE"] ?? "Plaintext message"; +const EXPECTED_LOCATIONS = Number.parseInt( + process.env["EXPECTED_LOCATIONS"] ?? "1", +); + +function checkStatusPageNotification(n: sarif.Notification) { + const expectedMessage = EXPECTED_MESSAGE.trim(); + if (!n.message.text?.includes(expectedMessage)) { + if (n.message.text !== undefined) { + core.setFailed( + `Expected the status page diagnostic to contain the message '${expectedMessage}', but found '${n.message.text.replaceAll("\n", "\\n")}'.`, + ); + } else { + core.setFailed( + `Expected the status page diagnostic to contain the message '${expectedMessage}', but found nothing.`, + ); + } } - if (n.locations.length !== 1) { + + const locationCount = n.locations?.length ?? 0; + if (locationCount !== EXPECTED_LOCATIONS) { core.setFailed( - `Expected the status page diagnostic to have exactly 1 location, but found ${n.locations.length}.`, + `Expected the status page diagnostic to have exactly ${EXPECTED_LOCATIONS} location(s), but found ${locationCount}.`, ); } } @@ -40,8 +56,7 @@ if (toolExecutionNotifications === undefined) { const statusPageNotifications = toolExecutionNotifications.filter( (n) => - n.descriptor?.id === "lang/diagnostics/example" && - n.properties?.visibility?.statusPage, + n.descriptor?.id === DIAGNOSTIC_ID && n.properties?.visibility?.statusPage, ); if (statusPageNotifications.length !== 1) { core.setFailed( @@ -61,9 +76,9 @@ if (notifications === undefined) { const diagnosticNotification = notifications.filter( (n) => - n.id === "lang/diagnostics/example" && - n.name === "lang/diagnostics/example" && - n.fullDescription?.text === "Diagnostic name", + n.id === DIAGNOSTIC_ID && + n.name === DIAGNOSTIC_ID && + n.fullDescription?.text === DIAGNOSTIC_NAME, ); if (diagnosticNotification.length !== 1) { core.setFailed( diff --git a/pr-checks/checks/diagnostics-without-db.yml b/pr-checks/checks/diagnostics-without-db.yml new file mode 100644 index 000000000..f4bdfc451 --- /dev/null +++ b/pr-checks/checks/diagnostics-without-db.yml @@ -0,0 +1,67 @@ +name: "Diagnostic export without database" +description: "Tests that temporary diagnostics are correctly included in the failed SARIF." +versions: + # Overridden by the FF below. + - linked +installNode: true + +env: + # Allow the failed SARIF file to be generated. + CODEQL_ACTION_SKIP_GENERATE_FAILED_SARIF: false + # But don't actually upload it. + CODEQL_ACTION_SKIP_SARIF_UPLOAD: true + # Mark telemetry for this workflow so it can be treated separately. + CODEQL_ACTION_TESTING_ENVIRONMENT: codeql-action-pr-checks + +steps: + # We deliberately need to make this step fail before the database is initialised, but after + # some diagnostic has been produced. The simplest way to do that right now is to force the + # use of a nightly release and then fail because we are attempting to analyse Swift on + # a non-macOS runner. + - name: Initialise failing analysis + id: init + uses: ./../action/init + continue-on-error: true + env: + CODEQL_ACTION_FORCE_NIGHTLY: true + with: + languages: swift + tools: ${{ steps.prepare-test.outputs.tools-url }} + + - name: Analyze + uses: ./analyze + # In a real workflow, this step wouldn't run. Since we used `continue-on-error` + # above, we manually disable it with an `if` condition. + if: false + with: + category: "/test-codeql-version:${{ matrix.version }}" + + - name: Check diagnostics appear on disk + env: + SARIF_PATH: "${{ runner.temp }}/codeql-action-temp-diagnostics/" + run: | + cd ../action + npx tsx ./pr-checks/diagnostics-without-db.ts +validationJobs: + diagnostics-without-db-validation: + checkout: true + installNode: true + steps: + - name: Download workflow artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: test-mode-sarif-* + path: sarif + + - name: List artifact contents + run: ls -laR ./sarif + + - name: Check diagnostics appear in SARIF + env: + SARIF_PATH: "./sarif/codeql-failed-run.sarif" + DIAGNOSTIC_ID: "codeql-action/forced-nightly-cli" + DIAGNOSTIC_NAME: "A nightly release of CodeQL was used" + EXPECTED_LOCATIONS: "0" + EXPECTED_MESSAGE: | + GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release. + run: npx tsx ./pr-checks/check-diagnostics.ts diff --git a/pr-checks/diagnostics-without-db.ts b/pr-checks/diagnostics-without-db.ts new file mode 100644 index 000000000..b742c10d6 --- /dev/null +++ b/pr-checks/diagnostics-without-db.ts @@ -0,0 +1,48 @@ +import * as fs from "fs"; +import * as path from "path"; + +import * as core from "@actions/core"; + +import type * as diagnostics from "../src/diagnostics"; + +const DIAGNOSTIC_ID = "codeql-action/forced-nightly-cli"; +const sarifDirectory = process.env["SARIF_PATH"]; + +if (sarifDirectory === undefined) { + throw new Error("SARIF_PATH is undefined."); +} + +const paths = fs + .readdirSync(sarifDirectory, { + recursive: true, + encoding: "utf-8", + }) + .filter((p) => p.endsWith(".json")); + +core.info(`Found ${paths.length} paths in '${sarifDirectory}'.`); +core.info(`Searching for a diagnostic with id '${DIAGNOSTIC_ID}'...`); + +let found = false; +for (const diagnosticPath of paths) { + if (path.extname(diagnosticPath) !== ".json") { + core.info(`Skipping ${diagnosticPath}.`); + continue; + } + core.info(`Found ${diagnosticPath}.`); + + const contents = JSON.parse( + fs.readFileSync(path.resolve(sarifDirectory, diagnosticPath), "utf-8"), + ) as diagnostics.DiagnosticMessage; + + if (contents.source.id === DIAGNOSTIC_ID) { + core.info( + `Found '${DIAGNOSTIC_ID}' in '${diagnosticPath}'. Skipping remaining files.`, + ); + found = true; + break; + } +} + +if (!found) { + throw new Error(`Didn't find '${DIAGNOSTIC_ID}' diagnostic.`); +} diff --git a/pr-checks/sync.ts b/pr-checks/sync.ts index d6382786c..57fa3cab3 100755 --- a/pr-checks/sync.ts +++ b/pr-checks/sync.ts @@ -90,7 +90,7 @@ interface Specification extends JobSpecification { services?: any; /** Additional jobs to run after the main PR check job. */ - validationJobs?: Record; + validationJobs?: Record; /** If set, this check is part of a named collection that gets its own caller workflow. */ collection?: string; @@ -122,6 +122,10 @@ interface JobSpecification { installYq?: boolean; } +type ValidationJobSpecification = JobSpecification & { + checkout?: boolean; +}; + /** Describes language/framework-specific steps and inputs. */ interface LanguageSetup { specProperty: keyof JobSpecification; @@ -462,6 +466,20 @@ function generateJobMatrix( return matrix; } +/** + * Gets the `actions/checkout` step. + */ +function getCheckoutStep(): Step { + return { + name: "Check out repository", + uses: pinnedUses( + "actions/checkout", + "3d3c42e5aac5ba805825da76410c181273ba90b1", + "v7.0.1", + ), + }; +} + /** * Retrieves setup steps and additional input definitions based on specific languages or frameworks * that are requested by the `checkSpecification`. @@ -531,14 +549,7 @@ function generateJob( // Construct the workflow steps needed for this check. const steps: Step[] = [ - { - name: "Check out repository", - uses: pinnedUses( - "actions/checkout", - "3d3c42e5aac5ba805825da76410c181273ba90b1", - "v7.0.1", - ), - }, + getCheckoutStep(), ...setupInfo.steps, { name: "Prepare test", @@ -609,13 +620,16 @@ function generateJob( /** Generates a validation job. */ function generateValidationJob( specDocument: yaml.Document, - jobSpecification: JobSpecification, + jobSpecification: ValidationJobSpecification, checkName: string, name: string, ) { // Determine which languages or frameworks have to be installed. const { inputs, steps } = getSetupSteps(jobSpecification); + // Construct a checkout step if requested. + const checkoutStep = jobSpecification.checkout ? [getCheckoutStep()] : []; + // Extract the sequence of steps from the YAML document to persist as much formatting as possible. const specSteps = specDocument.getIn([ "validationJobs", @@ -624,7 +638,7 @@ function generateValidationJob( ]) as yaml.YAMLSeq; // Add the generated steps in front of the ones from the specification. - specSteps.items.unshift(...steps); + specSteps.items.unshift(...checkoutStep, ...steps); const validationJob: Record = { name: jobSpecification.name, @@ -635,7 +649,7 @@ function generateValidationJob( "security-events": "read", }, "timeout-minutes": 5, - "runs-on": "ubuntu-slim", + "runs-on": "ubuntu-latest", steps: specSteps, };