From ed5bb3c5a346ef40d30263a830dce8458aa92152 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Fri, 9 Oct 2026 19:29:16 +0100 Subject: [PATCH] Write diagnostics to temporary location when database isn't ready --- lib/entry-points.js | 65 +++++++++++------- src/config/db-config.test.ts | 126 ++++++++++++++++++--------------- src/diagnostics.ts | 130 +++++++++++++++++++++++------------ src/init-action.ts | 9 +-- src/setup-codeql.test.ts | 86 +++++++++++++---------- 5 files changed, 253 insertions(+), 163 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index 5bfa98c25..7796bd070 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -172981,8 +172981,7 @@ var semver4 = __toESM(require_semver2()); // src/diagnostics.ts var import_fs = require("fs"); var import_path2 = __toESM(require("path")); -var unwrittenDiagnostics = []; -var unwrittenDefaultLanguageDiagnostics = []; +var temporaryDiagnostics = []; var diagnosticCounter = 0; function makeDiagnostic(id, name, data = void 0) { return { @@ -172991,6 +172990,16 @@ function makeDiagnostic(id, name, data = void 0) { source: { ...data?.source, id, name } }; } +function getTempDiagnosticPath(tmpDir, language) { + const root = import_path2.default.join(tmpDir, "codeql-action-temp-diagnostics"); + return getDiagnosticsPath(root, language); +} +function getDiagnosticsPath(basePath, language) { + if (language !== void 0) { + return import_path2.default.resolve(basePath, language, "diagnostic", "codeql-action"); + } + return basePath; +} function addDiagnostic(config, language, diagnostic) { const logger2 = getActionsLogger(); const databasePath = language ? getCodeQLDatabasePath(config, language) : config.dbLocation; @@ -173000,7 +173009,9 @@ function addDiagnostic(config, language, diagnostic) { logger2.debug( `Writing a diagnostic for ${language}, but the database at ${databasePath} does not exist yet.` ); - unwrittenDiagnostics.push({ diagnostic, language }); + const tempDiagnosticsPath = getTempDiagnosticPath(config.tempDir, language); + const diagnosticPath = writeDiagnosticFile(tempDiagnosticsPath, diagnostic); + temporaryDiagnostics.push({ path: diagnosticPath, language }); } } function addNoLanguageDiagnostic(config, diagnostic) { @@ -173013,7 +173024,9 @@ function addNoLanguageDiagnostic(config, diagnostic) { diagnostic ); } else { - unwrittenDefaultLanguageDiagnostics.push(diagnostic); + const tempDiagnosticsPath = getTempDiagnosticPath(getTemporaryDirectory()); + const diagnosticPath = writeDiagnosticFile(tempDiagnosticsPath, diagnostic); + temporaryDiagnostics.push({ path: diagnosticPath }); } } function writeDiagnosticFile(diagnosticsPath, diagnostic) { @@ -173028,15 +173041,14 @@ function writeDiagnosticFile(diagnosticsPath, diagnostic) { `codeql-action-${sanitizedTimestamp}-${uniqueSuffix}.json` ); (0, import_fs.writeFileSync)(jsonPath, JSON.stringify(diagnostic)); + return jsonPath; +} +function getDatabaseDiagnosticsPath(config, language) { + return getDiagnosticsPath(config.dbLocation, language); } function writeDiagnostic(config, language, diagnostic) { const logger2 = getActionsLogger(); - const databasePath = language ? getCodeQLDatabasePath(config, language) : config.dbLocation; - const diagnosticsPath = import_path2.default.resolve( - databasePath, - "diagnostic", - "codeql-action" - ); + const diagnosticsPath = getDatabaseDiagnosticsPath(config, language); try { writeDiagnosticFile(diagnosticsPath, diagnostic); } catch (err) { @@ -173044,30 +173056,34 @@ function writeDiagnostic(config, language, diagnostic) { logger2.debug(JSON.stringify(diagnostic)); } } -function logUnwrittenDiagnostics() { +function logTemporaryDiagnostics() { const logger2 = getActionsLogger(); - const num = unwrittenDiagnostics.length; + const num = temporaryDiagnostics.length; if (num > 0) { logger2.warning( `${num} diagnostic(s) could not be written to the database and will not appear on the Tool Status Page.` ); - for (const unwritten of unwrittenDiagnostics) { - logger2.debug(JSON.stringify(unwritten.diagnostic)); + for (const temporary of temporaryDiagnostics) { + logger2.debug((0, import_fs.readFileSync)(temporary.path, "utf-8")); } } } function flushDiagnostics(config) { const logger2 = getActionsLogger(); - const diagnosticsCount = unwrittenDiagnostics.length + unwrittenDefaultLanguageDiagnostics.length; - logger2.debug(`Writing ${diagnosticsCount} diagnostic(s) to database.`); - for (const unwritten of unwrittenDiagnostics) { - writeDiagnostic(config, unwritten.language, unwritten.diagnostic); + const diagnosticsCount = temporaryDiagnostics.length; + logger2.debug( + `Moving ${diagnosticsCount} diagnostic(s) to their respective databases.` + ); + for (const temporary of temporaryDiagnostics) { + const directory = getDatabaseDiagnosticsPath( + config, + temporary.language ?? config.languages[0] + ); + const filename = import_path2.default.basename(temporary.path); + const destination = import_path2.default.join(directory, filename); + (0, import_fs.renameSync)(temporary.path, destination); } - for (const unwritten of unwrittenDefaultLanguageDiagnostics) { - addNoLanguageDiagnostic(config, unwritten); - } - unwrittenDiagnostics = []; - unwrittenDefaultLanguageDiagnostics = []; + temporaryDiagnostics = []; } function makeTelemetryDiagnostic(id, name, attributes, tags) { return makeDiagnostic(id, name, { @@ -186577,6 +186593,7 @@ async function run3(actionState) { if (statusReportBase !== void 0) { await sendStatusReport(statusReportBase); } + logTemporaryDiagnostics(); return; } let overlayBaseDatabaseStats; @@ -186786,7 +186803,7 @@ exec ${goBinaryPath} "$@"` ); return; } finally { - logUnwrittenDiagnostics(); + logTemporaryDiagnostics(); } await sendCompletedStatusReport2( startedAt, diff --git a/src/config/db-config.test.ts b/src/config/db-config.test.ts index 76fa91801..386556f3d 100644 --- a/src/config/db-config.test.ts +++ b/src/config/db-config.test.ts @@ -9,8 +9,9 @@ import { LoggedMessage, makeMacro, RecordingLogger, + setupActionsVars, } from "../testing-utils"; -import { ConfigurationError, prettyPrintPack } from "../util"; +import { ConfigurationError, prettyPrintPack, withTmpDir } from "../util"; import * as dbConfig from "./db-config"; @@ -562,69 +563,84 @@ test("mergeDefaultSetupAndUserConfigs - keeps other properties from user-supplie t.deepEqual(result, configFile); }); -test("mergeDefaultSetupAndUserConfigs - ignores, but warns about, unknown keys from Default Setup", async (t) => { - const logger = new RecordingLogger(); - const configFile: dbConfig.UserConfig = { - "query-filters": [{ exclude: { a: "b" } }], - "paths-ignore": ["path"], - }; +test.serial( + "mergeDefaultSetupAndUserConfigs - ignores, but warns about, unknown keys from Default Setup", + async (t) => { + const logger = new RecordingLogger(); + const configFile: dbConfig.UserConfig = { + "query-filters": [{ exclude: { a: "b" } }], + "paths-ignore": ["path"], + }; - const result = dbConfig.mergeDefaultSetupAndUserConfigs( - logger, - { - "default-setup": { - borg: [], - org: { - unknown: "foo", - "model-packs": [], + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = dbConfig.mergeDefaultSetupAndUserConfigs( + logger, + { + "default-setup": { + borg: [], + org: { + unknown: "foo", + "model-packs": [], + }, + } as unknown as dbConfig.DefaultSetupConfig, + "paths-ignore": ["other-path"], }, - } as unknown as dbConfig.DefaultSetupConfig, - "paths-ignore": ["other-path"], - }, - configFile, - ); + configFile, + ); - t.deepEqual(result, { - ...configFile, - "default-setup": { org: { "model-packs": [] } }, - }); + t.deepEqual(result, { + ...configFile, + "default-setup": { org: { "model-packs": [] } }, + }); - const expectedUnrecognisedKeys = [ - ".default-setup.org.unknown", - ".default-setup.borg", - ".paths-ignore", - ].join(", "); - checkExpectedLogMessages(t, logger.messages, [ - `Unrecognised keys in Default Setup configuration: ${expectedUnrecognisedKeys}`, - ]); -}); + const expectedUnrecognisedKeys = [ + ".default-setup.org.unknown", + ".default-setup.borg", + ".paths-ignore", + ].join(", "); + checkExpectedLogMessages(t, logger.messages, [ + `Unrecognised keys in Default Setup configuration: ${expectedUnrecognisedKeys}`, + ]); + }); + }, +); -test("mergeDefaultSetupAndUserConfigs - warns about invalid keys from Default Setup", async (t) => { - const logger = new RecordingLogger(); - const configFile: dbConfig.UserConfig = {}; +test.serial( + "mergeDefaultSetupAndUserConfigs - warns about invalid keys from Default Setup", + async (t) => { + const logger = new RecordingLogger(); + const configFile: dbConfig.UserConfig = {}; - const result = dbConfig.mergeDefaultSetupAndUserConfigs( - logger, - { - "default-setup": { - org: { - "model-packs": [123], + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + + const result = dbConfig.mergeDefaultSetupAndUserConfigs( + logger, + { + "default-setup": { + org: { + "model-packs": [123], + }, + } as unknown as dbConfig.DefaultSetupConfig, }, - } as unknown as dbConfig.DefaultSetupConfig, - }, - configFile, - ); + configFile, + ); - t.deepEqual(result, { - ...configFile, - "default-setup": { org: { "model-packs": [123] } }, - }); + t.deepEqual(result, { + ...configFile, + "default-setup": { org: { "model-packs": [123] } }, + }); - const expectedInvalidKeys = [".default-setup.org.model-packs[0]"].join(", "); - checkExpectedLogMessages(t, logger.messages, [ - `Invalid keys in Default Setup configuration: ${expectedInvalidKeys}`, - ]); -}); + const expectedInvalidKeys = [".default-setup.org.model-packs[0]"].join( + ", ", + ); + checkExpectedLogMessages(t, logger.messages, [ + `Invalid keys in Default Setup configuration: ${expectedInvalidKeys}`, + ]); + }); + }, +); /** Parses `contents` as a configuration without validating it. */ function parseUnvalidatedConfig(contents: string): dbConfig.UserConfig { diff --git a/src/diagnostics.ts b/src/diagnostics.ts index caab84934..f08c83c42 100644 --- a/src/diagnostics.ts +++ b/src/diagnostics.ts @@ -1,6 +1,13 @@ -import { existsSync, mkdirSync, writeFileSync } from "fs"; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from "fs"; import path from "path"; +import { getTemporaryDirectory } from "./actions-util"; import type { Config } from "./config-utils"; import { Language } from "./languages"; import { getActionsLogger } from "./logging"; @@ -84,22 +91,16 @@ export type DiagnosticMessage = DiagnosticMessageOptions & { source: DiagnosticSource; }; -/** Represents a diagnostic message that has not yet been written to the database. */ -interface UnwrittenDiagnostic { - /** The diagnostic message that has not yet been written. */ - diagnostic: DiagnosticMessage; - /** The language the diagnostic is for. */ - language: Language; +/** Represents a diagnostic message that has not yet been saved to the database. */ +interface TemporaryDiagnostic { + /** The path to which the diagnostic has temporarily been written to. */ + path: string; + /** The language the diagnostic is for, if any. */ + language?: Language; } -/** A list of diagnostics which have not yet been written to disk. */ -let unwrittenDiagnostics: UnwrittenDiagnostic[] = []; - -/** - * A list of diagnostics which have not yet been written to disk, - * and where the language does not matter. - */ -let unwrittenDefaultLanguageDiagnostics: DiagnosticMessage[] = []; +/** A list of diagnostics which have not yet been saved to the database. */ +let temporaryDiagnostics: TemporaryDiagnostic[] = []; /** * Counter used to generate a unique suffix for each diagnostic filename, so that @@ -128,6 +129,29 @@ export function makeDiagnostic( }; } +/** + * Gets a path relative to {@link tmpDir} where diagnostics for {@link language} + * can temporarily be stored at before the database is initialised. If {@link language} + * is `undefined`, then the common base path for all languages relative to {@link tmpDir} + * is returned. + */ +export function getTempDiagnosticPath(tmpDir: string, language?: Language) { + const root = path.join(tmpDir, "codeql-action-temp-diagnostics"); + return getDiagnosticsPath(root, language); +} + +/** + * Gets a path relative to {@link basePath} where diagnostics for {@link language} + * should be stored at. If {@link language} is `undefined`, then {@link baseBath} + * is returned instead. + */ +export function getDiagnosticsPath(basePath: string, language?: Language) { + if (language !== undefined) { + return path.resolve(basePath, language, "diagnostic", "codeql-action"); + } + return basePath; +} + /** * Adds the given diagnostic to the database. If the database does not yet exist, * the diagnostic will be written to it once it has been created. @@ -155,7 +179,12 @@ export function addDiagnostic( `Writing a diagnostic for ${language}, but the database at ${databasePath} does not exist yet.`, ); - unwrittenDiagnostics.push({ diagnostic, language }); + // Write the diagnostic to a temporary location. + const tempDiagnosticsPath = getTempDiagnosticPath(config.tempDir, language); + const diagnosticPath = writeDiagnosticFile(tempDiagnosticsPath, diagnostic); + + // Track the temporary file. + temporaryDiagnostics.push({ path: diagnosticPath, language }); } } @@ -173,7 +202,11 @@ export function addNoLanguageDiagnostic( diagnostic, ); } else { - unwrittenDefaultLanguageDiagnostics.push(diagnostic); + const tempDiagnosticsPath = getTempDiagnosticPath(getTemporaryDirectory()); + const diagnosticPath = writeDiagnosticFile(tempDiagnosticsPath, diagnostic); + + // Track the temporary file. + temporaryDiagnostics.push({ path: diagnosticPath }); } } @@ -183,7 +216,7 @@ export function addNoLanguageDiagnostic( function writeDiagnosticFile( diagnosticsPath: string, diagnostic: DiagnosticMessage, -) { +): string { // Create the directory if it doesn't exist yet. mkdirSync(diagnosticsPath, { recursive: true }); @@ -202,6 +235,16 @@ function writeDiagnosticFile( ); writeFileSync(jsonPath, JSON.stringify(diagnostic)); + + return jsonPath; +} + +/** Gets the path where diagnostics for {@link language} should be stored in the database. */ +export function getDatabaseDiagnosticsPath( + config: Config, + language: Language | undefined, +) { + return getDiagnosticsPath(config.dbLocation, language); } /** @@ -217,14 +260,7 @@ function writeDiagnostic( diagnostic: DiagnosticMessage, ) { const logger = getActionsLogger(); - const databasePath = language - ? getCodeQLDatabasePath(config, language) - : config.dbLocation; - const diagnosticsPath = path.resolve( - databasePath, - "diagnostic", - "codeql-action", - ); + const diagnosticsPath = getDatabaseDiagnosticsPath(config, language); try { writeDiagnosticFile(diagnosticsPath, diagnostic); @@ -234,39 +270,47 @@ function writeDiagnostic( } } -/** Report if there are unwritten diagnostics and write them to the log. */ -export function logUnwrittenDiagnostics() { +/** Report if there are temporary diagnostics and write them to the log. */ +export function logTemporaryDiagnostics() { const logger = getActionsLogger(); - const num = unwrittenDiagnostics.length; + const num = temporaryDiagnostics.length; + if (num > 0) { logger.warning( `${num} diagnostic(s) could not be written to the database and will not appear on the Tool Status Page.`, ); - for (const unwritten of unwrittenDiagnostics) { - logger.debug(JSON.stringify(unwritten.diagnostic)); + for (const temporary of temporaryDiagnostics) { + logger.debug(readFileSync(temporary.path, "utf-8")); } } } -/** Writes all unwritten diagnostics to disk. */ +/** Relocates all temporary diagnostics to the respective databases. */ export function flushDiagnostics(config: Config) { const logger = getActionsLogger(); - const diagnosticsCount = - unwrittenDiagnostics.length + unwrittenDefaultLanguageDiagnostics.length; - logger.debug(`Writing ${diagnosticsCount} diagnostic(s) to database.`); + const diagnosticsCount = temporaryDiagnostics.length; + logger.debug( + `Moving ${diagnosticsCount} diagnostic(s) to their respective databases.`, + ); - for (const unwritten of unwrittenDiagnostics) { - writeDiagnostic(config, unwritten.language, unwritten.diagnostic); - } - for (const unwritten of unwrittenDefaultLanguageDiagnostics) { - addNoLanguageDiagnostic(config, unwritten); + for (const temporary of temporaryDiagnostics) { + // If `temporary.language` is `undefined`, then we didn't have a `config` at the time that + // `addNoLanguageDiagnostic` was called. In that case, we arbitrarily choose the first + // configured language here like we would have done in `addNoLanguageDiagnostic`. + const directory = getDatabaseDiagnosticsPath( + config, + temporary.language ?? config.languages[0], + ); + const filename = path.basename(temporary.path); + const destination = path.join(directory, filename); + + renameSync(temporary.path, destination); } - // Reset the unwritten diagnostics arrays. - unwrittenDiagnostics = []; - unwrittenDefaultLanguageDiagnostics = []; + // Reset the temporary diagnostics arrays. + temporaryDiagnostics = []; } /** diff --git a/src/init-action.ts b/src/init-action.ts index 5376763b5..73ffcd6f1 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -40,7 +40,7 @@ import { addDiagnostic, addNoLanguageDiagnostic, flushDiagnostics, - logUnwrittenDiagnostics, + logTemporaryDiagnostics, makeDiagnostic, makeTelemetryDiagnostic, } from "./diagnostics"; @@ -484,6 +484,7 @@ async function run( if (statusReportBase !== undefined) { await sendStatusReport(statusReportBase); } + logTemporaryDiagnostics(); return; } @@ -766,8 +767,8 @@ async function run( ); } - // Write diagnostics to the database that we previously stored in memory because the database - // did not exist until now. + // Move diagnostics to the database that we previously kept in a temporary location + // because the database did not exist until now flushDiagnostics(config); // We save the config here instead of at the end of `initConfig` because we @@ -797,7 +798,7 @@ async function run( ); return; } finally { - logUnwrittenDiagnostics(); + logTemporaryDiagnostics(); } await sendCompletedStatusReport( startedAt, diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index 3fef11e63..70a459d93 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -1647,16 +1647,20 @@ test.serial( }, ]); - const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases( - overlayMatchEnabledVersions, - ["javascript"], - createFeatures([Feature.OverlayAnalysisMatchCodeqlVersion]), - getRunnerLogger(true), - ); - t.deepEqual(result, [ - { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" }, - { cliVersion: "2.20.0", tagName: "codeql-bundle-v2.20.0" }, - ]); + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = + await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases( + overlayMatchEnabledVersions, + ["javascript"], + createFeatures([Feature.OverlayAnalysisMatchCodeqlVersion]), + getRunnerLogger(true), + ); + t.deepEqual(result, [ + { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" }, + { cliVersion: "2.20.0", tagName: "codeql-bundle-v2.20.0" }, + ]); + }); }, ); @@ -1773,21 +1777,25 @@ test.serial( }, ]); - const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases( - overlayMatchEnabledVersions, - ["javascript"], - createFeatures([Feature.OverlayAnalysisMatchCodeqlVersionDryRun]), - getRunnerLogger(true), - ); - t.deepEqual( - result, - [], - "Dry-run should return an empty list so the caller falls back.", - ); - t.assert( - listStub.calledOnce, - "Dry-run should still list Actions caches to populate the diagnostic.", - ); + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = + await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases( + overlayMatchEnabledVersions, + ["javascript"], + createFeatures([Feature.OverlayAnalysisMatchCodeqlVersionDryRun]), + getRunnerLogger(true), + ); + t.deepEqual( + result, + [], + "Dry-run should return an empty list so the caller falls back.", + ); + t.assert( + listStub.calledOnce, + "Dry-run should still list Actions caches to populate the diagnostic.", + ); + }); }, ); @@ -1801,18 +1809,22 @@ test.serial( }, ]); - const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases( - overlayMatchEnabledVersions, - ["javascript"], - createFeatures([ - Feature.OverlayAnalysisMatchCodeqlVersion, - Feature.OverlayAnalysisMatchCodeqlVersionDryRun, - ]), - getRunnerLogger(true), - ); - t.deepEqual(result, [ - { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" }, - ]); + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + const result = + await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases( + overlayMatchEnabledVersions, + ["javascript"], + createFeatures([ + Feature.OverlayAnalysisMatchCodeqlVersion, + Feature.OverlayAnalysisMatchCodeqlVersionDryRun, + ]), + getRunnerLogger(true), + ); + t.deepEqual(result, [ + { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" }, + ]); + }); }, );