github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-10-03 17:41:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into redsun82-linux-arm64-support

Browse Source
This commit is contained in:
Paolo Tranquilli
2026-08-21 17:43:09 +02:00
parent b4c390c362 486fec2a3e
commit 97b7459e09
32 changed files with 2613 additions and 1684 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/analyze-action-post.ts
View File

@@ -38,7 +38,7 @@ export async function runWrapper() {
logger,
);
if (config !== undefined) {
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(logger, config.codeQLCmd);
const version = await codeql.getVersion();
await debugArtifacts.uploadCombinedSarifArtifacts(
logger,

2
src/analyze-action.ts
View File

@@ -255,7 +255,7 @@ async function run({ startedAt, logger }: ActionState<["Base", "Logger"]>) {
);
}
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(logger, config.codeQLCmd);
if (hasBadExpectErrorInput()) {
throw new util.ConfigurationError(

2
src/api-compatibility.json
View File

@@ -1 +1 @@
{"maximumVersion": "3.22", "minimumVersion": "3.17"}
{"maximumVersion":"3.23","minimumVersion":"3.17"}

2
src/autobuild-action.ts
View File

@@ -99,7 +99,7 @@ async function run({ startedAt, logger }: ActionState<["Base", "Logger"]>) {
);
}
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(logger, config.codeQLCmd);
languages = await determineAutobuildLanguages(codeql, config, logger);
if (languages !== undefined) {

2
src/autobuild.ts
View File

@@ -155,7 +155,7 @@ export async function runAutobuild(
logger: Logger,
) {
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = await getCodeQL(config.codeQLCmd);
const codeQL = await getCodeQL(logger, config.codeQLCmd);
if (language === BuiltInLanguage.cpp) {
await setupCppAutobuild(codeQL, logger);
}

128
src/cli/output-cache.test.ts Normal file
View File

@@ -0,0 +1,128 @@
import * as fs from "fs";
import path from "path";
import test from "ava";
import { EnvVar } from "../environment";
import { getRunnerLogger } from "../logging";
import { getTestEnv, setupTests } from "../testing-utils";
import * as util from "../util";
import * as outputCache from "./output-cache";
setupTests(test);
const logger = getRunnerLogger(true);
test.serial(
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "codeql-action-command-cache.json");
fs.writeFileSync(
cacheFile,
JSON.stringify({
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0" } },
}),
"utf8",
);
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.deepEqual(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
{
version: "2.20.0",
},
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
fs.writeFileSync(
cacheFile,
JSON.stringify({
cmd: "/path/to/other-codeql",
version: { version: "2.20.0" },
}),
"utf8",
);
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a malformed persisted value",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
fs.writeFileSync(cacheFile, "not valid json", "utf8");
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
const testValues = [
{ cmd: "/path/to/codeql" },
{ entries: { version: { version: "2.20.0" } } },
{ cmd: "/path/to/codeql", entries: {} },
{ cmd: "/path/to/codeql", entries: null },
{ cmd: "/path/to/codeql", entries: { version: {} } },
{ cmd: "/path/to/codeql", entries: { version: null } },
{ cmd: "/path/to/codeql", entries: { version: "2.20.0" } },
{ cmd: "/path/to/codeql", entries: { version: { version: null } } },
{ cmd: "/path/to/codeql", entries: { version: { version: 2.2 } } },
{ cmd: "/path/to/codeql", entries: { version: { version: 2 } } },
{
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0", overlayVersion: "1" } },
},
{
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0", features: "nope" } },
},
].map((v) => JSON.stringify(v));
for (const value of testValues) {
fs.writeFileSync(cacheFile, value, "utf8");
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
value,
);
}
});
},
);
test.serial("getCachedCodeQlVersion ignores non-existent file", async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.notThrows(() => {
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
});
});

156
src/cli/output-cache.ts Normal file
View File

@@ -0,0 +1,156 @@
import * as fs from "fs";
import path from "path";
import { getTemporaryDirectory } from "../actions-util";
import { Env } from "../environment";
import { Logger } from "../logging";
import type { VersionInfo } from "./types";
/**
* The keys of the command cache. Each key corresponds to a command whose output we cache.
*/
export type CommandCacheKey = string;
/**
* The type of the command cache that is persisted to disk.
*/
export interface OutputCache {
cmd: string;
entries: Record<CommandCacheKey, unknown>;
}
/**
* The name of the temporary file that backs the on-disk cache of
* CLI responses between workflow steps.
*/
const COMMAND_CACHE_FILENAME = "codeql-action-command-cache.json";
/**
* The module-global variable that caches the CodeQL CLI version in-memory.
*/
let cachedCodeQlVersion: undefined | VersionInfo = undefined;
/**
* Resets the in-process cache of the CodeQL CLI version. Only for use in tests,
* which exercise multiple "steps" within a single process.
*/
export function resetCachedCodeQlVersion(): void {
cachedCodeQlVersion = undefined;
}
/**
* Returns the path to the temporary file that backs the
* on-disk cache of CLI responses between workflow steps.
*/
function getCommandCacheFilePath(env: Env): string {
return path.join(getTemporaryDirectory(env), COMMAND_CACHE_FILENAME);
}
/**
* Caches the CodeQL CLI version both in-memory and on disk.
* @param env The environment variables to use.
* @param cmd The path to the CodeQL CLI.
* @param version The version information to cache.
*/
export function cacheCodeQlVersion(
env: Env,
cmd: string,
version: VersionInfo,
): void {
if (cachedCodeQlVersion !== undefined) {
throw new Error("cacheCodeQlVersion() should be called only once");
}
cachedCodeQlVersion = version;
const outputCache = {
cmd,
entries: { version },
} satisfies OutputCache;
// Persist the version so that subsequent Actions steps, which run in separate
// processes, can reuse it rather than invoking `codeql version` again. We
// record the CLI path so that a different step using a different CodeQL bundle
// doesn't pick up a stale version.
fs.writeFileSync(
getCommandCacheFilePath(env),
JSON.stringify(outputCache),
"utf8",
);
}
/**
* Returns the cached CodeQL CLI version, if any.
* @param logger The logger to use for logging messages.
* @param env The environment variables to use.
* @param cmd The path to the CodeQL CLI.
*/
export function getCachedCodeQlVersion(
logger: Logger,
env: Env,
cmd?: string,
): undefined | VersionInfo {
if (cachedCodeQlVersion !== undefined) {
return cachedCodeQlVersion;
}
// Fall back to the value persisted by an earlier Actions step, if any. This is
// best-effort: any malformed or mismatched value is ignored so that the caller
// invokes `codeql version` instead.
let serialized: string;
try {
serialized = fs.readFileSync(getCommandCacheFilePath(env), "utf8");
} catch (e) {
logger.debug(
`Cannot read CLI-cache file ${getCommandCacheFilePath(env)}: ${e}`,
);
return undefined;
}
let persisted: unknown;
try {
persisted = JSON.parse(serialized);
} catch (e) {
logger.debug(`Cannot parse CLI-cache data as JSON: ${e}`);
return undefined;
}
if (
!isOutputCache(persisted) ||
(cmd !== undefined && persisted.cmd !== cmd)
) {
return undefined;
}
// Memoize the parsed value so that subsequent calls in this process don't
// re-parse the environment variable.
cachedCodeQlVersion = persisted.entries.version as VersionInfo;
return cachedCodeQlVersion;
}
/**
* Determines whether a value is a `VersionInfo` object.
* @param x The value to test
*/
function isVersionInfo(x: unknown): x is VersionInfo {
const candidate = x as Partial<VersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.version === "string" &&
(candidate.features === undefined ||
(typeof candidate.features === "object" &&
candidate.features !== null)) &&
(candidate.overlayVersion === undefined ||
typeof candidate.overlayVersion === "number")
);
}
/**
* Determines whether a value is a `OutputCache` object.
* @param x The value to test
*/
function isOutputCache(x: unknown): x is OutputCache {
const candidate = x as Partial<OutputCache> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.cmd === "string" &&
candidate.entries !== undefined &&
isVersionInfo(candidate.entries.version)
);
}

13
src/cli/types.ts Normal file
View File

@@ -0,0 +1,13 @@
export interface VersionInfo {
version: string;
features?: { [name: string]: boolean };
/**
* The overlay version helps deal with backward incompatible changes for
* overlay analysis. When a precompiled query pack reports the same overlay
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
* analysis with that pack. Otherwise, if the overlay versions are different,
* or if either the pack or the CLI does not report an overlay version,
* we need to revert to non-overlay analysis.
*/
overlayVersion?: number;
}

29
src/codeql.test.ts
View File

@@ -51,6 +51,31 @@ test.beforeEach(() => {
});
});
test("isDiskConfigurationError - true for expected errors", async (t) => {
t.true(
codeql.isDiskConfigurationError(new Error("ENOSPC: Out of disk space")),
);
t.true(
codeql.isDiskConfigurationError(
new Error(
"EACCES: permission denied, mkdir /opt/hostedtoolcache/CodeQL/",
),
),
);
});
test("isDiskConfigurationError - false for other errors", async (t) => {
t.false(codeql.isDiskConfigurationError("Not an Error instance"));
const otherMessages = [
"Does not contain an error code we test for",
"ENOSP: Not quite the full error code",
];
for (const otherMessage of otherMessages) {
t.false(codeql.isDiskConfigurationError(new Error(otherMessage)));
}
});
async function installIntoToolcache({
apiDetails = SAMPLE_DOTCOM_API_DETAILS,
cliVersion,
@@ -580,7 +605,6 @@ const injectedConfigMacro = makeMacro({
"",
undefined,
undefined,
getRunnerLogger(true),
);
const args = runnerConstructorStub.firstCall.args[1] as string[];
@@ -856,7 +880,6 @@ test.serial(
"",
undefined,
"/path/to/qlconfig.yml",
getRunnerLogger(true),
);
const args = runnerConstructorStub.firstCall.args[1] as string[];
@@ -887,7 +910,6 @@ test.serial(
"",
undefined,
undefined, // undefined qlconfigFile
getRunnerLogger(true),
);
const args = runnerConstructorStub.firstCall.args[1] as any[];
@@ -1066,7 +1088,6 @@ test.serial(
"sourceRoot",
undefined,
undefined,
getRunnerLogger(false),
);
t.true(runnerConstructorStub.calledOnce);

59
src/codeql.ts
View File

@@ -12,10 +12,12 @@ import {
runTool,
} from "./actions-util";
import * as api from "./api-client";
import * as outputCache from "./cli/output-cache";
import type { VersionInfo } from "./cli/types";
import { CliError, wrapCliConfigurationError } from "./cli-errors";
import { appendExtraQueryExclusions, type Config } from "./config-utils";
import { DocUrl } from "./doc-url";
import { EnvVar } from "./environment";
import { EnvVar, getEnv } from "./environment";
import {
CodeQLDefaultVersionInfo,
Feature,
@@ -23,7 +25,7 @@ import {
} from "./feature-flags";
import { isAnalyzingDefaultBranch } from "./git-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import { getRunnerLogger, Logger } from "./logging";
import { writeBaseDatabaseOidsFile, writeOverlayChangesFile } from "./overlay";
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
import * as setupCodeql from "./setup-codeql";
@@ -91,7 +93,6 @@ export interface CodeQL {
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
logger: Logger,
): Promise<void>;
/**
* Runs the autobuilder for the given language.
@@ -215,20 +216,6 @@ export interface CodeQL {
): Promise<void>;
}
export interface VersionInfo {
version: string;
features?: { [name: string]: boolean };
/**
* The overlay version helps deal with backward incompatible changes for
* overlay analysis. When a precompiled query pack reports the same overlay
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
* analysis with that pack. Otherwise, if the overlay versions are different,
* or if either the pack or the CLI does not report an overlay version,
* we need to revert to non-overlay analysis.
*/
overlayVersion?: number;
}
export interface ResolveDatabaseOutput {
overlayBaseSpecifier?: string;
}
@@ -286,6 +273,26 @@ const GHES_MOST_RECENT_DEPRECATION_DATE = "2026-07-01";
/** The CLI verbosity level to use for extraction in debug mode. */
const EXTRACTION_DEBUG_MODE_VERBOSITY = "progress++";
/**
* Decides whether `e` is a disk-related error outside of our control
* that should be classified as a `ConfigurationError`.
*
* @param e The error to check.
* @returns True if the error should be treated as a `ConfigurationError` or false if not.
*/
export function isDiskConfigurationError(e: unknown): boolean {
if (!(e instanceof Error)) {
return false;
}
return (
// out of disk space
e.message.includes("ENOSPC") ||
// access denied
e.message.includes("EACCES")
);
}
/**
* Set up CodeQL CLI access.
*
@@ -346,7 +353,7 @@ export async function setupCodeQL(
);
}
cachedCodeQL = await getCodeQLForCmd(codeqlCmd, checkVersion);
cachedCodeQL = await getCodeQLForCmd(logger, codeqlCmd, checkVersion);
return {
codeql: cachedCodeQL,
toolsDownloadStatusReport,
@@ -356,8 +363,7 @@ export async function setupCodeQL(
} catch (rawError) {
const e = api.wrapApiConfigurationError(rawError);
const ErrorClass =
e instanceof util.ConfigurationError ||
(e instanceof Error && e.message.includes("ENOSPC")) // out of disk space
e instanceof util.ConfigurationError || isDiskConfigurationError(e)
? util.ConfigurationError
: Error;
@@ -372,9 +378,9 @@ export async function setupCodeQL(
/**
* Use the CodeQL executable located at the given path.
*/
export async function getCodeQL(cmd: string): Promise<CodeQL> {
export async function getCodeQL(logger: Logger, cmd: string): Promise<CodeQL> {
if (cachedCodeQL === undefined) {
cachedCodeQL = await getCodeQLForCmd(cmd, true);
cachedCodeQL = await getCodeQLForCmd(logger, cmd, true);
}
return cachedCodeQL;
}
@@ -481,8 +487,9 @@ export function createStubCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
*/
export async function getCodeQLForTesting(
cmd = "codeql-for-testing",
logger: Logger = getRunnerLogger(true),
): Promise<CodeQL> {
return getCodeQLForCmd(cmd, false);
return getCodeQLForCmd(logger, cmd, false);
}
/**
@@ -494,6 +501,7 @@ export async function getCodeQLForTesting(
* @returns A new CodeQL object
*/
async function getCodeQLForCmd(
logger: Logger,
cmd: string,
checkVersion: boolean,
): Promise<CodeQL> {
@@ -502,7 +510,7 @@ async function getCodeQLForCmd(
return cmd;
},
async getVersion() {
let result = util.getCachedCodeQlVersion(cmd);
let result = outputCache.getCachedCodeQlVersion(logger, getEnv(), cmd);
if (result === undefined) {
result = await runCliJson<VersionInfo>(
cmd,
@@ -511,7 +519,7 @@ async function getCodeQLForCmd(
noStreamStdout: true,
},
);
util.cacheCodeQlVersion(cmd, result);
outputCache.cacheCodeQlVersion(getEnv(), cmd, result);
}
return result;
},
@@ -539,7 +547,6 @@ async function getCodeQLForCmd(
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
logger: Logger,
) {
const extraArgs = config.languages.map(
(language) => `--language=${language}`,

86
src/config-utils.test.ts
View File

@@ -1295,13 +1295,12 @@ checkOverlayEnablementMacro.serial(
);
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if runner disk space is below v2 limit and v2 resource checks enabled",
"No overlay-base database on default branch if runner disk space is below minimum",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisResourceChecksV2,
],
isDefaultBranch: true,
diskUsage: {
@@ -1315,13 +1314,12 @@ checkOverlayEnablementMacro.serial(
);
checkOverlayEnablementMacro.serial(
"Overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks enabled",
"Overlay-base database on default branch if runner disk space is above minimum",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisResourceChecksV2,
],
isDefaultBranch: true,
diskUsage: {
@@ -1335,17 +1333,93 @@ checkOverlayEnablementMacro.serial(
},
);
// Check that each feature flag lowers the limit to the threshold that its name
// declares. Both sides of the boundary are needed to pin the threshold down: a
// mapping to a lower value would still pass the case at the limit, and one to a
// higher value would still fail the case below it.
for (const [feature, thresholdGb] of [
[Feature.OverlayAnalysisMinDisk8Gb, 8],
[Feature.OverlayAnalysisMinDisk9Gb, 9],
[Feature.OverlayAnalysisMinDisk10Gb, 10],
[Feature.OverlayAnalysisMinDisk11Gb, 11],
[Feature.OverlayAnalysisMinDisk12Gb, 12],
[Feature.OverlayAnalysisMinDisk13Gb, 13],
] as Array<[Feature, number]>) {
const features = [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
feature,
];
checkOverlayEnablementMacro.serial(
`Overlay-base database on default branch if ${feature} is enabled and runner disk space is at its limit`,
{
languages: [BuiltInLanguage.javascript],
features,
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: thresholdGb * 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
checkOverlayEnablementMacro.serial(
`No overlay-base database on default branch if ${feature} is enabled and runner disk space is below its limit`,
{
languages: [BuiltInLanguage.javascript],
features,
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: thresholdGb * 1_000_000_000 - 1_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
},
);
}
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks not enabled",
"Overlay-base database on default branch if runner disk space is exactly at the lowest limit enabled by a feature flag",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisMinDisk9Gb,
Feature.OverlayAnalysisMinDisk12Gb,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 15_000_000_000,
numAvailableBytes: 9_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if runner disk space is below the lowest limit enabled by a feature flag",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisMinDisk9Gb,
Feature.OverlayAnalysisMinDisk12Gb,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 8_500_000_000,
numTotalBytes: 100_000_000_000,
},
},

70
src/config-utils.ts
View File

@@ -48,7 +48,7 @@ import {
import { prepareDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import * as errorMessages from "./error-messages";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Feature, FeatureEnablement, FeatureWithoutCLI } from "./feature-flags";
import {
RepositoryProperties,
RepositoryPropertyName,
@@ -101,19 +101,23 @@ export { type Config } from "./config/action-config";
* whether to perform overlay analysis, then the action will not perform overlay
* analysis unless overlay analysis has been explicitly enabled via environment
* variable.
*
* This threshold can be lowered by the feature flags in
* `OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE`.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14000;
/**
* The v2 minimum available disk space (in MB) required to perform overlay
* analysis. This is a lower threshold than the v1 limit, allowing overlay
* analysis to run on runners with less available disk space.
* Minimum available disk space (in MB) enabled by each overlay feature flag.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB = 14000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB * 1_000_000;
const OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE = {
[Feature.OverlayAnalysisMinDisk8Gb]: 8000,
[Feature.OverlayAnalysisMinDisk9Gb]: 9000,
[Feature.OverlayAnalysisMinDisk10Gb]: 10000,
[Feature.OverlayAnalysisMinDisk11Gb]: 11000,
[Feature.OverlayAnalysisMinDisk12Gb]: 12000,
[Feature.OverlayAnalysisMinDisk13Gb]: 13000,
} satisfies Partial<Record<FeatureWithoutCLI, number>>;
/**
* The minimum memory (in MB) that must be available for CodeQL to perform overlay analysis. If
@@ -588,24 +592,44 @@ async function checkOverlayAnalysisFeatureEnabled(
return new Success(undefined);
}
/**
* Returns the minimum available disk space (in MB) required to perform overlay
* analysis, which is the lowest threshold enabled by a feature flag, or the
* default threshold if no such feature flag is enabled.
*/
async function getMinimumDiskSpaceMb(
features: FeatureEnablement,
): Promise<number> {
let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB;
for (const [feature, thresholdMb] of Object.entries(
OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE,
)) {
if (await features.getValue(feature as FeatureWithoutCLI)) {
minimumMb = Math.min(minimumMb, thresholdMb);
}
}
return minimumMb;
}
/** Checks if the runner has enough disk space for overlay analysis. */
function runnerHasSufficientDiskSpace(
diskUsage: DiskUsage,
logger: Logger,
useV2ResourceChecks: boolean,
minimumDiskSpaceMb: number,
): boolean {
const minimumDiskSpaceBytes = useV2ResourceChecks
? OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES
: OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES;
if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) {
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000);
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1_000_000) {
logger.info(
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
`due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`,
);
return false;
}
logger.debug(
`Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the ` +
`minimum of ${minimumDiskSpaceMb} MB.`,
);
return true;
}
@@ -637,7 +661,7 @@ async function runnerHasSufficientMemory(
}
logger.debug(
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
);
return true;
}
@@ -648,12 +672,13 @@ async function runnerHasSufficientMemory(
*/
async function checkRunnerResources(
codeql: CodeQL,
features: FeatureEnablement,
diskUsage: DiskUsage,
ramInput: string | undefined,
logger: Logger,
useV2ResourceChecks: boolean,
): Promise<Result<void, OverlayDisabledReason>> {
if (!runnerHasSufficientDiskSpace(diskUsage, logger, useV2ResourceChecks)) {
const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features);
if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) {
return new Failure(OverlayDisabledReason.InsufficientDiskSpace);
}
if (!(await runnerHasSufficientMemory(codeql, ramInput, logger))) {
@@ -752,9 +777,6 @@ export async function checkOverlayEnablement(
Feature.OverlayAnalysisSkipResourceChecks,
codeql,
));
const useV2ResourceChecks = await features.getValue(
Feature.OverlayAnalysisResourceChecksV2,
);
const checkOverlayStatus = await features.getValue(
Feature.OverlayAnalysisStatusCheck,
);
@@ -770,10 +792,10 @@ export async function checkOverlayEnablement(
performResourceChecks && diskUsage !== undefined
? await checkRunnerResources(
codeql,
features,
diskUsage,
ramInput,
logger,
useV2ResourceChecks,
)
: new Success<void>(undefined);
if (resourceResult.isFailure()) {

8
src/defaults.json
View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.26.2",
"cliVersion": "2.26.2",
"priorBundleVersion": "codeql-bundle-v2.26.1",
"priorCliVersion": "2.26.1"
"bundleVersion": "codeql-bundle-v2.26.3",
"cliVersion": "2.26.3",
"priorBundleVersion": "codeql-bundle-v2.26.2",
"priorCliVersion": "2.26.2"
}

6
src/environment.ts
View File

@@ -39,12 +39,6 @@ export enum EnvVar {
*/
CODE_SCANNING_REF = "CODE_SCANNING_REF",
/**
* `PersistedVersionInfo` for the CodeQL CLI, so later Actions steps can reuse it instead of
* invoking `codeql version` again.
*/
CODEQL_VERSION_INFO = "CODEQL_ACTION_CLI_VERSION_INFO",
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",

62
src/feature-flags.ts
View File

@@ -121,12 +121,37 @@ export enum Feature {
* `OverlayAnalysisMatchCodeqlVersion` overrides this flag.
*/
OverlayAnalysisMatchCodeqlVersionDryRun = "overlay_analysis_match_codeql_version_dry_run",
OverlayAnalysisPython = "overlay_analysis_python",
/**
* Controls whether lower disk space requirements are used for overlay hardware checks.
* Has no effect if `OverlayAnalysisSkipResourceChecks` is enabled.
* Lowers the overlay minimum available disk space to 8 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisResourceChecksV2 = "overlay_analysis_resource_checks_v2",
OverlayAnalysisMinDisk8Gb = "overlay_analysis_min_disk_8_gb",
/**
* Lowers the overlay minimum available disk space to 9 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk9Gb = "overlay_analysis_min_disk_9_gb",
/**
* Lowers the overlay minimum available disk space to 10 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk10Gb = "overlay_analysis_min_disk_10_gb",
/**
* Lowers the overlay minimum available disk space to 11 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk11Gb = "overlay_analysis_min_disk_11_gb",
/**
* Lowers the overlay minimum available disk space to 12 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk12Gb = "overlay_analysis_min_disk_12_gb",
/**
* Lowers the overlay minimum available disk space to 13 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk13Gb = "overlay_analysis_min_disk_13_gb",
OverlayAnalysisPython = "overlay_analysis_python",
OverlayAnalysisRuby = "overlay_analysis_ruby",
/** Controls whether hardware checks are skipped for overlay analysis. */
OverlayAnalysisSkipResourceChecks = "overlay_analysis_skip_resource_checks",
@@ -354,9 +379,34 @@ export const featureConfig = {
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisResourceChecksV2]: {
[Feature.OverlayAnalysisMinDisk8Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2",
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk9Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk10Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk11Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk12Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk13Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisStatusCheck]: {

5
src/init-action-post-helper.ts
View File

@@ -123,6 +123,7 @@ async function prepareFailedSarif(
const category = `/language:${language}`;
const checkoutPath = ".";
const result = await generateFailedSarif(
logger,
features,
config,
category,
@@ -146,6 +147,7 @@ async function prepareFailedSarif(
const checkoutPath = getCheckoutPathInputOrThrow(workflow, jobName, matrix);
const result = await generateFailedSarif(
logger,
features,
config,
category,
@@ -156,6 +158,7 @@ async function prepareFailedSarif(
}
async function generateFailedSarif(
logger: Logger,
features: FeatureEnablement,
config: Config,
category: string | undefined,
@@ -163,7 +166,7 @@ async function generateFailedSarif(
sarifFile?: string,
) {
const databasePath = config.dbLocation;
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(logger, config.codeQLCmd);
// Set the filename for the SARIF file if not already set.
if (sarifFile === undefined) {

2
src/init-action-post.ts
View File

@@ -75,7 +75,7 @@ async function run(startedAt: Date) {
"Debugging artifacts are unavailable since the 'init' Action failed before it could produce any.",
);
} else {
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(logger, config.codeQLCmd);
uploadFailedSarifResult = await initActionPostHelper.uploadFailureInfo(
debugArtifacts.tryUploadAllAvailableDebugArtifacts,

2
src/init-action.ts
View File

@@ -689,7 +689,6 @@ async function run(
sourceRoot,
"Runner.Worker.exe",
qlconfigFile,
logger,
);
// To check custom query packs for compatibility with overlay analysis, we
@@ -718,7 +717,6 @@ async function run(
sourceRoot,
"Runner.Worker.exe",
qlconfigFile,
logger,
);
}

2
src/init.ts
View File

@@ -89,7 +89,6 @@ export async function runDatabaseInitCluster(
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
logger: Logger,
): Promise<void> {
fs.mkdirSync(config.dbLocation, { recursive: true });
await configUtils.wrapEnvironment(
@@ -100,7 +99,6 @@ export async function runDatabaseInitCluster(
sourceRoot,
processName,
qlconfigFile,
logger,
),
);
}

2
src/resolve-environment.ts
View File

@@ -9,7 +9,7 @@ export async function runResolveBuildEnvironment(
) {
logger.startGroup(`Attempting to resolve build environment for ${language}`);
const codeql = await getCodeQL(cmd);
const codeql = await getCodeQL(logger, cmd);
if (workingDir !== undefined) {
logger.info(`Using ${workingDir} as the working directory.`);

4
src/status-report.ts
View File

@@ -14,6 +14,7 @@ import {
isSelfHostedRunner,
} from "./actions-util";
import { getAnalysisKey, getApiClient } from "./api-client";
import { getCachedCodeQlVersion } from "./cli/output-cache";
import type { Config } from "./config/action-config";
import type { ComputedInput, InputName } from "./config/inputs";
import { parseRegistriesWithoutCredentials } from "./config/pack-registries";
@@ -30,7 +31,6 @@ import { registryBaseSchema } from "./start-proxy/types";
import {
ConfigurationError,
getRequiredEnvParam,
getCachedCodeQlVersion,
isInTestMode,
GITHUB_DOTCOM_URL,
DiskUsage,
@@ -376,7 +376,7 @@ export async function createStatusReportBase(
core.exportVariable(EnvVar.WORKFLOW_STARTED_AT, workflowStartedAt);
}
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const codeQlCliVersion = getCachedCodeQlVersion();
const codeQlCliVersion = getCachedCodeQlVersion(logger, getEnv());
const actionRef = process.env["GITHUB_ACTION_REF"] || "";
const testingEnvironment = getTestingEnvironment();
// re-export the testing environment variable so that it is available to subsequent steps,

5
src/testing-utils.ts
View File

@@ -18,6 +18,8 @@ import { AnalysisKind } from "./analyses";
import * as apiClient from "./api-client";
import { GitHubApiDetails } from "./api-client";
import { CachingKind } from "./caching-utils";
import { resetCachedCodeQlVersion } from "./cli/output-cache";
import type { VersionInfo } from "./cli/types";
import * as codeql from "./codeql";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json";
@@ -39,7 +41,6 @@ import {
GitHubVariant,
GitHubVersion,
HTTPError,
resetCachedCodeQlVersion,
Result,
Success,
} from "./util";
@@ -872,7 +873,7 @@ export const makeVersionInfo = (
version: string,
features?: { [name: string]: boolean },
overlayVersion?: number,
): codeql.VersionInfo => ({
): VersionInfo => ({
version,
features,
overlayVersion,

2
src/tools-features.ts
View File

@@ -1,6 +1,6 @@
import * as semver from "semver";
import type { VersionInfo } from "./codeql";
import type { VersionInfo } from "./cli/types";
export enum ToolsFeature {
BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries",

2
src/upload-lib.ts
View File

@@ -140,7 +140,7 @@ async function combineSarifFilesUsingCLI(
const config = await getConfig(tempDir, logger);
if (config !== undefined) {
codeQL = await getCodeQL(config.codeQLCmd);
codeQL = await getCodeQL(logger, config.codeQLCmd);
tempDir = config.tempDir;
} else {
logger.info(

55
src/util.test.ts
View File

@@ -532,58 +532,3 @@ test("Failure.orElse returns the default value for a failure result", (t) => {
const result = new util.Failure(new Error("test error"));
t.is(result.orElse("default value"), "default value");
});
test.serial(
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0" },
});
t.deepEqual(util.getCachedCodeQlVersion("/path/to/codeql"), {
version: "2.20.0",
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = JSON.stringify({
cmd: "/path/to/other-codeql",
version: { version: "2.20.0" },
});
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined);
},
);
test.serial(
"getCachedCodeQlVersion ignores a malformed persisted value",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = "not valid json";
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined);
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
(t) => {
for (const value of [
JSON.stringify({ cmd: "/path/to/codeql" }),
JSON.stringify({ cmd: "/path/to/codeql", version: {} }),
JSON.stringify({ cmd: "/path/to/codeql", version: { version: 2 } }),
JSON.stringify({ version: { version: "2.20.0" } }),
JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0", overlayVersion: "1" },
}),
JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0", features: "nope" },
}),
]) {
process.env[EnvVar.CODEQL_VERSION_INFO] = value;
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined, value);
}
},
);

86
src/util.ts
View File

@@ -10,7 +10,7 @@ import * as yaml from "js-yaml";
import * as semver from "semver";
import * as apiCompatibility from "./api-compatibility.json";
import type { CodeQL, VersionInfo } from "./codeql";
import type { CodeQL } from "./codeql";
import type { Pack } from "./config/db-config";
import type { Config } from "./config-utils";
import { EnvVar, getRequiredEnvParam } from "./environment";
@@ -598,90 +598,6 @@ export function asHTTPError(arg: any): HTTPError | undefined {
return undefined;
}
let cachedCodeQlVersion: undefined | VersionInfo = undefined;
/**
* Resets the in-process cache of the CodeQL CLI version. Only for use in tests,
* which exercise multiple "steps" within a single process.
*/
export function resetCachedCodeQlVersion(): void {
cachedCodeQlVersion = undefined;
}
/** The persisted version together with the CLI path it was obtained from. */
interface PersistedVersionInfo {
cmd: string;
version: VersionInfo;
}
function isVersionInfo(x: unknown): x is VersionInfo {
const candidate = x as Partial<VersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.version === "string" &&
(candidate.features === undefined ||
(typeof candidate.features === "object" &&
candidate.features !== null)) &&
(candidate.overlayVersion === undefined ||
typeof candidate.overlayVersion === "number")
);
}
function isPersistedVersionInfo(x: unknown): x is PersistedVersionInfo {
const candidate = x as Partial<PersistedVersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.cmd === "string" &&
isVersionInfo(candidate.version)
);
}
export function cacheCodeQlVersion(cmd: string, version: VersionInfo): void {
if (cachedCodeQlVersion !== undefined) {
throw new Error("cacheCodeQlVersion() should be called only once");
}
cachedCodeQlVersion = version;
// Persist the version so that subsequent Actions steps, which run in separate
// processes, can reuse it rather than invoking `codeql version` again. We
// record the CLI path so that a different step using a different CodeQL bundle
// doesn't pick up a stale version.
core.exportVariable(
EnvVar.CODEQL_VERSION_INFO,
JSON.stringify({ cmd, version }),
);
}
export function getCachedCodeQlVersion(cmd?: string): undefined | VersionInfo {
if (cachedCodeQlVersion !== undefined) {
return cachedCodeQlVersion;
}
// Fall back to the value persisted by an earlier Actions step, if any. This is
// best-effort: any malformed or mismatched value is ignored so that the caller
// invokes `codeql version` instead.
const serialized = process.env[EnvVar.CODEQL_VERSION_INFO];
if (!serialized) {
return undefined;
}
let persisted: unknown;
try {
persisted = JSON.parse(serialized);
} catch {
return undefined;
}
if (
!isPersistedVersionInfo(persisted) ||
(cmd !== undefined && persisted.cmd !== cmd)
) {
return undefined;
}
// Memoize the parsed value so that subsequent calls in this process don't
// re-parse the environment variable.
cachedCodeQlVersion = persisted.version;
return cachedCodeQlVersion;
}
export async function codeQlVersionAtLeast(
codeql: CodeQL,
requiredVersion: string,