Compare commits

...

40 Commits

Author SHA1 Message Date
Henry Mercer
db488ddef3 Merge pull request #4102 from github/update-v4.37.8-9ee088e13
Merge main into releases/v4
2026-08-21 13:24:15 +01:00
github-actions[bot]
1845f5ba8b Update changelog for v4.37.8 2026-08-21 12:09:34 +00:00
Henry Mercer
9ee088e136 Merge pull request #4080 from github/henrymercer/studious-giggle
Determine the overlay minimum disk space requirement from feature flags
2026-08-18 12:25:05 +00:00
Henry Mercer
1aef003397 Address review feedback on overlay disk flags
Document each minimum disk feature flag individually and replace the tuple list with an explicit feature-to-threshold mapping.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-08-18 12:09:33 +01:00
Henry Mercer
508b83bc41 Merge main into overlay minimum disk feature branch
Resolve the overlap with the separately shipped promotion of the overlay resource checks while preserving the feature-flagged minimum disk thresholds.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-08-18 12:09:09 +01:00
Michael B. Gale
d97b3428e8 Merge pull request #4098 from github/mbg/permission-error-as-configuration-error
Make `EACCES` when installing CodeQL CLI a `ConfigurationError`
2026-08-14 11:56:43 +00:00
Michael B. Gale
47fa622223 Make EACCES a ConfigurationError 2026-08-14 11:56:26 +01:00
Michael B. Gale
45693cc688 Refactor ENOSPC check into isDiskConfigurationError function 2026-08-14 11:53:32 +01:00
Mario Campos
c2fd8f54d1 Merge pull request #4081 from github/mario-campos/version-cache-to-disk
Persist CodeQL version output to file rather than environment
2026-08-13 14:27:41 +00:00
Mario Campos
c56f48e9bd Log unexpected conditions during caching CLI output 2026-08-13 09:14:00 -05:00
Mario Campos
aa0eadc572 Merge branch 'main' into mario-campos/version-cache-to-disk
# Conflicts:
#	lib/entry-points.js
2026-08-13 09:02:15 -05:00
Mario Campos
43250d671a Change cache key to string type to include CLI args 2026-08-13 08:59:09 -05:00
Michael B. Gale
5008effa71 Merge pull request #4094 from github/mergeback/v4.37.7-to-main-ff2f1c62
Mergeback v4.37.7 refs/heads/releases/v4 into main
2026-08-13 13:48:05 +00:00
github-actions[bot]
053d41e61e Rebuild 2026-08-13 13:30:41 +00:00
github-actions[bot]
1158e1c92a Update changelog and version after v4.37.7 2026-08-13 13:30:28 +00:00
Mario Campos
2d49edbac6 Re-order env to be first argument for consistency 2026-08-13 08:13:49 -05:00
Mario Campos
6dc633238e Bolster output-cache unit tests with more test cases 2026-08-12 13:45:22 -05:00
Mario Campos
6c0d9018d4 Change OutputCache to use object for entries
This will ensure it works nicely with `JSON.stringify`. And, then we can validate the type before marshaling.
2026-08-12 12:01:20 -05:00
Mario Campos
bf96b0df93 Expand test to ensure it does not throw an exception 2026-08-12 11:39:32 -05:00
Mario Campos
337136ab8a Rename CommandCacheRecord -> OutputCache 2026-08-12 11:29:11 -05:00
Mario Campos
a9baab8dee Export CLI cache types 2026-08-12 11:26:52 -05:00
Mario Campos
33d70867d5 Pass environment explicitly to CLI caching functions 2026-08-12 11:24:37 -05:00
Mario Campos
40f80a8df0 Rename type to better match generic intention 2026-08-11 18:09:09 -05:00
Mario Campos
b222c3aaea Generalize file cache data structure 2026-08-11 18:09:09 -05:00
Mario Campos
11569df0a1 Update JSDoc of getCachedCodeQlVersion 2026-08-11 15:55:16 -05:00
Mario Campos
0a99875ae5 Move VersionInfo-related types to cli/output-cache.ts
This brings them out of the crowded all-purpose `util.ts` and into `cli/output-cache.ts` where they are exclusively used.
2026-08-11 15:55:15 -05:00
Mario Campos
246018e041 Move VersionInfo to dedicated module 2026-08-11 15:55:15 -05:00
Mario Campos
1332611f51 Move cache-related util functions into dedicated module 2026-08-11 15:55:15 -05:00
Mario Campos
4dc327a942 Introduce basic cli/output-cache.ts module 2026-08-11 15:36:17 -05:00
Mario Campos
bb19330c5e Add test of getCachedCodeQlVersion with no file 2026-08-11 15:36:17 -05:00
Mario Campos
0e85c0e99c Refactor unit test to extract testing values 2026-08-11 15:36:17 -05:00
Mario Campos
bfcd769ba1 Fix JSDoc of env param 2026-08-11 15:36:17 -05:00
Mario Campos
208a88adc7 Simplify JSDoc of getCachedCodeQlVersion
Co-authored-by: Michael B. Gale <mbg@github.com>
2026-08-11 11:06:17 -05:00
Mario Campos
9183a7b6e1 Handle file-read errors as cache misses
This is particularly important for the first time that `getCachedCodeQlVersion` is invoked, as this cache file will not yet exist.
2026-08-10 10:44:48 -05:00
Mario Campos
5f8c44ba62 Persist CodeQL version output to file rather than environment 2026-08-07 15:11:20 -05:00
Henry Mercer
794f5bc385 Fix the memory check debug message at equality
The comparison accepts exactly the minimum, so say "at or above", to
match the wording of the disk space check.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-08-06 16:48:44 +01:00
Henry Mercer
54109818e0 Address review feedback on the disk space check
Say "at or above" in the debug message logged when the check passes,
since the comparison accepts exactly the minimum.

Check each feature flag against the threshold its name declares, rather
than only exercising a few of them, so that a mistake in one of the
mappings cannot go unnoticed. Both sides of the boundary are needed to
pin a threshold down: a mapping to a lower value would still pass the
case at the limit, and one to a higher value would still fail the case
below it.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-08-06 16:41:48 +01:00
Henry Mercer
99caaa8b90 Remove the overlay_analysis_resource_checks_v2 feature flag
The flag no longer has any effect now that its 14 GB threshold is the
unconditional default, so remove it. Setting
CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2 no longer does
anything.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-08-06 16:38:34 +01:00
Henry Mercer
6117bb503a Derive overlay minimum disk space from feature flags
Overlay analysis required 20 GB of available disk space, lowered to
14 GB when overlay_analysis_resource_checks_v2 was enabled. That gave
us a single step to roll out, and any further reduction needed another
flag and another release.

Determine the threshold from the new overlay_analysis_min_disk_N_gb
flags instead, taking the lowest one that is enabled so that a lower
limit can be rolled out to a subset of repositories without first
disabling the flag above it. When none are enabled, the 14 GB limit
now applies unconditionally, replacing the 20 GB default.

Thresholds remain in decimal MB, matching the bytes-per-MB convention
the disk check already used, so the effective byte values are unchanged
from the previous 14 GB path. Also log the available and required space
at debug level when the check passes, so that run logs show which
threshold took effect.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-08-06 16:38:33 +01:00
Henry Mercer
af767ec1f6 Add overlay_analysis_min_disk_N_gb feature flags
Add six feature flags, overlay_analysis_min_disk_8_gb through
overlay_analysis_min_disk_13_gb, which will be used to control the
amount of available disk space that overlay analysis requires.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-08-06 16:38:33 +01:00
18 changed files with 1566 additions and 1093 deletions

View File

@@ -2,6 +2,10 @@
See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs.
## 4.37.8 - 21 Aug 2026
No user facing changes.
## 4.37.7 - 13 Aug 2026
- Update default CodeQL bundle version to [2.26.3](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.26.3). [#4085](https://github.com/github/codeql-action/pull/4085)

1905
lib/entry-points.js generated

File diff suppressed because it is too large Load Diff

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "codeql",
"version": "4.37.7",
"version": "4.37.8",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "codeql",
"version": "4.37.7",
"version": "4.37.8",
"license": "MIT",
"workspaces": [
"pr-checks"

View File

@@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "4.37.7",
"version": "4.37.8",
"private": true,
"description": "CodeQL action",
"scripts": {

View File

@@ -0,0 +1,128 @@
import * as fs from "fs";
import path from "path";
import test from "ava";
import { EnvVar } from "../environment";
import { getRunnerLogger } from "../logging";
import { getTestEnv, setupTests } from "../testing-utils";
import * as util from "../util";
import * as outputCache from "./output-cache";
setupTests(test);
const logger = getRunnerLogger(true);
test.serial(
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "codeql-action-command-cache.json");
fs.writeFileSync(
cacheFile,
JSON.stringify({
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0" } },
}),
"utf8",
);
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.deepEqual(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
{
version: "2.20.0",
},
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
fs.writeFileSync(
cacheFile,
JSON.stringify({
cmd: "/path/to/other-codeql",
version: { version: "2.20.0" },
}),
"utf8",
);
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a malformed persisted value",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
fs.writeFileSync(cacheFile, "not valid json", "utf8");
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
const testValues = [
{ cmd: "/path/to/codeql" },
{ entries: { version: { version: "2.20.0" } } },
{ cmd: "/path/to/codeql", entries: {} },
{ cmd: "/path/to/codeql", entries: null },
{ cmd: "/path/to/codeql", entries: { version: {} } },
{ cmd: "/path/to/codeql", entries: { version: null } },
{ cmd: "/path/to/codeql", entries: { version: "2.20.0" } },
{ cmd: "/path/to/codeql", entries: { version: { version: null } } },
{ cmd: "/path/to/codeql", entries: { version: { version: 2.2 } } },
{ cmd: "/path/to/codeql", entries: { version: { version: 2 } } },
{
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0", overlayVersion: "1" } },
},
{
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0", features: "nope" } },
},
].map((v) => JSON.stringify(v));
for (const value of testValues) {
fs.writeFileSync(cacheFile, value, "utf8");
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
value,
);
}
});
},
);
test.serial("getCachedCodeQlVersion ignores non-existent file", async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.notThrows(() => {
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
});
});

156
src/cli/output-cache.ts Normal file
View File

@@ -0,0 +1,156 @@
import * as fs from "fs";
import path from "path";
import { getTemporaryDirectory } from "../actions-util";
import { Env } from "../environment";
import { Logger } from "../logging";
import type { VersionInfo } from "./types";
/**
* The keys of the command cache. Each key corresponds to a command whose output we cache.
*/
export type CommandCacheKey = string;
/**
* The type of the command cache that is persisted to disk.
*/
export interface OutputCache {
cmd: string;
entries: Record<CommandCacheKey, unknown>;
}
/**
* The name of the temporary file that backs the on-disk cache of
* CLI responses between workflow steps.
*/
const COMMAND_CACHE_FILENAME = "codeql-action-command-cache.json";
/**
* The module-global variable that caches the CodeQL CLI version in-memory.
*/
let cachedCodeQlVersion: undefined | VersionInfo = undefined;
/**
* Resets the in-process cache of the CodeQL CLI version. Only for use in tests,
* which exercise multiple "steps" within a single process.
*/
export function resetCachedCodeQlVersion(): void {
cachedCodeQlVersion = undefined;
}
/**
* Returns the path to the temporary file that backs the
* on-disk cache of CLI responses between workflow steps.
*/
function getCommandCacheFilePath(env: Env): string {
return path.join(getTemporaryDirectory(env), COMMAND_CACHE_FILENAME);
}
/**
* Caches the CodeQL CLI version both in-memory and on disk.
* @param env The environment variables to use.
* @param cmd The path to the CodeQL CLI.
* @param version The version information to cache.
*/
export function cacheCodeQlVersion(
env: Env,
cmd: string,
version: VersionInfo,
): void {
if (cachedCodeQlVersion !== undefined) {
throw new Error("cacheCodeQlVersion() should be called only once");
}
cachedCodeQlVersion = version;
const outputCache = {
cmd,
entries: { version },
} satisfies OutputCache;
// Persist the version so that subsequent Actions steps, which run in separate
// processes, can reuse it rather than invoking `codeql version` again. We
// record the CLI path so that a different step using a different CodeQL bundle
// doesn't pick up a stale version.
fs.writeFileSync(
getCommandCacheFilePath(env),
JSON.stringify(outputCache),
"utf8",
);
}
/**
* Returns the cached CodeQL CLI version, if any.
* @param logger The logger to use for logging messages.
* @param env The environment variables to use.
* @param cmd The path to the CodeQL CLI.
*/
export function getCachedCodeQlVersion(
logger: Logger,
env: Env,
cmd?: string,
): undefined | VersionInfo {
if (cachedCodeQlVersion !== undefined) {
return cachedCodeQlVersion;
}
// Fall back to the value persisted by an earlier Actions step, if any. This is
// best-effort: any malformed or mismatched value is ignored so that the caller
// invokes `codeql version` instead.
let serialized: string;
try {
serialized = fs.readFileSync(getCommandCacheFilePath(env), "utf8");
} catch (e) {
logger.debug(
`Cannot read CLI-cache file ${getCommandCacheFilePath(env)}: ${e}`,
);
return undefined;
}
let persisted: unknown;
try {
persisted = JSON.parse(serialized);
} catch (e) {
logger.debug(`Cannot parse CLI-cache data as JSON: ${e}`);
return undefined;
}
if (
!isOutputCache(persisted) ||
(cmd !== undefined && persisted.cmd !== cmd)
) {
return undefined;
}
// Memoize the parsed value so that subsequent calls in this process don't
// re-parse the environment variable.
cachedCodeQlVersion = persisted.entries.version as VersionInfo;
return cachedCodeQlVersion;
}
/**
* Determines whether a value is a `VersionInfo` object.
* @param x The value to test
*/
function isVersionInfo(x: unknown): x is VersionInfo {
const candidate = x as Partial<VersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.version === "string" &&
(candidate.features === undefined ||
(typeof candidate.features === "object" &&
candidate.features !== null)) &&
(candidate.overlayVersion === undefined ||
typeof candidate.overlayVersion === "number")
);
}
/**
* Determines whether a value is a `OutputCache` object.
* @param x The value to test
*/
function isOutputCache(x: unknown): x is OutputCache {
const candidate = x as Partial<OutputCache> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.cmd === "string" &&
candidate.entries !== undefined &&
isVersionInfo(candidate.entries.version)
);
}

13
src/cli/types.ts Normal file
View File

@@ -0,0 +1,13 @@
export interface VersionInfo {
version: string;
features?: { [name: string]: boolean };
/**
* The overlay version helps deal with backward incompatible changes for
* overlay analysis. When a precompiled query pack reports the same overlay
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
* analysis with that pack. Otherwise, if the overlay versions are different,
* or if either the pack or the CLI does not report an overlay version,
* we need to revert to non-overlay analysis.
*/
overlayVersion?: number;
}

View File

@@ -51,6 +51,31 @@ test.beforeEach(() => {
});
});
test("isDiskConfigurationError - true for expected errors", async (t) => {
t.true(
codeql.isDiskConfigurationError(new Error("ENOSPC: Out of disk space")),
);
t.true(
codeql.isDiskConfigurationError(
new Error(
"EACCES: permission denied, mkdir /opt/hostedtoolcache/CodeQL/",
),
),
);
});
test("isDiskConfigurationError - false for other errors", async (t) => {
t.false(codeql.isDiskConfigurationError("Not an Error instance"));
const otherMessages = [
"Does not contain an error code we test for",
"ENOSP: Not quite the full error code",
];
for (const otherMessage of otherMessages) {
t.false(codeql.isDiskConfigurationError(new Error(otherMessage)));
}
});
async function installIntoToolcache({
apiDetails = SAMPLE_DOTCOM_API_DETAILS,
cliVersion,

View File

@@ -12,10 +12,12 @@ import {
runTool,
} from "./actions-util";
import * as api from "./api-client";
import * as outputCache from "./cli/output-cache";
import type { VersionInfo } from "./cli/types";
import { CliError, wrapCliConfigurationError } from "./cli-errors";
import { appendExtraQueryExclusions, type Config } from "./config-utils";
import { DocUrl } from "./doc-url";
import { EnvVar } from "./environment";
import { EnvVar, getEnv } from "./environment";
import {
CodeQLDefaultVersionInfo,
Feature,
@@ -214,20 +216,6 @@ export interface CodeQL {
): Promise<void>;
}
export interface VersionInfo {
version: string;
features?: { [name: string]: boolean };
/**
* The overlay version helps deal with backward incompatible changes for
* overlay analysis. When a precompiled query pack reports the same overlay
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
* analysis with that pack. Otherwise, if the overlay versions are different,
* or if either the pack or the CLI does not report an overlay version,
* we need to revert to non-overlay analysis.
*/
overlayVersion?: number;
}
export interface ResolveDatabaseOutput {
overlayBaseSpecifier?: string;
}
@@ -285,6 +273,26 @@ const GHES_MOST_RECENT_DEPRECATION_DATE = "2026-07-01";
/** The CLI verbosity level to use for extraction in debug mode. */
const EXTRACTION_DEBUG_MODE_VERBOSITY = "progress++";
/**
* Decides whether `e` is a disk-related error outside of our control
* that should be classified as a `ConfigurationError`.
*
* @param e The error to check.
* @returns True if the error should be treated as a `ConfigurationError` or false if not.
*/
export function isDiskConfigurationError(e: unknown): boolean {
if (!(e instanceof Error)) {
return false;
}
return (
// out of disk space
e.message.includes("ENOSPC") ||
// access denied
e.message.includes("EACCES")
);
}
/**
* Set up CodeQL CLI access.
*
@@ -355,8 +363,7 @@ export async function setupCodeQL(
} catch (rawError) {
const e = api.wrapApiConfigurationError(rawError);
const ErrorClass =
e instanceof util.ConfigurationError ||
(e instanceof Error && e.message.includes("ENOSPC")) // out of disk space
e instanceof util.ConfigurationError || isDiskConfigurationError(e)
? util.ConfigurationError
: Error;
@@ -503,7 +510,7 @@ async function getCodeQLForCmd(
return cmd;
},
async getVersion() {
let result = util.getCachedCodeQlVersion(cmd);
let result = outputCache.getCachedCodeQlVersion(logger, getEnv(), cmd);
if (result === undefined) {
result = await runCliJson<VersionInfo>(
cmd,
@@ -512,7 +519,7 @@ async function getCodeQLForCmd(
noStreamStdout: true,
},
);
util.cacheCodeQlVersion(cmd, result);
outputCache.cacheCodeQlVersion(getEnv(), cmd, result);
}
return result;
},

View File

@@ -1333,6 +1333,101 @@ checkOverlayEnablementMacro.serial(
},
);
// Check that each feature flag lowers the limit to the threshold that its name
// declares. Both sides of the boundary are needed to pin the threshold down: a
// mapping to a lower value would still pass the case at the limit, and one to a
// higher value would still fail the case below it.
for (const [feature, thresholdGb] of [
[Feature.OverlayAnalysisMinDisk8Gb, 8],
[Feature.OverlayAnalysisMinDisk9Gb, 9],
[Feature.OverlayAnalysisMinDisk10Gb, 10],
[Feature.OverlayAnalysisMinDisk11Gb, 11],
[Feature.OverlayAnalysisMinDisk12Gb, 12],
[Feature.OverlayAnalysisMinDisk13Gb, 13],
] as Array<[Feature, number]>) {
const features = [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
feature,
];
checkOverlayEnablementMacro.serial(
`Overlay-base database on default branch if ${feature} is enabled and runner disk space is at its limit`,
{
languages: [BuiltInLanguage.javascript],
features,
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: thresholdGb * 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
checkOverlayEnablementMacro.serial(
`No overlay-base database on default branch if ${feature} is enabled and runner disk space is below its limit`,
{
languages: [BuiltInLanguage.javascript],
features,
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: thresholdGb * 1_000_000_000 - 1_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
},
);
}
checkOverlayEnablementMacro.serial(
"Overlay-base database on default branch if runner disk space is exactly at the lowest limit enabled by a feature flag",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisMinDisk9Gb,
Feature.OverlayAnalysisMinDisk12Gb,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 9_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if runner disk space is below the lowest limit enabled by a feature flag",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisMinDisk9Gb,
Feature.OverlayAnalysisMinDisk12Gb,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 8_500_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
},
);
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if memory flag is too low",
{

View File

@@ -48,7 +48,7 @@ import {
import { prepareDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import * as errorMessages from "./error-messages";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Feature, FeatureEnablement, FeatureWithoutCLI } from "./feature-flags";
import {
RepositoryProperties,
RepositoryPropertyName,
@@ -101,10 +101,23 @@ export { type Config } from "./config/action-config";
* whether to perform overlay analysis, then the action will not perform overlay
* analysis unless overlay analysis has been explicitly enabled via environment
* variable.
*
* This threshold can be lowered by the feature flags in
* `OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE`.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
/**
* Minimum available disk space (in MB) enabled by each overlay feature flag.
*/
const OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE = {
[Feature.OverlayAnalysisMinDisk8Gb]: 8000,
[Feature.OverlayAnalysisMinDisk9Gb]: 9000,
[Feature.OverlayAnalysisMinDisk10Gb]: 10000,
[Feature.OverlayAnalysisMinDisk11Gb]: 11000,
[Feature.OverlayAnalysisMinDisk12Gb]: 12000,
[Feature.OverlayAnalysisMinDisk13Gb]: 13000,
} satisfies Partial<Record<FeatureWithoutCLI, number>>;
/**
* The minimum memory (in MB) that must be available for CodeQL to perform overlay analysis. If
@@ -579,21 +592,44 @@ async function checkOverlayAnalysisFeatureEnabled(
return new Success(undefined);
}
/**
* Returns the minimum available disk space (in MB) required to perform overlay
* analysis, which is the lowest threshold enabled by a feature flag, or the
* default threshold if no such feature flag is enabled.
*/
async function getMinimumDiskSpaceMb(
features: FeatureEnablement,
): Promise<number> {
let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB;
for (const [feature, thresholdMb] of Object.entries(
OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE,
)) {
if (await features.getValue(feature as FeatureWithoutCLI)) {
minimumMb = Math.min(minimumMb, thresholdMb);
}
}
return minimumMb;
}
/** Checks if the runner has enough disk space for overlay analysis. */
function runnerHasSufficientDiskSpace(
diskUsage: DiskUsage,
logger: Logger,
minimumDiskSpaceMb: number,
): boolean {
const minimumDiskSpaceBytes = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES;
if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) {
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000);
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1_000_000) {
logger.info(
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
`due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`,
);
return false;
}
logger.debug(
`Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the ` +
`minimum of ${minimumDiskSpaceMb} MB.`,
);
return true;
}
@@ -625,7 +661,7 @@ async function runnerHasSufficientMemory(
}
logger.debug(
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
);
return true;
}
@@ -636,11 +672,13 @@ async function runnerHasSufficientMemory(
*/
async function checkRunnerResources(
codeql: CodeQL,
features: FeatureEnablement,
diskUsage: DiskUsage,
ramInput: string | undefined,
logger: Logger,
): Promise<Result<void, OverlayDisabledReason>> {
if (!runnerHasSufficientDiskSpace(diskUsage, logger)) {
const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features);
if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) {
return new Failure(OverlayDisabledReason.InsufficientDiskSpace);
}
if (!(await runnerHasSufficientMemory(codeql, ramInput, logger))) {
@@ -752,7 +790,13 @@ export async function checkOverlayEnablement(
}
const resourceResult =
performResourceChecks && diskUsage !== undefined
? await checkRunnerResources(codeql, diskUsage, ramInput, logger)
? await checkRunnerResources(
codeql,
features,
diskUsage,
ramInput,
logger,
)
: new Success<void>(undefined);
if (resourceResult.isFailure()) {
return resourceResult;

View File

@@ -39,12 +39,6 @@ export enum EnvVar {
*/
CODE_SCANNING_REF = "CODE_SCANNING_REF",
/**
* `PersistedVersionInfo` for the CodeQL CLI, so later Actions steps can reuse it instead of
* invoking `codeql version` again.
*/
CODEQL_VERSION_INFO = "CODEQL_ACTION_CLI_VERSION_INFO",
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",

View File

@@ -121,6 +121,36 @@ export enum Feature {
* `OverlayAnalysisMatchCodeqlVersion` overrides this flag.
*/
OverlayAnalysisMatchCodeqlVersionDryRun = "overlay_analysis_match_codeql_version_dry_run",
/**
* Lowers the overlay minimum available disk space to 8 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk8Gb = "overlay_analysis_min_disk_8_gb",
/**
* Lowers the overlay minimum available disk space to 9 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk9Gb = "overlay_analysis_min_disk_9_gb",
/**
* Lowers the overlay minimum available disk space to 10 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk10Gb = "overlay_analysis_min_disk_10_gb",
/**
* Lowers the overlay minimum available disk space to 11 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk11Gb = "overlay_analysis_min_disk_11_gb",
/**
* Lowers the overlay minimum available disk space to 12 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk12Gb = "overlay_analysis_min_disk_12_gb",
/**
* Lowers the overlay minimum available disk space to 13 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk13Gb = "overlay_analysis_min_disk_13_gb",
OverlayAnalysisPython = "overlay_analysis_python",
OverlayAnalysisRuby = "overlay_analysis_ruby",
/** Controls whether hardware checks are skipped for overlay analysis. */
@@ -349,6 +379,36 @@ export const featureConfig = {
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk8Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk9Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk10Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk11Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk12Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk13Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisStatusCheck]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK",

View File

@@ -14,6 +14,7 @@ import {
isSelfHostedRunner,
} from "./actions-util";
import { getAnalysisKey, getApiClient } from "./api-client";
import { getCachedCodeQlVersion } from "./cli/output-cache";
import type { Config } from "./config/action-config";
import type { ComputedInput, InputName } from "./config/inputs";
import { parseRegistriesWithoutCredentials } from "./config/pack-registries";
@@ -30,7 +31,6 @@ import { registryBaseSchema } from "./start-proxy/types";
import {
ConfigurationError,
getRequiredEnvParam,
getCachedCodeQlVersion,
isInTestMode,
GITHUB_DOTCOM_URL,
DiskUsage,
@@ -376,7 +376,7 @@ export async function createStatusReportBase(
core.exportVariable(EnvVar.WORKFLOW_STARTED_AT, workflowStartedAt);
}
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const codeQlCliVersion = getCachedCodeQlVersion();
const codeQlCliVersion = getCachedCodeQlVersion(logger, getEnv());
const actionRef = process.env["GITHUB_ACTION_REF"] || "";
const testingEnvironment = getTestingEnvironment();
// re-export the testing environment variable so that it is available to subsequent steps,

View File

@@ -18,6 +18,8 @@ import { AnalysisKind } from "./analyses";
import * as apiClient from "./api-client";
import { GitHubApiDetails } from "./api-client";
import { CachingKind } from "./caching-utils";
import { resetCachedCodeQlVersion } from "./cli/output-cache";
import type { VersionInfo } from "./cli/types";
import * as codeql from "./codeql";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json";
@@ -39,7 +41,6 @@ import {
GitHubVariant,
GitHubVersion,
HTTPError,
resetCachedCodeQlVersion,
Result,
Success,
} from "./util";
@@ -872,7 +873,7 @@ export const makeVersionInfo = (
version: string,
features?: { [name: string]: boolean },
overlayVersion?: number,
): codeql.VersionInfo => ({
): VersionInfo => ({
version,
features,
overlayVersion,

View File

@@ -1,6 +1,6 @@
import * as semver from "semver";
import type { VersionInfo } from "./codeql";
import type { VersionInfo } from "./cli/types";
export enum ToolsFeature {
BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries",

View File

@@ -532,58 +532,3 @@ test("Failure.orElse returns the default value for a failure result", (t) => {
const result = new util.Failure(new Error("test error"));
t.is(result.orElse("default value"), "default value");
});
test.serial(
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0" },
});
t.deepEqual(util.getCachedCodeQlVersion("/path/to/codeql"), {
version: "2.20.0",
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = JSON.stringify({
cmd: "/path/to/other-codeql",
version: { version: "2.20.0" },
});
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined);
},
);
test.serial(
"getCachedCodeQlVersion ignores a malformed persisted value",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = "not valid json";
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined);
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
(t) => {
for (const value of [
JSON.stringify({ cmd: "/path/to/codeql" }),
JSON.stringify({ cmd: "/path/to/codeql", version: {} }),
JSON.stringify({ cmd: "/path/to/codeql", version: { version: 2 } }),
JSON.stringify({ version: { version: "2.20.0" } }),
JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0", overlayVersion: "1" },
}),
JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0", features: "nope" },
}),
]) {
process.env[EnvVar.CODEQL_VERSION_INFO] = value;
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined, value);
}
},
);

View File

@@ -10,7 +10,7 @@ import * as yaml from "js-yaml";
import * as semver from "semver";
import * as apiCompatibility from "./api-compatibility.json";
import type { CodeQL, VersionInfo } from "./codeql";
import type { CodeQL } from "./codeql";
import type { Pack } from "./config/db-config";
import type { Config } from "./config-utils";
import { EnvVar, getRequiredEnvParam } from "./environment";
@@ -598,90 +598,6 @@ export function asHTTPError(arg: any): HTTPError | undefined {
return undefined;
}
let cachedCodeQlVersion: undefined | VersionInfo = undefined;
/**
* Resets the in-process cache of the CodeQL CLI version. Only for use in tests,
* which exercise multiple "steps" within a single process.
*/
export function resetCachedCodeQlVersion(): void {
cachedCodeQlVersion = undefined;
}
/** The persisted version together with the CLI path it was obtained from. */
interface PersistedVersionInfo {
cmd: string;
version: VersionInfo;
}
function isVersionInfo(x: unknown): x is VersionInfo {
const candidate = x as Partial<VersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.version === "string" &&
(candidate.features === undefined ||
(typeof candidate.features === "object" &&
candidate.features !== null)) &&
(candidate.overlayVersion === undefined ||
typeof candidate.overlayVersion === "number")
);
}
function isPersistedVersionInfo(x: unknown): x is PersistedVersionInfo {
const candidate = x as Partial<PersistedVersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.cmd === "string" &&
isVersionInfo(candidate.version)
);
}
export function cacheCodeQlVersion(cmd: string, version: VersionInfo): void {
if (cachedCodeQlVersion !== undefined) {
throw new Error("cacheCodeQlVersion() should be called only once");
}
cachedCodeQlVersion = version;
// Persist the version so that subsequent Actions steps, which run in separate
// processes, can reuse it rather than invoking `codeql version` again. We
// record the CLI path so that a different step using a different CodeQL bundle
// doesn't pick up a stale version.
core.exportVariable(
EnvVar.CODEQL_VERSION_INFO,
JSON.stringify({ cmd, version }),
);
}
export function getCachedCodeQlVersion(cmd?: string): undefined | VersionInfo {
if (cachedCodeQlVersion !== undefined) {
return cachedCodeQlVersion;
}
// Fall back to the value persisted by an earlier Actions step, if any. This is
// best-effort: any malformed or mismatched value is ignored so that the caller
// invokes `codeql version` instead.
const serialized = process.env[EnvVar.CODEQL_VERSION_INFO];
if (!serialized) {
return undefined;
}
let persisted: unknown;
try {
persisted = JSON.parse(serialized);
} catch {
return undefined;
}
if (
!isPersistedVersionInfo(persisted) ||
(cmd !== undefined && persisted.cmd !== cmd)
) {
return undefined;
}
// Memoize the parsed value so that subsequent calls in this process don't
// re-parse the environment variable.
cachedCodeQlVersion = persisted.version;
return cachedCodeQlVersion;
}
export async function codeQlVersionAtLeast(
codeql: CodeQL,
requiredVersion: string,