github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-10-30 17:39:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into henrymercer/dead-code-elimination

Browse Source
This commit is contained in:
Henry Mercer
2025-11-19 14:56:28 +00:00
parent cac5926de5 0b4317954f
commit 112cd075bd
24 changed files with 548 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/analyze-action.ts
View File

@@ -25,7 +25,7 @@ import {
isCodeQualityEnabled,
isCodeScanningEnabled,
} from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { cleanupAndUploadDatabases } from "./database-upload";
import {
DependencyCacheUploadStatusReport,
uploadDependencyCaches,
@@ -35,7 +35,7 @@ import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
import { getRepositoryNwo } from "./repository";
import * as statusReport from "./status-report";
import {
@@ -417,12 +417,21 @@ async function run() {
}
// Possibly upload the overlay-base database to actions cache.
// If databases are to be uploaded, they will first be cleaned up at the overlay level.
await uploadOverlayBaseDatabaseToCache(codeql, config, logger);
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` level.
await cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger);
// Possibly upload the database bundles for remote queries.
// If databases are to be uploaded, they will first be cleaned up at the clear level.
await uploadDatabases(repositoryNwo, codeql, config, apiDetails, logger);
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` or `clear` level.
await cleanupAndUploadDatabases(
repositoryNwo,
codeql,
config,
apiDetails,
features,
logger,
);
// Possibly upload the TRAP caches for later re-use
const trapCacheUploadStartTime = performance.now();

9
src/codeql.ts
View File

@@ -35,7 +35,7 @@ import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
import { BuildMode, getErrorMessage } from "./util";
import { BuildMode, CleanupLevel, getErrorMessage } from "./util";
type Options = Array<string | number | boolean>;
@@ -141,7 +141,10 @@ export interface CodeQL {
/**
* Clean up all the databases within a database cluster.
*/
databaseCleanupCluster(config: Config, cleanupLevel: string): Promise<void>;
databaseCleanupCluster(
config: Config,
cleanupLevel: CleanupLevel,
): Promise<void>;
/**
* Run 'codeql database bundle'.
*/
@@ -878,7 +881,7 @@ async function getCodeQLForCmd(
},
async databaseCleanupCluster(
config: Config,
cleanupLevel: string,
cleanupLevel: CleanupLevel,
): Promise<void> {
const cacheCleanupFlag = (await util.codeQlVersionAtLeast(
this,

100
src/config-utils.test.ts
View File

@@ -37,7 +37,9 @@ import {
ConfigurationError,
withTmpDir,
BuildMode,
DiskUsage,
} from "./util";
import * as util from "./util";
setupTests(test);
@@ -976,6 +978,7 @@ interface OverlayDatabaseModeTestSetup {
codeqlVersion: string;
gitRoot: string | undefined;
codeScanningConfig: configUtils.UserConfig;
diskUsage: DiskUsage | undefined;
}
const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
@@ -988,6 +991,10 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
codeqlVersion: CODEQL_OVERLAY_MINIMUM_VERSION,
gitRoot: "/some/git/root",
codeScanningConfig: {},
diskUsage: {
numAvailableBytes: 50_000_000_000,
numTotalBytes: 100_000_000_000,
},
};
const getOverlayDatabaseModeMacro = test.macro({
@@ -1020,6 +1027,8 @@ const getOverlayDatabaseModeMacro = test.macro({
setup.overlayDatabaseEnvVar;
}
sinon.stub(util, "checkDiskUsage").resolves(setup.diskUsage);
// Mock feature flags
const features = createFeatures(setup.features);
@@ -1177,6 +1186,45 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if runner disk space is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if we can't determine runner disk space",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
diskUsage: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
@@ -1347,6 +1395,45 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if runner disk space is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if we can't determine runner disk space",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
diskUsage: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with disable-default-queries",
@@ -1481,6 +1568,19 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by env on a runner with low disk space",
{
overlayDatabaseEnvVar: "overlay",
diskUsage: { numAvailableBytes: 0, numTotalBytes: 100_000_000_000 },
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by feature flag",

51
src/config-utils.ts
View File

@@ -43,10 +43,22 @@ import {
codeQlVersionAtLeast,
cloneObject,
isDefined,
checkDiskUsage,
} from "./util";
export * from "./config/db-config";
/**
* The minimum available disk space (in MB) required to perform overlay analysis.
* If the available disk space on the runner is below the threshold when deciding
* whether to perform overlay analysis, then the action will not perform overlay
* analysis unless overlay analysis has been explicitly enabled via environment
* variable.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & {
// Token to use when downloading packs from this registry.
token: string;
@@ -679,20 +691,37 @@ export async function getOverlayDatabaseMode(
codeScanningConfig,
)
) {
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
const diskUsage = await checkDiskUsage(logger);
if (
diskUsage === undefined ||
diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES
) {
const diskSpaceMb =
diskUsage === undefined
? 0
: Math.round(diskUsage.numAvailableBytes / 1_000_000);
overlayDatabaseMode = OverlayDatabaseMode.None;
useOverlayDatabaseCaching = false;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
`due to insufficient disk space (${diskSpaceMb} MB).`,
);
} else {
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
);
}
}
}

24
src/database-upload.test.ts
View File

@@ -10,11 +10,12 @@ import { GitHubApiDetails } from "./api-client";
import * as apiClient from "./api-client";
import { createStubCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { cleanupAndUploadDatabases } from "./database-upload";
import * as gitUtils from "./git-utils";
import { KnownLanguage } from "./languages";
import { RepositoryNwo } from "./repository";
import {
createFeatures,
createTestConfig,
getRecordingLogger,
LoggedMessage,
@@ -91,11 +92,12 @@ test("Abort database upload if 'upload-database' input set to false", async (t)
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -121,7 +123,7 @@ test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
await mockHttpRequests(201);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
{
@@ -129,6 +131,7 @@ test("Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
analysisKinds: [AnalysisKind.CodeQuality],
},
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -155,11 +158,12 @@ test("Abort database upload if running against GHES", async (t) => {
config.gitHubVersion = { type: GitHubVariant.GHES, version: "3.0" };
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
config,
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -183,11 +187,12 @@ test("Abort database upload if not analyzing default branch", async (t) => {
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(false);
const loggedMessages = [];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -212,11 +217,12 @@ test("Don't crash if uploading a database fails", async (t) => {
await mockHttpRequests(500);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
@@ -243,11 +249,12 @@ test("Successfully uploading a database to github.com", async (t) => {
await mockHttpRequests(201);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(
@@ -272,7 +279,7 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
const databaseUploadSpy = await mockHttpRequests(201);
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
await cleanupAndUploadDatabases(
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
@@ -281,6 +288,7 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
url: "https://tenant.ghe.com",
apiURL: undefined,
},
createFeatures([]),
getRecordingLogger(loggedMessages),
);
t.assert(

15
src/database-upload.ts
View File

@@ -5,17 +5,20 @@ import { AnalysisKind } from "./analyses";
import { getApiClient, GitHubApiDetails } from "./api-client";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
import { bundleDb, parseGitHubUrl } from "./util";
import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
export async function uploadDatabases(
export async function cleanupAndUploadDatabases(
repositoryNwo: RepositoryNwo,
codeql: CodeQL,
config: Config,
apiDetails: GitHubApiDetails,
features: FeatureEnablement,
logger: Logger,
): Promise<void> {
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
@@ -50,10 +53,16 @@ export async function uploadDatabases(
return;
}
const cleanupLevel =
config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase &&
(await features.getValue(Feature.UploadOverlayDbToApi))
? CleanupLevel.Overlay
: CleanupLevel.Clear;
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "clear");
await codeql.databaseCleanupCluster(config, cleanupLevel);
});
const client = getApiClient();

22
src/feature-flags.ts
View File

@@ -77,6 +77,7 @@ export enum Feature {
OverlayAnalysisSwift = "overlay_analysis_swift",
PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib",
QaTelemetryEnabled = "qa_telemetry_enabled",
UploadOverlayDbToApi = "upload_overlay_db_to_api",
UseRepositoryProperties = "use_repository_properties",
ValidateDbConfig = "validate_db_config",
}
@@ -166,6 +167,11 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: undefined,
},
[Feature.JavaMinimizeDependencyJars]: {
defaultValue: false,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
},
[Feature.OverlayAnalysis]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -277,21 +283,21 @@ export const featureConfig: Record<
minimumVersion: undefined,
toolsFeature: ToolsFeature.PythonDefaultIsToNotExtractStdlib,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
},
[Feature.QaTelemetryEnabled]: {
defaultValue: false,
envVar: "CODEQL_ACTION_QA_TELEMETRY",
legacyApi: true,
minimumVersion: undefined,
},
[Feature.JavaMinimizeDependencyJars]: {
[Feature.UploadOverlayDbToApi]: {
defaultValue: false,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
envVar: "CODEQL_ACTION_UPLOAD_OVERLAY_DB_TO_API",
minimumVersion: undefined,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
envVar: "CODEQL_ACTION_USE_REPOSITORY_PROPERTIES",
minimumVersion: undefined,
},
[Feature.ValidateDbConfig]: {
defaultValue: false,

7
src/overlay-database-utils.ts
View File

@@ -16,6 +16,7 @@ import { type Config } from "./config-utils";
import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import {
CleanupLevel,
getErrorMessage,
isInTestMode,
tryGetFolderBytes,
@@ -28,7 +29,7 @@ export enum OverlayDatabaseMode {
None = "none",
}
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.4";
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.23.5";
/**
* The maximum (uncompressed) size of the overlay base database that we will
@@ -204,7 +205,7 @@ function checkOverlayBaseDatabase(
* @returns A promise that resolves to true if the upload was performed and
* successfully completed, or false otherwise
*/
export async function uploadOverlayBaseDatabaseToCache(
export async function cleanupAndUploadOverlayBaseDatabaseToCache(
codeql: CodeQL,
config: Config,
logger: Logger,
@@ -242,7 +243,7 @@ export async function uploadOverlayBaseDatabaseToCache(
// Clean up the database using the overlay cleanup level.
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, "overlay");
await codeql.databaseCleanupCluster(config, CleanupLevel.Overlay);
});
const dbLocation = config.dbLocation;

5
src/util.ts
View File

@@ -1231,3 +1231,8 @@ export function unsafeEntriesInvariant<T extends Record<string, any>>(
([_, val]) => val !== undefined,
) as Array<[keyof T, Exclude<T[keyof T], undefined>]>;
}
export enum CleanupLevel {
Clear = "clear",
Overlay = "overlay",
}