github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-10-03 09:14:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor cleanupAndUploadDatabases querying checkout_path itself

Browse Source
This commit is contained in:
Michael B. Gale
2026-09-16 12:57:20 +01:00
parent ada4e83349
commit 8a88af6849
4 changed files with 95 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/entry-points.js generated
View File

@@ -154172,7 +154172,8 @@ async function warnIfGoInstalledAfterInit(config, logger) {
// src/database-upload.ts
var fs18 = __toESM(require("fs"));
async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetails, features, logger) {
async function cleanupAndUploadDatabases(action, repositoryNwo, codeql, config, apiDetails, checkoutPath) {
const logger = action.logger;
if (getRequiredInput("upload-database") !== "true") {
logger.debug("Database upload disabled in workflow. Skipping upload.");
return [];
@@ -154195,7 +154196,7 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
logger.debug("Not analyzing default branch. Skipping upload.");
return [];
}
const shouldUploadOverlayBase = config.overlayDatabaseMode === "overlay-base" /* OverlayBase */ && await features.getValue("upload_overlay_db_to_api" /* UploadOverlayDbToApi */, codeql);
const shouldUploadOverlayBase = config.overlayDatabaseMode === "overlay-base" /* OverlayBase */ && await action.features.getValue("upload_overlay_db_to_api" /* UploadOverlayDbToApi */, codeql);
const cleanupLevel = shouldUploadOverlayBase ? "overlay" /* Overlay */ : "clear" /* Clear */;
await withGroupAsync("Cleaning up databases", async () => {
await codeql.databaseCleanupCluster(config, cleanupLevel);
@@ -154208,9 +154209,7 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
includeDiagnostics: false
});
bundledDbSize = fs18.statSync(bundledDb).size;
const commitOid = await getCommitOid(
getRequiredInput("checkout_path")
);
const commitOid = await getCommitOid(checkoutPath);
const maxAttempts = 4;
let uploadDurationMs;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
@@ -156691,12 +156690,12 @@ async function run({
checkoutPath
);
databaseUploadResults = await cleanupAndUploadDatabases(
{ logger, features },
repositoryNwo,
codeql,
config,
apiDetails,
features,
logger
checkoutPath
);
const trapCacheUploadStartTime = import_perf_hooks6.performance.now();
didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger);

4
src/analyze-action.ts
View File

@@ -407,12 +407,12 @@ async function run({
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` or `clear` level.
databaseUploadResults = await cleanupAndUploadDatabases(
{ logger, features },
repositoryNwo,
codeql,
config,
apiDetails,
features,
logger,
checkoutPath,
);
// Possibly upload the TRAP caches for later re-use

134
src/database-upload.test.ts
View File

@@ -20,8 +20,9 @@ import {
checkExpectedLogMessages,
createFeatures,
createTestConfig,
getRecordingLogger,
LoggedMessage,
getTestEnv,
initAllState,
RecordingLogger,
setupActionsVars,
setupTests,
} from "./testing-utils";
@@ -90,23 +91,24 @@ test.serial(
"Abort database upload if 'upload-database' input set to false",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("false");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
const loggedMessages: LoggedMessage[] = [];
const logger = new RecordingLogger();
await cleanupAndUploadDatabases(
initAllState({ env, logger }),
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
"",
);
checkExpectedLogMessages(t, loggedMessages, [
checkExpectedLogMessages(t, logger.messages, [
"Database upload disabled in workflow. Skipping upload.",
]);
});
@@ -117,7 +119,8 @@ test.serial(
"Abort database upload if 'analysis-kinds: code-scanning' is not enabled",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -126,8 +129,9 @@ test.serial(
await mockHttpRequests(201);
const loggedMessages: LoggedMessage[] = [];
const logger = new RecordingLogger();
await cleanupAndUploadDatabases(
initAllState({ env, logger }),
testRepoName,
getCodeQL(),
{
@@ -135,10 +139,9 @@ test.serial(
analysisKinds: [AnalysisKind.CodeQuality],
},
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
"",
);
checkExpectedLogMessages(t, loggedMessages, [
checkExpectedLogMessages(t, logger.messages, [
"Not uploading database because 'analysis-kinds: code-scanning' is not enabled.",
]);
});
@@ -147,7 +150,8 @@ test.serial(
test.serial("Abort database upload if running against GHES", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -157,16 +161,16 @@ test.serial("Abort database upload if running against GHES", async (t) => {
const config = getTestConfig(tmpDir);
config.gitHubVersion = { type: GitHubVariant.GHES, version: "3.0" };
const loggedMessages: LoggedMessage[] = [];
const logger = new RecordingLogger();
await cleanupAndUploadDatabases(
initAllState({ env, logger }),
testRepoName,
getCodeQL(),
config,
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
"",
);
checkExpectedLogMessages(t, loggedMessages, [
checkExpectedLogMessages(t, logger.messages, [
"Not running against github.com or GHEC-DR. Skipping upload.",
]);
});
@@ -176,23 +180,24 @@ test.serial(
"Abort database upload if not analyzing default branch",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(false);
const loggedMessages: LoggedMessage[] = [];
const logger = new RecordingLogger();
await cleanupAndUploadDatabases(
initAllState({ env, logger }),
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
"",
);
checkExpectedLogMessages(t, loggedMessages, [
checkExpectedLogMessages(t, logger.messages, [
"Not analyzing default branch. Skipping upload.",
]);
});
@@ -203,7 +208,8 @@ test.serial(
"Don't crash if uploading a database fails with a non-retryable error",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -212,17 +218,17 @@ test.serial(
const databaseUploadSpy = await mockHttpRequests(422);
const loggedMessages: LoggedMessage[] = [];
const logger = new RecordingLogger();
await cleanupAndUploadDatabases(
initAllState({ env, logger }),
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
"",
);
checkExpectedLogMessages(t, loggedMessages, [
checkExpectedLogMessages(t, logger.messages, [
"Failed to upload database for javascript: some error message",
]);
@@ -236,7 +242,8 @@ test.serial(
"Don't crash if uploading a database fails with a retryable error",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -251,17 +258,17 @@ test.serial(
.stub(global, "setTimeout")
.callsFake((fn: () => void) => originalSetTimeout(fn, 0));
const loggedMessages: LoggedMessage[] = [];
const logger = new RecordingLogger();
await cleanupAndUploadDatabases(
initAllState({ env, logger }),
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
"",
);
checkExpectedLogMessages(t, loggedMessages, [
checkExpectedLogMessages(t, logger.messages, [
"Failed to upload database for javascript: some error message",
]);
@@ -279,7 +286,8 @@ test.serial(
test.serial("Successfully uploading a database to github.com", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -288,16 +296,16 @@ test.serial("Successfully uploading a database to github.com", async (t) => {
await mockHttpRequests(201);
const loggedMessages: LoggedMessage[] = [];
const logger = new RecordingLogger();
await cleanupAndUploadDatabases(
initAllState({ env, logger }),
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
testApiDetails,
createFeatures([]),
getRecordingLogger(loggedMessages),
"",
);
checkExpectedLogMessages(t, loggedMessages, [
checkExpectedLogMessages(t, logger.messages, [
"Successfully uploaded database for javascript",
]);
});
@@ -305,7 +313,8 @@ test.serial("Successfully uploading a database to github.com", async (t) => {
test.serial("Successfully uploading a database to GHEC-DR", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -314,8 +323,9 @@ test.serial("Successfully uploading a database to GHEC-DR", async (t) => {
const databaseUploadSpy = await mockHttpRequests(201);
const loggedMessages: LoggedMessage[] = [];
const logger = new RecordingLogger();
await cleanupAndUploadDatabases(
initAllState({ env, logger }),
testRepoName,
getCodeQL(),
getTestConfig(tmpDir),
@@ -324,10 +334,9 @@ test.serial("Successfully uploading a database to GHEC-DR", async (t) => {
url: "https://tenant.ghe.com",
apiURL: undefined,
},
createFeatures([]),
getRecordingLogger(loggedMessages),
"",
);
checkExpectedLogMessages(t, loggedMessages, [
checkExpectedLogMessages(t, logger.messages, [
"Successfully uploaded database for javascript",
]);
t.assert(
@@ -343,7 +352,8 @@ test.serial(
"Records overlay and clear cleanup sizes when uploading an overlay-base database",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -375,14 +385,16 @@ test.serial(
const config = getTestConfig(tmpDir);
config.overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
const loggedMessages: LoggedMessage[] = [];
const results = await cleanupAndUploadDatabases(
initAllState({
env,
features: createFeatures([Feature.UploadOverlayDbToApi]),
}),
testRepoName,
codeql,
config,
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger(loggedMessages),
"",
);
// The database should be cleaned up at the `overlay` level for the upload
@@ -402,7 +414,8 @@ test.serial(
"Does not measure clear cleanup size for a regular (non-overlay-base) upload",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -422,12 +435,15 @@ test.serial(
});
const results = await cleanupAndUploadDatabases(
initAllState({
env,
features: createFeatures([Feature.UploadOverlayDbToApi]),
}),
testRepoName,
codeql,
getTestConfig(tmpDir),
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger([]),
"",
);
// A regular upload is cleaned only once, at the `clear` level.
@@ -441,7 +457,8 @@ test.serial(
test.serial("Does not measure clear cleanup size in debug mode", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -465,12 +482,15 @@ test.serial("Does not measure clear cleanup size in debug mode", async (t) => {
config.debugMode = true;
const results = await cleanupAndUploadDatabases(
initAllState({
env,
features: createFeatures([Feature.UploadOverlayDbToApi]),
}),
testRepoName,
codeql,
config,
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger([]),
"",
);
// In debug mode we clean up at the `overlay` level for the upload but skip
@@ -486,7 +506,8 @@ test.serial(
"Does not record a clear cleanup duration when the clear cleanup fails",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const env = getTestEnv();
setupActionsVars(tmpDir, tmpDir, {}, env);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
@@ -510,12 +531,15 @@ test.serial(
config.overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
const results = await cleanupAndUploadDatabases(
initAllState({
env,
features: createFeatures([Feature.UploadOverlayDbToApi]),
}),
testRepoName,
codeql,
config,
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger([]),
"",
);
// When the `clear` cleanup fails, no size is measured, so we should not

15
src/database-upload.ts
View File

@@ -1,5 +1,6 @@
import * as fs from "fs";
import { ActionState } from "./action-common";
import * as actionsUtil from "./actions-util";
import { AnalysisKind } from "./analyses";
import {
@@ -9,7 +10,7 @@ import {
} from "./api-client";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Feature } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
@@ -45,13 +46,15 @@ export interface DatabaseUploadResult {
}
export async function cleanupAndUploadDatabases(
action: ActionState<["Logger", "FeatureFlags"]>,
repositoryNwo: RepositoryNwo,
codeql: CodeQL,
config: Config,
apiDetails: GitHubApiDetails,
features: FeatureEnablement,
logger: Logger,
checkoutPath: string,
): Promise<DatabaseUploadResult[]> {
const logger = action.logger;
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
logger.debug("Database upload disabled in workflow. Skipping upload.");
return [];
@@ -87,7 +90,7 @@ export async function cleanupAndUploadDatabases(
// If config.overlayDatabaseMode is OverlayBase, then we have overlay base databases for all languages.
const shouldUploadOverlayBase =
config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase &&
(await features.getValue(Feature.UploadOverlayDbToApi, codeql));
(await action.features.getValue(Feature.UploadOverlayDbToApi, codeql));
const cleanupLevel = shouldUploadOverlayBase
? CleanupLevel.Overlay
: CleanupLevel.Clear;
@@ -110,9 +113,7 @@ export async function cleanupAndUploadDatabases(
includeDiagnostics: false,
});
bundledDbSize = fs.statSync(bundledDb).size;
const commitOid = await gitUtils.getCommitOid(
actionsUtil.getRequiredInput("checkout_path"),
);
const commitOid = await gitUtils.getCommitOid(checkoutPath);
// Upload with manual retry logic. We disable Octokit's built-in retries
// because the request body is a ReadStream, which can only be consumed
// once.