From 6045ee80fc41ff6e3612d0742ba6c7fbb2f730b2 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 19 Aug 2026 17:28:03 +0100 Subject: [PATCH] Disable overlay analysis when pull request analyses fail Once branch selection has settled on an overlay-base build, make a single non-paginating request for the most recent pull request failure marker. In the happy path no markers exist and the list is empty. On detection, save the persistent overlay status cache entry, which is what makes pull requests skip overlay analysis too, and disable overlay analysis for this run. Any failure is treated as if no marker was found, so the check never disables overlay analysis on its own errors. `overlay_analysis_status_check_pr_dry_run` performs the same request and emits the same telemetry, but leaves overlay analysis enabled. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: ff943063-d9be-440f-9cac-5e44c4fcfbaa --- lib/entry-points.js | 157 +++++++++++++++++++++++++++- src/config-utils.test.ts | 62 ++++++++++++ src/config-utils.ts | 33 +++++- src/feature-flags.ts | 23 +++++ src/overlay/diagnostics.ts | 52 ++++++++++ src/overlay/status.test.ts | 202 +++++++++++++++++++++++++++++++++++++ src/overlay/status.ts | 141 ++++++++++++++++++++++++++ 7 files changed, 667 insertions(+), 3 deletions(-) diff --git a/lib/entry-points.js b/lib/entry-points.js index c6aee0c60..94dc73eee 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -146671,6 +146671,27 @@ async function listActionsCaches(keyPrefix, ref) { } ); } +async function listActionsCachesPage({ + keyPrefix, + sort, + direction, + perPage +}) { + const repositoryNwo = getRepositoryNwo(); + const response = await getApiClient().request( + "GET /repos/{owner}/{repo}/actions/caches", + { + owner: repositoryNwo.owner, + repo: repositoryNwo.repo, + key: keyPrefix, + sort, + direction, + per_page: perPage, + request: { retries: 0 } + } + ); + return response.data.actions_caches; +} async function deleteActionsCache(id) { const repositoryNwo = getRepositoryNwo(); await getApiClient().rest.actions.deleteActionsCacheById({ @@ -147917,6 +147938,16 @@ var featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK", minimumVersion: void 0 }, + ["overlay_analysis_status_check_pr" /* OverlayAnalysisStatusCheckPr */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK_PR", + minimumVersion: void 0 + }, + ["overlay_analysis_status_check_pr_dry_run" /* OverlayAnalysisStatusCheckPrDryRun */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK_PR_DRY_RUN", + minimumVersion: void 0 + }, ["overlay_analysis_status_save" /* OverlayAnalysisStatusSave */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_SAVE", @@ -149786,6 +149817,29 @@ Improved incremental analysis will be automatically retried when the next versio ) ); } + if (overlayDisabledReason === "pull-request-analysis-failed" /* PullRequestAnalysisFailed */) { + addNoLanguageDiagnostic( + config, + makeDiagnostic( + "codeql-action/overlay-disabled-due-to-pull-request-failure", + "Skipped improved incremental analysis because pull request analyses did not complete successfully", + { + attributes: { + languages: config.languages + }, + markdownMessage: `Improved incremental analysis was skipped because a pull request analysis for this repository did not complete successfully with CodeQL version ${(await codeql.getVersion()).version} on a runner with similar hardware resources. One possible reason for this is that improved incremental analysis can require a significant amount of disk space for some repositories. If you want to try re-enabling improved incremental analysis, increase the disk space available to the runner. If that doesn't help, contact GitHub Support for further assistance. + +Improved incremental analysis will be automatically retried when the next version of CodeQL is released. You can also manually trigger a retry by [removing](${"https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manage-caches#deleting-cache-entries" /* DELETE_ACTIONS_CACHE_ENTRIES */}) \`codeql-overlay-status-*\` and \`codeql-overlay-pr-status-*\` entries from the Actions cache.`, + severity: "note", + visibility: { + cliSummaryTable: true, + statusPage: true, + telemetry: false + } + } + ) + ); + } if (overlayDisabledReason === "disabled-by-repository-property" /* DisabledByRepositoryProperty */) { addNoLanguageDiagnostic( config, @@ -149808,6 +149862,16 @@ Improved incremental analysis will be automatically retried when the next versio ); } } +function addPullRequestAnalysisFailedTelemetryDiagnostic(languages, isDryRun) { + addNoLanguageDiagnostic( + void 0, + makeTelemetryDiagnostic( + "codeql-action/overlay-pull-request-analysis-failed", + "Pull request analysis using overlay analysis did not complete successfully", + { languages, isDryRun } + ) + ); +} // src/overlay/status.ts var fs8 = __toESM(require("fs")); @@ -149960,6 +150024,83 @@ async function savePullRequestFailureMarker(codeql, languages, diskUsage, checkR return false; } } +async function getPullRequestFailureCheck(features) { + if (await features.getValue("overlay_analysis_status_check_pr" /* OverlayAnalysisStatusCheckPr */)) { + return "enforce" /* Enforce */; + } + if (await features.getValue("overlay_analysis_status_check_pr_dry_run" /* OverlayAnalysisStatusCheckPrDryRun */)) { + return "dry-run" /* DryRun */; + } + return "none" /* None */; +} +async function shouldSkipOverlayAnalysisAfterPullRequestFailure(codeql, languages, diskUsage, check, logger) { + if (check === "none" /* None */) { + return false; + } + const marker = await findPullRequestFailureMarker( + codeql, + languages, + diskUsage, + logger + ); + if (marker === void 0) { + return false; + } + const isDryRun = check === "dry-run" /* DryRun */; + const foundMarker = `Found the Actions cache entry ${marker.key}, which indicates that a pull request analysis using improved incremental analysis did not complete successfully.`; + addPullRequestAnalysisFailedTelemetryDiagnostic(languages, isDryRun); + if (isDryRun) { + logger.debug( + `${foundMarker} Improved incremental analysis would have been disabled, but this check is running in dry-run mode.` + ); + return false; + } + logger.info(foundMarker); + const saved = await saveOverlayStatus( + codeql, + languages, + diskUsage, + createOverlayStatus({ + attemptedToBuildOverlayBaseDatabase: true, + builtOverlayBaseDatabase: false + }), + logger + ); + if (!saved) { + logger.warning( + "Failed to record that pull request analyses using improved incremental analysis are not completing successfully. Improved incremental analysis is disabled for this analysis, but the Action will check for failed pull request analyses again on the next analysis." + ); + } + return true; +} +async function findPullRequestFailureMarker(codeql, languages, diskUsage, logger) { + const keyPrefix = await getPullRequestMarkerCacheKeyPrefix( + codeql, + languages, + diskUsage + ); + let marker; + try { + [marker] = await listActionsCachesPage({ + keyPrefix, + sort: "created_at", + direction: "desc", + perPage: 1 + }); + } catch (error3) { + logger.warning( + `Failed to check the Actions cache for pull request analyses that did not complete successfully: ${getErrorMessage(error3)}` + ); + return void 0; + } + if (marker === void 0) { + logger.debug( + `Found no Actions cache entries with the prefix ${keyPrefix}, so no pull request analysis using improved incremental analysis has recently failed.` + ); + return void 0; + } + return marker; +} async function getCacheKey(codeql, languages, diskUsage) { return `codeql-overlay-status-${await getCacheKeySuffix(codeql, languages, diskUsage)}`; } @@ -150559,8 +150700,10 @@ async function checkOverlayEnablement(codeql, features, languages, sourceRoot, b const checkOverlayStatus = await features.getValue( "overlay_analysis_status_check" /* OverlayAnalysisStatusCheck */ ); + const pullRequestFailureCheck = await getPullRequestFailureCheck(features); const needDiskUsage = performResourceChecks || checkOverlayStatus; - const diskUsage = needDiskUsage ? await checkDiskUsage(logger) : void 0; + const wantDiskUsage = needDiskUsage || pullRequestFailureCheck !== "none" /* None */ && !isAnalyzingPullRequest(); + const diskUsage = wantDiskUsage ? await checkDiskUsage(logger) : void 0; if (needDiskUsage && diskUsage === void 0) { logger.warning( `Unable to determine disk usage, therefore setting overlay database mode to ${"none" /* None */}.` @@ -150590,6 +150733,18 @@ async function checkOverlayEnablement(codeql, features, languages, sourceRoot, b `Setting overlay database mode to ${overlayDatabaseMode} with caching because we are analyzing a pull request.` ); } else if (await isAnalyzingDefaultBranch()) { + if (diskUsage !== void 0 && await shouldSkipOverlayAnalysisAfterPullRequestFailure( + codeql, + languages, + diskUsage, + pullRequestFailureCheck, + logger + )) { + logger.info( + `Setting overlay database mode to ${"none" /* None */} because a pull request analysis using overlay analysis did not complete successfully.` + ); + return new Failure("pull-request-analysis-failed" /* PullRequestAnalysisFailed */); + } overlayDatabaseMode = "overlay-base" /* OverlayBase */; logger.info( `Setting overlay database mode to ${overlayDatabaseMode} with caching because we are analyzing the default branch.` diff --git a/src/config-utils.test.ts b/src/config-utils.test.ts index 29d72f3af..2c8736c93 100644 --- a/src/config-utils.test.ts +++ b/src/config-utils.test.ts @@ -1008,6 +1008,7 @@ interface OverlayDatabaseModeTestSetup { diskUsage: DiskUsage | undefined; memoryFlagValue: number; shouldSkipOverlayAnalysisDueToCachedStatus: boolean; + shouldSkipOverlayAnalysisDueToPullRequestFailure: boolean; repositoryProperties: RepositoryProperties; } @@ -1029,6 +1030,7 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = { }, memoryFlagValue: 6920, shouldSkipOverlayAnalysisDueToCachedStatus: false, + shouldSkipOverlayAnalysisDueToPullRequestFailure: false, repositoryProperties: {}, }; @@ -1072,6 +1074,13 @@ const checkOverlayEnablementMacro = makeMacro({ .stub(overlayStatus, "shouldSkipOverlayAnalysis") .resolves(setup.shouldSkipOverlayAnalysisDueToCachedStatus); + sinon + .stub( + overlayStatus, + "shouldSkipOverlayAnalysisAfterPullRequestFailure", + ) + .resolves(setup.shouldSkipOverlayAnalysisDueToPullRequestFailure); + // Mock feature flags const features = createFeatures(setup.features); @@ -1206,6 +1215,59 @@ checkOverlayEnablementMacro.serial( }, ); +checkOverlayEnablementMacro.serial( + "No overlay-base database on default branch if a pull request analysis failed", + { + languages: [BuiltInLanguage.javascript], + features: [ + Feature.OverlayAnalysis, + Feature.OverlayAnalysisJavascript, + Feature.OverlayAnalysisStatusCheckPr, + ], + isDefaultBranch: true, + shouldSkipOverlayAnalysisDueToPullRequestFailure: true, + }, + { + disabledReason: OverlayDisabledReason.PullRequestAnalysisFailed, + }, +); + +checkOverlayEnablementMacro.serial( + "Overlay-base database on default branch if no pull request analysis failed", + { + languages: [BuiltInLanguage.javascript], + features: [ + Feature.OverlayAnalysis, + Feature.OverlayAnalysisJavascript, + Feature.OverlayAnalysisStatusCheckPr, + ], + isDefaultBranch: true, + shouldSkipOverlayAnalysisDueToPullRequestFailure: false, + }, + { + overlayDatabaseMode: OverlayDatabaseMode.OverlayBase, + useOverlayDatabaseCaching: true, + }, +); + +checkOverlayEnablementMacro.serial( + "Pull request analyses do not consult the pull request failure status", + { + languages: [BuiltInLanguage.javascript], + features: [ + Feature.OverlayAnalysis, + Feature.OverlayAnalysisJavascript, + Feature.OverlayAnalysisStatusCheckPr, + ], + isPullRequest: true, + shouldSkipOverlayAnalysisDueToPullRequestFailure: true, + }, + { + overlayDatabaseMode: OverlayDatabaseMode.Overlay, + useOverlayDatabaseCaching: true, + }, +); + checkOverlayEnablementMacro.serial( "Overlay-base database on default branch when feature enabled with custom analysis", { diff --git a/src/config-utils.ts b/src/config-utils.ts index 0a6ced00a..3727f252f 100644 --- a/src/config-utils.ts +++ b/src/config-utils.ts @@ -70,7 +70,12 @@ import { OverlayDisabledReason, } from "./overlay/diagnostics"; import { OverlayDatabaseMode } from "./overlay/overlay-database-mode"; -import { shouldSkipOverlayAnalysis } from "./overlay/status"; +import { + getPullRequestFailureCheck, + PullRequestFailureCheck, + shouldSkipOverlayAnalysis, + shouldSkipOverlayAnalysisAfterPullRequestFailure, +} from "./overlay/status"; import { RepositoryNwo } from "./repository"; import { ToolsFeature } from "./tools-features"; import { downloadTrapCaches } from "./trap-caching"; @@ -780,8 +785,16 @@ export async function checkOverlayEnablement( const checkOverlayStatus = await features.getValue( Feature.OverlayAnalysisStatusCheck, ); + const pullRequestFailureCheck = await getPullRequestFailureCheck(features); + // The pull request failure check needs the disk usage to compute its cache key prefix, but it + // only applies when analyzing the default branch, and it fails open if the disk usage is + // unavailable rather than disabling overlay analysis. const needDiskUsage = performResourceChecks || checkOverlayStatus; - const diskUsage = needDiskUsage ? await checkDiskUsage(logger) : undefined; + const wantDiskUsage = + needDiskUsage || + (pullRequestFailureCheck !== PullRequestFailureCheck.None && + !isAnalyzingPullRequest()); + const diskUsage = wantDiskUsage ? await checkDiskUsage(logger) : undefined; if (needDiskUsage && diskUsage === undefined) { logger.warning( `Unable to determine disk usage, therefore setting overlay database mode to ${OverlayDatabaseMode.None}.`, @@ -822,6 +835,22 @@ export async function checkOverlayEnablement( "with caching because we are analyzing a pull request.", ); } else if (await isAnalyzingDefaultBranch()) { + if ( + diskUsage !== undefined && + (await shouldSkipOverlayAnalysisAfterPullRequestFailure( + codeql, + languages, + diskUsage, + pullRequestFailureCheck, + logger, + )) + ) { + logger.info( + `Setting overlay database mode to ${OverlayDatabaseMode.None} ` + + "because a pull request analysis using overlay analysis did not complete successfully.", + ); + return new Failure(OverlayDisabledReason.PullRequestAnalysisFailed); + } overlayDatabaseMode = OverlayDatabaseMode.OverlayBase; logger.info( `Setting overlay database mode to ${overlayDatabaseMode} ` + diff --git a/src/feature-flags.ts b/src/feature-flags.ts index 7abccf60c..c242246b0 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -157,6 +157,19 @@ export enum Feature { OverlayAnalysisSkipResourceChecks = "overlay_analysis_skip_resource_checks", /** Controls whether the Actions cache is checked for overlay build outcomes. */ OverlayAnalysisStatusCheck = "overlay_analysis_status_check", + /** + * Controls whether the Actions cache is checked for pull request analyses that failed while + * using overlay analysis, and overlay analysis disabled if any are found. + * + * Requires `OverlayAnalysisStatusCheck` to be enabled as well: disabling overlay analysis for + * subsequent runs works by writing the cache entry that flag controls the reading of. + */ + OverlayAnalysisStatusCheckPr = "overlay_analysis_status_check_pr", + /** + * Like `OverlayAnalysisStatusCheckPr`, but only logs a diagnostic instead of disabling overlay + * analysis. `OverlayAnalysisStatusCheckPr` overrides this flag. + */ + OverlayAnalysisStatusCheckPrDryRun = "overlay_analysis_status_check_pr_dry_run", /** Controls whether overlay build failures on the default branch are stored in the Actions cache. */ OverlayAnalysisStatusSave = "overlay_analysis_status_save", QaTelemetryEnabled = "qa_telemetry_enabled", @@ -414,6 +427,16 @@ export const featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK", minimumVersion: undefined, }, + [Feature.OverlayAnalysisStatusCheckPr]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK_PR", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisStatusCheckPrDryRun]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK_PR_DRY_RUN", + minimumVersion: undefined, + }, [Feature.OverlayAnalysisStatusSave]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_SAVE", diff --git a/src/overlay/diagnostics.ts b/src/overlay/diagnostics.ts index 4b716c3df..beb839a74 100644 --- a/src/overlay/diagnostics.ts +++ b/src/overlay/diagnostics.ts @@ -39,6 +39,8 @@ export enum OverlayDisabledReason { NotPullRequestOrDefaultBranch = "not-pull-request-or-default-branch", /** The top-level overlay analysis feature flag is not enabled. */ OverallFeatureNotEnabled = "overall-feature-not-enabled", + /** Overlay analysis was disabled because pull request analyses that used it failed. */ + PullRequestAnalysisFailed = "pull-request-analysis-failed", /** * Overlay analysis was selected for a pull request, but diff-informed * analysis was not enabled for the run (for example, because the @@ -108,6 +110,38 @@ export async function addOverlayDisablementDiagnostics( ); } + if ( + overlayDisabledReason === OverlayDisabledReason.PullRequestAnalysisFailed + ) { + addNoLanguageDiagnostic( + config, + makeDiagnostic( + "codeql-action/overlay-disabled-due-to-pull-request-failure", + "Skipped improved incremental analysis because pull request analyses did not complete successfully", + { + attributes: { + languages: config.languages, + }, + markdownMessage: + "Improved incremental analysis was skipped because a pull request analysis for this " + + `repository did not complete successfully with CodeQL version ${(await codeql.getVersion()).version} ` + + "on a runner with similar hardware resources. " + + "One possible reason for this is that improved incremental analysis can require a significant amount of disk space for some repositories. " + + "If you want to try re-enabling improved incremental analysis, increase the disk space available " + + "to the runner. If that doesn't help, contact GitHub Support for further assistance.\n\n" + + "Improved incremental analysis will be automatically retried when the next version of CodeQL is released. " + + `You can also manually trigger a retry by [removing](${DocUrl.DELETE_ACTIONS_CACHE_ENTRIES}) \`codeql-overlay-status-*\` and \`codeql-overlay-pr-status-*\` entries from the Actions cache.`, + severity: "note", + visibility: { + cliSummaryTable: true, + statusPage: true, + telemetry: false, + }, + }, + ), + ); + } + if ( overlayDisabledReason === OverlayDisabledReason.DisabledByRepositoryProperty ) { @@ -135,3 +169,21 @@ export async function addOverlayDisablementDiagnostics( ); } } + +/** + * Add a telemetry diagnostic recording that we found a pull request analysis that ran with overlay + * analysis and did not complete successfully. + */ +export function addPullRequestAnalysisFailedTelemetryDiagnostic( + languages: string[], + isDryRun: boolean, +) { + addNoLanguageDiagnostic( + undefined, + makeTelemetryDiagnostic( + "codeql-action/overlay-pull-request-analysis-failed", + "Pull request analysis using overlay analysis did not complete successfully", + { languages, isDryRun }, + ), + ); +} diff --git a/src/overlay/status.test.ts b/src/overlay/status.test.ts index 557ca1bb8..25793b5da 100644 --- a/src/overlay/status.test.ts +++ b/src/overlay/status.test.ts @@ -5,7 +5,10 @@ import * as actionsCache from "@actions/cache"; import test from "ava"; import * as sinon from "sinon"; +import * as apiClient from "../api-client"; +import { Feature } from "../feature-flags"; import { + createFeatures, getRecordingLogger, LoggedMessage, mockCodeQLVersion, @@ -16,10 +19,13 @@ import { DiskUsage, withTmpDir } from "../util"; import { getCacheKey, + getPullRequestFailureCheck, getPullRequestMarkerCacheKey, getPullRequestMarkerCacheKeyPrefix, + PullRequestFailureCheck, savePullRequestFailureMarker, shouldSkipOverlayAnalysis, + shouldSkipOverlayAnalysisAfterPullRequestFailure, } from "./status"; setupTests(test); @@ -230,6 +236,202 @@ test.serial( }, ); +test("getPullRequestFailureCheck prefers enforcement over dry run", async (t) => { + t.is( + await getPullRequestFailureCheck(createFeatures([])), + PullRequestFailureCheck.None, + ); + t.is( + await getPullRequestFailureCheck( + createFeatures([Feature.OverlayAnalysisStatusCheckPrDryRun]), + ), + PullRequestFailureCheck.DryRun, + ); + t.is( + await getPullRequestFailureCheck( + createFeatures([Feature.OverlayAnalysisStatusCheckPr]), + ), + PullRequestFailureCheck.Enforce, + ); + t.is( + await getPullRequestFailureCheck( + createFeatures([ + Feature.OverlayAnalysisStatusCheckPr, + Feature.OverlayAnalysisStatusCheckPrDryRun, + ]), + ), + PullRequestFailureCheck.Enforce, + ); +}); + +test.serial( + "shouldSkipOverlayAnalysisAfterPullRequestFailure makes no request when the check is disabled", + async (t) => { + const listStub = sinon + .stub(apiClient, "listActionsCachesPage") + .resolves([]); + + t.false( + await shouldSkipOverlayAnalysisAfterPullRequestFailure( + mockCodeQLVersion("2.20.0"), + ["javascript"], + makeDiskUsage(50), + PullRequestFailureCheck.None, + getRecordingLogger([]), + ), + ); + t.true(listStub.notCalled); + }, +); + +test.serial( + "shouldSkipOverlayAnalysisAfterPullRequestFailure makes a single request for the most recent marker", + async (t) => { + const listStub = sinon + .stub(apiClient, "listActionsCachesPage") + .resolves([]); + + t.false( + await shouldSkipOverlayAnalysisAfterPullRequestFailure( + mockCodeQLVersion("2.20.0"), + ["javascript"], + makeDiskUsage(50), + PullRequestFailureCheck.Enforce, + getRecordingLogger([]), + ), + ); + t.true(listStub.calledOnce); + t.deepEqual(listStub.firstCall.args[0], { + keyPrefix: "codeql-overlay-pr-status-javascript-2.20.0-runner-50GB-", + sort: "created_at", + direction: "desc", + perPage: 1, + }); + }, +); + +test.serial( + "shouldSkipOverlayAnalysisAfterPullRequestFailure saves the overlay status when enforcing", + async (t) => { + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + sinon.stub(apiClient, "listActionsCachesPage").resolves([ + { + key: "codeql-overlay-pr-status-javascript-2.20.0-runner-50GB-1-1-2", + }, + ]); + const saveCacheStub = sinon.stub(actionsCache, "saveCache").resolves(1); + + t.true( + await shouldSkipOverlayAnalysisAfterPullRequestFailure( + mockCodeQLVersion("2.20.0"), + ["javascript"], + makeDiskUsage(50), + PullRequestFailureCheck.Enforce, + getRecordingLogger([]), + ), + ); + t.true(saveCacheStub.calledOnce); + t.is( + saveCacheStub.firstCall.args[1], + "codeql-overlay-status-javascript-2.20.0-runner-50GB", + ); + const savedStatus = JSON.parse( + await fs.promises.readFile(saveCacheStub.firstCall.args[0][0], "utf-8"), + ) as Record; + t.true(savedStatus["attemptedToBuildOverlayBaseDatabase"]); + t.false(savedStatus["builtOverlayBaseDatabase"]); + }); + }, +); + +test.serial( + "shouldSkipOverlayAnalysisAfterPullRequestFailure does not skip or save when performing a dry run", + async (t) => { + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + sinon.stub(apiClient, "listActionsCachesPage").resolves([ + { + key: "codeql-overlay-pr-status-javascript-2.20.0-runner-50GB-1-1-2", + }, + ]); + const saveCacheStub = sinon.stub(actionsCache, "saveCache").resolves(1); + + t.false( + await shouldSkipOverlayAnalysisAfterPullRequestFailure( + mockCodeQLVersion("2.20.0"), + ["javascript"], + makeDiskUsage(50), + PullRequestFailureCheck.DryRun, + getRecordingLogger([]), + ), + ); + t.true(saveCacheStub.notCalled); + }); + }, +); + +test.serial( + "shouldSkipOverlayAnalysisAfterPullRequestFailure fails open when the request fails", + async (t) => { + sinon.stub(apiClient, "listActionsCachesPage").rejects(new Error("kaboom")); + const messages: LoggedMessage[] = []; + + t.false( + await shouldSkipOverlayAnalysisAfterPullRequestFailure( + mockCodeQLVersion("2.20.0"), + ["javascript"], + makeDiskUsage(50), + PullRequestFailureCheck.Enforce, + getRecordingLogger(messages), + ), + ); + t.true( + messages.some( + (m) => + m.type === "warning" && + typeof m.message === "string" && + m.message.includes("kaboom"), + ), + ); + }, +); + +test.serial( + "shouldSkipOverlayAnalysisAfterPullRequestFailure still skips when the overlay status cannot be saved", + async (t) => { + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir); + sinon.stub(apiClient, "listActionsCachesPage").resolves([ + { + key: "codeql-overlay-pr-status-javascript-2.20.0-runner-50GB-1-1-2", + }, + ]); + // `saveCache` reports most failures by returning -1 rather than by throwing. + sinon.stub(actionsCache, "saveCache").resolves(-1); + const messages: LoggedMessage[] = []; + + t.true( + await shouldSkipOverlayAnalysisAfterPullRequestFailure( + mockCodeQLVersion("2.20.0"), + ["javascript"], + makeDiskUsage(50), + PullRequestFailureCheck.Enforce, + getRecordingLogger(messages), + ), + ); + t.true( + messages.some( + (m) => + m.type === "warning" && + typeof m.message === "string" && + m.message.includes("Failed to record"), + ), + ); + }); + }, +); + test.serial( "savePullRequestFailureMarker reports failure when the cache entry is not saved", async (t) => { diff --git a/src/overlay/status.ts b/src/overlay/status.ts index 4210ce38d..7a693e473 100644 --- a/src/overlay/status.ts +++ b/src/overlay/status.ts @@ -18,7 +18,9 @@ import { getWorkflowRunAttempt, getWorkflowRunID, } from "../actions-util"; +import { listActionsCachesPage, type ActionsCacheItem } from "../api-client"; import { type CodeQL } from "../codeql"; +import { Feature, FeatureEnablement } from "../feature-flags"; import * as json from "../json"; import { Logger } from "../logging"; import { @@ -28,6 +30,8 @@ import { waitForResultWithTimeLimit, } from "../util"; +import { addPullRequestAnalysisFailedTelemetryDiagnostic } from "./diagnostics"; + /** The maximum time to wait for a cache operation to complete. */ const MAX_CACHE_OPERATION_MS = 30_000; @@ -284,6 +288,143 @@ export async function savePullRequestFailureMarker( } } +/** + * How to react to a pull request analysis that ran with overlay analysis and did not complete + * successfully. + */ +export enum PullRequestFailureCheck { + /** Do not check for failed pull request analyses. */ + None = "none", + /** Check for failed pull request analyses, but do not disable overlay analysis. */ + DryRun = "dry-run", + /** Check for failed pull request analyses and disable overlay analysis if one is found. */ + Enforce = "enforce", +} + +/** Determines how to react to failed pull request analyses, based on the enabled features. */ +export async function getPullRequestFailureCheck( + features: FeatureEnablement, +): Promise { + if (await features.getValue(Feature.OverlayAnalysisStatusCheckPr)) { + return PullRequestFailureCheck.Enforce; + } + if (await features.getValue(Feature.OverlayAnalysisStatusCheckPrDryRun)) { + return PullRequestFailureCheck.DryRun; + } + return PullRequestFailureCheck.None; +} + +/** + * Whether overlay analysis should be skipped because a pull request analysis that ran with overlay + * analysis did not complete successfully. + * + * When enforcing, this also records the failure in the persistent overlay status cache entry, so + * that subsequent analyses skip overlay analysis without repeating this check. Reading that entry + * back is gated on `Feature.OverlayAnalysisStatusCheck`, which therefore needs to be enabled too + * for enforcement to take effect beyond the current analysis. + * + * This makes at most one API request. Any failure is treated as if no failed pull request analysis + * was found. + */ +export async function shouldSkipOverlayAnalysisAfterPullRequestFailure( + codeql: CodeQL, + languages: string[], + diskUsage: DiskUsage, + check: PullRequestFailureCheck, + logger: Logger, +): Promise { + if (check === PullRequestFailureCheck.None) { + return false; + } + + const marker = await findPullRequestFailureMarker( + codeql, + languages, + diskUsage, + logger, + ); + if (marker === undefined) { + return false; + } + + const isDryRun = check === PullRequestFailureCheck.DryRun; + const foundMarker = + `Found the Actions cache entry ${marker.key}, which indicates that a pull request analysis ` + + "using improved incremental analysis did not complete successfully."; + addPullRequestAnalysisFailedTelemetryDiagnostic(languages, isDryRun); + + if (isDryRun) { + logger.debug( + `${foundMarker} Improved incremental analysis would have been disabled, but this check is ` + + "running in dry-run mode.", + ); + return false; + } + + logger.info(foundMarker); + + const saved = await saveOverlayStatus( + codeql, + languages, + diskUsage, + createOverlayStatus({ + attemptedToBuildOverlayBaseDatabase: true, + builtOverlayBaseDatabase: false, + }), + logger, + ); + if (!saved) { + logger.warning( + "Failed to record that pull request analyses using improved incremental analysis are not " + + "completing successfully. Improved incremental analysis is disabled for this analysis, but " + + "the Action will check for failed pull request analyses again on the next analysis.", + ); + } + return true; +} + +/** + * Look for the most recently created cache entry marking a pull request analysis that ran with + * overlay analysis and did not complete successfully. + */ +async function findPullRequestFailureMarker( + codeql: CodeQL, + languages: string[], + diskUsage: DiskUsage, + logger: Logger, +): Promise { + const keyPrefix = await getPullRequestMarkerCacheKeyPrefix( + codeql, + languages, + diskUsage, + ); + + let marker: ActionsCacheItem | undefined; + try { + [marker] = await listActionsCachesPage({ + keyPrefix, + sort: "created_at", + direction: "desc", + perPage: 1, + }); + } catch (error) { + logger.warning( + "Failed to check the Actions cache for pull request analyses that did not complete " + + `successfully: ${getErrorMessage(error)}`, + ); + return undefined; + } + + if (marker === undefined) { + logger.debug( + `Found no Actions cache entries with the prefix ${keyPrefix}, so no pull request analysis ` + + "using improved incremental analysis has recently failed.", + ); + return undefined; + } + return marker; +} + export async function getCacheKey( codeql: CodeQL, languages: string[],