Record an overlay status only for conclusive job statuses

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

lib/entry-points.js

@@ -162344,13 +162344,22 @@ function didCodeQlReportError(env) {
   const jobStatus = env.getOptional("CODEQL_ACTION_JOB_STATUS" /* JOB_STATUS */);
   return jobStatus === "JOB_STATUS_FAILURE" /* FailureStatus */ || jobStatus === "JOB_STATUS_CONFIGURATION_ERROR" /* ConfigErrorStatus */;
 }
+function isConclusiveJobStatus(jobStatus) {
+  switch (jobStatus?.trim().toLowerCase()) {
+    case "failure":
+    case "success":
+      return true;
+    default:
+      return false;
+  }
+}
 async function recordOverlayStatus(codeql, config, features, jobStatus, env, logger) {
   if (config.overlayDatabaseMode !== "overlay-base" /* OverlayBase */ || env.getOptional("CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY" /* ANALYZE_DID_COMPLETE_SUCCESSFULLY */) === "true" || !await features.getValue("overlay_analysis_status_save" /* OverlayAnalysisStatusSave */)) {
     return;
   }
-  if (jobStatus?.trim().toLowerCase() === "cancelled" && !didCodeQlReportError(env)) {
+  if (!isConclusiveJobStatus(jobStatus) && !didCodeQlReportError(env)) {
     logger.info(
-      "Not recording an improved incremental analysis failure for this job because the workflow run was cancelled."
+      `Not recording an improved incremental analysis failure for this job because the job status (${jobStatus ?? "unset"}) does not tell us whether the analysis itself failed.`
     );
     return;
   }

src/init-action-post-helper.test.ts

@@ -548,16 +548,16 @@ test.serial(
 );
 
 /**
- * Runs `uploadFailureInfo` for an overlay-base job that did not complete successfully, for a job
- * that the Actions runtime environment reports as cancelled.
+ * Runs `uploadFailureInfo` for an overlay-base job that did not complete successfully, with the
+ * given job status from the Actions runtime environment.
  */
-async function testCancelledOverlayJob({
-  jobStatus = "cancelled",
+async function runOverlayPostStep({
+  jobStatus,
   codeQlReportedError = false,
 }: {
-  jobStatus?: string;
+  jobStatus: string | undefined;
   codeQlReportedError?: boolean;
-} = {}) {
+}) {
   return await util.withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
     delete process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY];
@@ -599,7 +599,9 @@ async function testCancelledOverlayJob({
 test.serial(
   "does not save overlay status when the job was cancelled",
   async (t) => {
-    const { saveOverlayStatusStub } = await testCancelledOverlayJob();
+    const { saveOverlayStatusStub } = await runOverlayPostStep({
+      jobStatus: "cancelled",
+    });
 
     t.true(
       saveOverlayStatusStub.notCalled,
@@ -609,23 +611,63 @@ test.serial(
 );
 
 test.serial(
-  "saves overlay status when the job failed rather than being cancelled",
+  "does not save overlay status when the job status is not recognised",
   async (t) => {
-    const { saveOverlayStatusStub } = await testCancelledOverlayJob({
-      jobStatus: "failure",
+    const { saveOverlayStatusStub } = await runOverlayPostStep({
+      jobStatus: "some-new-status",
     });
 
     t.true(
-      saveOverlayStatusStub.calledOnce,
-      "only cancellations are treated as unrelated to the analysis",
+      saveOverlayStatusStub.notCalled,
+      "a status we do not recognise tells us nothing about whether the analysis would have succeeded",
     );
   },
 );
 
+test.serial(
+  "does not save overlay status when the job status is unavailable",
+  async (t) => {
+    const { saveOverlayStatusStub } = await runOverlayPostStep({
+      jobStatus: undefined,
+    });
+
+    t.true(
+      saveOverlayStatusStub.notCalled,
+      "without a job status we cannot tell whether the analysis would have succeeded",
+    );
+  },
+);
+
+test.serial(
+  "saves overlay status when the job failed rather than being cancelled",
+  async (t) => {
+    const { saveOverlayStatusStub } = await runOverlayPostStep({
+      jobStatus: "failure",
+    });
+
+    t.true(
+      saveOverlayStatusStub.calledOnce,
+      "a failed job indicates that the analysis itself failed",
+    );
+  },
+);
+
+test.serial("saves overlay status when the job succeeded", async (t) => {
+  const { saveOverlayStatusStub } = await runOverlayPostStep({
+    jobStatus: "success",
+  });
+
+  t.true(
+    saveOverlayStatusStub.calledOnce,
+    "the analysis did not complete successfully even though the job as a whole succeeded",
+  );
+});
+
 test.serial(
   "saves overlay status when a CodeQL Action reported an error before the run was cancelled",
   async (t) => {
-    const { saveOverlayStatusStub } = await testCancelledOverlayJob({
+    const { saveOverlayStatusStub } = await runOverlayPostStep({
+      jobStatus: "cancelled",
       codeQlReportedError: true,
     });
 

src/init-action-post-helper.ts

@@ -431,6 +431,22 @@ function didCodeQlReportError(env: ReadOnlyEnv): boolean {
   );
 }
 
+/**
+ * Whether the job status tells us anything about whether the analysis itself would have succeeded.
+ *
+ * We check for the statuses we know to be meaningful rather than excluding the ones that are not,
+ * so that a status we do not recognise is treated as inconclusive.
+ */
+function isConclusiveJobStatus(jobStatus: string | undefined): boolean {
+  switch (jobStatus?.trim().toLowerCase()) {
+    case "failure":
+    case "success":
+      return true;
+    default:
+      return false;
+  }
+}
+
 /**
  * If overlay base database creation was attempted but the analysis did not complete
  * successfully, save the failure status to the Actions cache so that subsequent runs
@@ -452,16 +468,14 @@ async function recordOverlayStatus(
     return;
   }
 
-  // A cancelled run tells us nothing about whether the analysis would have succeeded, so recording
-  // a failure would disable overlay analysis needlessly. Note that we still record a failure if one
-  // of our own Actions reported an error before the run was cancelled.
-  if (
-    jobStatus?.trim().toLowerCase() === "cancelled" &&
-    !didCodeQlReportError(env)
-  ) {
+  // Only record a failure when the job outcome tells us something about the analysis. A cancelled
+  // job, or a status we do not recognise, says nothing about whether the analysis would have
+  // succeeded, so recording a failure would disable overlay analysis needlessly. We still record
+  // one if a CodeQL Action reported an error before the job ended.
+  if (!isConclusiveJobStatus(jobStatus) && !didCodeQlReportError(env)) {
     logger.info(
-      "Not recording an improved incremental analysis failure for this job because the workflow " +
-        "run was cancelled.",
+      "Not recording an improved incremental analysis failure for this job because the job " +
+        `status (${jobStatus ?? "unset"}) does not tell us whether the analysis itself failed.`,
     );
     return;
   }