Merge branch 'main' into henrymercer/remove-legacy-tracing

src/codeql.test.ts

@@ -554,13 +554,13 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
       tagName: "codeql-bundle-20230203",
     });
     mockDownloadApi({
-      repo: "dsp-testing/codeql-cli-nightlies",
+      repo: "codeql-testing/codeql-cli-nightlies",
       platformSpecific: false,
       tagName: "codeql-bundle-20230203",
     });
 
     const result = await codeql.setupCodeQL(
-      "https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz",
+      "https://github.com/codeql-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz",
       sampleApiDetails,
       tmpDir,
       util.GitHubVariant.DOTCOM,

src/codeql.ts

@@ -287,6 +287,12 @@ export const CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
  */
 export const CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.4";
 
+/**
+ * Versions 2.12.6+ of the CodeQL CLI fix a bug where duplicate notification objects could be produced,
+ * leading to an invalid SARIF output.
+ */
+export const CODEQL_VERSION_DUPLICATE_NOTIFICATIONS_FIXED = "2.12.6";
+
 /**
  * Set up CodeQL CLI access.
  *
@@ -756,7 +762,13 @@ export async function getCodeQLForCmd(
         Feature.ExportDiagnosticsEnabled,
         this
       );
-      const codeqlOutputFile = shouldExportDiagnostics
+      const shouldWorkaroundInvalidNotifications =
+        shouldExportDiagnostics &&
+        !(await util.codeQlVersionAbove(
+          this,
+          CODEQL_VERSION_DUPLICATE_NOTIFICATIONS_FIXED
+        ));
+      const codeqlOutputFile = shouldWorkaroundInvalidNotifications
         ? path.join(config.tempDir, "codeql-intermediate-results.sarif")
         : sarifFile;
       const codeqlArgs = [
@@ -788,6 +800,8 @@ export async function getCodeQLForCmd(
       }
       if (shouldExportDiagnostics) {
         codeqlArgs.push("--sarif-include-diagnostics");
+      } else if (await util.codeQlVersionAbove(this, "2.12.4")) {
+        codeqlArgs.push("--no-sarif-include-diagnostics");
       }
       codeqlArgs.push(databasePath);
       if (querySuitePaths) {
@@ -800,7 +814,7 @@ export async function getCodeQLForCmd(
         errorMatchers
       );
 
-      if (shouldExportDiagnostics) {
+      if (shouldWorkaroundInvalidNotifications) {
         util.fixInvalidNotificationsInFile(codeqlOutputFile, sarifFile, logger);
       }
 
@@ -903,17 +917,21 @@ export async function getCodeQLForCmd(
       tempDir: string,
       logger: Logger
     ): Promise<void> {
-      const intermediateSarifFile = path.join(
-        tempDir,
-        "codeql-intermediate-results.sarif"
-      );
+      const shouldWorkaroundInvalidNotifications =
+        !(await util.codeQlVersionAbove(
+          this,
+          CODEQL_VERSION_DUPLICATE_NOTIFICATIONS_FIXED
+        ));
+      const codeqlOutputFile = shouldWorkaroundInvalidNotifications
+        ? path.join(tempDir, "codeql-intermediate-results.sarif")
+        : sarifFile;
       const args = [
         "database",
         "export-diagnostics",
         `${databasePath}`,
         "--db-cluster", // Database is always a cluster for CodeQL versions that support diagnostics.
         "--format=sarif-latest",
-        `--output=${intermediateSarifFile}`,
+        `--output=${codeqlOutputFile}`,
         "--sarif-include-diagnostics", // ExportDiagnosticsEnabled is always true if this command is run.
         "-vvv",
         ...getExtraOptionsFromEnv(["diagnostics", "export"]),
@@ -923,12 +941,10 @@ export async function getCodeQLForCmd(
       }
       await new toolrunner.ToolRunner(cmd, args).exec();
 
-      // Fix invalid notifications in the SARIF file output by CodeQL.
-      util.fixInvalidNotificationsInFile(
-        intermediateSarifFile,
-        sarifFile,
-        logger
-      );
+      if (shouldWorkaroundInvalidNotifications) {
+        // Fix invalid notifications in the SARIF file output by CodeQL.
+        util.fixInvalidNotificationsInFile(codeqlOutputFile, sarifFile, logger);
+      }
     },
     async diagnosticsExport(
       sarifFile: string,

src/config-utils.test.ts

@@ -2307,7 +2307,7 @@ test("downloadPacks-with-registries", async (t) => {
       {
         // no slash
         url: "http://ghcr.io",
-        packages: ["codeql/*", "dsp-testing/*"],
+        packages: ["codeql/*", "codeql-testing/*"],
         token: "not-a-token",
       },
       {
@@ -2397,7 +2397,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
     const registriesInput = yaml.dump([
       {
         url: "http://ghcr.io",
-        packages: ["codeql/*", "dsp-testing/*"],
+        packages: ["codeql/*", "codeql-testing/*"],
         token: "not-a-token",
       },
       {
@@ -2439,7 +2439,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
     const registriesInput = yaml.dump([
       {
         // missing url property
-        packages: ["codeql/*", "dsp-testing/*"],
+        packages: ["codeql/*", "codeql-testing/*"],
         token: "not-a-token",
       },
       {
@@ -2478,7 +2478,7 @@ test("no generateRegistries when CLI is too old", async (t) => {
       {
         // no slash
         url: "http://ghcr.io",
-        packages: ["codeql/*", "dsp-testing/*"],
+        packages: ["codeql/*", "codeql-testing/*"],
         token: "not-a-token",
       },
     ]);
@@ -2527,7 +2527,7 @@ test("generateRegistries prefers original CODEQL_REGISTRIES_AUTH", async (t) =>
     const registriesInput = yaml.dump([
       {
         url: "http://ghcr.io",
-        packages: ["codeql/*", "dsp-testing/*"],
+        packages: ["codeql/*", "codeql-testing/*"],
         token: "not-a-token",
       },
     ]);

src/defaults.json

@@ -1,6 +1,6 @@
 {
-  "bundleVersion": "codeql-bundle-20230317",
-  "cliVersion": "2.12.5",
-  "priorBundleVersion": "codeql-bundle-20230304",
-  "priorCliVersion": "2.12.4"
+  "bundleVersion": "codeql-bundle-20230403",
+  "cliVersion": "2.12.6",
+  "priorBundleVersion": "codeql-bundle-20230317",
+  "priorCliVersion": "2.12.5"
 }

src/feature-flags.ts

@@ -62,12 +62,12 @@ export const featureConfig: Record<
   [Feature.ExportCodeScanningConfigEnabled]: {
     envVar: "CODEQL_ACTION_EXPORT_CODE_SCANNING_CONFIG",
     minimumVersion: "2.12.3",
-    defaultValue: false,
+    defaultValue: true,
   },
   [Feature.ExportDiagnosticsEnabled]: {
     envVar: "CODEQL_ACTION_EXPORT_DIAGNOSTICS",
     minimumVersion: "2.12.4",
-    defaultValue: false,
+    defaultValue: true,
   },
 
   [Feature.MlPoweredQueriesEnabled]: {
@@ -78,7 +78,7 @@ export const featureConfig: Record<
   [Feature.UploadFailedSarifEnabled]: {
     envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
     minimumVersion: "2.11.3",
-    defaultValue: false,
+    defaultValue: true,
   },
 };
 

src/util.test.ts

@@ -441,7 +441,11 @@ test("fixInvalidNotifications leaves notifications with unique locations alone",
     getRecordingLogger(messages)
   );
   t.deepEqual(result, createMockSarifWithNotification([stubLocation]));
-  t.is(messages.length, 0);
+  t.is(messages.length, 1);
+  t.deepEqual(messages[0], {
+    type: "debug",
+    message: "No duplicate locations found in SARIF notification objects.",
+  });
 });
 
 test("fixInvalidNotifications removes duplicate locations", (t) => {

src/util.ts

@@ -877,6 +877,8 @@ export function fixInvalidNotifications(
       `Removed ${numDuplicateLocationsRemoved} duplicate locations from SARIF notification ` +
         "objects."
     );
+  } else {
+    logger.debug("No duplicate locations found in SARIF notification objects.");
   }
   return newSarif;
 }

src/workflow.test.ts

@@ -56,17 +56,6 @@ test("getWorkflowErrors() when on.push is a valid superset", (t) => {
   t.deepEqual(...errorCodes(errors, []));
 });
 
-test("getWorkflowErrors() when on.push should not have a path", (t) => {
-  const errors = getWorkflowErrors({
-    on: {
-      push: { branches: ["main"], paths: ["test/*"] },
-      pull_request: { branches: ["main"] },
-    },
-  });
-
-  t.deepEqual(...errorCodes(errors, [WorkflowErrors.PathsSpecified]));
-});
-
 test("getWorkflowErrors() when on.push is a correct object", (t) => {
   const errors = getWorkflowErrors({
     on: { push: { branches: ["main"] }, pull_request: { branches: ["main"] } },
@@ -317,7 +306,7 @@ test("formatWorkflowErrors() when there is one error", (t) => {
 test("formatWorkflowErrors() when there are multiple errors", (t) => {
   const message = formatWorkflowErrors([
     WorkflowErrors.CheckoutWrongHead,
-    WorkflowErrors.PathsSpecified,
+    WorkflowErrors.MismatchedBranches,
   ]);
   t.true(message.startsWith("2 issues were detected with this workflow:"));
 });
@@ -331,10 +320,10 @@ test("formatWorkflowCause() with no errors", (t) => {
 test("formatWorkflowCause()", (t) => {
   const message = formatWorkflowCause([
     WorkflowErrors.CheckoutWrongHead,
-    WorkflowErrors.PathsSpecified,
+    WorkflowErrors.MismatchedBranches,
   ]);
 
-  t.deepEqual(message, "CheckoutWrongHead,PathsSpecified");
+  t.deepEqual(message, "CheckoutWrongHead,MismatchedBranches");
   t.deepEqual(formatWorkflowCause([]), undefined);
 });
 

src/workflow.ts

@@ -108,8 +108,6 @@ function toCodedErrors(errors: {
 export const WorkflowErrors = toCodedErrors({
   MismatchedBranches: `Please make sure that every branch in on.pull_request is also in on.push so that Code Scanning can compare pull requests against the state of the base branch.`,
   MissingPushHook: `Please specify an on.push hook so that Code Scanning can compare pull requests against the state of the base branch.`,
-  PathsSpecified: `Using on.push.paths can prevent Code Scanning annotating new alerts in your pull requests.`,
-  PathsIgnoreSpecified: `Using on.push.paths-ignore can prevent Code Scanning annotating new alerts in your pull requests.`,
   CheckoutWrongHead: `git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.`,
 });
 
@@ -162,19 +160,6 @@ export function getWorkflowErrors(doc: Workflow): CodedError[] {
     if (!hasPush && hasPullRequest) {
       missingPush = true;
     }
-    if (hasPush && hasPullRequest) {
-      const paths = doc.on.push?.paths;
-      // if you specify paths or paths-ignore you can end up with commits that have no baseline
-      // if they didn't change any files
-      // currently we cannot go back through the history and find the most recent baseline
-      if (Array.isArray(paths) && paths.length > 0) {
-        errors.push(WorkflowErrors.PathsSpecified);
-      }
-      const pathsIgnore = doc.on.push?.["paths-ignore"];
-      if (Array.isArray(pathsIgnore) && pathsIgnore.length > 0) {
-        errors.push(WorkflowErrors.PathsIgnoreSpecified);
-      }
-    }
 
     // if doc.on.pull_request is null that means 'all branches'
     // if doc.on.pull_request is undefined that means 'off'