diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 580719d47..96e7f52f6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ - + ### Risk assessment diff --git a/.github/workflows/__multi-language-autodetect.yml b/.github/workflows/__multi-language-autodetect.yml index e5f157881..8bedb73e8 100644 --- a/.github/workflows/__multi-language-autodetect.yml +++ b/.github/workflows/__multi-language-autodetect.yml @@ -164,4 +164,5 @@ jobs: exit 1 fi env: + CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI: true CODEQL_ACTION_TEST_MODE: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 82679e074..1baf1076d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th ## [UNRELEASED] +- We have improved the CodeQL Action's ability to validate that the workflow it is used in does not use different versions of the CodeQL Action for different workflow steps. Mixing different versions of the CodeQL Action in the same workflow is unsupported and can lead to unpredictable results. A warning will now be emitted from the `codeql-action/init` step if different versions of the CodeQL Action are detected in the workflow file. Additionally, an error will now be thrown by the other CodeQL Action steps if they load a configuration file that was generated by a different version of the `codeql-action/init` step. [#3099](https://github.com/github/codeql-action/pull/3099) and [#3100](https://github.com/github/codeql-action/pull/3100) + +## 3.30.3 - 10 Sep 2025 + No user facing changes. ## 3.30.2 - 09 Sep 2025 diff --git a/lib/analyze-action-post.js b/lib/analyze-action-post.js index f6a82b731..6f5aecb52 100644 --- a/lib/analyze-action-post.js +++ b/lib/analyze-action-post.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -117686,7 +117686,7 @@ function withGroup(groupName, f) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -117796,6 +117796,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -117960,7 +117966,18 @@ async function getConfig(tempDir, logger) { const configString = fs3.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString); + const config = JSON.parse(configString); + if (config.version === void 0) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.` + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'` + ); + } + return config; } function appendExtraQueryExclusions(extraQueryExclusions, cliConfig) { const augmentedConfig = cloneObject(cliConfig); @@ -118206,13 +118223,16 @@ async function getCodeQLForCmd(cmd, checkVersion) { ); } }, - async betterResolveLanguages() { + async betterResolveLanguages({ + filterToLanguagesWithQueries + } = { filterToLanguagesWithQueries: false }) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [], ...getExtraOptionsFromEnv(["resolve", "languages"]) ]; const output = await runCli(cmd, codeqlArgs); @@ -118251,7 +118271,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/lib/analyze-action.js b/lib/analyze-action.js index d79056a20..fd7a163ae 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -32287,7 +32287,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -90811,7 +90811,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -91030,6 +91030,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -91620,7 +91626,18 @@ async function getConfig(tempDir, logger) { const configString = fs9.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString); + const config = JSON.parse(configString); + if (config.version === void 0) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.` + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'` + ); + } + return config; } function appendExtraQueryExclusions(extraQueryExclusions, cliConfig) { const augmentedConfig = cloneObject(cliConfig); @@ -92769,13 +92786,16 @@ async function getCodeQLForCmd(cmd, checkVersion) { ); } }, - async betterResolveLanguages() { + async betterResolveLanguages({ + filterToLanguagesWithQueries + } = { filterToLanguagesWithQueries: false }) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [], ...getExtraOptionsFromEnv(["resolve", "languages"]) ]; const output = await runCli(cmd, codeqlArgs); @@ -92814,7 +92834,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", @@ -93707,7 +93726,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, async function runInterpretResultsFor(analysis, language, queries, enableDebugLogging) { logger.info(`Interpreting ${analysis.name} results for ${language}`); let category = automationDetailsId; - if (dbAnalysisConfig.kind === "code-quality" /* CodeQuality */) { + if (analysis.kind === "code-quality" /* CodeQuality */) { category = fixCodeQualityCategory(logger, automationDetailsId); } const sarifFile = path16.join( diff --git a/lib/autobuild-action.js b/lib/autobuild-action.js index a885e9107..bf8b988bb 100644 --- a/lib/autobuild-action.js +++ b/lib/autobuild-action.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -78426,7 +78426,7 @@ function getActionsLogger() { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -78534,6 +78534,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -78962,7 +78968,18 @@ async function getConfig(tempDir, logger) { const configString = fs4.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString); + const config = JSON.parse(configString); + if (config.version === void 0) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.` + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'` + ); + } + return config; } function appendExtraQueryExclusions(extraQueryExclusions, cliConfig) { const augmentedConfig = cloneObject(cliConfig); @@ -79241,13 +79258,16 @@ async function getCodeQLForCmd(cmd, checkVersion) { ); } }, - async betterResolveLanguages() { + async betterResolveLanguages({ + filterToLanguagesWithQueries + } = { filterToLanguagesWithQueries: false }) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [], ...getExtraOptionsFromEnv(["resolve", "languages"]) ]; const output = await runCli(cmd, codeqlArgs); @@ -79286,7 +79306,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/lib/init-action-post.js b/lib/init-action-post.js index f761edef2..c5c5b87bf 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -32287,7 +32287,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -129016,7 +129016,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -129129,6 +129129,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -129575,7 +129581,18 @@ async function getConfig(tempDir, logger) { const configString = fs9.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString); + const config = JSON.parse(configString); + if (config.version === void 0) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.` + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'` + ); + } + return config; } function appendExtraQueryExclusions(extraQueryExclusions, cliConfig) { const augmentedConfig = cloneObject(cliConfig); @@ -130679,13 +130696,16 @@ async function getCodeQLForCmd(cmd, checkVersion) { ); } }, - async betterResolveLanguages() { + async betterResolveLanguages({ + filterToLanguagesWithQueries + } = { filterToLanguagesWithQueries: false }) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [], ...getExtraOptionsFromEnv(["resolve", "languages"]) ]; const output = await runCli(cmd, codeqlArgs); @@ -130724,7 +130744,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", @@ -133301,7 +133320,8 @@ function toCodedErrors(errors) { } var WorkflowErrors = toCodedErrors({ MissingPushHook: `Please specify an on.push hook to analyze and see code scanning alerts from the default branch on the Security tab.`, - CheckoutWrongHead: `git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.` + CheckoutWrongHead: `git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.`, + InconsistentActionVersion: `Not all workflow steps that use \`github/codeql-action\` actions use the same version. Please ensure that all such steps use the same version to avoid compatibility issues.` }); async function getWorkflow(logger) { const maybeWorkflow = process.env["CODE_SCANNING_WORKFLOW_FILE"]; diff --git a/lib/init-action.js b/lib/init-action.js index d26067571..f790d4749 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -32287,7 +32287,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -86417,7 +86417,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -86629,6 +86629,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -87196,11 +87202,22 @@ function getNoLanguagesError() { function getUnknownLanguagesError(languages) { return `Did not recognize the following languages: ${languages.join(", ")}`; } -async function getSupportedLanguageMap(codeql) { - const resolveResult = await codeql.betterResolveLanguages(); +async function getSupportedLanguageMap(codeql, features, logger) { + const resolveSupportedLanguagesUsingCli = await features.getValue( + "resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */, + codeql + ); + const resolveResult = await codeql.betterResolveLanguages({ + filterToLanguagesWithQueries: resolveSupportedLanguagesUsingCli + }); + if (resolveSupportedLanguagesUsingCli) { + logger.debug( + `The CodeQL CLI supports the following languages: ${Object.keys(resolveResult.extractors).join(", ")}` + ); + } const supportedLanguages = {}; for (const extractor of Object.keys(resolveResult.extractors)) { - if (KnownLanguage[extractor] !== void 0) { + if (resolveSupportedLanguagesUsingCli || KnownLanguage[extractor] !== void 0) { supportedLanguages[extractor] = extractor; } } @@ -87236,14 +87253,14 @@ async function getRawLanguagesInRepo(repository, sourceRoot, logger) { logger.debug(`Raw languages in repository: ${result.join(", ")}`); return result; } -async function getLanguages(codeql, languagesInput, repository, sourceRoot, logger) { +async function getLanguages(codeql, languagesInput, repository, sourceRoot, features, logger) { const { rawLanguages, autodetected } = await getRawLanguages( languagesInput, repository, sourceRoot, logger ); - const languageMap = await getSupportedLanguageMap(codeql); + const languageMap = await getSupportedLanguageMap(codeql, features, logger); const languagesSet = /* @__PURE__ */ new Set(); const unknownLanguages = []; for (const language of rawLanguages) { @@ -87311,6 +87328,7 @@ async function initActionState({ languagesInput, repository, sourceRoot, + features, logger ); const buildMode = await parseBuildModeInput( @@ -87335,6 +87353,7 @@ async function initActionState({ augmentationProperties ); return { + version: getActionVersion(), analysisKinds, languages, buildMode, @@ -87686,7 +87705,6 @@ async function initConfig(inputs) { exclude: { tags: "exclude-from-incremental" } }); } - await saveConfig(config, logger); return config; } function parseRegistries(registriesInput) { @@ -89363,13 +89381,16 @@ async function getCodeQLForCmd(cmd, checkVersion) { ); } }, - async betterResolveLanguages() { + async betterResolveLanguages({ + filterToLanguagesWithQueries + } = { filterToLanguagesWithQueries: false }) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [], ...getExtraOptionsFromEnv(["resolve", "languages"]) ]; const output = await runCli(cmd, codeqlArgs); @@ -89408,7 +89429,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", @@ -90110,7 +90130,8 @@ function toCodedErrors(errors) { } var WorkflowErrors = toCodedErrors({ MissingPushHook: `Please specify an on.push hook to analyze and see code scanning alerts from the default branch on the Security tab.`, - CheckoutWrongHead: `git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.` + CheckoutWrongHead: `git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.`, + InconsistentActionVersion: `Not all workflow steps that use \`github/codeql-action\` actions use the same version. Please ensure that all such steps use the same version to avoid compatibility issues.` }); async function groupLanguagesByExtractor(languages, codeql) { const resolveResult = await codeql.betterResolveLanguages(); @@ -90164,6 +90185,22 @@ async function getWorkflowErrors(doc, codeql) { } } } + const codeqlStepRefs = []; + for (const job of Object.values(doc?.jobs || {})) { + if (Array.isArray(job.steps)) { + for (const step of job.steps) { + if (step.uses?.startsWith("github/codeql-action/")) { + const parts = step.uses.split("@"); + if (parts.length >= 2) { + codeqlStepRefs.push(parts[parts.length - 1]); + } + } + } + } + } + if (codeqlStepRefs.length > 0 && !codeqlStepRefs.every((ref) => ref === codeqlStepRefs[0])) { + errors.push(WorkflowErrors.InconsistentActionVersion); + } const hasPushTrigger = hasWorkflowTrigger("push", doc); const hasPullRequestTrigger = hasWorkflowTrigger("pull_request", doc); const hasWorkflowCallTrigger = hasWorkflowTrigger("workflow_call", doc); @@ -90684,6 +90721,7 @@ exec ${goBinaryPath} "$@"` } finally { logUnwrittenDiagnostics(); } + await saveConfig(config, logger); await sendCompletedStatusReport( startedAt, config, diff --git a/lib/resolve-environment-action.js b/lib/resolve-environment-action.js index d7a9a4ade..a71808c9b 100644 --- a/lib/resolve-environment-action.js +++ b/lib/resolve-environment-action.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -78419,7 +78419,7 @@ function getActionsLogger() { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -78525,6 +78525,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -78689,7 +78695,18 @@ async function getConfig(tempDir, logger) { const configString = fs3.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString); + const config = JSON.parse(configString); + if (config.version === void 0) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.` + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'` + ); + } + return config; } function appendExtraQueryExclusions(extraQueryExclusions, cliConfig) { const augmentedConfig = cloneObject(cliConfig); @@ -78941,13 +78958,16 @@ async function getCodeQLForCmd(cmd, checkVersion) { ); } }, - async betterResolveLanguages() { + async betterResolveLanguages({ + filterToLanguagesWithQueries + } = { filterToLanguagesWithQueries: false }) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [], ...getExtraOptionsFromEnv(["resolve", "languages"]) ]; const output = await runCli(cmd, codeqlArgs); @@ -78986,7 +79006,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/lib/start-proxy-action-post.js b/lib/start-proxy-action-post.js index 8bb47e562..92b143742 100644 --- a/lib/start-proxy-action-post.js +++ b/lib/start-proxy-action-post.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -117158,7 +117158,7 @@ function getActionsLogger() { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; // src/tools-features.ts @@ -117205,6 +117205,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -117369,7 +117375,18 @@ async function getConfig(tempDir, logger) { const configString = fs.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString); + const config = JSON.parse(configString); + if (config.version === void 0) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.` + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'` + ); + } + return config; } // src/debug-artifacts.ts diff --git a/lib/start-proxy-action.js b/lib/start-proxy-action.js index ad4e5c882..6e121d96a 100644 --- a/lib/start-proxy-action.js +++ b/lib/start-proxy-action.js @@ -44966,7 +44966,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { diff --git a/lib/upload-lib.js b/lib/upload-lib.js index e312beccb..53f1ceedf 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -33584,7 +33584,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -89115,7 +89115,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -89222,6 +89222,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -89404,7 +89410,18 @@ async function getConfig(tempDir, logger) { const configString = fs7.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString); + const config = JSON.parse(configString); + if (config.version === void 0) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.` + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'` + ); + } + return config; } function appendExtraQueryExclusions(extraQueryExclusions, cliConfig) { const augmentedConfig = cloneObject(cliConfig); @@ -90508,13 +90525,16 @@ async function getCodeQLForCmd(cmd, checkVersion) { ); } }, - async betterResolveLanguages() { + async betterResolveLanguages({ + filterToLanguagesWithQueries + } = { filterToLanguagesWithQueries: false }) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [], ...getExtraOptionsFromEnv(["resolve", "languages"]) ]; const output = await runCli(cmd, codeqlArgs); @@ -90553,7 +90573,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/lib/upload-sarif-action-post.js b/lib/upload-sarif-action-post.js index 59847a4a8..1a87ff27c 100644 --- a/lib/upload-sarif-action-post.js +++ b/lib/upload-sarif-action-post.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -117319,7 +117319,7 @@ function withGroup(groupName, f) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; // src/tools-features.ts @@ -117370,6 +117370,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js index 624b843fd..06080a84b 100644 --- a/lib/upload-sarif-action.js +++ b/lib/upload-sarif-action.js @@ -32287,7 +32287,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.3", + version: "3.30.4", private: true, description: "CodeQL action", scripts: { @@ -89108,7 +89108,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -89217,6 +89217,12 @@ var featureConfig = { legacyApi: true, minimumVersion: void 0 }, + ["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: void 0, + toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */ + }, ["overlay_analysis" /* OverlayAnalysis */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", @@ -89677,7 +89683,18 @@ async function getConfig(tempDir, logger) { const configString = fs8.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString); + const config = JSON.parse(configString); + if (config.version === void 0) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.` + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'` + ); + } + return config; } function appendExtraQueryExclusions(extraQueryExclusions, cliConfig) { const augmentedConfig = cloneObject(cliConfig); @@ -91208,13 +91225,16 @@ async function getCodeQLForCmd(cmd, checkVersion) { ); } }, - async betterResolveLanguages() { + async betterResolveLanguages({ + filterToLanguagesWithQueries + } = { filterToLanguagesWithQueries: false }) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [], ...getExtraOptionsFromEnv(["resolve", "languages"]) ]; const output = await runCli(cmd, codeqlArgs); @@ -91253,7 +91273,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/package-lock.json b/package-lock.json index 12ab1133b..a50a405a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "codeql", - "version": "3.30.3", + "version": "3.30.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "codeql", - "version": "3.30.3", + "version": "3.30.4", "license": "MIT", "dependencies": { "@actions/artifact": "^2.3.1", diff --git a/package.json b/package.json index dacadc732..42850ef00 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeql", - "version": "3.30.3", + "version": "3.30.4", "private": true, "description": "CodeQL action", "scripts": { diff --git a/pr-checks/checks/multi-language-autodetect.yml b/pr-checks/checks/multi-language-autodetect.yml index e663c4f8f..794da3567 100644 --- a/pr-checks/checks/multi-language-autodetect.yml +++ b/pr-checks/checks/multi-language-autodetect.yml @@ -1,6 +1,8 @@ name: "Multi-language repository" description: "An end-to-end integration test of a multi-language repository using automatic language detection for macOS" operatingSystems: ["macos", "ubuntu"] +env: + CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI: true installGo: true steps: - name: Use Xcode 16 diff --git a/src/analyze.ts b/src/analyze.ts index 153b00a1f..b22e9c23d 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -780,7 +780,7 @@ export async function runQueries( // If this is a Code Quality analysis, correct the category to one // accepted by the Code Quality backend. let category = automationDetailsId; - if (dbAnalysisConfig.kind === analyses.AnalysisKind.CodeQuality) { + if (analysis.kind === analyses.AnalysisKind.CodeQuality) { category = fixCodeQualityCategory(logger, automationDetailsId); } diff --git a/src/codeql.test.ts b/src/codeql.test.ts index ece6ab878..36775f653 100644 --- a/src/codeql.test.ts +++ b/src/codeql.test.ts @@ -953,7 +953,7 @@ test("runTool recognizes fatal internal errors", async (t) => { await codeqlObject.databaseRunQueries(stubConfig.dbLocation, []), { instanceOf: CliError, - message: `Encountered a fatal error while running "codeql-for-testing database run-queries --intra-layer-parallelism --min-disk-free=1024 -v". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details: + message: `Encountered a fatal error while running "codeql-for-testing database run-queries --min-disk-free=1024 -v". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details: com.semmle.util.exception.CatastrophicError: An error occurred while evaluating ControlFlowGraph::ControlFlow::Root.isRootOf/1#dispred#f610e6ed/2@86282cc8 Severe disk cache trouble (corruption or out of space) at /home/runner/work/_temp/codeql_databases/go/db-go/default/cache/pages/28/33.pack: Failed to write item to disk. See the logs for more details.`, }, diff --git a/src/codeql.ts b/src/codeql.ts index 567eb8087..16b105a3d 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -127,7 +127,9 @@ export interface CodeQL { /** * Run 'codeql resolve languages' with '--format=betterjson'. */ - betterResolveLanguages(): Promise; + betterResolveLanguages(options?: { + filterToLanguagesWithQueries: boolean; + }): Promise; /** * Run 'codeql resolve build-environment' */ @@ -736,13 +738,22 @@ export async function getCodeQLForCmd( ); } }, - async betterResolveLanguages() { + async betterResolveLanguages( + { + filterToLanguagesWithQueries, + }: { + filterToLanguagesWithQueries: boolean; + } = { filterToLanguagesWithQueries: false }, + ) { const codeqlArgs = [ "resolve", "languages", "--format=betterjson", "--extractor-options-verbosity=4", "--extractor-include-aliases", + ...(filterToLanguagesWithQueries + ? ["--filter-to-languages-with-queries"] + : []), ...getExtraOptionsFromEnv(["resolve", "languages"]), ]; const output = await runCli(cmd, codeqlArgs); @@ -789,7 +800,6 @@ export async function getCodeQLForCmd( "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", ...queries, diff --git a/src/config-utils.test.ts b/src/config-utils.test.ts index 3bb9e1e11..b5ef77717 100644 --- a/src/config-utils.test.ts +++ b/src/config-utils.test.ts @@ -199,6 +199,7 @@ test("load code quality config", async (t) => { // And the config we expect it to result in const expectedConfig: configUtils.Config = { + version: actionsUtil.getActionVersion(), analysisKinds: [AnalysisKind.CodeQuality], languages: [KnownLanguage.actions], buildMode: undefined, @@ -228,7 +229,7 @@ test("load code quality config", async (t) => { }); }); -test("loading config saves config", async (t) => { +test("loading a saved config produces the same config", async (t) => { return await withTmpDir(async (tempDir) => { const logger = getRunnerLogger(true); @@ -258,6 +259,7 @@ test("loading config saves config", async (t) => { logger, }), ); + await configUtils.saveConfig(config1, logger); // The saved config file should now exist t.true(fs.existsSync(configUtils.getPathToParsedConfigFile(tempDir))); @@ -273,6 +275,57 @@ test("loading config saves config", async (t) => { }); }); +test("loading config with version mismatch throws", async (t) => { + return await withTmpDir(async (tempDir) => { + const logger = getRunnerLogger(true); + + const codeql = createStubCodeQL({ + async betterResolveLanguages() { + return { + extractors: { + javascript: [{ extractor_root: "" }], + python: [{ extractor_root: "" }], + }, + }; + }, + }); + + // Sanity check the saved config file does not already exist + t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tempDir))); + + // Sanity check that getConfig returns undefined before we have called initConfig + t.deepEqual(await configUtils.getConfig(tempDir, logger), undefined); + + // Stub `getActionVersion` to return some nonsense. + const getActionVersionStub = sinon + .stub(actionsUtil, "getActionVersion") + .returns("does-not-exist"); + + const config = await configUtils.initConfig( + createTestInitConfigInputs({ + languagesInput: "javascript,python", + tempDir, + codeql, + workspacePath: tempDir, + logger, + }), + ); + // initConfig does not save the config, so we do it here. + await configUtils.saveConfig(config, logger); + + // Restore `getActionVersion`. + getActionVersionStub.restore(); + + // The saved config file should now exist + t.true(fs.existsSync(configUtils.getPathToParsedConfigFile(tempDir))); + + // Trying to read the configuration should now throw an error. + await t.throwsAsync(configUtils.getConfig(tempDir, logger), { + instanceOf: ConfigurationError, + }); + }); +}); + test("load input outside of workspace", async (t) => { return await withTmpDir(async (tempDir) => { try { @@ -389,6 +442,7 @@ test("load non-empty input", async (t) => { // And the config we expect it to parse to const expectedConfig: configUtils.Config = { + version: actionsUtil.getActionVersion(), analysisKinds: [AnalysisKind.CodeScanning], languages: [KnownLanguage.javascript], buildMode: BuildMode.None, @@ -1038,6 +1092,13 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo"); expectedLanguages: ["javascript", "csharp", "cpp"], expectedApiCall: true, }, + { + name: "unsupported languages from github api", + languagesInput: "", + languagesInRepository: ["html"], + expectedApiCall: true, + expectedError: configUtils.getNoLanguagesError(), + }, { name: "no languages", languagesInput: "", @@ -1067,57 +1128,71 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo"); expectedLanguages: ["javascript"], }, ].forEach((args) => { - test(`getLanguages: ${args.name}`, async (t) => { - const mockRequest = mockLanguagesInRepo(args.languagesInRepository); - const stubExtractorEntry = { - extractor_root: "", - }; - const codeQL = createStubCodeQL({ - betterResolveLanguages: () => - Promise.resolve({ - aliases: { - "c#": KnownLanguage.csharp, - c: KnownLanguage.cpp, - kotlin: KnownLanguage.java, - typescript: KnownLanguage.javascript, - }, - extractors: { - cpp: [stubExtractorEntry], - csharp: [stubExtractorEntry], - java: [stubExtractorEntry], - javascript: [stubExtractorEntry], - python: [stubExtractorEntry], - }, - }), + for (const resolveSupportedLanguagesUsingCli of [true, false]) { + test(`getLanguages${resolveSupportedLanguagesUsingCli ? " (supported languages via CLI)" : ""}: ${args.name}`, async (t) => { + const features = createFeatures( + resolveSupportedLanguagesUsingCli + ? [Feature.ResolveSupportedLanguagesUsingCli] + : [], + ); + const mockRequest = mockLanguagesInRepo(args.languagesInRepository); + const stubExtractorEntry = { + extractor_root: "", + }; + const codeQL = createStubCodeQL({ + betterResolveLanguages: (options) => + Promise.resolve({ + aliases: { + "c#": KnownLanguage.csharp, + c: KnownLanguage.cpp, + kotlin: KnownLanguage.java, + typescript: KnownLanguage.javascript, + }, + extractors: { + cpp: [stubExtractorEntry], + csharp: [stubExtractorEntry], + java: [stubExtractorEntry], + javascript: [stubExtractorEntry], + python: [stubExtractorEntry], + ...(options?.filterToLanguagesWithQueries + ? {} + : { + html: [stubExtractorEntry], + }), + }, + }), + }); + + if (args.expectedLanguages) { + // happy path + const actualLanguages = await configUtils.getLanguages( + codeQL, + args.languagesInput, + mockRepositoryNwo, + ".", + features, + mockLogger, + ); + + t.deepEqual(actualLanguages.sort(), args.expectedLanguages.sort()); + } else { + // there is an error + await t.throwsAsync( + async () => + await configUtils.getLanguages( + codeQL, + args.languagesInput, + mockRepositoryNwo, + ".", + features, + mockLogger, + ), + { message: args.expectedError }, + ); + } + t.deepEqual(mockRequest.called, args.expectedApiCall); }); - - if (args.expectedLanguages) { - // happy path - const actualLanguages = await configUtils.getLanguages( - codeQL, - args.languagesInput, - mockRepositoryNwo, - ".", - mockLogger, - ); - - t.deepEqual(actualLanguages.sort(), args.expectedLanguages.sort()); - } else { - // there is an error - await t.throwsAsync( - async () => - await configUtils.getLanguages( - codeQL, - args.languagesInput, - mockRepositoryNwo, - ".", - mockLogger, - ), - { message: args.expectedError }, - ); - } - t.deepEqual(mockRequest.called, args.expectedApiCall); - }); + } }); for (const { displayName, language, feature } of [ diff --git a/src/config-utils.ts b/src/config-utils.ts index 92d36920c..538c366e8 100644 --- a/src/config-utils.ts +++ b/src/config-utils.ts @@ -5,7 +5,7 @@ import { performance } from "perf_hooks"; import * as yaml from "js-yaml"; import * as semver from "semver"; -import { isAnalyzingPullRequest } from "./actions-util"; +import { getActionVersion, isAnalyzingPullRequest } from "./actions-util"; import { AnalysisConfig, AnalysisKind, @@ -102,6 +102,10 @@ interface IncludeQueryFilter { * Format of the parsed config file. */ export interface Config { + /** + * The version of the CodeQL Action that the configuration is for. + */ + version: string; /** * Set of analysis kinds that are enabled. */ @@ -312,16 +316,31 @@ export function getUnknownLanguagesError(languages: string[]): string { export async function getSupportedLanguageMap( codeql: CodeQL, + features: FeatureEnablement, + logger: Logger, ): Promise> { - const resolveResult = await codeql.betterResolveLanguages(); + const resolveSupportedLanguagesUsingCli = await features.getValue( + Feature.ResolveSupportedLanguagesUsingCli, + codeql, + ); + const resolveResult = await codeql.betterResolveLanguages({ + filterToLanguagesWithQueries: resolveSupportedLanguagesUsingCli, + }); + if (resolveSupportedLanguagesUsingCli) { + logger.debug( + `The CodeQL CLI supports the following languages: ${Object.keys(resolveResult.extractors).join(", ")}`, + ); + } const supportedLanguages: Record = {}; // Populate canonical language names for (const extractor of Object.keys(resolveResult.extractors)) { - // Require the language to be a known language. - // This is a temporary workaround since we have extractors that are not - // supported languages, such as `csv`, `html`, `properties`, `xml`, and - // `yaml`. We should replace this with a more robust solution in the future. - if (KnownLanguage[extractor] !== undefined) { + // If the CLI supports resolving languages with default queries, use these + // as the set of supported languages. Otherwise, require the language to be + // a known language. + if ( + resolveSupportedLanguagesUsingCli || + KnownLanguage[extractor] !== undefined + ) { supportedLanguages[extractor] = extractor; } } @@ -403,6 +422,7 @@ export async function getLanguages( languagesInput: string | undefined, repository: RepositoryNwo, sourceRoot: string, + features: FeatureEnablement, logger: Logger, ): Promise { // Obtain languages without filtering them. @@ -413,7 +433,7 @@ export async function getLanguages( logger, ); - const languageMap = await getSupportedLanguageMap(codeql); + const languageMap = await getSupportedLanguageMap(codeql, features, logger); const languagesSet = new Set(); const unknownLanguages: string[] = []; @@ -560,6 +580,7 @@ export async function initActionState( languagesInput, repository, sourceRoot, + features, logger, ); @@ -591,6 +612,7 @@ export async function initActionState( ); return { + version: getActionVersion(), analysisKinds, languages, buildMode, @@ -1184,9 +1206,6 @@ export async function initConfig(inputs: InitConfigInputs): Promise { exclude: { tags: "exclude-from-incremental" }, }); } - - // Save the config so we can easily access it again in the future - await saveConfig(config, logger); return config; } @@ -1284,7 +1303,7 @@ export function getPathToParsedConfigFile(tempDir: string): string { /** * Store the given config to the path returned from getPathToParsedConfigFile. */ -async function saveConfig(config: Config, logger: Logger) { +export async function saveConfig(config: Config, logger: Logger) { const configString = JSON.stringify(config); const configFile = getPathToParsedConfigFile(config.tempDir); fs.mkdirSync(path.dirname(configFile), { recursive: true }); @@ -1308,7 +1327,21 @@ export async function getConfig( const configString = fs.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); - return JSON.parse(configString) as Config; + + const config = JSON.parse(configString) as Partial; + + if (config.version === undefined) { + throw new ConfigurationError( + `Loaded configuration file, but it does not contain the expected 'version' field.`, + ); + } + if (config.version !== getActionVersion()) { + throw new ConfigurationError( + `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'`, + ); + } + + return config as Config; } /** diff --git a/src/feature-flags.ts b/src/feature-flags.ts index d9f1a1cc3..ef6747d6d 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -73,6 +73,7 @@ export enum Feature { OverlayAnalysisSwift = "overlay_analysis_swift", PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib", QaTelemetryEnabled = "qa_telemetry_enabled", + ResolveSupportedLanguagesUsingCli = "resolve_supported_languages_using_cli", } export const featureConfig: Record< @@ -145,6 +146,12 @@ export const featureConfig: Record< legacyApi: true, minimumVersion: undefined, }, + [Feature.ResolveSupportedLanguagesUsingCli]: { + defaultValue: false, + envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI", + minimumVersion: undefined, + toolsFeature: ToolsFeature.BuiltinExtractorsSpecifyDefaultQueries, + }, [Feature.OverlayAnalysis]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS", diff --git a/src/init-action.ts b/src/init-action.ts index 8c9e2f36b..aa97b65fc 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -680,6 +680,12 @@ async function run() { } finally { logUnwrittenDiagnostics(); } + + // We save the config here instead of at the end of `initConfig` because we + // may have updated the config returned from `initConfig`, e.g. to revert to + // `OverlayDatabaseMode.None` if we failed to download an overlay-base + // database. + await configUtils.saveConfig(config, logger); await sendCompletedStatusReport( startedAt, config, diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index c4f6ae7c0..ea43abcaa 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -22,18 +22,23 @@ export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; /** * The maximum (uncompressed) size of the overlay base database that we will - * upload. Actions Cache has an overall capacity of 10 GB, and the Actions Cache - * client library uses zstd compression. + * upload. By default, the Actions Cache has an overall capacity of 10 GB, and + * the Actions Cache client library uses zstd compression. * * Ideally we would apply a size limit to the compressed overlay-base database, * but we cannot do so because compression is handled transparently by the * Actions Cache client library. Instead we place a limit on the uncompressed * size of the overlay-base database. * - * Assuming 2.5:1 compression ratio, the 6 GB limit on uncompressed data would - * translate to a limit of around 2.4 GB after compression. + * Assuming 2.5:1 compression ratio, the 15 GB limit on uncompressed data would + * translate to a limit of around 6 GB after compression. This is a high limit + * compared to the default 10GB Actions Cache capacity, but enforcement of Actions + * Cache quotas is not immediate. + * + * TODO: revisit this limit before removing the restriction for overlay analysis + * to the `github` and `dsp-testing` orgs. */ -const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6000; +const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15000; const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000; diff --git a/src/testing-utils.ts b/src/testing-utils.ts index a12c99585..c930d5350 100644 --- a/src/testing-utils.ts +++ b/src/testing-utils.ts @@ -6,6 +6,7 @@ import { TestFn } from "ava"; import nock from "nock"; import * as sinon from "sinon"; +import { getActionVersion } from "./actions-util"; import { AnalysisKind } from "./analyses"; import * as apiClient from "./api-client"; import { GitHubApiDetails } from "./api-client"; @@ -356,6 +357,7 @@ export function createTestConfig(overrides: Partial): Config { return Object.assign( {}, { + version: getActionVersion(), analysisKinds: [AnalysisKind.CodeScanning], languages: [], buildMode: undefined, diff --git a/src/tools-features.ts b/src/tools-features.ts index 7db27edf1..0e88ccd92 100644 --- a/src/tools-features.ts +++ b/src/tools-features.ts @@ -4,11 +4,12 @@ import type { VersionInfo } from "./codeql"; export enum ToolsFeature { AnalysisSummaryV2IsDefault = "analysisSummaryV2Default", + BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries", DatabaseInterpretResultsSupportsSarifRunProperty = "databaseInterpretResultsSupportsSarifRunProperty", - IndirectTracingSupportsStaticBinaries = "indirectTracingSupportsStaticBinaries", - SarifMergeRunsFromEqualCategory = "sarifMergeRunsFromEqualCategory", ForceOverwrite = "forceOverwrite", + IndirectTracingSupportsStaticBinaries = "indirectTracingSupportsStaticBinaries", PythonDefaultIsToNotExtractStdlib = "pythonDefaultIsToNotExtractStdlib", + SarifMergeRunsFromEqualCategory = "sarifMergeRunsFromEqualCategory", } /** diff --git a/src/workflow.test.ts b/src/workflow.test.ts index 21a5874f0..9af81459e 100644 --- a/src/workflow.test.ts +++ b/src/workflow.test.ts @@ -655,6 +655,65 @@ test("getWorkflowErrors() should not report a warning if there is a workflow_cal t.deepEqual(...errorCodes(errors, [])); }); +test("getWorkflowErrors() should report a warning if different versions of the CodeQL Action are used", async (t) => { + const errors = await getWorkflowErrors( + yaml.load(` + name: "CodeQL" + on: + push: + branches: [main] + jobs: + analyze: + steps: + - uses: github/codeql-action/init@v2 + - uses: github/codeql-action/analyze@v3 + `) as Workflow, + await getCodeQLForTesting(), + ); + + t.deepEqual( + ...errorCodes(errors, [WorkflowErrors.InconsistentActionVersion]), + ); +}); + +test("getWorkflowErrors() should not report a warning if the same versions of the CodeQL Action are used", async (t) => { + const errors = await getWorkflowErrors( + yaml.load(` + name: "CodeQL" + on: + push: + branches: [main] + jobs: + analyze: + steps: + - uses: github/codeql-action/init@v3 + - uses: github/codeql-action/analyze@v3 + `) as Workflow, + await getCodeQLForTesting(), + ); + + t.deepEqual(...errorCodes(errors, [])); +}); + +test("getWorkflowErrors() should not report a warning involving versions of other actions", async (t) => { + const errors = await getWorkflowErrors( + yaml.load(` + name: "CodeQL" + on: + push: + branches: [main] + jobs: + analyze: + steps: + - uses: actions/checkout@v5 + - uses: github/codeql-action/init@v3 + `) as Workflow, + await getCodeQLForTesting(), + ); + + t.deepEqual(...errorCodes(errors, [])); +}); + test("getCategoryInputOrThrow returns category for simple workflow with category", (t) => { process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository"; t.is( diff --git a/src/workflow.ts b/src/workflow.ts index 37094154e..ee95c337f 100644 --- a/src/workflow.ts +++ b/src/workflow.ts @@ -72,6 +72,7 @@ function toCodedErrors(errors: { export const WorkflowErrors = toCodedErrors({ MissingPushHook: `Please specify an on.push hook to analyze and see code scanning alerts from the default branch on the Security tab.`, CheckoutWrongHead: `git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.`, + InconsistentActionVersion: `Not all workflow steps that use \`github/codeql-action\` actions use the same version. Please ensure that all such steps use the same version to avoid compatibility issues.`, }); /** @@ -161,6 +162,29 @@ export async function getWorkflowErrors( } } + // Check that all `github/codeql-action` steps use the same ref, i.e. the same version. + // Mixing different versions of the actions can lead to unpredictable behaviour. + const codeqlStepRefs: string[] = []; + for (const job of Object.values(doc?.jobs || {})) { + if (Array.isArray(job.steps)) { + for (const step of job.steps) { + if (step.uses?.startsWith("github/codeql-action/")) { + const parts = step.uses.split("@"); + if (parts.length >= 2) { + codeqlStepRefs.push(parts[parts.length - 1]); + } + } + } + } + } + + if ( + codeqlStepRefs.length > 0 && + !codeqlStepRefs.every((ref) => ref === codeqlStepRefs[0]) + ) { + errors.push(WorkflowErrors.InconsistentActionVersion); + } + // If there is no push trigger, we will not be able to analyze the default branch. // So add a warning to the user to add a push trigger. // If there is a workflow_call trigger, we don't need a push trigger since we assume