Merge pull request #3083 from github/henrymercer/resolve-languages-default-queries

Resolve supported languages using CodeQL CLI
This commit is contained in:
Henry Mercer
2025-09-12 10:12:15 +01:00
committed by GitHub
17 changed files with 230 additions and 74 deletions

View File

@@ -164,4 +164,5 @@ jobs:
exit 1
fi
env:
CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI: true
CODEQL_ACTION_TEST_MODE: true

View File

@@ -117796,6 +117796,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -118217,13 +118223,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [],
...getExtraOptionsFromEnv(["resolve", "languages"])
];
const output = await runCli(cmd, codeqlArgs);

11
lib/analyze-action.js generated
View File

@@ -91030,6 +91030,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -92780,13 +92786,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [],
...getExtraOptionsFromEnv(["resolve", "languages"])
];
const output = await runCli(cmd, codeqlArgs);

View File

@@ -78534,6 +78534,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -79252,13 +79258,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [],
...getExtraOptionsFromEnv(["resolve", "languages"])
];
const output = await runCli(cmd, codeqlArgs);

View File

@@ -129129,6 +129129,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -130690,13 +130696,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [],
...getExtraOptionsFromEnv(["resolve", "languages"])
];
const output = await runCli(cmd, codeqlArgs);

33
lib/init-action.js generated
View File

@@ -86629,6 +86629,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -87196,11 +87202,22 @@ function getNoLanguagesError() {
function getUnknownLanguagesError(languages) {
return `Did not recognize the following languages: ${languages.join(", ")}`;
}
async function getSupportedLanguageMap(codeql) {
const resolveResult = await codeql.betterResolveLanguages();
async function getSupportedLanguageMap(codeql, features, logger) {
const resolveSupportedLanguagesUsingCli = await features.getValue(
"resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */,
codeql
);
const resolveResult = await codeql.betterResolveLanguages({
filterToLanguagesWithQueries: resolveSupportedLanguagesUsingCli
});
if (resolveSupportedLanguagesUsingCli) {
logger.debug(
`The CodeQL CLI supports the following languages: ${Object.keys(resolveResult.extractors).join(", ")}`
);
}
const supportedLanguages = {};
for (const extractor of Object.keys(resolveResult.extractors)) {
if (KnownLanguage[extractor] !== void 0) {
if (resolveSupportedLanguagesUsingCli || KnownLanguage[extractor] !== void 0) {
supportedLanguages[extractor] = extractor;
}
}
@@ -87236,14 +87253,14 @@ async function getRawLanguagesInRepo(repository, sourceRoot, logger) {
logger.debug(`Raw languages in repository: ${result.join(", ")}`);
return result;
}
async function getLanguages(codeql, languagesInput, repository, sourceRoot, logger) {
async function getLanguages(codeql, languagesInput, repository, sourceRoot, features, logger) {
const { rawLanguages, autodetected } = await getRawLanguages(
languagesInput,
repository,
sourceRoot,
logger
);
const languageMap = await getSupportedLanguageMap(codeql);
const languageMap = await getSupportedLanguageMap(codeql, features, logger);
const languagesSet = /* @__PURE__ */ new Set();
const unknownLanguages = [];
for (const language of rawLanguages) {
@@ -87311,6 +87328,7 @@ async function initActionState({
languagesInput,
repository,
sourceRoot,
features,
logger
);
const buildMode = await parseBuildModeInput(
@@ -89363,13 +89381,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [],
...getExtraOptionsFromEnv(["resolve", "languages"])
];
const output = await runCli(cmd, codeqlArgs);

View File

@@ -78525,6 +78525,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -78952,13 +78958,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [],
...getExtraOptionsFromEnv(["resolve", "languages"])
];
const output = await runCli(cmd, codeqlArgs);

View File

@@ -117205,6 +117205,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",

11
lib/upload-lib.js generated
View File

@@ -89221,6 +89221,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -90518,13 +90524,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [],
...getExtraOptionsFromEnv(["resolve", "languages"])
];
const output = await runCli(cmd, codeqlArgs);

View File

@@ -117370,6 +117370,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",

View File

@@ -89217,6 +89217,12 @@ var featureConfig = {
legacyApi: true,
minimumVersion: void 0
},
["resolve_supported_languages_using_cli" /* ResolveSupportedLanguagesUsingCli */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: void 0,
toolsFeature: "builtinExtractorsSpecifyDefaultQueries" /* BuiltinExtractorsSpecifyDefaultQueries */
},
["overlay_analysis" /* OverlayAnalysis */]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -91219,13 +91225,16 @@ async function getCodeQLForCmd(cmd, checkVersion) {
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...filterToLanguagesWithQueries ? ["--filter-to-languages-with-queries"] : [],
...getExtraOptionsFromEnv(["resolve", "languages"])
];
const output = await runCli(cmd, codeqlArgs);

View File

@@ -1,6 +1,8 @@
name: "Multi-language repository"
description: "An end-to-end integration test of a multi-language repository using automatic language detection for macOS"
operatingSystems: ["macos", "ubuntu"]
env:
CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI: true
installGo: true
steps:
- name: Use Xcode 16

View File

@@ -127,7 +127,9 @@ export interface CodeQL {
/**
* Run 'codeql resolve languages' with '--format=betterjson'.
*/
betterResolveLanguages(): Promise<BetterResolveLanguagesOutput>;
betterResolveLanguages(options?: {
filterToLanguagesWithQueries: boolean;
}): Promise<BetterResolveLanguagesOutput>;
/**
* Run 'codeql resolve build-environment'
*/
@@ -736,13 +738,22 @@ export async function getCodeQLForCmd(
);
}
},
async betterResolveLanguages() {
async betterResolveLanguages(
{
filterToLanguagesWithQueries,
}: {
filterToLanguagesWithQueries: boolean;
} = { filterToLanguagesWithQueries: false },
) {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...(filterToLanguagesWithQueries
? ["--filter-to-languages-with-queries"]
: []),
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runCli(cmd, codeqlArgs);

View File

@@ -1092,6 +1092,13 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
expectedLanguages: ["javascript", "csharp", "cpp"],
expectedApiCall: true,
},
{
name: "unsupported languages from github api",
languagesInput: "",
languagesInRepository: ["html"],
expectedApiCall: true,
expectedError: configUtils.getNoLanguagesError(),
},
{
name: "no languages",
languagesInput: "",
@@ -1121,57 +1128,71 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
expectedLanguages: ["javascript"],
},
].forEach((args) => {
test(`getLanguages: ${args.name}`, async (t) => {
const mockRequest = mockLanguagesInRepo(args.languagesInRepository);
const stubExtractorEntry = {
extractor_root: "",
};
const codeQL = createStubCodeQL({
betterResolveLanguages: () =>
Promise.resolve({
aliases: {
"c#": KnownLanguage.csharp,
c: KnownLanguage.cpp,
kotlin: KnownLanguage.java,
typescript: KnownLanguage.javascript,
},
extractors: {
cpp: [stubExtractorEntry],
csharp: [stubExtractorEntry],
java: [stubExtractorEntry],
javascript: [stubExtractorEntry],
python: [stubExtractorEntry],
},
}),
for (const resolveSupportedLanguagesUsingCli of [true, false]) {
test(`getLanguages${resolveSupportedLanguagesUsingCli ? " (supported languages via CLI)" : ""}: ${args.name}`, async (t) => {
const features = createFeatures(
resolveSupportedLanguagesUsingCli
? [Feature.ResolveSupportedLanguagesUsingCli]
: [],
);
const mockRequest = mockLanguagesInRepo(args.languagesInRepository);
const stubExtractorEntry = {
extractor_root: "",
};
const codeQL = createStubCodeQL({
betterResolveLanguages: (options) =>
Promise.resolve({
aliases: {
"c#": KnownLanguage.csharp,
c: KnownLanguage.cpp,
kotlin: KnownLanguage.java,
typescript: KnownLanguage.javascript,
},
extractors: {
cpp: [stubExtractorEntry],
csharp: [stubExtractorEntry],
java: [stubExtractorEntry],
javascript: [stubExtractorEntry],
python: [stubExtractorEntry],
...(options?.filterToLanguagesWithQueries
? {}
: {
html: [stubExtractorEntry],
}),
},
}),
});
if (args.expectedLanguages) {
// happy path
const actualLanguages = await configUtils.getLanguages(
codeQL,
args.languagesInput,
mockRepositoryNwo,
".",
features,
mockLogger,
);
t.deepEqual(actualLanguages.sort(), args.expectedLanguages.sort());
} else {
// there is an error
await t.throwsAsync(
async () =>
await configUtils.getLanguages(
codeQL,
args.languagesInput,
mockRepositoryNwo,
".",
features,
mockLogger,
),
{ message: args.expectedError },
);
}
t.deepEqual(mockRequest.called, args.expectedApiCall);
});
if (args.expectedLanguages) {
// happy path
const actualLanguages = await configUtils.getLanguages(
codeQL,
args.languagesInput,
mockRepositoryNwo,
".",
mockLogger,
);
t.deepEqual(actualLanguages.sort(), args.expectedLanguages.sort());
} else {
// there is an error
await t.throwsAsync(
async () =>
await configUtils.getLanguages(
codeQL,
args.languagesInput,
mockRepositoryNwo,
".",
mockLogger,
),
{ message: args.expectedError },
);
}
t.deepEqual(mockRequest.called, args.expectedApiCall);
});
}
});
for (const { displayName, language, feature } of [

View File

@@ -316,16 +316,31 @@ export function getUnknownLanguagesError(languages: string[]): string {
export async function getSupportedLanguageMap(
codeql: CodeQL,
features: FeatureEnablement,
logger: Logger,
): Promise<Record<string, string>> {
const resolveResult = await codeql.betterResolveLanguages();
const resolveSupportedLanguagesUsingCli = await features.getValue(
Feature.ResolveSupportedLanguagesUsingCli,
codeql,
);
const resolveResult = await codeql.betterResolveLanguages({
filterToLanguagesWithQueries: resolveSupportedLanguagesUsingCli,
});
if (resolveSupportedLanguagesUsingCli) {
logger.debug(
`The CodeQL CLI supports the following languages: ${Object.keys(resolveResult.extractors).join(", ")}`,
);
}
const supportedLanguages: Record<string, string> = {};
// Populate canonical language names
for (const extractor of Object.keys(resolveResult.extractors)) {
// Require the language to be a known language.
// This is a temporary workaround since we have extractors that are not
// supported languages, such as `csv`, `html`, `properties`, `xml`, and
// `yaml`. We should replace this with a more robust solution in the future.
if (KnownLanguage[extractor] !== undefined) {
// If the CLI supports resolving languages with default queries, use these
// as the set of supported languages. Otherwise, require the language to be
// a known language.
if (
resolveSupportedLanguagesUsingCli ||
KnownLanguage[extractor] !== undefined
) {
supportedLanguages[extractor] = extractor;
}
}
@@ -407,6 +422,7 @@ export async function getLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
sourceRoot: string,
features: FeatureEnablement,
logger: Logger,
): Promise<Language[]> {
// Obtain languages without filtering them.
@@ -417,7 +433,7 @@ export async function getLanguages(
logger,
);
const languageMap = await getSupportedLanguageMap(codeql);
const languageMap = await getSupportedLanguageMap(codeql, features, logger);
const languagesSet = new Set<Language>();
const unknownLanguages: string[] = [];
@@ -564,6 +580,7 @@ export async function initActionState(
languagesInput,
repository,
sourceRoot,
features,
logger,
);

View File

@@ -73,6 +73,7 @@ export enum Feature {
OverlayAnalysisSwift = "overlay_analysis_swift",
PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib",
QaTelemetryEnabled = "qa_telemetry_enabled",
ResolveSupportedLanguagesUsingCli = "resolve_supported_languages_using_cli",
}
export const featureConfig: Record<
@@ -145,6 +146,12 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: undefined,
},
[Feature.ResolveSupportedLanguagesUsingCli]: {
defaultValue: false,
envVar: "CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI",
minimumVersion: undefined,
toolsFeature: ToolsFeature.BuiltinExtractorsSpecifyDefaultQueries,
},
[Feature.OverlayAnalysis]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",

View File

@@ -4,11 +4,12 @@ import type { VersionInfo } from "./codeql";
export enum ToolsFeature {
AnalysisSummaryV2IsDefault = "analysisSummaryV2Default",
BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries",
DatabaseInterpretResultsSupportsSarifRunProperty = "databaseInterpretResultsSupportsSarifRunProperty",
IndirectTracingSupportsStaticBinaries = "indirectTracingSupportsStaticBinaries",
SarifMergeRunsFromEqualCategory = "sarifMergeRunsFromEqualCategory",
ForceOverwrite = "forceOverwrite",
IndirectTracingSupportsStaticBinaries = "indirectTracingSupportsStaticBinaries",
PythonDefaultIsToNotExtractStdlib = "pythonDefaultIsToNotExtractStdlib",
SarifMergeRunsFromEqualCategory = "sarifMergeRunsFromEqualCategory",
}
/**