Move AugmentationProperties out of config-utils

This commit is contained in:
Michael B. Gale
2025-09-19 17:06:48 +01:00
parent 6222edff53
commit ddc6d540f0
12 changed files with 971 additions and 938 deletions

View File

@@ -3,7 +3,6 @@ import * as path from "path";
import { performance } from "perf_hooks";
import * as yaml from "js-yaml";
import * as semver from "semver";
import { getActionVersion, isAnalyzingPullRequest } from "./actions-util";
import {
@@ -17,7 +16,12 @@ import {
import * as api from "./api-client";
import { CachingKind, getCachingKind } from "./caching-utils";
import { type CodeQL } from "./codeql";
import { ExcludeQueryFilter, UserConfig } from "./config/db-config";
import {
calculateAugmentation,
ExcludeQueryFilter,
generateCodeScanningConfig,
UserConfig,
} from "./config/db-config";
import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import * as errorMessages from "./error-messages";
import { Feature, FeatureEnablement } from "./feature-flags";
@@ -32,7 +36,6 @@ import { RepositoryNwo } from "./repository";
import { downloadTrapCaches } from "./trap-caching";
import {
GitHubVersion,
prettyPrintPack,
ConfigurationError,
BuildMode,
codeQlVersionAtLeast,
@@ -166,56 +169,6 @@ export interface Config {
useOverlayDatabaseCaching: boolean;
}
/**
* Describes how to augment the user config with inputs from the action.
*
* When running a CodeQL analysis, the user can supply a config file. When
* running a CodeQL analysis from a GitHub action, the user can supply a
* config file _and_ a set of inputs.
*
* The inputs from the action are used to augment the user config before
* passing the user config to the CodeQL CLI invocation.
*/
export interface AugmentationProperties {
/**
* Whether or not the queries input combines with the queries in the config.
*/
queriesInputCombines: boolean;
/**
* The queries input from the `with` block of the action declaration
*/
queriesInput?: Array<{ uses: string }>;
/**
* Whether or not the packs input combines with the packs in the config.
*/
packsInputCombines: boolean;
/**
* The packs input from the `with` block of the action declaration
*/
packsInput?: string[];
}
/**
* The default, empty augmentation properties. This is most useful
* for tests.
*/
export const defaultAugmentationProperties: AugmentationProperties = {
queriesInputCombines: false,
packsInputCombines: false,
packsInput: undefined,
queriesInput: undefined,
};
export type Packs = Partial<Record<Language, string[]>>;
export interface Pack {
name: string;
version?: string;
path?: string;
}
export async function getSupportedLanguageMap(
codeql: CodeQL,
features: FeatureEnablement,
@@ -580,73 +533,6 @@ async function loadUserConfig(
}
}
/**
* Calculates how the codeql config file needs to be augmented before passing
* it to the CLI. The reason this is necessary is the codeql-action can be called
* with extra inputs from the workflow. These inputs are not part of the config
* and the CLI does not know about these inputs so we need to inject them into
* the config file sent to the CLI.
*
* @param rawPacksInput The packs input from the action configuration.
* @param rawQueriesInput The queries input from the action configuration.
* @param languages The languages that the config file is for. If the packs input
* is non-empty, then there must be exactly one language. Otherwise, an
* error is thrown.
*
* @returns The properties that need to be augmented in the config file.
*
* @throws An error if the packs input is non-empty and the languages input does
* not have exactly one language.
*/
// exported for testing.
export async function calculateAugmentation(
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
languages: Language[],
): Promise<AugmentationProperties> {
const packsInputCombines = shouldCombine(rawPacksInput);
const packsInput = parsePacksFromInput(
rawPacksInput,
languages,
packsInputCombines,
);
const queriesInputCombines = shouldCombine(rawQueriesInput);
const queriesInput = parseQueriesFromInput(
rawQueriesInput,
queriesInputCombines,
);
return {
packsInputCombines,
packsInput: packsInput?.[languages[0]],
queriesInput,
queriesInputCombines,
};
}
function parseQueriesFromInput(
rawQueriesInput: string | undefined,
queriesInputCombines: boolean,
) {
if (!rawQueriesInput) {
return undefined;
}
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: (rawQueriesInput?.trim() ?? "");
if (queriesInputCombines && trimmedInput.length === 0) {
throw new ConfigurationError(
errorMessages.getConfigFilePropertyError(
undefined,
"queries",
"A '+' was used in the 'queries' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs.",
),
);
}
return trimmedInput.split(",").map((query) => ({ uses: query.trim() }));
}
const OVERLAY_ANALYSIS_FEATURES: Record<Language, Feature> = {
actions: Feature.OverlayAnalysisActions,
cpp: Feature.OverlayAnalysisCpp,
@@ -842,161 +728,6 @@ export async function getOverlayDatabaseMode(
};
}
/**
* Pack names must be in the form of `scope/name`, with only alpha-numeric characters,
* and `-` allowed as long as not the first or last char.
**/
const PACK_IDENTIFIER_PATTERN = (function () {
const alphaNumeric = "[a-z0-9]";
const alphaNumericDash = "[a-z0-9-]";
const component = `${alphaNumeric}(${alphaNumericDash}*${alphaNumeric})?`;
return new RegExp(`^${component}/${component}$`);
})();
// Exported for testing
export function parsePacksFromInput(
rawPacksInput: string | undefined,
languages: Language[],
packsInputCombines: boolean,
): Packs | undefined {
if (!rawPacksInput?.trim()) {
return undefined;
}
if (languages.length > 1) {
throw new ConfigurationError(
"Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by language.",
);
} else if (languages.length === 0) {
throw new ConfigurationError(
"No languages specified. Cannot process the packs input.",
);
}
rawPacksInput = rawPacksInput.trim();
if (packsInputCombines) {
rawPacksInput = rawPacksInput.trim().substring(1).trim();
if (!rawPacksInput) {
throw new ConfigurationError(
errorMessages.getConfigFilePropertyError(
undefined,
"packs",
"A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs.",
),
);
}
}
return {
[languages[0]]: rawPacksInput.split(",").reduce((packs, pack) => {
packs.push(validatePackSpecification(pack));
return packs;
}, [] as string[]),
};
}
/**
* Validates that this package specification is syntactically correct.
* It may not point to any real package, but after this function returns
* without throwing, we are guaranteed that the package specification
* is roughly correct.
*
* The CLI itself will do a more thorough validation of the package
* specification.
*
* A package specification looks like this:
*
* `scope/name@version:path`
*
* Version and path are optional.
*
* @param packStr the package specification to verify.
* @param configFile Config file to use for error reporting
*/
export function parsePacksSpecification(packStr: string): Pack {
if (typeof packStr !== "string") {
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
packStr = packStr.trim();
const atIndex = packStr.indexOf("@");
const colonIndex = packStr.indexOf(":", atIndex);
const packStart = 0;
const versionStart = atIndex + 1 || undefined;
const pathStart = colonIndex + 1 || undefined;
const packEnd = Math.min(
atIndex > 0 ? atIndex : Infinity,
colonIndex > 0 ? colonIndex : Infinity,
packStr.length,
);
const versionEnd = versionStart
? Math.min(colonIndex > 0 ? colonIndex : Infinity, packStr.length)
: undefined;
const pathEnd = pathStart ? packStr.length : undefined;
const packName = packStr.slice(packStart, packEnd).trim();
const version = versionStart
? packStr.slice(versionStart, versionEnd).trim()
: undefined;
const packPath = pathStart
? packStr.slice(pathStart, pathEnd).trim()
: undefined;
if (!PACK_IDENTIFIER_PATTERN.test(packName)) {
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
if (version) {
try {
new semver.Range(version);
} catch {
// The range string is invalid. OK to ignore the caught error
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
}
if (
packPath &&
(path.isAbsolute(packPath) ||
// Permit using "/" instead of "\" on Windows
// Use `x.split(y).join(z)` as a polyfill for `x.replaceAll(y, z)` since
// if we used a regex we'd need to escape the path separator on Windows
// which seems more awkward.
path.normalize(packPath).split(path.sep).join("/") !==
packPath.split(path.sep).join("/"))
) {
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
if (!packPath && pathStart) {
// 0 length path
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
return {
name: packName,
version,
path: packPath,
};
}
export function validatePackSpecification(pack: string) {
return prettyPrintPack(parsePacksSpecification(pack));
}
/**
* The convention in this action is that an input value that is prefixed with a '+' will
* be combined with the corresponding value in the config file.
*
* Without a '+', an input value will override the corresponding value in the config file.
*
* @param inputValue The input value to process.
* @returns true if the input value should replace the corresponding value in the config file,
* false if it should be appended.
*/
function shouldCombine(inputValue?: string): boolean {
return !!inputValue?.trim().startsWith("+");
}
function dbLocationOrDefault(
dbLocation: string | undefined,
tempDir: string,
@@ -1402,56 +1133,6 @@ export async function parseBuildModeInput(
return input as BuildMode;
}
export function generateCodeScanningConfig(
originalUserInput: UserConfig,
augmentationProperties: AugmentationProperties,
): UserConfig {
// make a copy so we can modify it
const augmentedConfig = cloneObject(originalUserInput);
// Inject the queries from the input
if (augmentationProperties.queriesInput) {
if (augmentationProperties.queriesInputCombines) {
augmentedConfig.queries = (augmentedConfig.queries || []).concat(
augmentationProperties.queriesInput,
);
} else {
augmentedConfig.queries = augmentationProperties.queriesInput;
}
}
if (augmentedConfig.queries?.length === 0) {
delete augmentedConfig.queries;
}
// Inject the packs from the input
if (augmentationProperties.packsInput) {
if (augmentationProperties.packsInputCombines) {
// At this point, we already know that this is a single-language analysis
if (Array.isArray(augmentedConfig.packs)) {
augmentedConfig.packs = (augmentedConfig.packs || []).concat(
augmentationProperties.packsInput,
);
} else if (!augmentedConfig.packs) {
augmentedConfig.packs = augmentationProperties.packsInput;
} else {
// At this point, we know there is only one language.
// If there were more than one language, an error would already have been thrown.
const language = Object.keys(augmentedConfig.packs)[0];
augmentedConfig.packs[language] = augmentedConfig.packs[
language
].concat(augmentationProperties.packsInput);
}
} else {
augmentedConfig.packs = augmentationProperties.packsInput;
}
}
if (Array.isArray(augmentedConfig.packs) && !augmentedConfig.packs.length) {
delete augmentedConfig.packs;
}
return augmentedConfig;
}
/**
* Appends `extraQueryExclusions` to `cliConfig`'s `query-filters`.
*

View File

@@ -1,3 +1,11 @@
import * as path from "path";
import * as semver from "semver";
import * as errorMessages from "../error-messages";
import { Language } from "../languages";
import { cloneObject, ConfigurationError, prettyPrintPack } from "../util";
export interface ExcludeQueryFilter {
exclude: Record<string, string[] | string>;
}
@@ -30,3 +38,326 @@ export interface UserConfig {
// codeql query suite `include` and `exclude` properties
"query-filters"?: QueryFilter[];
}
/**
* Describes how to augment the user config with inputs from the action.
*
* When running a CodeQL analysis, the user can supply a config file. When
* running a CodeQL analysis from a GitHub action, the user can supply a
* config file _and_ a set of inputs.
*
* The inputs from the action are used to augment the user config before
* passing the user config to the CodeQL CLI invocation.
*/
export interface AugmentationProperties {
/**
* Whether or not the queries input combines with the queries in the config.
*/
queriesInputCombines: boolean;
/**
* The queries input from the `with` block of the action declaration
*/
queriesInput?: Array<{ uses: string }>;
/**
* Whether or not the packs input combines with the packs in the config.
*/
packsInputCombines: boolean;
/**
* The packs input from the `with` block of the action declaration
*/
packsInput?: string[];
}
/**
* The default, empty augmentation properties. This is most useful
* for tests.
*/
export const defaultAugmentationProperties: AugmentationProperties = {
queriesInputCombines: false,
packsInputCombines: false,
packsInput: undefined,
queriesInput: undefined,
};
/**
* The convention in this action is that an input value that is prefixed with a '+' will
* be combined with the corresponding value in the config file.
*
* Without a '+', an input value will override the corresponding value in the config file.
*
* @param inputValue The input value to process.
* @returns true if the input value should replace the corresponding value in the config file,
* false if it should be appended.
*/
function shouldCombine(inputValue?: string): boolean {
return !!inputValue?.trim().startsWith("+");
}
export type Packs = Partial<Record<Language, string[]>>;
export interface Pack {
name: string;
version?: string;
path?: string;
}
/**
* Pack names must be in the form of `scope/name`, with only alpha-numeric characters,
* and `-` allowed as long as not the first or last char.
**/
const PACK_IDENTIFIER_PATTERN = (function () {
const alphaNumeric = "[a-z0-9]";
const alphaNumericDash = "[a-z0-9-]";
const component = `${alphaNumeric}(${alphaNumericDash}*${alphaNumeric})?`;
return new RegExp(`^${component}/${component}$`);
})();
/**
* Validates that this package specification is syntactically correct.
* It may not point to any real package, but after this function returns
* without throwing, we are guaranteed that the package specification
* is roughly correct.
*
* The CLI itself will do a more thorough validation of the package
* specification.
*
* A package specification looks like this:
*
* `scope/name@version:path`
*
* Version and path are optional.
*
* @param packStr the package specification to verify.
* @param configFile Config file to use for error reporting
*/
export function parsePacksSpecification(packStr: string): Pack {
if (typeof packStr !== "string") {
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
packStr = packStr.trim();
const atIndex = packStr.indexOf("@");
const colonIndex = packStr.indexOf(":", atIndex);
const packStart = 0;
const versionStart = atIndex + 1 || undefined;
const pathStart = colonIndex + 1 || undefined;
const packEnd = Math.min(
atIndex > 0 ? atIndex : Infinity,
colonIndex > 0 ? colonIndex : Infinity,
packStr.length,
);
const versionEnd = versionStart
? Math.min(colonIndex > 0 ? colonIndex : Infinity, packStr.length)
: undefined;
const pathEnd = pathStart ? packStr.length : undefined;
const packName = packStr.slice(packStart, packEnd).trim();
const version = versionStart
? packStr.slice(versionStart, versionEnd).trim()
: undefined;
const packPath = pathStart
? packStr.slice(pathStart, pathEnd).trim()
: undefined;
if (!PACK_IDENTIFIER_PATTERN.test(packName)) {
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
if (version) {
try {
new semver.Range(version);
} catch {
// The range string is invalid. OK to ignore the caught error
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
}
if (
packPath &&
(path.isAbsolute(packPath) ||
// Permit using "/" instead of "\" on Windows
// Use `x.split(y).join(z)` as a polyfill for `x.replaceAll(y, z)` since
// if we used a regex we'd need to escape the path separator on Windows
// which seems more awkward.
path.normalize(packPath).split(path.sep).join("/") !==
packPath.split(path.sep).join("/"))
) {
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
if (!packPath && pathStart) {
// 0 length path
throw new ConfigurationError(errorMessages.getPacksStrInvalid(packStr));
}
return {
name: packName,
version,
path: packPath,
};
}
export function validatePackSpecification(pack: string) {
return prettyPrintPack(parsePacksSpecification(pack));
}
// Exported for testing
export function parsePacksFromInput(
rawPacksInput: string | undefined,
languages: Language[],
packsInputCombines: boolean,
): Packs | undefined {
if (!rawPacksInput?.trim()) {
return undefined;
}
if (languages.length > 1) {
throw new ConfigurationError(
"Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by language.",
);
} else if (languages.length === 0) {
throw new ConfigurationError(
"No languages specified. Cannot process the packs input.",
);
}
rawPacksInput = rawPacksInput.trim();
if (packsInputCombines) {
rawPacksInput = rawPacksInput.trim().substring(1).trim();
if (!rawPacksInput) {
throw new ConfigurationError(
errorMessages.getConfigFilePropertyError(
undefined,
"packs",
"A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs.",
),
);
}
}
return {
[languages[0]]: rawPacksInput.split(",").reduce((packs, pack) => {
packs.push(validatePackSpecification(pack));
return packs;
}, [] as string[]),
};
}
/**
* Calculates how the codeql config file needs to be augmented before passing
* it to the CLI. The reason this is necessary is the codeql-action can be called
* with extra inputs from the workflow. These inputs are not part of the config
* and the CLI does not know about these inputs so we need to inject them into
* the config file sent to the CLI.
*
* @param rawPacksInput The packs input from the action configuration.
* @param rawQueriesInput The queries input from the action configuration.
* @param languages The languages that the config file is for. If the packs input
* is non-empty, then there must be exactly one language. Otherwise, an
* error is thrown.
*
* @returns The properties that need to be augmented in the config file.
*
* @throws An error if the packs input is non-empty and the languages input does
* not have exactly one language.
*/
// exported for testing.
export async function calculateAugmentation(
rawPacksInput: string | undefined,
rawQueriesInput: string | undefined,
languages: Language[],
): Promise<AugmentationProperties> {
const packsInputCombines = shouldCombine(rawPacksInput);
const packsInput = parsePacksFromInput(
rawPacksInput,
languages,
packsInputCombines,
);
const queriesInputCombines = shouldCombine(rawQueriesInput);
const queriesInput = parseQueriesFromInput(
rawQueriesInput,
queriesInputCombines,
);
return {
packsInputCombines,
packsInput: packsInput?.[languages[0]],
queriesInput,
queriesInputCombines,
};
}
function parseQueriesFromInput(
rawQueriesInput: string | undefined,
queriesInputCombines: boolean,
) {
if (!rawQueriesInput) {
return undefined;
}
const trimmedInput = queriesInputCombines
? rawQueriesInput.trim().slice(1).trim()
: (rawQueriesInput?.trim() ?? "");
if (queriesInputCombines && trimmedInput.length === 0) {
throw new ConfigurationError(
errorMessages.getConfigFilePropertyError(
undefined,
"queries",
"A '+' was used in the 'queries' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs.",
),
);
}
return trimmedInput.split(",").map((query) => ({ uses: query.trim() }));
}
export function generateCodeScanningConfig(
originalUserInput: UserConfig,
augmentationProperties: AugmentationProperties,
): UserConfig {
// make a copy so we can modify it
const augmentedConfig = cloneObject(originalUserInput);
// Inject the queries from the input
if (augmentationProperties.queriesInput) {
if (augmentationProperties.queriesInputCombines) {
augmentedConfig.queries = (augmentedConfig.queries || []).concat(
augmentationProperties.queriesInput,
);
} else {
augmentedConfig.queries = augmentationProperties.queriesInput;
}
}
if (augmentedConfig.queries?.length === 0) {
delete augmentedConfig.queries;
}
// Inject the packs from the input
if (augmentationProperties.packsInput) {
if (augmentationProperties.packsInputCombines) {
// At this point, we already know that this is a single-language analysis
if (Array.isArray(augmentedConfig.packs)) {
augmentedConfig.packs = (augmentedConfig.packs || []).concat(
augmentationProperties.packsInput,
);
} else if (!augmentedConfig.packs) {
augmentedConfig.packs = augmentationProperties.packsInput;
} else {
// At this point, we know there is only one language.
// If there were more than one language, an error would already have been thrown.
const language = Object.keys(augmentedConfig.packs)[0];
augmentedConfig.packs[language] = augmentedConfig.packs[
language
].concat(augmentationProperties.packsInput);
}
} else {
augmentedConfig.packs = augmentationProperties.packsInput;
}
}
if (Array.isArray(augmentedConfig.packs) && !augmentedConfig.packs.length) {
delete augmentedConfig.packs;
}
return augmentedConfig;
}