github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-10-30 01:27:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into mario-campos/cache-cli-resolve-langs

Browse Source 
# Conflicts:
#	lib/entry-points.js
#	src/codeql.ts
#	src/environment.ts
#	src/status-report.ts
This commit is contained in:
Mario Campos
2026-07-09 15:02:03 -05:00
parent 5110948a58 0c7580ae89
commit d4b99ad098
126 changed files with 18417 additions and 16116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/action-common.ts Normal file
View File

@@ -0,0 +1,95 @@
import * as core from "@actions/core";
import { ActionsEnv, getActionsEnv } from "./actions-util";
import { Env } from "./environment";
import { FeatureEnablement } from "./feature-flags";
import { getActionsLogger, Logger } from "./logging";
import {
ActionName,
getDisplayActionName,
sendUnhandledErrorStatusReport,
} from "./status-report";
import { getEnv, getErrorMessage } from "./util";
/** Common state that is always available in `ActionState`. */
export interface BaseState {
/** The name of the Action. */
name: ActionName;
/** When the Action was started. */
startedAt: Date;
}
/** Describes different state features that an Action may have. */
export interface FeatureState {
Logger: {
/** The logger that is in use. */
logger: Logger;
};
Env: {
/** Information about environment variables. */
env: Env;
};
Actions: {
/** Access to Actions-related functionality. */
actions: ActionsEnv;
};
FeatureFlags: {
/** Information about enabled feature flags. */
features: FeatureEnablement;
};
}
/** Identifies a type of state an Action may have. */
export type StateFeature = keyof FeatureState;
/** Constructs the intersection of all state types identifies by `Fs`. */
export type FieldsOf<Fs extends readonly StateFeature[]> = Fs extends []
? BaseState
: Fs extends [
infer Head extends StateFeature,
...infer Tail extends readonly StateFeature[],
]
? FeatureState[Head] & FieldsOf<Tail>
: never;
/** Describes the state of an Action that has access to the state corresponding to `Fs`. */
export type ActionState<Fs extends readonly StateFeature[]> = FieldsOf<Fs>;
/** The type of an Action's main entry point. This is a function that is provided
* with a basic `ActionState` object with features that are always available.
* Each Action can then augment the `state` further if additional features are required.
*/
export type ActionMain = (
state: ActionState<["Logger", "Env", "Actions"]>,
) => Promise<void>;
/** A specification for a CodeQL Action step. */
export interface Action {
/** The name of the Action. */
name: ActionName;
/** The entry point for the Action. */
run: ActionMain;
}
/** A generic entry point that sets up the basic environment for the `action` and runs it. */
export async function runInActions(action: Action) {
const startedAt = new Date();
const logger = getActionsLogger();
const env = getEnv();
const actionsEnv = getActionsEnv();
try {
await action.run({
name: action.name,
startedAt,
logger,
env,
actions: actionsEnv,
});
} catch (error) {
core.setFailed(
`${getDisplayActionName(action.name)} action failed: ${getErrorMessage(error)}`,
);
await sendUnhandledErrorStatusReport(action.name, startedAt, error, logger);
}
}

91
src/actions-util.ts
View File

@@ -7,12 +7,13 @@ import * as github from "@actions/github";
import * as io from "@actions/io";
import type { Config } from "./config-utils";
import { Env, EnvVar, ActionsEnvVars } from "./environment";
import { Logger } from "./logging";
import {
doesDirectoryExist,
getCodeQLDatabasePath,
getRequiredEnvParam,
ConfigurationError,
getEnv,
} from "./util";
/**
@@ -61,17 +62,21 @@ export const getOptionalInput = function (name: string): string | undefined {
return value.length > 0 ? value : undefined;
};
export function getTemporaryDirectory(): string {
const value = process.env["CODEQL_ACTION_TEMP"];
return value !== undefined && value !== ""
? value
: getRequiredEnvParam("RUNNER_TEMP");
/**
* Gets the temporary directory used by the CodeQL Action. This will either be the temporary
* directory that has been set in `CODEQL_ACTION_TEMP` by e.g. a previous step, or the
* value of `RUNNER_TEMP` otherwise.
*/
export function getTemporaryDirectory(env: Env = getEnv()): string {
return (
env.getOptional(EnvVar.TEMP) ?? env.getRequired(ActionsEnvVars.RUNNER_TEMP)
);
}
const PR_DIFF_RANGE_JSON_FILENAME = "pr-diff-range.json";
export function getDiffRangesJsonFilePath(): string {
return path.join(getTemporaryDirectory(), PR_DIFF_RANGE_JSON_FILENAME);
export function getDiffRangesJsonFilePath(env: Env = getEnv()): string {
return path.join(getTemporaryDirectory(env), PR_DIFF_RANGE_JSON_FILENAME);
}
export function getActionVersion(): string {
@@ -83,16 +88,16 @@ export function getActionVersion(): string {
*
* This will be "dynamic" for default setup workflow runs.
*/
export function getWorkflowEventName() {
return getRequiredEnvParam("GITHUB_EVENT_NAME");
export function getWorkflowEventName(env: Env = getEnv()) {
return env.getRequired(ActionsEnvVars.GITHUB_EVENT_NAME);
}
/**
* Returns whether the current workflow is executing a local copy of the Action, e.g. we're running
* a workflow on the codeql-action repo itself.
*/
export function isRunningLocalAction(): boolean {
const relativeScriptPath = getRelativeScriptPath();
export function isRunningLocalAction(env: Env = getEnv()): boolean {
const relativeScriptPath = getRelativeScriptPath(env);
return (
relativeScriptPath.startsWith("..") || path.isAbsolute(relativeScriptPath)
);
@@ -103,15 +108,15 @@ export function isRunningLocalAction(): boolean {
*
* This can be used to get the Action's name or tell if we're running a local Action.
*/
function getRelativeScriptPath(): string {
const runnerTemp = getRequiredEnvParam("RUNNER_TEMP");
function getRelativeScriptPath(env: Env): string {
const runnerTemp = env.getRequired(ActionsEnvVars.RUNNER_TEMP);
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
return path.relative(actionsDirectory, __filename);
}
/** Returns the contents of `GITHUB_EVENT_PATH` as a JSON object. */
export function getWorkflowEvent(): any {
const eventJsonFile = getRequiredEnvParam("GITHUB_EVENT_PATH");
export function getWorkflowEvent(env: Env = getEnv()): any {
const eventJsonFile = env.getRequired(ActionsEnvVars.GITHUB_EVENT_PATH);
try {
return JSON.parse(fs.readFileSync(eventJsonFile, "utf-8"));
} catch (e) {
@@ -180,17 +185,17 @@ export function getUploadValue(input: string | undefined): UploadKind {
/**
* Get the workflow run ID.
*/
export function getWorkflowRunID(): number {
const workflowRunIdString = getRequiredEnvParam("GITHUB_RUN_ID");
export function getWorkflowRunID(env: Env = getEnv()): number {
const workflowRunIdString = env.getRequired(ActionsEnvVars.GITHUB_RUN_ID);
const workflowRunID = parseInt(workflowRunIdString, 10);
if (Number.isNaN(workflowRunID)) {
throw new Error(
`GITHUB_RUN_ID must define a non NaN workflow run ID. Current value is ${workflowRunIdString}`,
`${ActionsEnvVars.GITHUB_RUN_ID} must define a non NaN workflow run ID. Current value is ${workflowRunIdString}`,
);
}
if (workflowRunID < 0) {
throw new Error(
`GITHUB_RUN_ID must be a non-negative integer. Current value is ${workflowRunIdString}`,
`${ActionsEnvVars.GITHUB_RUN_ID} must be a non-negative integer. Current value is ${workflowRunIdString}`,
);
}
return workflowRunID;
@@ -199,17 +204,19 @@ export function getWorkflowRunID(): number {
/**
* Get the workflow run attempt number.
*/
export function getWorkflowRunAttempt(): number {
const workflowRunAttemptString = getRequiredEnvParam("GITHUB_RUN_ATTEMPT");
export function getWorkflowRunAttempt(env: Env = getEnv()): number {
const workflowRunAttemptString = env.getRequired(
ActionsEnvVars.GITHUB_RUN_ATTEMPT,
);
const workflowRunAttempt = parseInt(workflowRunAttemptString, 10);
if (Number.isNaN(workflowRunAttempt)) {
throw new Error(
`GITHUB_RUN_ATTEMPT must define a non NaN workflow run attempt. Current value is ${workflowRunAttemptString}`,
`${ActionsEnvVars.GITHUB_RUN_ATTEMPT} must define a non NaN workflow run attempt. Current value is ${workflowRunAttemptString}`,
);
}
if (workflowRunAttempt <= 0) {
throw new Error(
`GITHUB_RUN_ATTEMPT must be a positive integer. Current value is ${workflowRunAttemptString}`,
`${ActionsEnvVars.GITHUB_RUN_ATTEMPT} must be a positive integer. Current value is ${workflowRunAttemptString}`,
);
}
return workflowRunAttempt;
@@ -266,18 +273,18 @@ export const getFileType = async (filePath: string): Promise<string> => {
}
};
export function isSelfHostedRunner() {
return process.env.RUNNER_ENVIRONMENT === "self-hosted";
export function isSelfHostedRunner(env: Env = getEnv()) {
return env.getOptional(ActionsEnvVars.RUNNER_ENVIRONMENT) === "self-hosted";
}
/** Determines whether the workflow trigger is `dynamic`. */
export function isDynamicWorkflow(): boolean {
return getWorkflowEventName() === "dynamic";
export function isDynamicWorkflow(env: Env = getEnv()): boolean {
return getWorkflowEventName(env) === "dynamic";
}
/** Determines whether we are running in default setup. */
export function isDefaultSetup(): boolean {
return isDynamicWorkflow();
export function isDefaultSetup(env: Env = getEnv()): boolean {
return isDynamicWorkflow(env);
}
export function prettyPrintInvocation(cmd: string, args: string[]): string {
@@ -375,9 +382,10 @@ const persistedInputsKey = "persisted_inputs";
* This would be simplified if actions/runner#3514 is addressed.
* https://github.com/actions/runner/issues/3514
*/
export const persistInputs = function () {
const inputEnvironmentVariables = Object.entries(process.env).filter(
([name]) => name.startsWith("INPUT_"),
export const persistInputs = function (env: Env = getEnv()) {
const entries = env.entries();
const inputEnvironmentVariables = entries.filter(([name]) =>
name.startsWith("INPUT_"),
);
core.saveState(persistedInputsKey, JSON.stringify(inputEnvironmentVariables));
};
@@ -405,7 +413,9 @@ export interface PullRequestBranches {
* @returns the base and head branches of the pull request, or undefined if
* we are not analyzing a pull request.
*/
export function getPullRequestBranches(): PullRequestBranches | undefined {
export function getPullRequestBranches(
env: Env = getEnv(),
): PullRequestBranches | undefined {
const pullRequest = github.context.payload.pull_request;
if (pullRequest) {
return {
@@ -419,8 +429,10 @@ export function getPullRequestBranches(): PullRequestBranches | undefined {
// PR analysis under Default Setup does not have the pull_request context,
// but it should set CODE_SCANNING_REF and CODE_SCANNING_BASE_BRANCH.
const codeScanningRef = process.env.CODE_SCANNING_REF;
const codeScanningBaseBranch = process.env.CODE_SCANNING_BASE_BRANCH;
const codeScanningRef = env.getOptional(EnvVar.CODE_SCANNING_REF);
const codeScanningBaseBranch = env.getOptional(
EnvVar.CODE_SCANNING_BASE_BRANCH,
);
if (codeScanningRef && codeScanningBaseBranch) {
return {
base: codeScanningBaseBranch,
@@ -435,8 +447,8 @@ export function getPullRequestBranches(): PullRequestBranches | undefined {
/**
* Returns whether we are analyzing a pull request.
*/
export function isAnalyzingPullRequest(): boolean {
return getPullRequestBranches() !== undefined;
export function isAnalyzingPullRequest(env: Env = getEnv()): boolean {
return getPullRequestBranches(env) !== undefined;
}
/**
@@ -460,13 +472,14 @@ const qualityCategoryMapping: Record<string, string> = {
export function fixCodeQualityCategory(
logger: Logger,
category?: string,
env: Env = getEnv(),
): string | undefined {
// The `category` should always be set by Default Setup. We perform this check
// to avoid potential issues if Code Quality supports Advanced Setup in the future
// and before this workaround is removed.
if (
category !== undefined &&
isDefaultSetup() &&
isDefaultSetup(env) &&
category.startsWith("/language:")
) {
const language = category.substring("/language:".length);

25
src/analyze-action.ts
View File

@@ -4,6 +4,7 @@ import { performance } from "perf_hooks";
import * as core from "@actions/core";
import { Action, ActionState, runInActions } from "./action-common";
import * as actionsUtil from "./actions-util";
import * as analyses from "./analyses";
import {
@@ -40,7 +41,6 @@ import {
createStatusReportBase,
DatabaseCreationTimings,
getActionsStatus,
sendUnhandledErrorStatusReport,
StatusReportBase,
} from "./status-report";
import {
@@ -212,7 +212,7 @@ async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
await runAutobuild(config, BuiltInLanguage.go, logger);
}
async function run(startedAt: Date) {
async function run({ startedAt, logger }: ActionState<["Logger"]>) {
// To capture errors appropriately, keep as much code within the try-catch as
// possible, and only use safe functions outside.
@@ -228,7 +228,6 @@ async function run(startedAt: Date) {
let didUploadTrapCaches = false;
let dependencyCacheResults: DependencyCacheUploadStatusReport | undefined;
let databaseUploadResults: DatabaseUploadResult[] = [];
const logger = getActionsLogger();
try {
util.initializeEnvironment(actionsUtil.getActionVersion());
@@ -523,19 +522,13 @@ async function run(startedAt: Date) {
}
}
/** Defines the `analyze` Action. */
const analyze: Action = {
name: ActionName.Analyze,
run,
};
export async function runWrapper() {
const startedAt = new Date();
const logger = getActionsLogger();
try {
await run(startedAt);
} catch (error) {
core.setFailed(`analyze action failed: ${util.getErrorMessage(error)}`);
await sendUnhandledErrorStatusReport(
ActionName.Analyze,
startedAt,
error,
logger,
);
}
await runInActions(analyze);
await util.checkForTimeout();
}

4
src/analyze.ts
View File

@@ -281,11 +281,11 @@ extensions:
.join(checkoutPath, range.path)
.replaceAll(path.sep, "/");
// Using yaml.dump() with `forceQuotes: true` ensures that all special
// Using yaml.dump() with `quoteStyle: "double"` ensures that all special
// characters are escaped, and that the path is always rendered as a
// quoted string on a single line.
return (
` - [${yaml.dump(filename, { forceQuotes: true }).trim()}, ` +
` - [${yaml.dump(filename, { forceQuotes: true, quoteStyle: "single" }).trim()}, ` +
`${range.startLine}, ${range.endLine}]\n`
);
})

19
src/api-client.test.ts
View File

@@ -6,7 +6,8 @@ import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as api from "./api-client";
import { DO_NOT_RETRY_STATUSES } from "./api-client";
import { setupTests } from "./testing-utils";
import { ActionsEnvVars } from "./environment";
import { getTestEnv, setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
@@ -20,16 +21,13 @@ test.serial("getApiClient", async (t) => {
const githubStub: sinon.SinonStub = sinon.stub();
pluginStub.returns(githubStub);
sinon.stub(actionsUtil, "getRequiredInput").withArgs("token").returns("xyz");
const requiredEnvParamStub = sinon.stub(util, "getRequiredEnvParam");
requiredEnvParamStub
.withArgs("GITHUB_SERVER_URL")
.returns("http://github.localhost");
requiredEnvParamStub
.withArgs("GITHUB_API_URL")
.returns("http://api.github.localhost");
const env = getTestEnv();
env.set(ActionsEnvVars.GITHUB_SERVER_URL, "http://github.localhost");
env.set(ActionsEnvVars.GITHUB_API_URL, "http://api.github.localhost");
api.getApiClient();
sinon.stub(actionsUtil, "getRequiredInput").withArgs("token").returns("xyz");
api.getApiClient(env);
t.assert(
githubStub.calledOnceWithExactly({
@@ -181,6 +179,7 @@ test.serial(
"Code Security must be enabled for this repository to use code scanning",
"Advanced Security must be enabled for this repository to use code scanning",
"Code Scanning is not enabled for this repository. Please enable code scanning in the repository settings.",
"Code quality is not enabled for this repository. Please enable code quality in the repository settings.",
];
const transforms = [
(msg: string) => msg,

13
src/api-client.ts
View File

@@ -3,7 +3,7 @@ import * as githubUtils from "@actions/github/lib/utils";
import * as retry from "@octokit/plugin-retry";
import { getActionVersion, getRequiredInput } from "./actions-util";
import { EnvVar } from "./environment";
import { EnvVar, ReadOnlyEnv, ActionsEnvVars, getEnv } from "./environment";
import { Logger } from "./logging";
import { getRepositoryNwo, RepositoryNwo } from "./repository";
import {
@@ -67,16 +67,16 @@ function createApiClientWithDetails(
);
}
export function getApiDetails(): GitHubApiDetails {
export function getApiDetails(env: ReadOnlyEnv = getEnv()): GitHubApiDetails {
return {
auth: getRequiredInput("token"),
url: getRequiredEnvParam("GITHUB_SERVER_URL"),
apiURL: getRequiredEnvParam("GITHUB_API_URL"),
url: env.getRequired(ActionsEnvVars.GITHUB_SERVER_URL),
apiURL: env.getRequired(ActionsEnvVars.GITHUB_API_URL),
};
}
export function getApiClient() {
return createApiClientWithDetails(getApiDetails());
export function getApiClient(env: ReadOnlyEnv = getEnv()) {
return createApiClientWithDetails(getApiDetails(env));
}
export function getApiClientWithExternalAuth(
@@ -311,6 +311,7 @@ function isEnablementError(msg: string) {
/Code Security must be enabled/i,
/Advanced Security must be enabled/i,
/Code Scanning is not enabled/i,
/Code Quality is not enabled/i,
].some((pattern) => pattern.test(msg));
}

2
src/api-compatibility.json
View File

@@ -1 +1 @@
{"maximumVersion": "3.22", "minimumVersion": "3.16"}
{"maximumVersion": "3.22", "minimumVersion": "3.17"}

28
src/autobuild-action.ts
View File

@@ -1,5 +1,6 @@
import * as core from "@actions/core";
import { Action, ActionState, runInActions } from "./action-common";
import {
getActionVersion,
getOptionalInput,
@@ -11,13 +12,12 @@ import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { EnvVar } from "./environment";
import { Language } from "./languages";
import { Logger, getActionsLogger } from "./logging";
import { Logger } from "./logging";
import {
StatusReportBase,
getActionsStatus,
createStatusReportBase,
sendStatusReport,
sendUnhandledErrorStatusReport,
ActionName,
} from "./status-report";
import { endTracingForCluster } from "./tracer-config";
@@ -26,7 +26,6 @@ import {
checkDiskUsage,
checkGitHubVersionInRange,
ConfigurationError,
getErrorMessage,
initializeEnvironment,
wrapError,
} from "./util";
@@ -69,11 +68,10 @@ async function sendCompletedStatusReport(
}
}
async function run(startedAt: Date) {
async function run({ startedAt, logger }: ActionState<["Logger"]>) {
// To capture errors appropriately, keep as much code within the try-catch as
// possible, and only use safe functions outside.
const logger = getActionsLogger();
let config: Config | undefined;
let currentLanguage: Language | undefined;
let languages: Language[] | undefined;
@@ -142,18 +140,12 @@ async function run(startedAt: Date) {
await sendCompletedStatusReport(config, logger, startedAt, languages ?? []);
}
/** Defines the `autobuild` Action. */
const autobuild: Action = {
name: ActionName.Autobuild,
run,
};
export async function runWrapper() {
const startedAt = new Date();
const logger = getActionsLogger();
try {
await run(startedAt);
} catch (error) {
core.setFailed(`autobuild action failed. ${getErrorMessage(error)}`);
await sendUnhandledErrorStatusReport(
ActionName.Autobuild,
startedAt,
error,
logger,
);
}
await runInActions(autobuild);
}

4
src/autobuild.ts
View File

@@ -5,7 +5,7 @@ import { getGitHubVersion } from "./api-client";
import { CodeQL, getCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { DocUrl } from "./doc-url";
import { EnvVar } from "./environment";
import { ActionsEnvVars, EnvVar } from "./environment";
import { Feature, featureConfig, initFeatures } from "./feature-flags";
import { BuiltInLanguage, Language } from "./languages";
import { Logger } from "./logging";
@@ -126,7 +126,7 @@ export async function setupCppAutobuild(codeql: CodeQL, logger: Logger) {
if (await features.getValue(Feature.CppDependencyInstallation, codeql)) {
// disable autoinstall on self-hosted runners unless explicitly requested
if (
process.env["RUNNER_ENVIRONMENT"] === "self-hosted" &&
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] === "self-hosted" &&
process.env[envVar] !== "true"
) {
logger.info(

91
src/codeql.ts
View File

@@ -254,17 +254,17 @@ const CODEQL_MINIMUM_VERSION = "2.19.4";
/**
* This version will shortly become the oldest version of CodeQL that the Action will run with.
*/
const CODEQL_NEXT_MINIMUM_VERSION = "2.19.4";
const CODEQL_NEXT_MINIMUM_VERSION = "2.20.7";
/**
* This is the version of GHES that was most recently deprecated.
*/
const GHES_VERSION_MOST_RECENTLY_DEPRECATED = "3.15";
const GHES_VERSION_MOST_RECENTLY_DEPRECATED = "3.16";
/**
* This is the deprecation date for the version of GHES that was most recently deprecated.
*/
const GHES_MOST_RECENT_DEPRECATION_DATE = "2026-04-09";
const GHES_MOST_RECENT_DEPRECATION_DATE = "2026-07-01";
/** The CLI verbosity level to use for extraction in debug mode. */
const EXTRACTION_DEBUG_MODE_VERBOSITY = "progress++";
@@ -517,17 +517,10 @@ async function getCodeQLForCmd(
return cmd;
},
async getVersion() {
return getCachedOrRun(CommandCacheKey.Version, cmd, async () => {
const output = await runCli(cmd, ["version", "--format=json"], {
return getCachedOrRun(CommandCacheKey.Version, cmd, () => {
return runCliJson<VersionInfo>(cmd, ["version", "--format=json"], {
noStreamStdout: true,
});
try {
return JSON.parse(output) as VersionInfo;
} catch {
throw Error(
`Invalid JSON output from \`version --format=json\`: ${output}`,
);
}
});
},
async printVersion() {
@@ -733,26 +726,20 @@ async function getCodeQLForCmd(
filterToLanguagesWithQueries: boolean;
} = { filterToLanguagesWithQueries: false },
) {
const codeqlArgs = [
return runCliJson<ResolveLanguagesOutput>(cmd, [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
// TODO: Unconditionally include `--filter-to-languages-with-queries`
// once CODEQL_MINIMUM_VERSION is at least v2.23.0
// — the first version to support this flag.
...(filterToLanguagesWithQueries
? ["--filter-to-languages-with-queries"]
: []),
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveLanguagesOutput;
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`,
);
}
]);
},
async resolveBuildEnvironment(
workingDir: string | undefined,
@@ -768,15 +755,7 @@ async function getCodeQLForCmd(
if (workingDir !== undefined) {
codeqlArgs.push("--working-dir", workingDir);
}
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveBuildEnvironmentOutput;
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve build-environment: ${e} in\n${output}`,
);
}
return await runCliJson<ResolveBuildEnvironmentOutput>(cmd, codeqlArgs);
},
async databaseRunQueries(
databasePath: string,
@@ -978,15 +957,9 @@ async function getCodeQLForCmd(
...getExtraOptionsFromEnv(["resolve", "queries"]),
...queries,
];
const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
try {
return JSON.parse(output) as string[];
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve queries --format=startingpacks: ${e}`,
);
}
return await runCliJson<string[]>(cmd, codeqlArgs, {
noStreamStdout: true,
});
},
async resolveDatabase(
databasePath: string,
@@ -998,15 +971,9 @@ async function getCodeQLForCmd(
"--format=json",
...getExtraOptionsFromEnv(["resolve", "database"]),
];
const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
try {
return JSON.parse(output) as ResolveDatabaseOutput;
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve database --format=json: ${e}`,
);
}
return await runCliJson<ResolveDatabaseOutput>(cmd, codeqlArgs, {
noStreamStdout: true,
});
},
async mergeResults(
sarifFiles: string[],
@@ -1162,6 +1129,30 @@ async function runCli(
}
}
/**
* Wraps the command executor {@link runCli} and tries to parse the output as JSON.
* @param cmd The command to run.
* @param args The arguments to pass to the command.
* @param opts The options for running the command.
* @param opts.stdin Optional string to pass to the command's standard input.
* @param opts.noStreamStdout Optional boolean to indicate whether to stream the command's standard output.
* @returns The parsed JSON output from the command.
*/
async function runCliJson<T>(
cmd: string,
args: string[] = [],
opts: { stdin?: string; noStreamStdout?: boolean } = {},
): Promise<T> {
const output = await runCli(cmd, args, opts);
try {
return JSON.parse(output) as T;
} catch (e) {
throw Error(
`Unexpected output from codeql ${args.join(" ")}: ${getErrorMessage(e)}`,
);
}
}
/**
* Writes the code scanning configuration that is to be used by the CLI.
*

71
src/config-utils.test.ts
View File

@@ -36,6 +36,7 @@ import {
mockCodeQLVersion,
createTestConfig,
makeMacro,
initAllState,
} from "./testing-utils";
import {
GitHubVariant,
@@ -160,8 +161,9 @@ test.serial("load empty config", async (t) => {
},
});
const state = initAllState({ logger });
const config = await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
languagesInput: languages,
repository: { owner: "github", repo: "example" },
@@ -202,8 +204,9 @@ test.serial("load code quality config", async (t) => {
},
});
const state = initAllState({ logger });
const config = await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
analysisKinds: [AnalysisKind.CodeQuality],
languagesInput: languages,
@@ -280,9 +283,10 @@ test.serial(
repositoryProperties,
});
const state = initAllState({ logger });
await t.notThrowsAsync(async () => {
const config = await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
analysisKinds: [AnalysisKind.CodeQuality],
languagesInput: languages,
@@ -321,8 +325,9 @@ test.serial("loading a saved config produces the same config", async (t) => {
// Sanity check that getConfig returns undefined before we have called initConfig
t.deepEqual(await configUtils.getConfig(tempDir, logger), undefined);
const state = initAllState({ logger });
const config1 = await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
languagesInput: "javascript,python",
tempDir,
@@ -373,8 +378,9 @@ test.serial("loading config with version mismatch throws", async (t) => {
.stub(actionsUtil, "getActionVersion")
.returns("does-not-exist");
const state = initAllState({ logger });
const config = await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
languagesInput: "javascript,python",
tempDir,
@@ -402,8 +408,9 @@ test.serial("loading config with version mismatch throws", async (t) => {
test.serial("load input outside of workspace", async (t) => {
return await withTmpDir(async (tempDir) => {
try {
const state = initAllState();
await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
configFile: "../input",
tempDir,
@@ -424,34 +431,6 @@ test.serial("load input outside of workspace", async (t) => {
});
});
test.serial("load non-local input with invalid repo syntax", async (t) => {
return await withTmpDir(async (tempDir) => {
// no filename given, just a repo
const configFile = "octo-org/codeql-config@main";
try {
await configUtils.initConfig(
createFeatures([]),
createTestInitConfigInputs({
configFile,
tempDir,
workspacePath: tempDir,
}),
);
throw new Error("initConfig did not throw error");
} catch (err) {
t.deepEqual(
err,
new ConfigurationError(
errorMessages.getConfigFileRepoFormatInvalidMessage(
"octo-org/codeql-config@main",
),
),
);
}
});
});
test.serial("load non-existent input", async (t) => {
return await withTmpDir(async (tempDir) => {
const languagesInput = "javascript";
@@ -459,8 +438,9 @@ test.serial("load non-existent input", async (t) => {
t.false(fs.existsSync(path.join(tempDir, configFile)));
try {
const state = initAllState();
await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
languagesInput,
configFile,
@@ -536,8 +516,9 @@ test.serial("load non-empty input", async (t) => {
const languagesInput = "javascript";
const configFilePath = createConfigFile(inputFileContents, tempDir);
const state = initAllState();
const actualConfig = await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
languagesInput,
buildModeInput: "none",
@@ -595,8 +576,9 @@ test.serial(
// Only JS, python packs will be ignored
const languagesInput = "javascript";
const state = initAllState();
const config = await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
languagesInput,
configFile: configFilePath,
@@ -647,8 +629,9 @@ test.serial("API client used when reading remote config", async (t) => {
const configFile = "octo-org/codeql-config/config.yaml@main";
const languagesInput = "javascript";
const state = initAllState();
await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
languagesInput,
configFile,
@@ -669,9 +652,10 @@ test.serial(
mockGetContents(dummyResponse);
const repoReference = "octo-org/codeql-config/config.yaml@main";
const state = initAllState();
try {
await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
configFile: repoReference,
tempDir,
@@ -699,9 +683,10 @@ test.serial("Invalid format of remote config handled correctly", async (t) => {
mockGetContents(dummyResponse);
const repoReference = "octo-org/codeql-config/config.yaml@main";
const state = initAllState();
try {
await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
configFile: repoReference,
tempDir,
@@ -729,9 +714,10 @@ test.serial("No detected languages", async (t) => {
},
});
const state = initAllState();
try {
await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
tempDir,
codeql,
@@ -752,9 +738,10 @@ test.serial("Unknown languages", async (t) => {
return await withTmpDir(async (tempDir) => {
const languagesInput = "rubbish,english";
const state = initAllState();
try {
await configUtils.initConfig(
createFeatures([]),
state,
createTestInitConfigInputs({
languagesInput,
tempDir,

249
src/config-utils.ts
View File

@@ -5,6 +5,7 @@ import { performance } from "perf_hooks";
import * as core from "@actions/core";
import * as yaml from "js-yaml";
import { ActionState } from "./action-common";
import {
getActionVersion,
getOptionalInput,
@@ -18,8 +19,9 @@ import {
getAnalysisConfig,
} from "./analyses";
import * as api from "./api-client";
import { CachingKind, getCachingKind } from "./caching-utils";
import { getCachingKind } from "./caching-utils";
import { type CodeQL } from "./codeql";
import { type Config } from "./config/action-config";
import {
calculateAugmentation,
ExcludeQueryFilter,
@@ -27,6 +29,12 @@ import {
parseUserConfig,
UserConfig,
} from "./config/db-config";
import { getRemoteConfig } from "./config/file";
import {
parseRegistries,
type RegistryConfigNoCredentials,
type RegistryConfigWithCredentials,
} from "./config/pack-registries";
import {
addNoLanguageDiagnostic,
makeTelemetryDiagnostic,
@@ -79,6 +87,8 @@ import {
isHostedRunner,
} from "./util";
export { type Config } from "./config/action-config";
/**
* The minimum available disk space (in MB) required to perform overlay analysis.
* If the available disk space on the runner is below the threshold when deciding
@@ -117,148 +127,6 @@ const OVERLAY_MINIMUM_MEMORY_MB = 5 * 1024;
*/
const CODEQL_VERSION_REDUCED_OVERLAY_MEMORY_USAGE = "2.24.3";
export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & {
// Token to use when downloading packs from this registry.
token: string;
};
/**
* The list of registries and the associated pack globs that determine where each
* pack can be downloaded from.
*/
export interface RegistryConfigNoCredentials {
// URL of a package registry, eg- https://ghcr.io/v2/
url: string;
// List of globs that determine which packs are associated with this registry.
packages: string[] | string;
// Kind of registry, either "github" or "docker". Default is "docker".
// "docker" refers specifically to the GitHub Container Registry, which is the usual way of sharing CodeQL packs.
// "github" refers to packs published as content in a GitHub repository. This kind of registry is used in scenarios
// where GHCR is not available, such as certain GHES environments.
kind?: "github" | "docker";
}
/**
* Format of the parsed config file.
*/
export interface Config {
/**
* The version of the CodeQL Action that the configuration is for.
*/
version: string;
/**
* Set of analysis kinds that are enabled.
*/
analysisKinds: AnalysisKind[];
/**
* Set of languages to run analysis for.
*/
languages: Language[];
/**
* Build mode, if set. Currently only a single build mode is supported per job.
*/
buildMode: BuildMode | undefined;
/**
* A unaltered copy of the original user input.
* Mainly intended to be used for status reporting.
* If any field is useful for the actual processing
* of the action then consider pulling it out to a
* top-level field above.
*/
originalUserInput: UserConfig;
/**
* Directory to use for temporary files that should be
* deleted at the end of the job.
*/
tempDir: string;
/**
* Path of the CodeQL executable.
*/
codeQLCmd: string;
/**
* Version of GitHub we are talking to.
*/
gitHubVersion: GitHubVersion;
/**
* The location where CodeQL databases should be stored.
*/
dbLocation: string;
/**
* Specifies whether we are debugging mode and should try to produce extra
* output for debugging purposes when possible.
*/
debugMode: boolean;
/**
* Specifies the name of the debugging artifact if we are in debug mode.
*/
debugArtifactName: string;
/**
* Specifies the name of the database in the debugging artifact.
*/
debugDatabaseName: string;
/**
* The configuration we computed by combining `originalUserInput` with `augmentationProperties`,
* as well as adjustments made to it based on unsupported or required options.
*/
computedConfig: UserConfig;
/**
* Partial map from languages to locations of TRAP caches for that language.
* If a key is omitted, then TRAP caching should not be used for that language.
*/
trapCaches: { [language: Language]: string };
/**
* Time taken to download TRAP caches. Used for status reporting.
*/
trapCacheDownloadTime: number;
/** A value indicating how dependency caching should be used. */
dependencyCachingEnabled: CachingKind;
/** The keys of caches that we restored, if any. */
dependencyCachingRestoredKeys: string[];
/**
* Extra query exclusions to append to the config.
*/
extraQueryExclusions: ExcludeQueryFilter[];
/**
* The overlay database mode to use.
*/
overlayDatabaseMode: OverlayDatabaseMode;
/**
* Whether to use caching for overlay databases. If it is true, the action
* will upload the created overlay-base database to the actions cache, and
* download an overlay-base database from the actions cache before it creates
* a new overlay database. If it is false, the action assumes that the
* workflow will be responsible for managing database storage and retrieval.
*
* This property has no effect unless `overlayDatabaseMode` is `Overlay` or
* `OverlayBase`.
*/
useOverlayDatabaseCaching: boolean;
/**
* Whether the overlay database mode was set explicitly.
*/
overlayModeSetExplicitly: boolean;
/**
* A partial mapping from repository properties that affect us to their values.
*/
repositoryProperties: RepositoryProperties;
/**
* Whether to enable file coverage information.
*/
enableFileCoverageInformation: boolean;
}
async function getSupportedLanguageMap(
codeql: CodeQL,
logger: Logger,
@@ -597,12 +465,11 @@ async function downloadCacheWithTime(
}
async function loadUserConfig(
logger: Logger,
actionState: ActionState<["Logger", "Env", "FeatureFlags"]>,
configFile: string,
workspacePath: string,
apiDetails: api.GitHubApiCombinedDetails,
tempDir: string,
validateConfig: boolean,
): Promise<UserConfig> {
if (isLocal(configFile)) {
if (configFile !== userConfigFromActionPath(tempDir)) {
@@ -615,14 +482,12 @@ async function loadUserConfig(
);
}
}
return getLocalConfig(logger, configFile, validateConfig);
} else {
return await getRemoteConfig(
logger,
configFile,
apiDetails,
validateConfig,
const validateConfig = await actionState.features.getValue(
Feature.ValidateDbConfig,
);
return getLocalConfig(actionState.logger, configFile, validateConfig);
} else {
return await getRemoteConfig(actionState, configFile, apiDetails);
}
}
@@ -1136,10 +1001,11 @@ export async function applyIncrementalAnalysisSettings(
* a default config. The parsed config is then stored to a known location.
*/
export async function initConfig(
features: FeatureEnablement,
actionState: ActionState<["Logger", "Env", "FeatureFlags"]>,
inputs: InitConfigInputs,
): Promise<Config> {
const { logger, tempDir } = inputs;
const { logger, features } = actionState;
const { tempDir } = inputs;
// if configInput is set, it takes precedence over configFile
if (inputs.configInput) {
@@ -1158,14 +1024,12 @@ export async function initConfig(
logger.debug("No configuration file was provided");
} else {
logger.debug(`Using configuration file: ${inputs.configFile}`);
const validateConfig = await features.getValue(Feature.ValidateDbConfig);
userConfig = await loadUserConfig(
logger,
actionState,
inputs.configFile,
inputs.workspacePath,
inputs.apiDetails,
tempDir,
validateConfig,
);
}
@@ -1315,29 +1179,6 @@ export async function initConfig(
return config;
}
function parseRegistries(
registriesInput: string | undefined,
): RegistryConfigWithCredentials[] | undefined {
try {
return registriesInput
? (yaml.load(registriesInput) as RegistryConfigWithCredentials[])
: undefined;
} catch {
throw new ConfigurationError(
"Invalid registries input. Must be a YAML string.",
);
}
}
export function parseRegistriesWithoutCredentials(
registriesInput?: string,
): RegistryConfigNoCredentials[] | undefined {
return parseRegistries(registriesInput)?.map((r) => {
const { url, packages, kind } = r;
return { url, packages, kind };
});
}
function isLocal(configPath: string): boolean {
// If the path starts with ./, look locally
if (configPath.indexOf("./") === 0) {
@@ -1367,54 +1208,6 @@ function getLocalConfig(
);
}
async function getRemoteConfig(
logger: Logger,
configFile: string,
apiDetails: api.GitHubApiCombinedDetails,
validateConfig: boolean,
): Promise<UserConfig> {
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp(
"(?<owner>[^/]+)/(?<repo>[^/]+)/(?<path>[^@]+)@(?<ref>.*)",
);
const pieces = format.exec(configFile);
// 5 = 4 groups + the whole expression
if (pieces?.groups === undefined || pieces.length < 5) {
throw new ConfigurationError(
errorMessages.getConfigFileRepoFormatInvalidMessage(configFile),
);
}
const response = await api
.getApiClientWithExternalAuth(apiDetails)
.rest.repos.getContent({
owner: pieces.groups.owner,
repo: pieces.groups.repo,
path: pieces.groups.path,
ref: pieces.groups.ref,
});
let fileContents: string;
if ("content" in response.data && response.data.content !== undefined) {
fileContents = response.data.content;
} else if (Array.isArray(response.data)) {
throw new ConfigurationError(
errorMessages.getConfigFileDirectoryGivenMessage(configFile),
);
} else {
throw new ConfigurationError(
errorMessages.getConfigFileFormatInvalidMessage(configFile),
);
}
return parseUserConfig(
logger,
configFile,
Buffer.from(fileContents, "base64").toString("binary"),
validateConfig,
);
}
/**
* Get the file path where the parsed config will be stored.
*/

128
src/config/action-config.ts Normal file
View File

@@ -0,0 +1,128 @@
import type { AnalysisKind } from "../analyses";
import type { CachingKind } from "../caching-utils";
import type { RepositoryProperties } from "../feature-flags/properties";
import type { Language } from "../languages";
import type { OverlayDatabaseMode } from "../overlay/overlay-database-mode";
import type { BuildMode, GitHubVersion } from "../util";
import type { ExcludeQueryFilter, UserConfig } from "./db-config";
/**
* Format of the CodeQL Action configuration state that is persisted
* between steps of the CodeQL Action in a CodeQL workflow.
*/
export interface Config {
/**
* The version of the CodeQL Action that the configuration is for.
*/
version: string;
/**
* Set of analysis kinds that are enabled.
*/
analysisKinds: AnalysisKind[];
/**
* Set of languages to run analysis for.
*/
languages: Language[];
/**
* Build mode, if set. Currently only a single build mode is supported per job.
*/
buildMode: BuildMode | undefined;
/**
* A unaltered copy of the original user input.
* Mainly intended to be used for status reporting.
* If any field is useful for the actual processing
* of the action then consider pulling it out to a
* top-level field above.
*/
originalUserInput: UserConfig;
/**
* Directory to use for temporary files that should be
* deleted at the end of the job.
*/
tempDir: string;
/**
* Path of the CodeQL executable.
*/
codeQLCmd: string;
/**
* Version of GitHub we are talking to.
*/
gitHubVersion: GitHubVersion;
/**
* The location where CodeQL databases should be stored.
*/
dbLocation: string;
/**
* Specifies whether we are debugging mode and should try to produce extra
* output for debugging purposes when possible.
*/
debugMode: boolean;
/**
* Specifies the name of the debugging artifact if we are in debug mode.
*/
debugArtifactName: string;
/**
* Specifies the name of the database in the debugging artifact.
*/
debugDatabaseName: string;
/**
* The configuration we computed by combining `originalUserInput` with `augmentationProperties`,
* as well as adjustments made to it based on unsupported or required options.
*/
computedConfig: UserConfig;
/**
* Partial map from languages to locations of TRAP caches for that language.
* If a key is omitted, then TRAP caching should not be used for that language.
*/
trapCaches: { [language: Language]: string };
/**
* Time taken to download TRAP caches. Used for status reporting.
*/
trapCacheDownloadTime: number;
/** A value indicating how dependency caching should be used. */
dependencyCachingEnabled: CachingKind;
/** The keys of caches that we restored, if any. */
dependencyCachingRestoredKeys: string[];
/**
* Extra query exclusions to append to the config.
*/
extraQueryExclusions: ExcludeQueryFilter[];
/**
* The overlay database mode to use.
*/
overlayDatabaseMode: OverlayDatabaseMode;
/**
* Whether to use caching for overlay databases. If it is true, the action
* will upload the created overlay-base database to the actions cache, and
* download an overlay-base database from the actions cache before it creates
* a new overlay database. If it is false, the action assumes that the
* workflow will be responsible for managing database storage and retrieval.
*
* This property has no effect unless `overlayDatabaseMode` is `Overlay` or
* `OverlayBase`.
*/
useOverlayDatabaseCaching: boolean;
/**
* Whether the overlay database mode was set explicitly.
*/
overlayModeSetExplicitly: boolean;
/**
* A partial mapping from repository properties that affect us to their values.
*/
repositoryProperties: RepositoryProperties;
/**
* Whether to enable file coverage information.
*/
enableFileCoverageInformation: boolean;
}

97
src/config/file.test.ts
View File

@@ -1,22 +1,19 @@
import test from "ava";
import sinon from "sinon";
import { Feature } from "../feature-flags";
import { RepositoryPropertyName } from "../feature-flags/properties";
import {
getTestActionsEnv,
RecordingLogger,
setupTests,
} from "../testing-utils";
import { callee, setupTests } from "../testing-utils";
import { getConfigFileInput } from "./file";
setupTests(test);
test("getConfigFileInput returns undefined by default", async (t) => {
const logger = new RecordingLogger();
const actionsEnv = getTestActionsEnv();
const result = getConfigFileInput(logger, actionsEnv, {});
t.is(result, undefined);
await callee(getConfigFileInput)
.withArgs({})
.withFeatures([Feature.ConfigFileRepositoryProperty])
.passes(async (fn) => t.is(await fn(), undefined));
});
const repositoryProperties = {
@@ -24,9 +21,12 @@ const repositoryProperties = {
};
test("getConfigFileInput returns input value", async (t) => {
const logger = new RecordingLogger();
const actionsEnv = getTestActionsEnv();
const testInput = "/some/path";
const target = callee(getConfigFileInput).withFeatures([
Feature.ConfigFileRepositoryProperty,
]);
const actionsEnv = target.getState().actions;
sinon
.stub(actionsEnv, "getOptionalInput")
.withArgs("config-file")
@@ -34,36 +34,63 @@ test("getConfigFileInput returns input value", async (t) => {
// Even though both an input and repository property are configured,
// we prefer the direct input to the Action.
const result = getConfigFileInput(logger, actionsEnv, repositoryProperties);
t.is(result, testInput);
// Check for the expected log message.
t.true(logger.hasMessage("Using configuration file input from workflow"));
});
test("getConfigFileInput returns repository property value", async (t) => {
const logger = new RecordingLogger();
const actionsEnv = getTestActionsEnv();
// Since there is no direct input, we should use the repository property.
const result = getConfigFileInput(logger, actionsEnv, repositoryProperties);
t.is(result, repositoryProperties[RepositoryPropertyName.CONFIG_FILE]);
const targetWithArgs = target
.withActions(actionsEnv)
.withArgs(repositoryProperties);
await targetWithArgs.passes(async (fn) => t.is(await fn(), testInput));
// Check for the expected log message.
t.true(
logger.hasMessage(
"Using configuration file input from repository property",
),
targetWithArgs
.getLogger()
.hasMessage("Using configuration file input from workflow"),
);
});
test("getConfigFileInput returns repository property value", async (t) => {
// Since there is no direct input, we should use the repository property.
const target = callee(getConfigFileInput)
.withFeatures([Feature.ConfigFileRepositoryProperty])
.withArgs(repositoryProperties);
await target.passes(async (fn) =>
t.is(await fn(), repositoryProperties[RepositoryPropertyName.CONFIG_FILE]),
);
// Check for the expected log message.
t.true(
target
.getLogger()
.hasMessage("Using configuration file input from repository property"),
);
});
test("getConfigFileInput ignores empty repository property value", async (t) => {
const logger = new RecordingLogger();
const actionsEnv = getTestActionsEnv();
// Since the repository property value is an empty/whitespace string, we should ignore it.
const result = getConfigFileInput(logger, actionsEnv, {
[RepositoryPropertyName.CONFIG_FILE]: " ",
});
t.is(result, undefined);
await callee(getConfigFileInput)
.withFeatures([Feature.ConfigFileRepositoryProperty])
.withArgs({ [RepositoryPropertyName.CONFIG_FILE]: " " })
.passes(async (fn) => t.is(await fn(), undefined));
});
test("getConfigFileInput ignores repository property value when FF is off", async (t) => {
// Since the FF is off, we should ignore the repository property value.
const target = callee(getConfigFileInput)
.withFeatures([])
.withArgs(repositoryProperties);
await target.passes(async (fn) => t.is(await fn(), undefined));
t.false(
target
.getLogger()
.hasMessage("Using configuration file input from repository property"),
);
t.true(
target
.getLogger()
.hasMessage(
"Ignoring configuration file input from repository property, because the corresponding feature flag is disabled.",
),
);
});

87
src/config/file.ts
View File

@@ -1,18 +1,27 @@
import { ActionsEnv } from "../actions-util";
import { ActionState } from "../action-common";
import * as api from "../api-client";
import * as errorMessages from "../error-messages";
import { Feature } from "../feature-flags";
import {
RepositoryProperties,
RepositoryPropertyName,
} from "../feature-flags/properties";
import { Logger } from "../logging";
import { ConfigurationError } from "../util";
import { parseUserConfig, UserConfig } from "./db-config";
import { parseRemoteFileAddress } from "./remote-file";
/**
* Gets the value that is configured for the configuration file, if any.
*/
export function getConfigFileInput(
logger: Logger,
actions: ActionsEnv,
export async function getConfigFileInput(
{
logger,
actions,
features,
}: ActionState<["Logger", "Actions", "FeatureFlags"]>,
repositoryProperties: Partial<RepositoryProperties>,
): string | undefined {
): Promise<string | undefined> {
const input = actions.getOptionalInput("config-file");
if (input !== undefined) {
@@ -24,11 +33,71 @@ export function getConfigFileInput(
repositoryProperties[RepositoryPropertyName.CONFIG_FILE];
if (propertyValue !== undefined && propertyValue.trim().length > 0) {
logger.info(
`Using configuration file input from repository property: ${propertyValue}`,
// Only use the repository property value if the FF is enabled.
const useRepositoryProperty = await features.getValue(
Feature.ConfigFileRepositoryProperty,
);
return propertyValue;
if (useRepositoryProperty) {
logger.info(
`Using configuration file input from repository property: ${propertyValue}`,
);
return propertyValue;
} else {
logger.info(
"Ignoring configuration file input from repository property, because the corresponding feature flag is disabled.",
);
}
}
return undefined;
}
/**
* Attempts to fetch a `UserConfig` from a remote `address`.
*
* @param actionState The current Action state.
* @param configFile The remote address of the configuration file.
* @param apiDetails Information about how to connect to the API.
*
* @returns The `UserConfig`, if it could be fetched and parsed successfully.
*/
export async function getRemoteConfig(
actionState: ActionState<["Logger", "Env", "FeatureFlags"]>,
configFile: string,
apiDetails: api.GitHubApiCombinedDetails,
): Promise<UserConfig> {
const address = await parseRemoteFileAddress(actionState, configFile);
const response = await api
.getApiClientWithExternalAuth(apiDetails)
.rest.repos.getContent({
owner: address.owner,
repo: address.repo,
path: address.path,
ref: address.ref,
});
let fileContents: string;
if ("content" in response.data && response.data.content !== undefined) {
fileContents = response.data.content;
} else if (Array.isArray(response.data)) {
throw new ConfigurationError(
errorMessages.getConfigFileDirectoryGivenMessage(configFile),
);
} else {
throw new ConfigurationError(
errorMessages.getConfigFileFormatInvalidMessage(configFile),
);
}
const validateConfig = await actionState.features.getValue(
Feature.ValidateDbConfig,
);
return parseUserConfig(
actionState.logger,
configFile,
Buffer.from(fileContents, "base64").toString("binary"),
validateConfig,
);
}

49
src/config/pack-registries.ts Normal file
View File

@@ -0,0 +1,49 @@
import * as yaml from "js-yaml";
import { ConfigurationError } from "../util";
export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & {
// Token to use when downloading packs from this registry.
token: string;
};
/**
* The list of registries and the associated pack globs that determine where each
* pack can be downloaded from.
*/
export interface RegistryConfigNoCredentials {
// URL of a package registry, eg- https://ghcr.io/v2/
url: string;
// List of globs that determine which packs are associated with this registry.
packages: string[] | string;
// Kind of registry, either "github" or "docker". Default is "docker".
// "docker" refers specifically to the GitHub Container Registry, which is the usual way of sharing CodeQL packs.
// "github" refers to packs published as content in a GitHub repository. This kind of registry is used in scenarios
// where GHCR is not available, such as certain GHES environments.
kind?: "github" | "docker";
}
export function parseRegistries(
registriesInput: string | undefined,
): RegistryConfigWithCredentials[] | undefined {
try {
return registriesInput
? (yaml.load(registriesInput) as RegistryConfigWithCredentials[])
: undefined;
} catch {
throw new ConfigurationError(
"Invalid registries input. Must be a YAML string.",
);
}
}
export function parseRegistriesWithoutCredentials(
registriesInput?: string,
): RegistryConfigNoCredentials[] | undefined {
return parseRegistries(registriesInput)?.map((r) => {
const { url, packages, kind } = r;
return { url, packages, kind };
});
}

282
src/config/remote-file.test.ts Normal file
View File

@@ -0,0 +1,282 @@
import test from "ava";
import sinon from "sinon";
import { ActionsEnvVars } from "../environment";
import * as errors from "../error-messages";
import { Feature } from "../feature-flags";
import { callee, getTestEnv } from "../testing-utils";
import { ConfigurationError } from "../util";
import {
DEFAULT_CONFIG_FILE_NAME,
DEFAULT_CONFIG_FILE_REF,
parseRemoteFileAddress,
RemoteFileAddress,
} from "./remote-file";
type ParseRemoteFileAddressTest = {
input: string;
expected: RemoteFileAddress;
};
test("parseRemoteFileAddress accepts full remote addresses", async (t) => {
const target = callee(parseRemoteFileAddress);
const expected: RemoteFileAddress = {
owner: "owner",
repo: "repo",
path: "path",
ref: "ref",
};
const oldFormatInputs: ParseRemoteFileAddressTest[] = [
{ input: "owner/repo/path@ref", expected },
{ input: "owner /repo/path@ref", expected },
{ input: "owner/ repo/path@ref", expected },
{ input: "owner/repo /path@ref", expected },
{ input: "owner/repo/ path@ref", expected },
{ input: "owner/repo/path @ref", expected },
{ input: "owner/repo/path@ ref", expected },
{
input: "owner/repo/path/to/codeql.yml@ref/feature",
expected: { ...expected, path: "path/to/codeql.yml", ref: "ref/feature" },
},
{
input: " owner/repo/path/to/codeql.yml@ref/feature ",
expected: { ...expected, path: "path/to/codeql.yml", ref: "ref/feature" },
},
];
for (const oldFormatInput of oldFormatInputs) {
await target
.withArgs(oldFormatInput.input)
.passes(async (fn) => t.deepEqual(await fn(), oldFormatInput.expected));
}
// New format.
const newFormatInputs: ParseRemoteFileAddressTest[] = [
{ input: "owner/repo@ref:path", expected },
{ input: "owner /repo@ref:path", expected },
{ input: "owner/ repo@ref:path", expected },
{ input: "owner/repo @ref:path", expected },
{ input: "owner/repo@ ref:path", expected },
{ input: "owner/repo@ref :path", expected },
{ input: "owner/repo@ref: path", expected },
{
input: "owner/repo@ref/feature:path/to/codeql.yml",
expected: { ...expected, path: "path/to/codeql.yml", ref: "ref/feature" },
},
{
input: " owner/repo@ref/feature:path/to/codeql.yml ",
expected: { ...expected, path: "path/to/codeql.yml", ref: "ref/feature" },
},
];
for (const newFormatInput of newFormatInputs) {
const targetWithArgs = target.withArgs(newFormatInput.input);
// Should fail when the FF is not enabled.
await targetWithArgs
.withFeatures([])
.passes(async (fn) =>
t.throwsAsync(fn, { instanceOf: ConfigurationError }),
);
// And pass when the FF is enabled.
await targetWithArgs
.withFeatures([Feature.NewRemoteFileAddresses])
.passes(async (fn) => t.deepEqual(await fn(), newFormatInput.expected));
}
});
test("parseRemoteFileAddress accepts remote address without an owner", async (t) => {
const target = callee(parseRemoteFileAddress);
const env = target.getState().env;
const owner = "test-owner";
const getRequired = sinon.stub(env, "getRequired");
getRequired
.withArgs(ActionsEnvVars.GITHUB_REPOSITORY)
.returns(`${owner}/current-repo`);
const targetWithEnv = target.withEnv(env);
const testCases: ParseRemoteFileAddressTest[] = [
{
input: "repo@ref:path.yml",
expected: {
owner,
repo: "repo",
path: "path.yml",
ref: "ref",
},
},
{
input: "repo@ref",
expected: {
owner,
repo: "repo",
path: DEFAULT_CONFIG_FILE_NAME,
ref: "ref",
},
},
{
input: "repo:path.yml",
expected: {
owner,
repo: "repo",
path: "path.yml",
ref: DEFAULT_CONFIG_FILE_REF,
},
},
{
input: "repo",
expected: {
owner,
repo: "repo",
path: DEFAULT_CONFIG_FILE_NAME,
ref: DEFAULT_CONFIG_FILE_REF,
},
},
];
for (const testCase of testCases) {
const targetWithArgs = targetWithEnv.withArgs(testCase.input);
// Should fail when the FF is not enabled.
await targetWithArgs
.withFeatures([])
.passes(async (fn) =>
t.throwsAsync(fn, { instanceOf: ConfigurationError }),
);
// And pass when the FF is enabled.
await targetWithArgs
.withFeatures([Feature.NewRemoteFileAddresses])
.passes(async (fn) => t.deepEqual(await fn(), testCase.expected));
}
});
test("parseRemoteFileAddress throws for invalid `GITHUB_REPOSITORY`", async (t) => {
const target = callee(parseRemoteFileAddress).withArgs("repo@ref");
const env = target.getState().env;
const getRequired = sinon.stub(env, "getRequired");
getRequired.withArgs(ActionsEnvVars.GITHUB_REPOSITORY).returns(`not-valid`);
await target
.withEnv(env)
.withFeatures([Feature.NewRemoteFileAddresses])
.passes(async (fn) => t.throwsAsync(fn, { instanceOf: Error }));
t.assert(getRequired.calledOnceWith(ActionsEnvVars.GITHUB_REPOSITORY));
});
test("parseRemoteFileAddress accepts remote address without a path", async (t) => {
const target = callee(parseRemoteFileAddress);
const testCases: ParseRemoteFileAddressTest[] = [
{
input: "owner/repo@ref",
expected: {
owner: "owner",
repo: "repo",
path: DEFAULT_CONFIG_FILE_NAME,
ref: "ref",
},
},
{
input: "owner/repo",
expected: {
owner: "owner",
repo: "repo",
path: DEFAULT_CONFIG_FILE_NAME,
ref: DEFAULT_CONFIG_FILE_REF,
},
},
];
for (const testCase of testCases) {
const targetWithArgs = target.withArgs(testCase.input);
// Should fail when the FF is not enabled.
await targetWithArgs
.withFeatures([])
.passes(async (fn) =>
t.throwsAsync(fn, { instanceOf: ConfigurationError }),
);
// And pass when the FF is enabled.
await targetWithArgs
.withFeatures([Feature.NewRemoteFileAddresses])
.passes(async (fn) => t.deepEqual(await fn(), testCase.expected));
}
});
test("parseRemoteFileAddress accepts remote address without a ref", async (t) => {
const target = callee(parseRemoteFileAddress).withArgs("owner/repo:path");
// Should only accept the input if the FF is enabled.
await target.withFeatures([]).passes(t.throwsAsync);
await target
.withFeatures([Feature.NewRemoteFileAddresses])
.passes(async (fn) =>
t.deepEqual(await fn(), {
owner: "owner",
repo: "repo",
path: "path",
ref: DEFAULT_CONFIG_FILE_REF,
} satisfies RemoteFileAddress),
);
});
test("parseRemoteFileAddress rejects invalid values", async (t) => {
const env = getTestEnv();
const owner = "owner";
const getRequired = sinon.stub(env, "getRequired");
getRequired
.withArgs(ActionsEnvVars.GITHUB_REPOSITORY)
.returns(`${owner}/current-repo`);
const target = callee(parseRemoteFileAddress).withEnv(env);
const testInputs = [
" ",
"repo//absolute",
"repo:/absolute",
"/repo@ref",
" /repo@ref",
"repo@",
"repo:",
"repo/",
"/repo",
":path",
"@ref",
"@ref:path",
"owner/@ref:path",
"owner/@ref",
"owner/:path",
];
for (const testInput of testInputs) {
const targetWithArgs = target.withArgs(testInput);
// Should throw both when the new format is and isn't accepted.
await targetWithArgs.withFeatures([]).passes(async (fn) =>
t.throwsAsync(fn, {
instanceOf: ConfigurationError,
message: errors.getConfigFileRepoOldFormatInvalidMessage(testInput),
}),
);
await targetWithArgs
.withFeatures([Feature.NewRemoteFileAddresses])
.passes(async (fn) =>
t.throwsAsync(fn, {
// When the new format is accepted, there are some more specific
// errors in some cases. It is sufficient for us to check that
// an exception is thrown.
instanceOf: ConfigurationError,
}),
);
}
});

138
src/config/remote-file.ts Normal file
View File

@@ -0,0 +1,138 @@
import { ActionState } from "../action-common";
import { Env, ActionsEnvVars } from "../environment";
import * as errorMessages from "../error-messages";
import { Feature } from "../feature-flags";
import { ConfigurationError, Failure, Result, Success } from "../util";
/** Represents remote file addresses. */
export interface RemoteFileAddress {
/** The owner of the repository. */
owner: string;
/** The repository name. */
repo: string;
/** The path of the file. */
path: string;
/** The ref of the repository. */
ref: string;
}
/** The default file path to use in configuration file shorthands. */
export const DEFAULT_CONFIG_FILE_NAME = ".github/codeql-action.yaml";
/** The default ref to use in configuration file shorthands. */
export const DEFAULT_CONFIG_FILE_REF = "main";
/** Extracts the owner from the `GITHUB_REPOSITORY` environment variable. */
function getDefaultOwner(env: Env): string {
const currentRepoNwo = env.getRequired(ActionsEnvVars.GITHUB_REPOSITORY);
const nwoParts = currentRepoNwo.split("/");
if (nwoParts.length !== 2 || nwoParts[0].trim().length === 0) {
// This shouldn't happen, so we should throw if `GITHUB_REPOSITORY` doesn't match
// our expectations.
throw new Error(
`Expected ${ActionsEnvVars.GITHUB_REPOSITORY} to contain a name with owner, but got '${currentRepoNwo}'.`,
);
}
return nwoParts[0].trim();
}
/**
* The old remote address format that's always been supported for the `config-file` input.
* All the components are required. Unchanged from the previous implementation.
*/
const OLD_REMOTE_ADDRESS_FORMAT = new RegExp(
"(?<owner>[^/]+)/(?<repo>[^/]+)/(?<path>[^@]+)@(?<ref>.*)",
);
/**
* Attempts to parse `input` as a `RemoteFileAddress` using the old format.
*
* @param input The input to try and parse.
* @returns A `RemoteFileAddress` value if successful or `undefined` otherwise.
*/
function parseOldRemoteFileAddress(
input: string,
): Result<RemoteFileAddress, undefined> {
const pieces = OLD_REMOTE_ADDRESS_FORMAT.exec(input);
// 5 = 4 groups + the whole expression
if (pieces?.groups === undefined || pieces.length < 5) {
return new Failure(undefined);
}
return new Success({
owner: pieces.groups.owner.trim(),
repo: pieces.groups.repo.trim(),
path: pieces.groups.path.trim(),
ref: pieces.groups.ref.trim(),
});
}
/**
* Attempts to parse `configFile` into an array of `RemoteFileAddress` components.
*
* @param actionState The current Action state.
* @param configFile The string to try and parse.
* @returns The successful result of executing the regex.
* @throws `ConfigurationError` if the format of `configFile` is not valid.
*/
export async function parseRemoteFileAddress(
actionState: ActionState<["FeatureFlags", "Env"]>,
configFile: string,
): Promise<RemoteFileAddress> {
// Try to parse the input using the old format. If successful, return the
// resulting `RemoteFileAddress`. Otherwise, continue using the new format.
const oldFormatAddressResult = parseOldRemoteFileAddress(configFile);
if (oldFormatAddressResult.isSuccess()) {
return oldFormatAddressResult.value;
}
// If the FF for the new format is not enabled, throw the old format error.
const allowNewFormat = await actionState.features.getValue(
Feature.NewRemoteFileAddresses,
);
if (!allowNewFormat) {
throw new ConfigurationError(
errorMessages.getConfigFileRepoOldFormatInvalidMessage(configFile),
);
}
// retrieve the various parts of the config location, and ensure they're present
const format = new RegExp(
"^((?<owner>[^:@/]+)/)?(?<repo>[^:@/]+)(@(?<ref>[^:]+))?(:(?<path>.+))?$",
);
const pieces = format.exec(configFile.trim());
const repo: string | undefined = pieces?.groups?.repo?.trim();
// Check that the regular expression matched and that we have at least the repo name.
if (!pieces?.groups || !repo || repo.length === 0) {
// Neither the old format nor the new format worked. Throw an error that
// explains the format we accept. We only mention the new format, since that's
// what we want to be used going forward.
throw new ConfigurationError(
errorMessages.getConfigFileRepoFormatInvalidMessage(configFile),
);
}
const owner: string | undefined = pieces.groups.owner?.trim();
const path: string | undefined = pieces.groups.path?.trim();
const ref: string | undefined = pieces.groups.ref?.trim();
// Ensure that the path is a relative path.
if (path?.startsWith("/")) {
throw new ConfigurationError(
`The path component of '${configFile}' cannot be an absolute path.`,
);
}
return {
owner: owner || getDefaultOwner(actionState.env),
repo,
path: path || DEFAULT_CONFIG_FILE_NAME,
ref: ref || DEFAULT_CONFIG_FILE_REF,
};
}

191
src/database-upload.test.ts
View File

@@ -11,8 +11,10 @@ import * as apiClient from "./api-client";
import { createStubCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { cleanupAndUploadDatabases } from "./database-upload";
import { Feature } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { BuiltInLanguage } from "./languages";
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
import { RepositoryNwo } from "./repository";
import {
checkExpectedLogMessages,
@@ -24,6 +26,7 @@ import {
setupTests,
} from "./testing-utils";
import {
CleanupLevel,
GitHubVariant,
HTTPError,
initializeEnvironment,
@@ -335,3 +338,191 @@ test.serial("Successfully uploading a database to GHEC-DR", async (t) => {
);
});
});
test.serial(
"Records overlay and clear cleanup sizes when uploading an overlay-base database",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
await mockHttpRequests(201);
// Track the cleanup level passed to each cleanup so that the database
// bundle stub can write a differently-sized bundle for each level.
const cleanupLevels: CleanupLevel[] = [];
let lastCleanupLevel: CleanupLevel | undefined;
const overlaySizeBytes = 100;
const clearSizeBytes = 50;
const codeql = createStubCodeQL({
async databaseCleanupCluster(_config, cleanupLevel) {
cleanupLevels.push(cleanupLevel);
lastCleanupLevel = cleanupLevel;
},
async databaseBundle(_databasePath, outputFilePath) {
const sizeBytes =
lastCleanupLevel === CleanupLevel.Overlay
? overlaySizeBytes
: clearSizeBytes;
fs.writeFileSync(outputFilePath, "x".repeat(sizeBytes));
},
});
const config = getTestConfig(tmpDir);
config.overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
const loggedMessages: LoggedMessage[] = [];
const results = await cleanupAndUploadDatabases(
testRepoName,
codeql,
config,
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger(loggedMessages),
);
// The database should be cleaned up at the `overlay` level for the upload
// and then re-cleaned at the `clear` level to measure its size.
t.deepEqual(cleanupLevels, [CleanupLevel.Overlay, CleanupLevel.Clear]);
t.is(results.length, 1);
t.is(results[0].is_overlay_base, true);
t.is(results[0].zipped_upload_size_bytes, overlaySizeBytes);
t.is(results[0].clear_cleanup_zipped_size_bytes, clearSizeBytes);
t.is(typeof results[0].clear_cleanup_measurement_duration_ms, "number");
});
},
);
test.serial(
"Does not measure clear cleanup size for a regular (non-overlay-base) upload",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
await mockHttpRequests(201);
const cleanupLevels: CleanupLevel[] = [];
const codeql = createStubCodeQL({
async databaseCleanupCluster(_config, cleanupLevel) {
cleanupLevels.push(cleanupLevel);
},
async databaseBundle(_databasePath, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});
const results = await cleanupAndUploadDatabases(
testRepoName,
codeql,
getTestConfig(tmpDir),
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger([]),
);
// A regular upload is cleaned only once, at the `clear` level.
t.deepEqual(cleanupLevels, [CleanupLevel.Clear]);
t.is(results[0].is_overlay_base, false);
t.is(results[0].clear_cleanup_zipped_size_bytes, undefined);
t.is(results[0].clear_cleanup_measurement_duration_ms, undefined);
});
},
);
test.serial("Does not measure clear cleanup size in debug mode", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
await mockHttpRequests(201);
const cleanupLevels: CleanupLevel[] = [];
const codeql = createStubCodeQL({
async databaseCleanupCluster(_config, cleanupLevel) {
cleanupLevels.push(cleanupLevel);
},
async databaseBundle(_databasePath, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});
const config = getTestConfig(tmpDir);
config.overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
config.debugMode = true;
const results = await cleanupAndUploadDatabases(
testRepoName,
codeql,
config,
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger([]),
);
// In debug mode we clean up at the `overlay` level for the upload but skip
// the additional `clear` cleanup, to preserve the database for debugging.
t.deepEqual(cleanupLevels, [CleanupLevel.Overlay]);
t.is(results[0].is_overlay_base, true);
t.is(results[0].clear_cleanup_zipped_size_bytes, undefined);
t.is(results[0].clear_cleanup_measurement_duration_ms, undefined);
});
});
test.serial(
"Does not record a clear cleanup duration when the clear cleanup fails",
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
await mockHttpRequests(201);
const codeql = createStubCodeQL({
async databaseCleanupCluster(_config, cleanupLevel) {
if (cleanupLevel === CleanupLevel.Clear) {
throw new Error("clear cleanup failed");
}
},
async databaseBundle(_databasePath, outputFilePath) {
fs.writeFileSync(outputFilePath, "x".repeat(100));
},
});
const config = getTestConfig(tmpDir);
config.overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
const results = await cleanupAndUploadDatabases(
testRepoName,
codeql,
config,
testApiDetails,
createFeatures([Feature.UploadOverlayDbToApi]),
getRecordingLogger([]),
);
// When the `clear` cleanup fails, no size is measured, so we should not
// report a measurement duration either.
t.is(results[0].is_overlay_base, true);
t.is(results[0].clear_cleanup_zipped_size_bytes, undefined);
t.is(results[0].clear_cleanup_measurement_duration_ms, undefined);
});
},
);

93
src/database-upload.ts
View File

@@ -25,6 +25,19 @@ export interface DatabaseUploadResult {
zipped_upload_size_bytes?: number;
/** Whether the uploaded database is an overlay base. */
is_overlay_base?: boolean;
/**
* For overlay-base uploads only: the size in bytes that the zipped database
* would have been if it had been cleaned at the `clear` cleanup level instead
* of the `overlay` level.
*/
clear_cleanup_zipped_size_bytes?: number;
/**
* For overlay-base uploads only: the time in milliseconds spent measuring the
* `clear` cleanup size (cleaning up the cluster at the `clear` level and
* bundling each database). This is a cluster-wide measurement, so it is the
* same for every language in a run.
*/
clear_cleanup_measurement_duration_ms?: number;
/** Time taken to upload database in milliseconds. */
upload_duration_ms?: number;
/** If there was an error during database upload, this is its message. */
@@ -156,9 +169,89 @@ export async function cleanupAndUploadDatabases(
});
}
}
// When we upload an overlay-base database, we cleaned the databases at the `overlay` level, which
// retains more data than the `clear` level used for regular uploads. Measure what the zipped size
// would have been at the `clear` level too, so we can compare the storage cost of overlay-base
// databases against regular databases for the same repository.
//
// We skip this in debug mode, where the databases are preserved and uploaded as debug artifacts,
// since cleaning them up at the `clear` level would discard data that is useful for debugging.
if (shouldUploadOverlayBase && !config.debugMode) {
await withGroupAsync(
"Measuring database size at the clear cleanup level",
() => recordClearCleanupSizes(codeql, config, reports, logger),
);
}
return reports;
}
/**
* Cleans up the databases at the `clear` cleanup level and records the resulting zipped size for
* each language in `clear_cleanup_zipped_size_bytes`. If the cleanup succeeds, also records the
* time spent taking the measurement in `clear_cleanup_measurement_duration_ms`.
*
* This mutates the entries of `reports` in place. It must run only after all overlay-base uploads
* have completed, since the `clear` cleanup discards overlay data that the uploaded database
* depends on.
*
* Failures here are non-fatal: this is telemetry-only, so we log and move on rather than failing
* the workflow.
*/
async function recordClearCleanupSizes(
codeql: CodeQL,
config: Config,
reports: DatabaseUploadResult[],
logger: Logger,
): Promise<void> {
// Include both the cleanup and the re-bundling to record how much time taking this measurement adds
// to the run.
const startTime = performance.now();
try {
await codeql.databaseCleanupCluster(config, CleanupLevel.Clear);
} catch (e) {
// The cleanup didn't run, so there are no sizes to measure. Return without recording a
// duration, so that we don't report a measurement duration with no accompanying sizes.
logger.warning(
`Failed to clean up databases at the '${CleanupLevel.Clear}' level for ` +
`size measurement: ${util.getErrorMessage(e)}`,
);
return;
}
for (const language of config.languages) {
const report = reports.find((r) => r.language === language);
if (report === undefined) {
continue;
}
try {
const bundledDb = await bundleDb(config, language, codeql, language, {
includeDiagnostics: false,
});
report.clear_cleanup_zipped_size_bytes = fs.statSync(bundledDb).size;
logger.debug(
`Database for ${language} is ` +
`${report.clear_cleanup_zipped_size_bytes} bytes zipped at the ` +
`'${CleanupLevel.Clear}' cleanup level ` +
`(vs. ${report.zipped_upload_size_bytes ?? "unknown"} bytes at the ` +
`'${CleanupLevel.Overlay}' level).`,
);
} catch (e) {
logger.warning(
`Failed to measure the '${CleanupLevel.Clear}' cleanup database size ` +
`for ${language}: ${util.getErrorMessage(e)}`,
);
}
}
const durationMs = performance.now() - startTime;
for (const report of reports) {
report.clear_cleanup_measurement_duration_ms = durationMs;
}
}
/**
* Uploads a bundled database to the GitHub API.
*

4
src/debug-artifacts.ts
View File

@@ -4,7 +4,7 @@ import * as path from "path";
import * as artifact from "@actions/artifact";
import * as artifactLegacy from "@actions/artifact-legacy";
import * as core from "@actions/core";
import archiver from "archiver";
import { ZipArchive } from "archiver";
import { getOptionalInput, getTemporaryDirectory } from "./actions-util";
import { dbIsFinalized } from "./analyze";
@@ -397,7 +397,7 @@ async function createPartialDatabaseBundle(
await fs.promises.rm(databaseBundlePath, { force: true });
}
const output = fs.createWriteStream(databaseBundlePath);
const zip = archiver("zip");
const zip = new ZipArchive();
zip.on("error", (err) => {
throw err;

8
src/defaults.json
View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.25.6",
"cliVersion": "2.25.6",
"priorBundleVersion": "codeql-bundle-v2.25.5",
"priorCliVersion": "2.25.5"
"bundleVersion": "codeql-bundle-v2.26.0",
"cliVersion": "2.26.0",
"priorBundleVersion": "codeql-bundle-v2.25.6",
"priorCliVersion": "2.25.6"
}

135
src/environment.ts
View File

@@ -17,6 +17,18 @@ export enum EnvVar {
*/
CLI_VERBOSITY = "CODEQL_VERBOSITY",
/**
* Set by Default Setup to the base branch of the PR being analysed, if analysing a PR.
* This is needed because the `pull_request` context is not available for `dynamic` events.
*/
CODE_SCANNING_BASE_BRANCH = "CODE_SCANNING_BASE_BRANCH",
/**
* Set by Default Setup to the full ref being analysed, if analysing a PR.
* This is needed because the `pull_request` context is not available for `dynamic` events.
*/
CODE_SCANNING_REF = "CODE_SCANNING_REF",
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
@@ -77,6 +89,9 @@ export enum EnvVar {
/** Whether to suppress the warning if the current CLI will soon be unsupported. */
SUPPRESS_DEPRECATED_SOON_WARNING = "CODEQL_ACTION_SUPPRESS_DEPRECATED_SOON_WARNING",
/** Used to dictate or persist the temporary directory used by the CodeQL Action. */
TEMP = "CODEQL_ACTION_TEMP",
/** Whether to disable uploading SARIF results or status reports to the GitHub API */
TEST_MODE = "CODEQL_ACTION_TEST_MODE",
@@ -154,3 +169,123 @@ export enum EnvVar {
/** Used by Code Scanning Risk Assessment to communicate the assessment ID to the CodeQL Action. */
RISK_ASSESSMENT_ID = "CODEQL_ACTION_RISK_ASSESSMENT_ID",
}
/**
* Enumerates known GitHub Actions environment variables that we expect
* to be set in a GitHub Actions environment.
*/
export enum ActionsEnvVars {
GITHUB_ACTION_REPOSITORY = "GITHUB_ACTION_REPOSITORY",
GITHUB_API_URL = "GITHUB_API_URL",
GITHUB_EVENT_NAME = "GITHUB_EVENT_NAME",
GITHUB_EVENT_PATH = "GITHUB_EVENT_PATH",
GITHUB_JOB = "GITHUB_JOB",
GITHUB_REF = "GITHUB_REF",
GITHUB_REPOSITORY = "GITHUB_REPOSITORY",
GITHUB_RUN_ATTEMPT = "GITHUB_RUN_ATTEMPT",
GITHUB_RUN_ID = "GITHUB_RUN_ID",
GITHUB_SERVER_URL = "GITHUB_SERVER_URL",
GITHUB_SHA = "GITHUB_SHA",
GITHUB_WORKFLOW = "GITHUB_WORKFLOW",
RUNNER_ENVIRONMENT = "RUNNER_ENVIRONMENT",
RUNNER_NAME = "RUNNER_NAME",
RUNNER_OS = "RUNNER_OS",
RUNNER_TEMP = "RUNNER_TEMP",
}
/** A type representing all known environment variables. */
export type KnownEnvVar = EnvVar | ActionsEnvVars;
/**
* Gets an environment variable, but throws an error if it is not set.
*/
function getRequiredEnvVar(env: NodeJS.ProcessEnv, paramName: string): string {
const value = env[paramName];
if (value === undefined || value.length === 0) {
throw new Error(`${paramName} environment variable must be set`);
}
return value;
}
/**
* Get an environment parameter, but throw an error if it is not set.
*
* @deprecated Use `getRequired` of a `ReadOnlyEnv` or `Env` instance instead.
*/
export function getRequiredEnvParam(paramName: string): string {
return getRequiredEnvVar(process.env, paramName);
}
/**
* Gets an environment variable, but returns `undefined` if it is not set or empty.
*/
function getOptionalEnvVarFrom(
env: NodeJS.ProcessEnv,
paramName: string,
): string | undefined {
const value = env[paramName];
if (value?.trim().length === 0) {
return undefined;
}
return value;
}
/**
* Get an environment variable, but return `undefined` if it is not set or empty.
*
* @deprecated Use `getOptional` of a `ReadOnlyEnv` or `Env` instance instead.
*/
export function getOptionalEnvVar(paramName: string): string | undefined {
return getOptionalEnvVarFrom(process.env, paramName);
}
/**
* An abstraction around read-only environment variables, to allow abstracting away from `process.env`
* in tests, while clearly signalling in regular code that the consumer of the `ReadOnlyEnv` instance
* will only read from it.
*/
export class ReadOnlyEnv<T extends string | undefined = string | undefined> {
constructor(protected readonly vars: Record<string, T>) {}
/** Tries to get the value for `name` and throws if there isn't one. */
public getRequired(name: string): string {
return getRequiredEnvVar(this.vars, name);
}
/** Gets the value for `name`, or `undefined` if it isn't set or empty. */
public getOptional(name: string): string | undefined {
return getOptionalEnvVarFrom(this.vars, name);
}
/** Gets the entries of the underlying `ProcessEnv`. */
public entries(): Array<[string, T]> {
return Object.entries(this.vars);
}
}
/**
* A wrapper around an environment, to allow abstracting away from `process.env` in tests.
* Use `ReadOnlyEnv` instead if you only plan to read from the environment.
* This type allows writing to the environment.
*/
export class Env<
T extends string | undefined = string | undefined,
> extends ReadOnlyEnv<T> {
private changed: boolean = false;
/** Sets an environment variable. */
public set(name: string, value: T): void {
this.vars[name] = value;
this.changed = true;
}
/** Gets a value indicating whether `set` was called at least once. */
public hasChanged(): boolean {
return this.changed;
}
}
/** Gets an `Env` instance for `env`, which is `process.env` by default. */
export function getEnv(env: NodeJS.ProcessEnv = process.env): Env {
return new Env(env);
}

11
src/error-messages.ts
View File

@@ -30,7 +30,7 @@ export function getInvalidConfigFileMessage(
return `The configuration file "${configFile}" is invalid: ${messages.slice(0, 10).join(", ")}${andMore}`;
}
export function getConfigFileRepoFormatInvalidMessage(
export function getConfigFileRepoOldFormatInvalidMessage(
configFile: string,
): string {
let error = `The configuration file "${configFile}" is not a supported remote file reference.`;
@@ -39,6 +39,15 @@ export function getConfigFileRepoFormatInvalidMessage(
return error;
}
export function getConfigFileRepoFormatInvalidMessage(
configFile: string,
): string {
let error = `The configuration file "${configFile}" is not a supported remote file reference.`;
error += " Expected format [<owner>/]<repository>[@<ref>][:<file-path>]";
return error;
}
export function getConfigFileFormatInvalidMessage(configFile: string): string {
return `The configuration file "${configFile}" could not be read`;
}

14
src/feature-flags.ts
View File

@@ -74,6 +74,8 @@ export enum Feature {
AllowMultipleAnalysisKinds = "allow_multiple_analysis_kinds",
AllowToolcacheInput = "allow_toolcache_input",
CleanupTrapCaches = "cleanup_trap_caches",
/** Whether to allow the `config-file` input to be specified via a repository property. */
ConfigFileRepositoryProperty = "config_file_repository_property",
CppDependencyInstallation = "cpp_dependency_installation_enabled",
CsharpCacheBuildModeNone = "csharp_cache_bmn",
CsharpNewCacheKey = "csharp_new_cache_key",
@@ -90,6 +92,8 @@ export enum Feature {
ForceNightly = "force_nightly",
IgnoreGeneratedFiles = "ignore_generated_files",
JavaNetworkDebugging = "java_network_debugging",
/** Allow the new remote file address format. */
NewRemoteFileAddresses = "new_remote_file_addresses",
OverlayAnalysis = "overlay_analysis",
OverlayAnalysisCodeScanningCpp = "overlay_analysis_code_scanning_cpp",
OverlayAnalysisCodeScanningCsharp = "overlay_analysis_code_scanning_csharp",
@@ -184,6 +188,11 @@ export const featureConfig = {
envVar: "CODEQL_ACTION_CLEANUP_TRAP_CACHES",
minimumVersion: undefined,
},
[Feature.ConfigFileRepositoryProperty]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CONFIG_FILE_REPOSITORY_PROPERTY",
minimumVersion: undefined,
},
[Feature.CppDependencyInstallation]: {
defaultValue: false,
envVar: "CODEQL_EXTRACTOR_CPP_AUTOINSTALL_DEPENDENCIES",
@@ -248,6 +257,11 @@ export const featureConfig = {
envVar: "CODEQL_ACTION_JAVA_NETWORK_DEBUGGING",
minimumVersion: undefined,
},
[Feature.NewRemoteFileAddresses]: {
defaultValue: false,
envVar: "CODEQL_ACTION_NEW_REMOTE_FILE_ADDRESSES",
minimumVersion: undefined,
},
[Feature.OverlayAnalysis]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",

39
src/init-action.ts
View File

@@ -7,9 +7,9 @@ import * as io from "@actions/io";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { Action, ActionState, runInActions } from "./action-common";
import {
FileCmdNotFoundError,
getActionsEnv,
getActionVersion,
getFileType,
getOptionalInput,
@@ -57,7 +57,7 @@ import {
runDatabaseInitCluster,
} from "./init";
import { JavaEnvVars, BuiltInLanguage } from "./languages";
import { getActionsLogger, Logger, withGroupAsync } from "./logging";
import { Logger, withGroupAsync } from "./logging";
import {
downloadOverlayBaseDatabaseFromCache,
OverlayBaseDatabaseDownloadStats,
@@ -74,7 +74,6 @@ import {
createStatusReportBase,
getActionsStatus,
sendStatusReport,
sendUnhandledErrorStatusReport,
} from "./status-report";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
@@ -205,12 +204,12 @@ async function sendCompletedStatusReport(
}
}
async function run(startedAt: Date) {
async function run(actionState: ActionState<["Logger", "Env", "Actions"]>) {
// To capture errors appropriately, keep as much code within the try-catch as
// possible, and only use safe functions outside.
const logger = getActionsLogger();
const actionsEnv = getActionsEnv();
const startedAt = actionState.startedAt;
const logger = actionState.logger;
let apiDetails: GitHubApiCombinedDetails;
let config: configUtils.Config | undefined;
@@ -264,7 +263,11 @@ async function run(startedAt: Date) {
core.exportVariable(EnvVar.INIT_ACTION_HAS_RUN, "true");
configFile = getConfigFileInput(logger, actionsEnv, repositoryProperties);
const actionStateWithFeatures = { ...actionState, features };
configFile = await getConfigFileInput(
actionStateWithFeatures,
repositoryProperties,
);
// path.resolve() respects the intended semantics of source-root. If
// source-root is relative, it is relative to the GITHUB_WORKSPACE. If
@@ -367,7 +370,7 @@ async function run(startedAt: Date) {
repositoryProperties,
);
config = await initConfig(features, {
config = await initConfig(actionStateWithFeatures, {
analysisKinds,
languagesInput: getOptionalInput("languages"),
queriesInput: getOptionalInput("queries"),
@@ -855,19 +858,13 @@ async function recordZstdAvailability(
);
}
/** Defines the `init` Action. */
const init: Action = {
name: ActionName.Init,
run,
};
export async function runWrapper() {
const startedAt = new Date();
const logger = getActionsLogger();
try {
await run(startedAt);
} catch (error) {
core.setFailed(`init action failed: ${getErrorMessage(error)}`);
await sendUnhandledErrorStatusReport(
ActionName.Init,
startedAt,
error,
logger,
);
}
await runInActions(init);
await checkForTimeout();
}

3
src/init.test.ts
View File

@@ -8,6 +8,7 @@ import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { createStubCodeQL } from "./codeql";
import { ActionsEnvVars } from "./environment";
import { Feature } from "./feature-flags";
import {
checkPacksForOverlayCompatibility,
@@ -84,7 +85,7 @@ for (const { runnerEnv, ErrorConstructor, message } of [
`cleanupDatabaseClusterDirectory throws a ${ErrorConstructor.name} when cleanup fails on ${runnerEnv} runner`,
async (t) => {
await withTmpDir(async (tmpDir: string) => {
process.env["RUNNER_ENVIRONMENT"] = runnerEnv;
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = runnerEnv;
const dbLocation = path.resolve(tmpDir, "dbs");
fs.mkdirSync(dbLocation, { recursive: true });

5
src/init.ts
View File

@@ -7,6 +7,7 @@ import * as github from "@actions/github";
import * as io from "@actions/io";
import * as yaml from "js-yaml";
import { ActionState } from "./action-common";
import {
getOptionalInput,
isAnalyzingPullRequest,
@@ -81,11 +82,11 @@ export async function initCodeQL(
}
export async function initConfig(
features: FeatureEnablement,
actionState: ActionState<["Logger", "Env", "FeatureFlags"]>,
inputs: configUtils.InitConfigInputs,
): Promise<configUtils.Config> {
return await withGroupAsync("Load language configuration", async () => {
return await configUtils.initConfig(features, inputs);
return await configUtils.initConfig(actionState, inputs);
});
}

35
src/json/index.test.ts
View File

@@ -10,8 +10,8 @@ const testSchema = {
requiredKey: json.string,
};
const optionalSchema = {
optionalKey: json.optional(json.string),
const optionalOrNullSchema = {
optionalKey: json.optionalOrNull(json.string),
};
test("validateSchema - required properties are required", async (t) => {
@@ -28,13 +28,36 @@ test("validateSchema - required properties are required", async (t) => {
t.true(json.validateSchema(testSchema, { requiredKey: "foo" }));
});
test("validateSchema - optional properties are optional", async (t) => {
test("validateSchema - optionalOrNullSchema properties are optional or null", async (t) => {
// Optional fields may be absent
t.true(json.validateSchema(optionalSchema, {}));
t.true(json.validateSchema(optionalSchema, { optionalKey: undefined }));
t.true(json.validateSchema(optionalSchema, { optionalKey: null }));
t.true(json.validateSchema(optionalOrNullSchema, {}));
t.true(json.validateSchema(optionalOrNullSchema, { optionalKey: undefined }));
t.true(json.validateSchema(optionalOrNullSchema, { optionalKey: null }));
// But, if present, should have the expected type
t.false(json.validateSchema(optionalOrNullSchema, { optionalKey: 0 }));
t.false(json.validateSchema(optionalOrNullSchema, { optionalKey: 123 }));
t.false(json.validateSchema(optionalOrNullSchema, { optionalKey: false }));
t.false(json.validateSchema(optionalOrNullSchema, { optionalKey: true }));
t.false(json.validateSchema(optionalOrNullSchema, { optionalKey: [] }));
t.false(json.validateSchema(optionalOrNullSchema, { optionalKey: {} }));
t.true(json.validateSchema(optionalOrNullSchema, { optionalKey: "" }));
t.true(json.validateSchema(optionalOrNullSchema, { optionalKey: "foo" }));
});
const optionalSchema = {
optionalKey: json.optional(json.string),
};
test("validateSchema - optional properties are optional", async (t) => {
// Optional fields may be absent or explicitly undefined
t.true(json.validateSchema(optionalSchema, {}));
t.true(json.validateSchema(optionalSchema, { optionalKey: undefined }));
// But should reject null
t.false(json.validateSchema(optionalSchema, { optionalKey: null }));
// And, if present, should have the expected type
t.false(json.validateSchema(optionalSchema, { optionalKey: 0 }));
t.false(json.validateSchema(optionalSchema, { optionalKey: 123 }));
t.false(json.validateSchema(optionalSchema, { optionalKey: false }));

31
src/json/index.ts
View File

@@ -30,6 +30,11 @@ export function isString(value: unknown): value is string {
return typeof value === "string";
}
/** Asserts that `value` is a number. */
export function isNumber(value: unknown): value is number {
return typeof value === "number";
}
/** Asserts that `value` is either a string or undefined. */
export function isStringOrUndefined(
value: unknown,
@@ -55,8 +60,17 @@ export const string = {
required: true,
} as const satisfies Validator<string>;
/** Transforms a validator to be optional. */
export function optional<T>(validator: Validator<T>) {
/** A validator for number fields in schemas. */
export const number = {
validate: isNumber,
required: true,
} as const satisfies Validator<number>;
/**
* Transforms a validator to be optional, accepting `undefined` or `null` for an
* absent value.
*/
export function optionalOrNull<T>(validator: Validator<T>) {
return {
validate: (val: unknown) => {
return val === undefined || val === null || validator.validate(val);
@@ -65,6 +79,19 @@ export function optional<T>(validator: Validator<T>) {
} as const satisfies Validator<T | undefined | null>;
}
/**
* Transforms a validator to be optional, accepting `undefined` for an absent
* value but, unlike `optionalOrNull`, rejecting `null`.
*/
export function optional<T>(validator: Validator<T>) {
return {
validate: (val: unknown): val is T | undefined => {
return val === undefined || validator.validate(val);
},
required: false,
} as const satisfies Validator<T | undefined>;
}
/** Represents an arbitrary object schema. */
export type Schema = Record<string, Validator<any>>;

32
src/setup-codeql-action.ts
View File

@@ -1,6 +1,7 @@
import * as core from "@actions/core";
import { v4 as uuidV4 } from "uuid";
import { Action, ActionState, runInActions } from "./action-common";
import {
getActionVersion,
getOptionalInput,
@@ -14,7 +15,7 @@ import { getRawLanguagesNoAutodetect } from "./config-utils";
import { EnvVar } from "./environment";
import { initFeatures } from "./feature-flags";
import { initCodeQL } from "./init";
import { getActionsLogger, Logger } from "./logging";
import { Logger } from "./logging";
import { getRepositoryNwo } from "./repository";
import { ToolsSource } from "./setup-codeql";
import {
@@ -24,7 +25,6 @@ import {
createStatusReportBase,
getActionsStatus,
sendStatusReport,
sendUnhandledErrorStatusReport,
} from "./status-report";
import { ToolsDownloadStatusReport } from "./tools-download";
import {
@@ -36,7 +36,6 @@ import {
ConfigurationError,
wrapError,
checkActionVersion,
getErrorMessage,
} from "./util";
/**
@@ -88,12 +87,13 @@ async function sendCompletedStatusReport(
}
/** The main behaviour of this action. */
async function run(startedAt: Date): Promise<void> {
async function run({
startedAt,
logger,
}: ActionState<["Logger"]>): Promise<void> {
// To capture errors appropriately, keep as much code within the try-catch as
// possible, and only use safe functions outside.
const logger = getActionsLogger();
let codeql: CodeQL;
let toolsDownloadStatusReport: ToolsDownloadStatusReport | undefined;
let toolsFeatureFlagsValid: boolean | undefined;
@@ -195,20 +195,14 @@ async function run(startedAt: Date): Promise<void> {
);
}
/** Defines the `setup-codeql` Action. */
const setupCodeQL: Action = {
name: ActionName.SetupCodeQL,
run,
};
/** Run the action and catch any unhandled errors. */
export async function runWrapper(): Promise<void> {
const startedAt = new Date();
const logger = getActionsLogger();
try {
await run(startedAt);
} catch (error) {
core.setFailed(`setup-codeql action failed: ${getErrorMessage(error)}`);
await sendUnhandledErrorStatusReport(
ActionName.SetupCodeQL,
startedAt,
error,
logger,
);
}
await runInActions(setupCodeQL);
await checkForTimeout();
}

5
src/setup-codeql.test.ts
View File

@@ -7,6 +7,7 @@ import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as api from "./api-client";
import { EnvVar } from "./environment";
import { Feature } from "./feature-flags";
import { getRunnerLogger } from "./logging";
import { getCacheRestoreKeyPrefix } from "./overlay/caching";
@@ -637,8 +638,8 @@ test.serial(
async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
process.env["CODE_SCANNING_REF"] = "refs/heads/feature-branch";
process.env["CODE_SCANNING_BASE_BRANCH"] = "main";
process.env[EnvVar.CODE_SCANNING_REF] = "refs/heads/feature-branch";
process.env[EnvVar.CODE_SCANNING_BASE_BRANCH] = "main";
sinon.stub(api, "getAutomationID").resolves("test/");
const listStub = sinon.stub(api, "listActionsCaches").resolves([

16
src/start-proxy/types.ts
View File

@@ -12,7 +12,7 @@ export type RawCredential = UnvalidatedObject<Credential>;
/** A schema for credential objects with a username. */
export const usernameSchema = {
/** The username needed to authenticate to the package registry, if any. */
username: json.optional(json.string),
username: json.optionalOrNull(json.string),
} as const satisfies json.Schema;
/** Usernames may be present for both authentication with tokens or passwords. */
@@ -29,7 +29,7 @@ export function hasUsername(config: AuthConfig): config is Username {
/** A schema for credential objects with a username and password. */
export const usernamePasswordSchema = {
/** The password needed to authenticate to the package registry, if any. */
password: json.optional(json.string),
password: json.optionalOrNull(json.string),
...usernameSchema,
} as const satisfies json.Schema;
@@ -52,7 +52,7 @@ export function hasUsernameAndPassword(
/** A schema for credential objects for token-based authentication. */
export const tokenSchema = {
/** The token needed to authenticate to the package registry, if any. */
token: json.optional(json.string),
token: json.optionalOrNull(json.string),
...usernameSchema,
} as const satisfies json.Schema;
@@ -100,7 +100,7 @@ export const awsConfigSchema = {
"role-name": json.string,
domain: json.string,
"domain-owner": json.string,
audience: json.optional(json.string),
audience: json.optionalOrNull(json.string),
} as const satisfies json.Schema;
/** Configuration for AWS OIDC. */
@@ -116,8 +116,8 @@ export function isAWSConfig(
/** A schema for JFrog OIDC configurations. */
export const jfrogConfigSchema = {
"jfrog-oidc-provider-name": json.string,
audience: json.optional(json.string),
"identity-mapping-name": json.optional(json.string),
audience: json.optionalOrNull(json.string),
"identity-mapping-name": json.optionalOrNull(json.string),
} as const satisfies json.Schema;
/** Configuration for JFrog OIDC. */
@@ -150,8 +150,8 @@ export function isCloudsmithConfig(
/** A schema for GCP OIDC configurations. */
export const gcpConfigSchema = {
"workload-identity-provider": json.string,
"service-account": json.optional(json.string),
audience: json.optional(json.string),
"service-account": json.optionalOrNull(json.string),
audience: json.optionalOrNull(json.string),
} as const satisfies json.Schema;
/** Configuration for GCP OIDC. */

22
src/status-report.ts
View File

@@ -13,15 +13,16 @@ import {
} from "./actions-util";
import { getAnalysisKey, getApiClient } from "./api-client";
import { CommandCacheKey, getCachedCommandOutput } from "./cli/output-cache";
import { parseRegistriesWithoutCredentials, type Config } from "./config-utils";
import { DependencyCacheRestoreStatusReport } from "./dependency-caching";
import type { Config } from "./config/action-config";
import { parseRegistriesWithoutCredentials } from "./config/pack-registries";
import type { DependencyCacheRestoreStatusReport } from "./dependency-caching";
import { DocUrl } from "./doc-url";
import { EnvVar } from "./environment";
import { getRef } from "./git-utils";
import { Logger } from "./logging";
import { OverlayBaseDatabaseDownloadStats } from "./overlay/caching";
import type { Logger } from "./logging";
import type { OverlayBaseDatabaseDownloadStats } from "./overlay/caching";
import { getRepositoryNwo } from "./repository";
import { ToolsSource } from "./setup-codeql";
import type { ToolsSource } from "./setup-codeql";
import {
ConfigurationError,
getRequiredEnvParam,
@@ -46,6 +47,17 @@ export enum ActionName {
UploadSarif = "upload-sarif",
}
/**
* Maps an `ActionName` to its display name. Usually that is the same, except
* for `ActionName.Analyze` where it is `"analyze"` instead of `"finish"`.
*/
export function getDisplayActionName(actionName: ActionName): string {
if (actionName === ActionName.Analyze) {
return "analyze";
}
return actionName;
}
/**
* @returns a boolean indicating whether the analysis is considered to be first party.
*

129
src/testing-utils.ts
View File

@@ -10,6 +10,7 @@ import test, {
import nock from "nock";
import * as sinon from "sinon";
import { ActionState, StateFeature } from "./action-common";
import { ActionsEnv, getActionVersion } from "./actions-util";
import { AnalysisKind } from "./analyses";
import * as apiClient from "./api-client";
@@ -20,6 +21,7 @@ import { resetCachedCommandOutputs } from "./cli/output-cache";
import * as codeql from "./codeql";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json";
import { Env, ActionsEnvVars } from "./environment";
import {
CodeQLDefaultVersionInfo,
Feature,
@@ -28,9 +30,11 @@ import {
} from "./feature-flags";
import { Logger } from "./logging";
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
import { ActionName } from "./status-report";
import {
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
getEnv,
GitHubVariant,
GitHubVersion,
HTTPError,
@@ -173,6 +177,11 @@ export function makeMacro<Args extends unknown[]>(
return wrapper;
}
export function getTestEnv(): Env {
const testEnv: NodeJS.ProcessEnv = {};
return getEnv(testEnv);
}
/**
* Gets an `ActionsEnv` instance for use in tests.
*/
@@ -182,6 +191,116 @@ export function getTestActionsEnv(): ActionsEnv {
};
}
/** For testing purposes, we make all available state features accessible in `TestEnv`. */
type AllState = ["Logger", "Env", "Actions", "FeatureFlags"];
/** Initialise a fresh `ActionState<AllState>` value. */
export function initAllState(
overrides?: Partial<ActionState<AllState>>,
): ActionState<AllState> {
return {
name: ActionName.Init,
startedAt: new Date(),
logger: new RecordingLogger(),
env: getTestEnv(),
actions: getTestActionsEnv(),
features: createFeatures([]),
...overrides,
};
}
/**
* Wraps a function that accepts an `ActionState` for testing in different environments.
*/
export class TestEnv<
Args extends readonly any[],
R,
Fs extends ReadonlyArray<AllState[number]>,
> {
private readonly fn: (state: ActionState<Fs>, ...args: Args) => R;
private args?: Args;
private logger: RecordingLogger;
private state: ActionState<AllState>;
constructor(
fn: (state: ActionState<Fs>, ...args: Args) => R,
cloneFrom?: TestEnv<Args, R, Fs>,
) {
this.fn = fn;
this.args = cloneFrom?.args;
this.logger = new RecordingLogger();
this.state =
cloneFrom !== undefined
? { ...cloneFrom.state, logger: this.logger }
: initAllState({ logger: this.logger });
}
private clone(): TestEnv<Args, R, Fs> {
return new TestEnv(this.fn, this);
}
public getLogger(): RecordingLogger {
return this.logger;
}
public getState(): ActionState<AllState> {
return this.state;
}
public getArgs(): Args | undefined {
return this.args;
}
public withArgs(...args: Args) {
const result = this.clone();
result.args = args;
return result;
}
public withFeatures(enabled: Feature[]): TestEnv<Args, R, Fs> {
const result = this.clone();
result.state.features = createFeatures(enabled);
return result;
}
public withEnv(env: Env): TestEnv<Args, R, Fs> {
const result = this.clone();
result.state.env = env;
return result;
}
public withActions(actions: ActionsEnv): TestEnv<Args, R, Fs> {
const result = this.clone();
result.state.actions = actions;
return result;
}
call(): R {
if (!this.args) {
throw new Error("Trying to call function in TestEnv without arguments.");
}
return this.fn(this.state as unknown as ActionState<Fs>, ...this.args);
}
public passes<T>(
assertion: (makeCall: () => R) => T | Promise<T>,
): T | Promise<T> {
return assertion(() => {
const result = this.call();
return result;
});
}
}
/** Utility function to construct a `TestEnv`. */
export function callee<
Args extends readonly any[],
R,
Fs extends readonly StateFeature[],
>(fn: (state: ActionState<Fs>, ...args: Args) => R): TestEnv<Args, R, Fs> {
return new TestEnv(fn);
}
/**
* Default values for environment variables typically set in an Actions
* environment. Tests can override individual variables by passing them in the
@@ -201,7 +320,7 @@ export const DEFAULT_ACTIONS_VARS = {
GITHUB_WORKFLOW: "test-workflow",
RUNNER_NAME: "my-runner",
RUNNER_OS: "Linux",
} as const satisfies Record<string, string>;
} as const satisfies Partial<Record<ActionsEnvVars, string>>;
/** Partial mappings from GitHub Actions environment variables to values. */
export type ActionVarOverrides = Partial<
@@ -213,11 +332,15 @@ export type ActionVarOverrides = Partial<
* excluding some that are expected to be set to paths. See `setupActionsVars`.
*
* @param overrides Overrides for the defaults.
* @param env The environment to set the variables for.
*/
export function setupBaseActionsVars(overrides?: ActionVarOverrides) {
export function setupBaseActionsVars(
overrides?: ActionVarOverrides,
env: Env = getEnv(),
) {
const vars = { ...DEFAULT_ACTIONS_VARS, ...overrides };
for (const [key, value] of Object.entries(vars)) {
process.env[key] = value;
env.set(key, value);
}
}

32
src/upload-sarif-action.ts
View File

@@ -1,17 +1,17 @@
import * as core from "@actions/core";
import { Action, ActionState, runInActions } from "./action-common";
import * as actionsUtil from "./actions-util";
import { getActionVersion, getTemporaryDirectory } from "./actions-util";
import * as analyses from "./analyses";
import { getGitHubVersion } from "./api-client";
import { initFeatures } from "./feature-flags";
import { Logger, getActionsLogger } from "./logging";
import { Logger } from "./logging";
import { getRepositoryNwo } from "./repository";
import { InvalidSarifUploadError } from "./sarif";
import {
createStatusReportBase,
sendStatusReport,
sendUnhandledErrorStatusReport,
StatusReportBase,
getActionsStatus,
ActionName,
@@ -23,7 +23,6 @@ import {
ConfigurationError,
checkActionVersion,
checkDiskUsage,
getErrorMessage,
initializeEnvironment,
shouldSkipSarifUpload,
wrapError,
@@ -55,12 +54,9 @@ async function sendSuccessStatusReport(
}
}
async function run(startedAt: Date) {
async function run({ startedAt, logger }: ActionState<["Logger"]>) {
// To capture errors appropriately, keep as much code within the try-catch as
// possible, and only use safe functions outside.
const logger = getActionsLogger();
try {
initializeEnvironment(getActionVersion());
@@ -165,20 +161,12 @@ async function run(startedAt: Date) {
}
}
/** Defines the `upload-sarif` Action. */
const uploadSarif: Action = {
name: ActionName.UploadSarif,
run,
};
export async function runWrapper() {
const startedAt = new Date();
const logger = getActionsLogger();
try {
await run(startedAt);
} catch (error) {
core.setFailed(
`codeql/upload-sarif action failed: ${getErrorMessage(error)}`,
);
await sendUnhandledErrorStatusReport(
ActionName.UploadSarif,
startedAt,
error,
logger,
);
}
await runInActions(uploadSarif);
}

35
src/util.ts
View File

@@ -13,11 +13,14 @@ import * as apiCompatibility from "./api-compatibility.json";
import type { CodeQL } from "./codeql";
import type { Pack } from "./config/db-config";
import type { Config } from "./config-utils";
import { EnvVar } from "./environment";
import { EnvVar, getRequiredEnvParam } from "./environment";
import * as json from "./json";
import { Language } from "./languages";
import { Logger } from "./logging";
// Re-export for backwards compatibility to avoid updating a lot of imports elsewhere.
export { getRequiredEnvParam, getOptionalEnvVar, getEnv } from "./environment";
/**
* The name of the file containing the base database OIDs, as stored in the
* root of the database location.
@@ -566,28 +569,6 @@ export function initializeEnvironment(version: string) {
core.exportVariable(EnvVar.VERSION, version);
}
/**
* Get an environment parameter, but throw an error if it is not set.
*/
export function getRequiredEnvParam(paramName: string): string {
const value = process.env[paramName];
if (value === undefined || value.length === 0) {
throw new Error(`${paramName} environment variable must be set`);
}
return value;
}
/**
* Get an environment variable, but return `undefined` if it is not set or empty.
*/
export function getOptionalEnvVar(paramName: string): string | undefined {
const value = process.env[paramName];
if (value?.trim().length === 0) {
return undefined;
}
return value;
}
export class HTTPError extends Error {
public status: number;
@@ -628,7 +609,13 @@ export function getBaseDatabaseOidsFilePath(config: Config): string {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
// Create a bundle for the given DB, if it doesn't already exist
/**
* Bundles the database for the given language into a `.zip` file, returning the path to it.
*
* If a bundle for `dbName` already exists (e.g. from an earlier call), it is deleted and
* re-created, so each call produces a fresh bundle reflecting the current database contents and the
* given `includeDiagnostics` value.
*/
export async function bundleDb(
config: Config,
language: Language,