Compare commits

...

17 Commits

Author SHA1 Message Date
Henry Mercer
9bd4b5ea8f Add the PR link to the changelog entry
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 19:25:46 +01:00
Henry Mercer
c1e8d8d12c Reuse the toolcache for stable releases in the CodeQL Action repositories
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 18:38:59 +01:00
Henry Mercer
17f14786a5 Extract the default bundle sources from selectDefaultBundle
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 18:38:59 +01:00
Henry Mercer
0e574bc821 Accept GitHub release URLs in the tools input
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 18:38:58 +01:00
Henry Mercer
4cea925541 Look up releases requested by URL
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 18:38:58 +01:00
Henry Mercer
e8f5d72ffe Read the CLI version of a release from its assets
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 18:38:58 +01:00
Henry Mercer
4a4922e538 Parse GitHub release URLs
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 18:38:58 +01:00
Henry Mercer
78a5dc251b Extract the toolcache lookup from getCodeQLSource
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 18:38:58 +01:00
Henry Mercer
db99327672 Document how the CodeQL tools are versioned in the toolcache
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 18:38:58 +01:00
Henry Mercer
7f964a9e6e Select the default bundle from the first compatible release
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 17:20:28 +01:00
Henry Mercer
027ab3179c Test falling back to the public release for the default bundle
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 17:20:10 +01:00
Henry Mercer
caa2cae2fb Record per-language bundle fallbacks on the download source
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 17:19:53 +01:00
Henry Mercer
ffc453273e Select the latest nightly bundle with selectBundle
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 17:19:35 +01:00
Henry Mercer
303b251f60 Pass tarSupportsZstd to getLatestNightlyBundle
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 17:19:18 +01:00
Henry Mercer
f03d2dfd0b Add helpers for selecting a CodeQL bundle from a release
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 17:19:01 +01:00
Henry Mercer
6393cd425b Document how getCodeQLSource handles the tools input
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-25 17:18:42 +01:00
Henry Mercer
6ec2b18519 Rename logPerLanguageBundleFallback to logMissingPerLanguageBundle
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
2026-09-24 13:04:04 +01:00
10 changed files with 2004 additions and 577 deletions

View File

@@ -4,7 +4,7 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
## [UNRELEASED]
No user facing changes.
- The `tools` input to the `init` and `setup-codeql` actions now accepts GitHub release URLs. The Action selects a compatible bundle from the specified release, including a per-language bundle when eligible. [#4179](https://github.com/github/codeql-action/pull/4179)
## 4.38.2 - 24 Sept 2026

View File

@@ -10,6 +10,8 @@ inputs:
- A local path to a CodeQL Bundle tarball, or
- The URL of a CodeQL Bundle tarball GitHub release asset, or
- A release URL from GitHub.com or the current GitHub instance, such as
https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.27.0.
- A special value `linked` which uses the version of the CodeQL tools
that the Action has been bundled with.
- A special value `nightly` which uses the latest nightly version of the

857
lib/entry-points.js generated

File diff suppressed because it is too large Load Diff

View File

@@ -10,6 +10,8 @@ inputs:
- A local path to a CodeQL Bundle tarball, or
- The URL of a CodeQL Bundle tarball GitHub release asset, or
- A release URL from GitHub.com or the current GitHub instance, such as
https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.27.0.
- A special value `linked` which uses the version of the CodeQL tools
that the Action has been bundled with.
- A special value `nightly` which uses the latest nightly version of the

View File

@@ -27,6 +27,13 @@ export interface CodeQLDownloadSource {
cliVersion?: string;
/** Resolved version for telemetry, independent of whether the bundle can be cached. */
toolsVersion: string;
/** The release lacks the eligible per-language bundle, so we selected the combined bundle. */
perLanguageBundleFallback?: true;
/**
* The page of a requested release that may contain a different build than the bundle we cache for
* its version, so we don't cache it.
*/
customReleaseURL?: string;
}
/** Returns the exact bundle asset name for a platform and optional language. */

354
src/codeql-release.test.ts Normal file
View File

@@ -0,0 +1,354 @@
import * as github from "@actions/github";
import test from "ava";
import {
BundleSelectionOptions,
getPublicRelease,
getRelease,
getReleaseCliVersion,
getRequestedRelease,
parseCodeQLReleaseUrl,
selectBundle,
} from "./codeql-release";
import { ActionsEnvVars } from "./environment";
import { Feature } from "./feature-flags";
import { BuiltInLanguage } from "./languages";
import { BundlePlatform } from "./platform";
import {
createFeatures,
getRecordingLogger,
getTestEnv,
initAllState,
LoggedMessage,
SAMPLE_DOTCOM_API_DETAILS,
} from "./testing-utils";
import { ConfigurationError, GitHubVariant } from "./util";
const TAG = "codeql-bundle-v2.27.1";
const REFERENCE = {
serverURL: "https://github.com",
owner: "octo",
repo: "tools",
tagName: TAG,
};
const RELEASE_PAGE = `https://github.com/octo/tools/releases/tag/${TAG}`;
const API_URL = "https://api.github.com";
const COMBINED = "codeql-bundle-linux64.tar.zst";
const JAVA = "codeql-bundle-java-linux64.tar.zst";
const OPTIONS: BundleSelectionOptions = {
rawLanguages: ["java-kotlin"],
cliVersion: "2.27.1",
platform: BundlePlatform.Linux64,
variant: GitHubVariant.DOTCOM,
tarSupportsZstd: true,
};
/** Serves the release tagged `TAG` with the given assets from a stubbed API. */
function releaseFixture({ assetNames = [COMBINED, JAVA], status = 200 } = {}) {
const apiBase = `${API_URL}/repos/octo/tools/releases`;
const releaseAPIURL = `${apiBase}/tags/${TAG}`;
const assets = assetNames.map((name, index) => ({
name,
url: `${apiBase}/assets/${1000 + index}`,
}));
const requests: string[] = [];
const messages: LoggedMessage[] = [];
const state = initAllState({
env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted" }),
logger: getRecordingLogger(messages, { logToConsole: false }),
features: createFeatures([Feature.PerLanguageBundles]),
apiClient: github.getOctokit("123", {
baseUrl: API_URL,
request: {
fetch: async (url) => {
requests.push(String(url));
if (String(url) !== releaseAPIURL) {
throw new Error(`Unexpected API request: ${url}`);
}
return new Response(JSON.stringify({ tag_name: TAG, assets }), {
status,
headers: { "content-type": "application/json" },
});
},
},
}),
});
return {
assets,
messages,
releaseAPIURL,
requests,
state,
select: async (options: Partial<BundleSelectionOptions> = {}) =>
selectBundle(state, await getRelease(state, REFERENCE), {
...OPTIONS,
...options,
}),
};
}
test("selectBundle selects an eligible per-language bundle with a single release lookup", async (t) => {
const fixture = releaseFixture();
t.deepEqual(await fixture.select(), {
bundle: {
kind: "per-language",
url: fixture.assets[1].url,
language: BuiltInLanguage.java,
combinedBundleURL: fixture.assets[0].url,
},
compressionMethod: "zstd",
});
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
});
test("selectBundle falls back to the combined bundle from the same release", async (t) => {
const fixture = releaseFixture({ assetNames: [COMBINED] });
t.deepEqual(await fixture.select(), {
bundle: { kind: "combined", url: fixture.assets[0].url },
compressionMethod: "zstd",
perLanguageBundleFallback: true,
});
t.true(
fixture.messages.some(
(message) =>
message.type === "warning" &&
typeof message.message === "string" &&
message.message.includes(`'java' at ${RELEASE_PAGE}`),
),
);
});
test("selectBundle selects the combined bundle for jobs that aren't eligible for a per-language bundle", async (t) => {
const fixture = releaseFixture();
t.deepEqual(await fixture.select({ rawLanguages: ["java", "python"] }), {
bundle: { kind: "combined", url: fixture.assets[0].url },
compressionMethod: "zstd",
});
});
test("selectBundle uses the other compression method when the preferred one is missing", async (t) => {
const gzipOnly = releaseFixture({
assetNames: ["codeql-bundle-linux64.tar.gz", JAVA],
});
t.deepEqual(await gzipOnly.select(), {
bundle: { kind: "combined", url: gzipOnly.assets[0].url },
compressionMethod: "gzip",
});
for (const [platform, options] of [
[BundlePlatform.Win64, {}],
[BundlePlatform.Linux64, { cliVersion: "2.18.4" }],
[BundlePlatform.Linux64, { cliVersion: undefined }],
[BundlePlatform.Linux64, { tarSupportsZstd: false }],
] as const) {
const selection = await releaseFixture({
assetNames: [
`codeql-bundle-${platform}.tar.zst`,
`codeql-bundle-${platform}.tar.gz`,
],
}).select({ platform, ...options });
t.is(
selection.compressionMethod,
"gzip",
`${platform} ${JSON.stringify(options)}`,
);
}
});
test("selectBundle requires a combined bundle that we can extract", async (t) => {
await t.throwsAsync(releaseFixture({ assetNames: [JAVA] }).select(), {
instanceOf: ConfigurationError,
message: `No compatible CodeQL bundle was found in release ${RELEASE_PAGE}. Expected codeql-bundle-linux64.tar.zst or codeql-bundle-linux64.tar.gz.`,
});
await t.throwsAsync(
releaseFixture({ assetNames: [COMBINED] }).select({
tarSupportsZstd: false,
}),
{ instanceOf: ConfigurationError, message: /Expected [^ ]+\.tar\.gz\.$/ },
);
});
test("getRelease propagates API errors", async (t) => {
const fixture = releaseFixture({ status: 404 });
t.like(await t.throwsAsync(fixture.select()), { status: 404 });
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
});
test("getRequestedRelease names a requested release that can't be found", async (t) => {
const requested = { ...REFERENCE, isCurrentInstance: true };
await t.throwsAsync(
getRequestedRelease(releaseFixture({ status: 404 }).state, requested),
{
instanceOf: ConfigurationError,
message: `Could not find the CodeQL release ${RELEASE_PAGE}. Check that it exists and that the token has access to it.`,
},
);
t.like(
await t.throwsAsync(
getRequestedRelease(releaseFixture({ status: 500 }).state, requested),
),
{ status: 500 },
);
});
test("getRequestedRelease refers to a release on another instance by URL", async (t) => {
const fixture = releaseFixture();
const release = await getRequestedRelease(fixture.state, {
...REFERENCE,
isCurrentInstance: false,
});
t.is(
release.getAssetURL(COMBINED),
`https://github.com/octo/tools/releases/download/${TAG}/${COMBINED}`,
);
t.is(release.assetNames, undefined);
t.deepEqual(fixture.requests, []);
});
test("getPublicRelease constructs download URLs without looking up the release", async (t) => {
const fixture = releaseFixture();
const release = getPublicRelease({ ...REFERENCE, tagName: "nightly/v1+2" });
const baseURL =
"https://github.com/octo/tools/releases/download/nightly/v1%2B2";
t.is(
release.url,
"https://github.com/octo/tools/releases/tag/nightly/v1%2B2",
);
t.deepEqual(
await selectBundle(fixture.state, release, {
...OPTIONS,
cliVersion: undefined,
isLatestNightly: true,
}),
{
bundle: {
kind: "per-language",
url: `${baseURL}/${JAVA}`,
language: BuiltInLanguage.java,
combinedBundleURL: `${baseURL}/${COMBINED}`,
},
compressionMethod: "zstd",
},
);
t.deepEqual(fixture.requests, []);
});
test("parseCodeQLReleaseUrl accepts web and legacy links and decodes tags once", (t) => {
for (const [suffix, tagName] of [
[`tag/${TAG}`, TAG],
[TAG, TAG],
["codeql-bundle-20230120", "codeql-bundle-20230120"],
["codeql-bundle-v2.27.1-rc.1", "codeql-bundle-v2.27.1-rc.1"],
["tag/run-123", "run-123"],
["tag/build/123", "build/123"],
["tag/build%2F123%2Brc%231", "build/123+rc#1"],
["tag/build%252F123", "build%2F123"],
[`tag/${TAG}/?expanded=true#assets`, TAG],
]) {
t.deepEqual(
parseCodeQLReleaseUrl(
`https://github.com/octo/tools/releases/${suffix}`,
SAMPLE_DOTCOM_API_DETAILS,
),
{ ...REFERENCE, isCurrentInstance: true, tagName },
);
}
t.throws(
() =>
parseCodeQLReleaseUrl(
"https://github.com/octo/tools/releases/tag/%zz",
SAMPLE_DOTCOM_API_DETAILS,
),
{ instanceOf: ConfigurationError, message: /Invalid URL encoding/ },
);
});
test("parseCodeQLReleaseUrl matches the current instance and GitHub.com by origin", (t) => {
for (const [url, origin, serverURL] of [
["https://github.example.test", "https://github.com", "https://github.com"],
[
"https://github.example.test/",
"https://github.example.test",
"https://github.example.test",
],
[
"https://GitHub.Example.test",
"https://github.example.test",
"https://github.example.test",
],
[
"https://github.example.test:443",
"https://GITHUB.example.test:443",
"https://github.example.test",
],
]) {
t.deepEqual(
parseCodeQLReleaseUrl(`${origin}/octo/tools/releases/tag/${TAG}`, {
auth: "token",
url,
apiURL: undefined,
}),
{
...REFERENCE,
serverURL,
isCurrentInstance: serverURL !== "https://github.com",
},
`${url} ${origin}`,
);
}
});
test("parseCodeQLReleaseUrl excludes archives, REST references and untrusted URLs", (t) => {
for (const input of [
"/tmp/codeql-bundle.tar.zst",
"nightly",
`https://github.com/octo/tools/releases/download/${TAG}/${JAVA}`,
"https://api.github.com/repos/octo/tools/releases/assets/123",
"https://api.github.com/repos/octo/tools/releases/123",
`https://api.github.com/repos/octo/tools/releases/tags/${TAG}`,
"https://github.com/octo/tools/releases/latest",
"https://github.com/octo/tools/releases/tag/",
"https://github.com/octo/tools/releases/run-123",
`https://github.com.example.test/octo/tools/releases/tag/${TAG}`,
`https://github.com:8443/octo/tools/releases/tag/${TAG}`,
`https://github.com@example.test/octo/tools/releases/tag/${TAG}`,
`https://user@github.com/octo/tools/releases/tag/${TAG}`,
`http://github.com/octo/tools/releases/tag/${TAG}`,
]) {
t.is(
parseCodeQLReleaseUrl(input, SAMPLE_DOTCOM_API_DETAILS),
undefined,
input,
);
}
});
test("getRelease returns the names of the release's assets", async (t) => {
const assetNames = [COMBINED, "cli-version-2.27.2.txt"];
const fixture = releaseFixture({ assetNames });
const release = await getRelease(fixture.state, REFERENCE);
t.deepEqual(release.assetNames, assetNames);
});
test("getReleaseCliVersion prefers an unambiguous marker asset to the tag", (t) => {
for (const [tagName, markers, cliVersion] of [
[TAG, [], "2.27.1"],
[TAG, ["invalid"], "2.27.1"],
[TAG, ["2.27.2"], "2.27.2"],
[TAG, ["2.27.1", "2.28.0"], undefined],
["codeql-bundle-20260101", [], undefined],
["run-123", [], undefined],
["run-123", ["2.27.2+202601011200"], "2.27.2+202601011200"],
] as const) {
t.is(
getReleaseCliVersion(
tagName,
[COMBINED, ...markers.map((version) => `cli-version-${version}.txt`)],
getRecordingLogger([], { logToConsole: false }),
),
cliVersion,
`${tagName} ${markers.join(",")}`,
);
}
});

327
src/codeql-release.ts Normal file
View File

@@ -0,0 +1,327 @@
import * as semver from "semver";
import { ActionState } from "./action-common";
import type { GitHubApiDetails } from "./api-client";
import { CodeQLBundle, getCodeQLBundleName } from "./codeql-bundle";
import { CODEQL_VERSION_ZSTD_BUNDLE } from "./feature-flags";
import { Logger } from "./logging";
import {
getPerLanguageBundleLanguage,
logMissingPerLanguageBundle,
} from "./per-language-bundles";
import { BundlePlatform } from "./platform";
import type { CompressionMethod } from "./tar";
import {
asHTTPError,
ConfigurationError,
GITHUB_DOTCOM_URL,
GitHubVariant,
} from "./util";
/** Identifies a release on a GitHub instance. */
export interface CodeQLReleaseReference {
serverURL: string;
owner: string;
repo: string;
tagName: string;
}
/** A GitHub release that contains CodeQL bundles. */
export interface CodeQLRelease {
/** The release's web page, for messages. */
url: string;
/** Returns the download URL for an asset, or `undefined` if the release doesn't have it. */
getAssetURL(name: string): string | undefined;
/** Names of the release's assets, if we looked up the release. */
assetNames?: string[];
}
/** A release requested by URL, on this GitHub instance or on GitHub.com. */
export interface RequestedRelease extends CodeQLReleaseReference {
isCurrentInstance: boolean;
}
/**
* Recognizes release pages, including legacy bundle links, but never asset URLs.
*
* We only accept https URLs without credentials, on this GitHub instance, whose API we can use, or
* on GitHub.com, whose public releases we can download without credentials. The tag is
* percent-decoded once, and may contain `/`.
*/
export function parseCodeQLReleaseUrl(
input: string,
apiDetails: GitHubApiDetails,
): RequestedRelease | undefined {
let url: URL;
try {
url = new URL(input);
} catch {
return undefined;
}
if (url.protocol !== "https:" || url.username || url.password) {
return undefined;
}
// GitHub instances are served from the root of their origin.
const isCurrentInstance = url.origin === new URL(apiDetails.url).origin;
if (!isCurrentInstance && url.origin !== new URL(GITHUB_DOTCOM_URL).origin) {
return undefined;
}
// Older links to bundle releases omit "tag/", which GitHub still supports. We only accept this
// form for bundle tags, since other names can clash with routes like `releases/latest`.
const match = url.pathname
.replace(/\/$/, "")
.match(
/^\/([\w.-]+)\/([\w.-]+)\/releases\/(?:tag\/(.+)|(codeql-bundle-[^/]+))$/,
);
if (match === null) {
return undefined;
}
let tagName: string;
try {
tagName = decodeURIComponent(match[3] ?? match[4]);
} catch {
throw new ConfigurationError(
"Invalid URL encoding in the CodeQL release tag.",
);
}
return {
serverURL: url.origin,
isCurrentInstance,
owner: match[1],
repo: match[2],
tagName,
};
}
function parseCliVersion(value: string): string | undefined {
const parsed = semver.parse(value);
if (parsed === null) {
return undefined;
}
// semver's normalized version omits build metadata, which distinguishes nightly builds.
return (
parsed.version + (parsed.build.length ? `+${parsed.build.join(".")}` : "")
);
}
/**
* Returns the CLI version from the release's `cli-version-<version>.txt` asset if it has one, and
* otherwise the version in the tag. Returns `undefined` if the release has conflicting markers.
*/
export function getReleaseCliVersion(
tagName: string,
assetNames: string[],
logger: Logger,
): string | undefined {
const versions = new Set<string>();
for (const name of assetNames) {
const match = name.match(/^cli-version-(.+)\.txt$/);
if (match === null) {
continue;
}
const version = parseCliVersion(match[1]);
if (version !== undefined) {
versions.add(version);
} else {
logger.debug(`Ignoring invalid CLI version marker ${name}.`);
}
}
if (versions.size > 1) {
logger.warning(
`Release ${tagName} has conflicting CLI version markers. Using a combined CodeQL bundle.`,
);
return undefined;
}
if (versions.size === 1) {
return versions.values().next().value;
}
return parseCliVersion(tagName.replace(/^codeql-bundle-/, ""));
}
/**
* Encodes a tag for use in a URL path. Slashes stay as path separators, as in GitHub's release URLs
* for tags like `build/123`.
*/
function encodeTag(tagName: string): string {
return tagName.split("/").map(encodeURIComponent).join("/");
}
function getReleasePageURL(reference: CodeQLReleaseReference): string {
const { serverURL, owner, repo, tagName } = reference;
return `${serverURL}/${owner}/${repo}/releases/tag/${encodeTag(tagName)}`;
}
/**
* Looks up a release on the current GitHub instance, which works for private repositories.
*
* The API client determines which instance we query, so `reference.serverURL` is only used for the
* release page URL in messages. The asset URLs are REST API endpoints, which accept the token and
* return the file when requested with `Accept: application/octet-stream`, unlike browser download
* URLs.
*/
export async function getRelease(
{ apiClient }: ActionState<["Api"]>,
reference: CodeQLReleaseReference,
): Promise<CodeQLRelease> {
const { owner, repo, tagName } = reference;
const { data: release } = await apiClient.rest.repos.getReleaseByTag({
owner,
repo,
tag: tagName,
});
return {
url: getReleasePageURL(reference),
getAssetURL: (name) =>
release.assets.find((asset) => asset.name === name)?.url,
assetNames: release.assets.map((asset) => asset.name),
};
}
/**
* Refers to a public release without looking it up. Every asset gets a download URL, so a missing
* asset shows up as a failed download.
*/
export function getPublicRelease(
reference: CodeQLReleaseReference,
): CodeQLRelease {
const { serverURL, owner, repo, tagName } = reference;
return {
url: getReleasePageURL(reference),
getAssetURL: (name) =>
`${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}`,
};
}
/** Gets a release requested by URL, looking it up if it's on the current GitHub instance. */
export async function getRequestedRelease(
action: ActionState<["Api"]>,
requested: RequestedRelease,
): Promise<CodeQLRelease> {
if (!requested.isCurrentInstance) {
// We can't send the token to another instance, so only public releases are usable there, and we
// download them by URL rather than querying that instance's API. On the current instance we use
// the API instead, because release download URLs don't accept tokens and so would fail for
// private and internal repositories. Without the release's asset list, we assume the preferred
// compression is available and take the CLI version from the tag.
return getPublicRelease(requested);
}
try {
return await getRelease(action, requested);
} catch (e) {
if (asHTTPError(e)?.status === 404) {
throw new ConfigurationError(
`Could not find the CodeQL release ${getReleasePageURL(requested)}. Check that it ` +
"exists and that the token has access to it.",
);
}
throw e;
}
}
/** Describes the job and runner that we are selecting a bundle for. */
export interface BundleSelectionOptions {
/** Explicit `languages` input, which determines whether a per-language bundle is eligible. */
rawLanguages: string[] | undefined;
/** The CLI version in the release, if known. */
cliVersion: string | undefined;
platform: BundlePlatform | undefined;
variant: GitHubVariant;
tarSupportsZstd: boolean;
/** Whether the release is the latest nightly, whose CLI version we don't know yet. */
isLatestNightly?: boolean;
}
/** A bundle selected from a release. */
export interface BundleSelection {
bundle: CodeQLBundle;
compressionMethod: CompressionMethod;
/** The release lacks the eligible per-language bundle, so we selected the combined bundle. */
perLanguageBundleFallback?: true;
}
/** Returns the compression methods that we can extract, most preferred first. */
function getCompressionMethods({
cliVersion,
isLatestNightly,
platform,
tarSupportsZstd,
}: BundleSelectionOptions): CompressionMethod[] {
if (!tarSupportsZstd) {
return ["gzip"];
}
const preferZstd =
// In testing, gzip performs better than zstd on Windows.
platform !== BundlePlatform.Win64 &&
// Standard bundles have zstd archives from this version, and so does the latest nightly. For a
// release we looked up, gzip comes next if it lacks the zstd archive.
(isLatestNightly ||
(cliVersion !== undefined &&
semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE)));
return preferZstd ? ["zstd", "gzip"] : ["gzip", "zstd"];
}
/**
* Selects a per-language bundle if the job is eligible for one, and otherwise the combined bundle.
* We only select a per-language bundle from a release that also has a combined bundle with the same
* compression, so that we can fall back to it.
*
* If we looked up the release, we can tell that an eligible per-language bundle is missing, so we
* warn and select the combined bundle straight away. A public release gives every asset a URL, so a
* missing per-language bundle only shows up as a 404 when downloading, which is when we fall back.
*
* Throws a `ConfigurationError` if the release has no combined bundle that we can extract.
*/
export async function selectBundle(
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
release: CodeQLRelease,
options: BundleSelectionOptions,
): Promise<BundleSelection> {
const { logger } = action;
const compressionMethods = getCompressionMethods(options);
for (const compressionMethod of compressionMethods) {
const combinedBundleName = getCodeQLBundleName(
compressionMethod,
options.platform,
);
const combinedBundleURL = release.getAssetURL(combinedBundleName);
if (combinedBundleURL === undefined) {
continue;
}
const combined: BundleSelection = {
bundle: { kind: "combined", url: combinedBundleURL },
compressionMethod,
};
const language = await getPerLanguageBundleLanguage(action, {
...options,
compressionMethod,
});
if (language === undefined) {
logger.info(
`Selected CodeQL bundle ${combinedBundleName} from ${release.url}.`,
);
return combined;
}
const name = getCodeQLBundleName(
compressionMethod,
options.platform,
language,
);
const url = release.getAssetURL(name);
if (url === undefined) {
logMissingPerLanguageBundle(action, language, release.url);
return { ...combined, perLanguageBundleFallback: true };
}
logger.info(`Selected CodeQL bundle ${name} from ${release.url}.`);
return {
bundle: { kind: "per-language", url, language, combinedBundleURL },
compressionMethod,
};
}
throw new ConfigurationError(
`No compatible CodeQL bundle was found in release ${release.url}. Expected ${compressionMethods
.map((method) => getCodeQLBundleName(method, options.platform))
.join(" or ")}.`,
);
}

View File

@@ -132,15 +132,15 @@ export async function getPerLanguageBundleLanguage(
return language;
}
/** Explains why an eligible per-language bundle is being replaced by a combined bundle. */
export function logPerLanguageBundleFallback(
/** Logs that the selected per-language bundle is missing, so we're using the combined bundle. */
export function logMissingPerLanguageBundle(
{ logger }: ActionState<["Logger"]>,
language: BuiltInLanguage,
location: string,
): void {
logger.warning(
`No per-language CodeQL bundle for '${language}' was found at ${location}, so ` +
"falling back to the bundle that contains all languages. This analysis will still " +
"produce correct results, but will take longer to set up.",
`Expected a per-language CodeQL bundle for '${language}' at ${location}, but it wasn't ` +
"found, so using the bundle that contains all languages instead. This analysis will " +
"still produce correct results, but will take longer to set up.",
);
}

View File

@@ -38,6 +38,7 @@ import {
} from "./testing-utils";
import * as toolsDownload from "./tools-download";
import {
ConfigurationError,
getErrorMessage,
GitHubVariant,
HTTPError,
@@ -428,7 +429,7 @@ test.serial(
false, // useOverlayAwareDefaultCliVersion
SAMPLE_DOTCOM_API_DETAILS,
GitHubVariant.DOTCOM,
false,
true, // tarSupportsZstd
features,
logger,
);
@@ -498,7 +499,7 @@ test.serial(
false, // useOverlayAwareDefaultCliVersion
SAMPLE_DOTCOM_API_DETAILS,
GitHubVariant.DOTCOM,
false,
true, // tarSupportsZstd
features,
logger,
);
@@ -1214,9 +1215,72 @@ test.serial(
},
);
for (const fallback of [false, true]) {
for (const status of [200, 404]) {
test.serial(
`setupCodeQLBundle retains the selected release identity for an opaque asset URL${fallback ? " with fallback" : ""}`,
`getCodeQLSource falls back to the public release when the Action's release ${status === 200 ? "lacks a compatible bundle" : "can't be found"}`,
async (t) => {
sinon.stub(process, "platform").value("linux");
sinon.stub(process, "arch").value("x64");
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
const tag = PER_LANGUAGE_CLI_VERSION.enabledVersions[0].tagName;
const fetchRelease = sinon
.stub<Parameters<typeof fetch>, ReturnType<typeof fetch>>()
.callsFake(
async () =>
new Response(
JSON.stringify({
assets: [
{
name: "codeql-bundle-java-linux64.tar.zst",
url: "https://api.github.com/repos/codeql-testing/action-fork/releases/assets/1",
},
],
}),
{ status, headers: { "content-type": "application/json" } },
),
);
const client = github.getOctokit("123", {
request: { fetch: fetchRelease },
});
sinon.stub(api, "getApiClient").value(() => client);
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir, {
GITHUB_ACTION_REPOSITORY: "codeql-testing/action-fork",
});
const source = await setupCodeql.getCodeQLSource(
undefined,
PER_LANGUAGE_CLI_VERSION,
undefined, // rawLanguages
false, // useOverlayAwareDefaultCliVersion
SAMPLE_DOTCOM_API_DETAILS,
GitHubVariant.DOTCOM,
true, // tarSupportsZstd
createFeatures([]),
getRunnerLogger(true),
);
t.true(fetchRelease.calledOnce);
t.like(source, {
sourceType: "download",
bundle: {
kind: "combined",
url: `https://github.com/github/codeql-action/releases/download/${tag}/codeql-bundle-linux64.tar.zst`,
},
});
});
},
);
}
for (const scenario of ["per-language", "fallback", "missing"] as const) {
const suffix = {
"per-language": "",
fallback: " with fallback",
missing: " when the release lacks the per-language bundle",
}[scenario];
test.serial(
`setupCodeQLBundle retains the selected release identity for an opaque asset URL${suffix}`,
async (t) => {
sinon.stub(process, "platform").value("linux");
sinon.stub(process, "arch").value("x64");
@@ -1230,22 +1294,22 @@ for (const fallback of [false, true]) {
const assetURL =
"https://api.github.com/repos/codeql-testing/action-fork/releases/assets/123";
const combinedURL = `${assetURL}4`;
const assets = [
{ name: "codeql-bundle-linux64.tar.zst", url: combinedURL },
];
if (scenario !== "missing") {
assets.push({
name: "codeql-bundle-java-linux64.tar.zst",
url: assetURL,
});
}
const fetchRelease = sinon
.stub<Parameters<typeof fetch>, ReturnType<typeof fetch>>()
.callsFake(
async () =>
new Response(
JSON.stringify({
assets: [
{ name: "codeql-bundle-java-linux64.tar.zst", url: assetURL },
{
name: "codeql-bundle-linux64.tar.zst",
url: combinedURL,
},
],
}),
{ headers: { "content-type": "application/json" } },
),
new Response(JSON.stringify({ assets }), {
headers: { "content-type": "application/json" },
}),
);
const client = github.getOctokit("123", {
request: { fetch: fetchRelease },
@@ -1253,7 +1317,7 @@ for (const fallback of [false, true]) {
sinon.stub(api, "getApiClient").value(() => client);
const authorizationSpy = sinon.spy(api, "getAuthorizationHeaderFor");
const extractStub = stubDownloadAndExtract();
if (fallback) {
if (scenario === "fallback") {
extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404));
}
@@ -1273,47 +1337,407 @@ for (const fallback of [false, true]) {
getRunnerLogger(true),
);
t.true(fetchRelease.calledTwice);
const usesPerLanguageBundle = scenario === "per-language";
t.true(fetchRelease.calledOnce);
t.is(
fetchRelease.firstCall.args[0],
`https://api.github.com/repos/codeql-testing/action-fork/releases/tags/${tag}`,
);
t.is(extractStub.callCount, fallback ? 2 : 1);
t.is(extractStub.firstCall.args[0], assetURL);
t.is(extractStub.lastCall.args[0], fallback ? combinedURL : assetURL);
t.is(authorizationSpy.callCount, extractStub.callCount);
t.is(authorizationSpy.firstCall.args[2], assetURL);
t.is(extractStub.callCount, scenario === "fallback" ? 2 : 1);
t.is(
authorizationSpy.lastCall.args[2],
fallback ? combinedURL : assetURL,
extractStub.firstCall.args[0],
scenario === "missing" ? combinedURL : assetURL,
);
t.is(
extractStub.lastCall.args[0],
usesPerLanguageBundle ? assetURL : combinedURL,
);
t.deepEqual(
authorizationSpy.getCalls().map((call) => call.args[2]),
extractStub.getCalls().map((call) => call.args[0]),
);
t.is(extractStub.lastCall.args[3], "token token");
t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION);
t.is(
result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language,
fallback ? undefined : BuiltInLanguage.java,
usesPerLanguageBundle ? BuiltInLanguage.java : undefined,
);
t.is(
result.toolsDownloadStatusReport?.perLanguage
?.tools_per_language_bundle_fallback,
fallback ? true : undefined,
usesPerLanguageBundle ? undefined : true,
);
if (fallback) {
if (usesPerLanguageBundle) {
t.is(path.dirname(result.codeqlFolder), tmpDir);
t.deepEqual(toolcache.findAllVersions("CodeQL"), []);
t.false(fs.existsSync(`${result.codeqlFolder}.complete`));
} else {
t.is(
result.codeqlFolder,
toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION),
);
t.true(fs.existsSync(`${result.codeqlFolder}.complete`));
} else {
t.is(path.dirname(result.codeqlFolder), tmpDir);
t.deepEqual(toolcache.findAllVersions("CodeQL"), []);
t.false(fs.existsSync(`${result.codeqlFolder}.complete`));
}
});
},
);
}
const GHES_API_DETAILS = {
auth: "enterprise-token",
url: "https://github.example.test",
apiURL: "https://github.example.test/api/v3",
};
/** Models a hosted Linux runner with zstd and a release in `repository` on the instance `apiDetails` describes. */
function stubRequestedRelease({
apiDetails = SAMPLE_DOTCOM_API_DETAILS,
repository = "octo/tools",
tagName = `codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`,
assetNames = [
"codeql-bundle-linux64.tar.zst",
"codeql-bundle-java-linux64.tar.zst",
],
markers = [] as string[],
status = 200,
} = {}) {
sinon.stub(process, "platform").value("linux");
sinon.stub(process, "arch").value("x64");
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
sinon.stub(tar, "isZstdAvailable").resolves({
available: true,
foundZstdBinary: true,
});
const apiBase = `${apiDetails.apiURL}/repos/${repository}/releases`;
const assets = [
...assetNames,
...markers.map((version) => `cli-version-${version}.txt`),
].map((name, index) => ({ name, url: `${apiBase}/assets/${1000 + index}` }));
const requests: string[] = [];
const client = github.getOctokit("123", {
baseUrl: apiDetails.apiURL,
request: {
fetch: async (input) => {
const url = String(input);
requests.push(url);
if (url !== `${apiBase}/tags/${tagName}`) {
throw new Error(`Unexpected API request: ${url}`);
}
return new Response(JSON.stringify({ tag_name: tagName, assets }), {
status,
headers: { "content-type": "application/json" },
});
},
},
});
sinon.stub(api, "getApiClient").value(() => client);
return {
assets,
requests,
releaseURL: `${apiDetails.url}/${repository}/releases/tag/${tagName}`,
};
}
for (const scenario of ["combined", "per-language", "fallback"] as const) {
test.serial(
`setupCodeQLBundle downloads a ${scenario} bundle from a custom release without using the toolcache`,
async (t) => {
const fixture = stubRequestedRelease();
const extract = stubDownloadAndExtract();
if (scenario === "fallback") {
extract.onFirstCall().rejects(new HTTPError("Not Found", 404));
}
const find = sinon.spy(toolcache, "find");
sinon.stub(actionsUtil, "isDynamicWorkflow").returns(true);
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
createToolcacheEntry(
tmpDir,
"CodeQL",
MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
);
const result = await setupCodeql.setupCodeQLBundle(
fixture.releaseURL,
SAMPLE_DOTCOM_API_DETAILS,
tmpDir,
GitHubVariant.DOTCOM,
PER_LANGUAGE_CLI_VERSION,
scenario === "combined" ? undefined : ["java"],
false, // useOverlayAwareDefaultCliVersion
// The requested release takes precedence over forcing the nightly.
createFeatures([Feature.PerLanguageBundles, Feature.ForceNightly]),
getRunnerLogger(true),
);
t.is(fixture.requests.length, 1);
t.true(find.notCalled);
t.is(result.toolsSource, setupCodeql.ToolsSource.Download);
t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION);
t.is(extract.callCount, scenario === "fallback" ? 2 : 1);
t.is(
extract.lastCall.args[0],
fixture.assets[scenario === "per-language" ? 1 : 0].url,
);
t.is(extract.lastCall.args[3], "token token");
t.is(path.dirname(result.codeqlFolder), tmpDir);
t.false(fs.existsSync(`${result.codeqlFolder}.complete`));
});
},
);
}
test.serial(
"setupCodeQLBundle downloads the gzip bundle from a requested release that only has gzip bundles",
async (t) => {
const fixture = stubRequestedRelease({
assetNames: ["codeql-bundle-linux64.tar.gz"],
});
const extract = stubDownloadAndExtract();
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const result = await setupCodeql.setupCodeQLBundle(
fixture.releaseURL,
SAMPLE_DOTCOM_API_DETAILS,
tmpDir,
GitHubVariant.DOTCOM,
PER_LANGUAGE_CLI_VERSION,
["java"],
false, // useOverlayAwareDefaultCliVersion
createFeatures([Feature.PerLanguageBundles]),
getRunnerLogger(true),
);
t.true(extract.calledOnce);
t.is(extract.firstCall.args[0], fixture.assets[0].url);
t.is(extract.firstCall.args[1], "gzip");
t.is(result.toolsSource, setupCodeql.ToolsSource.Download);
t.is(result.toolsDownloadStatusReport?.perLanguage, undefined);
});
},
);
for (const [apiDetails, variant] of [
[SAMPLE_DOTCOM_API_DETAILS, GitHubVariant.DOTCOM],
[GHES_API_DETAILS, GitHubVariant.GHES],
] as const) {
test.serial(
`setupCodeQLBundle caches a stable release in the CodeQL Action repository on ${variant}`,
async (t) => {
const fixture = stubRequestedRelease({
apiDetails,
repository: "github/codeql-action",
});
const extract = stubDownloadAndExtract();
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const setup = () =>
setupCodeql.setupCodeQLBundle(
fixture.releaseURL,
apiDetails,
tmpDir,
variant,
PER_LANGUAGE_CLI_VERSION,
undefined, // rawLanguages
false, // useOverlayAwareDefaultCliVersion
createFeatures([]),
getRunnerLogger(true),
);
const downloaded = await setup();
t.deepEqual(fixture.requests, [
`${apiDetails.apiURL}/repos/github/codeql-action/releases/tags/codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`,
]);
t.is(downloaded.toolsSource, setupCodeql.ToolsSource.Download);
t.is(extract.firstCall.args[0], fixture.assets[0].url);
t.is(extract.firstCall.args[3], `token ${apiDetails.auth}`);
t.is(
downloaded.codeqlFolder,
toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION),
);
const cached = await setup();
t.is(cached.toolsSource, setupCodeql.ToolsSource.Toolcache);
t.is(cached.codeqlFolder, downloaded.codeqlFolder);
t.true(extract.calledOnce);
});
},
);
}
for (const { repository, tagName, markers } of [
{
repository: "github/codeql-action",
tagName: "codeql-bundle-v2.27.1-rc.1",
markers: [],
},
{
repository: "github/codeql-action",
tagName: "codeql-bundle-v2.27.1",
markers: ["2.27.1+202609241200"],
},
{
repository: "github/codeql-action",
tagName: "codeql-bundle-20260924",
markers: ["2.27.1+202609241200"],
},
{
repository: "github/codeql-action",
tagName: "codeql-bundle-acme-2.27.1",
markers: ["2.27.1"],
},
{
repository: "octo/tools",
tagName: "codeql-bundle-feature_branch",
markers: [],
},
]) {
const description =
markers.length === 0
? tagName
: `${tagName} with marker ${markers.join(", ")}`;
test.serial(
`setupCodeQLBundle doesn't share the toolcache with ${description} in ${repository}`,
async (t) => {
const fixture = stubRequestedRelease({ repository, tagName, markers });
const extract = stubDownloadAndExtract();
const find = sinon.spy(toolcache, "find");
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
createToolcacheEntry(
tmpDir,
"CodeQL",
MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
);
const result = await setupCodeql.setupCodeQLBundle(
fixture.releaseURL,
SAMPLE_DOTCOM_API_DETAILS,
tmpDir,
GitHubVariant.DOTCOM,
PER_LANGUAGE_CLI_VERSION,
undefined, // rawLanguages
false, // useOverlayAwareDefaultCliVersion
createFeatures([]),
getRunnerLogger(true),
);
t.true(find.notCalled);
t.is(result.toolsSource, setupCodeql.ToolsSource.Download);
t.is(extract.firstCall.args[0], fixture.assets[0].url);
t.is(path.dirname(result.codeqlFolder), tmpDir);
t.false(fs.existsSync(`${result.codeqlFolder}.complete`));
});
},
);
}
for (const repository of ["github/codeql-action", "octo/tools"]) {
test.serial(
`setupCodeQLBundle downloads a GitHub.com release in ${repository} from GHES by URL without credentials`,
async (t) => {
const fixture = stubRequestedRelease({ repository });
const extract = stubDownloadAndExtract();
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const result = await setupCodeql.setupCodeQLBundle(
fixture.releaseURL,
GHES_API_DETAILS,
tmpDir,
GitHubVariant.GHES,
PER_LANGUAGE_CLI_VERSION,
["java"],
false, // useOverlayAwareDefaultCliVersion
createFeatures([Feature.PerLanguageBundles]),
getRunnerLogger(true),
);
t.deepEqual(fixture.requests, []);
t.true(extract.calledOnce);
t.is(
extract.firstCall.args[0],
`https://github.com/${repository}/releases/download/codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}/codeql-bundle-linux64.tar.zst`,
);
t.is(extract.firstCall.args[3], undefined);
t.is(
fs.existsSync(`${result.codeqlFolder}.complete`),
repository === "github/codeql-action",
);
});
},
);
}
test.serial(
"setupCodeQLBundle doesn't substitute another release for a requested release that can't be found",
async (t) => {
const fixture = stubRequestedRelease({ status: 404 });
const extract = stubDownloadAndExtract();
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
await t.throwsAsync(
setupCodeql.setupCodeQLBundle(
fixture.releaseURL,
SAMPLE_DOTCOM_API_DETAILS,
tmpDir,
GitHubVariant.DOTCOM,
PER_LANGUAGE_CLI_VERSION,
undefined, // rawLanguages
false, // useOverlayAwareDefaultCliVersion
createFeatures([]),
getRunnerLogger(true),
),
{
instanceOf: ConfigurationError,
message: `Could not find the CodeQL release ${fixture.releaseURL}. Check that it exists and that the token has access to it.`,
},
);
t.is(fixture.requests.length, 1);
t.true(extract.notCalled);
});
},
);
test.serial(
"setupCodeQLBundle uses the CLI version marker of a requested release",
async (t) => {
const fixture = stubRequestedRelease({
tagName: "run-123",
markers: [MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION],
});
const extract = stubDownloadAndExtract();
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const result = await setupCodeql.setupCodeQLBundle(
fixture.releaseURL,
SAMPLE_DOTCOM_API_DETAILS,
tmpDir,
GitHubVariant.DOTCOM,
PER_LANGUAGE_CLI_VERSION,
["java"],
false, // useOverlayAwareDefaultCliVersion
createFeatures([Feature.PerLanguageBundles]),
getRunnerLogger(true),
);
// The tag doesn't give a version, so the job can only use the per-language bundle because of
// the marker.
t.is(extract.firstCall.args[0], fixture.assets[1].url);
t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION);
t.is(
result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language,
BuiltInLanguage.java,
);
});
},
);
for (const bundle of ["per-language", "combined", "fallback"] as const) {
test.serial(
`setupCodeQLBundle preserves the nightly version for a ${bundle} download`,
@@ -1420,9 +1844,10 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) {
t.deepEqual(downloadSpy.secondCall.args[0], {
...source,
bundle: { kind: "combined", url: combinedURL },
perLanguageBundleFallback: true,
});
checkExpectedLogMessages(t, loggedMessages, [
`No per-language CodeQL bundle for 'javascript' was found at ${perLanguageURL}`,
`Expected a per-language CodeQL bundle for 'javascript' at ${perLanguageURL}`,
]);
}
if (bundle === "per-language") {

View File

@@ -21,8 +21,19 @@ import {
CodeQLBundle,
CodeQLDownloadSource,
getCodeQLBundleFromUrl,
getCodeQLBundleName,
} from "./codeql-bundle";
import {
BundleSelection,
BundleSelectionOptions,
CodeQLRelease,
CodeQLReleaseReference,
getPublicRelease,
getRelease,
getReleaseCliVersion,
getRequestedRelease,
parseCodeQLReleaseUrl,
selectBundle,
} from "./codeql-release";
import * as defaults from "./defaults.json";
import {
addNoLanguageDiagnostic,
@@ -31,19 +42,14 @@ import {
} from "./diagnostics";
import { EnvVar, getEnv } from "./environment";
import {
CODEQL_VERSION_ZSTD_BUNDLE,
CodeQLDefaultVersionInfo,
CodeQLVersionInfo,
Feature,
FeatureEnablement,
} from "./feature-flags";
import { BuiltInLanguage } from "./languages";
import { Logger } from "./logging";
import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching";
import {
getPerLanguageBundleLanguage,
logPerLanguageBundleFallback,
} from "./per-language-bundles";
import { logMissingPerLanguageBundle } from "./per-language-bundles";
import { getBundlePlatform } from "./platform";
import * as tar from "./tar";
import {
@@ -88,14 +94,13 @@ export function getCodeQLActionRepository(logger: Logger): string {
return util.getRequiredEnvParam("GITHUB_ACTION_REPOSITORY");
}
async function getCodeQLBundleDownloadURL(
tagName: string,
/** Returns the repositories that release the default bundles, most preferred first. */
function getDefaultBundleSources(
apiDetails: api.GitHubApiDetails,
codeQLBundleName: string,
logger: Logger,
): Promise<string> {
): Array<[serverURL: string, repository: string]> {
const codeQLActionRepository = getCodeQLActionRepository(logger);
const potentialDownloadSources = [
const potentialDownloadSources: Array<[string, string]> = [
// This GitHub instance, and this Action.
[apiDetails.url, codeQLActionRepository],
// This GitHub instance, and the canonical Action.
@@ -105,42 +110,85 @@ async function getCodeQLBundleDownloadURL(
];
// We now filter out any duplicates.
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
const uniqueDownloadSources = potentialDownloadSources.filter(
(source, index, self) => {
return !self.slice(0, index).some((other) => deepEqual(source, other));
},
return potentialDownloadSources.filter((source, index, self) => {
return !self.slice(0, index).some((other) => deepEqual(source, other));
});
}
/**
* Whether a release requested by URL contains the same build as the toolcache entry for its CLI
* version. The toolcache is keyed by version alone, so we only assume this for stable releases
* tagged `codeql-bundle-v<version>` in the repositories we download the default bundles from.
*/
function isCacheableRelease(
reference: CodeQLReleaseReference,
cliVersion: string | undefined,
apiDetails: api.GitHubApiDetails,
logger: Logger,
): boolean {
if (
cliVersion === undefined ||
!/^\d+\.\d+\.\d+$/.test(cliVersion) ||
reference.tagName !== `codeql-bundle-v${cliVersion}`
) {
return false;
}
const repository = `${reference.owner}/${reference.repo}`.toLowerCase();
return getDefaultBundleSources(apiDetails, logger).some(
([serverURL, sourceRepository]) =>
new URL(serverURL).origin === new URL(reference.serverURL).origin &&
sourceRepository.toLowerCase() === repository,
);
for (const downloadSource of uniqueDownloadSources) {
const [apiURL, repository] = downloadSource;
}
/**
* Selects a bundle from the first release tagged `tagName` that has a compatible bundle, trying the
* Action repositories on this GitHub instance before the canonical Action on GitHub.com. If we
* can't look up a release, or it has no compatible bundle, we move on to the next repository. We
* assume that the public release on GitHub.com has every bundle.
*/
async function selectDefaultBundle(
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
tagName: string,
apiDetails: api.GitHubApiDetails,
options: BundleSelectionOptions,
): Promise<BundleSelection> {
const { logger } = action;
for (const [serverURL, repository] of getDefaultBundleSources(
apiDetails,
logger,
)) {
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
if (
apiURL === util.GITHUB_DOTCOM_URL &&
serverURL === util.GITHUB_DOTCOM_URL &&
repository === CODEQL_DEFAULT_ACTION_REPOSITORY
) {
break;
}
const [repositoryOwner, repositoryName] = repository.split("/");
const [owner, repo] = repository.split("/");
try {
const release = await api.getApiClient().rest.repos.getReleaseByTag({
owner: repositoryOwner,
repo: repositoryName,
tag: tagName,
});
for (const asset of release.data.assets) {
if (asset.name === codeQLBundleName) {
logger.info(
`Found CodeQL bundle ${codeQLBundleName} in ${repository} on ${apiURL} with URL ${asset.url}.`,
);
return asset.url;
}
}
const release = await getRelease(
{ apiClient: api.getApiClient() },
{ serverURL, owner, repo, tagName },
);
return await selectBundle(action, release, options);
} catch (e) {
logger.info(
`Looked for CodeQL bundle ${codeQLBundleName} in ${repository} on ${apiURL} but got error ${e}.`,
`Looked for CodeQL bundles in release ${tagName} of ${repository} on ${serverURL} but got error ${e}.`,
);
}
}
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
const [owner, repo] = CODEQL_DEFAULT_ACTION_REPOSITORY.split("/");
return selectBundle(
action,
getPublicRelease({
serverURL: util.GITHUB_DOTCOM_URL,
owner,
repo,
tagName,
}),
options,
);
}
function tryGetBundleVersionFromTagName(
@@ -182,6 +230,12 @@ export function tryGetTagNameFromUrl(
return match[1];
}
/**
* Converts a bundle version to a semantic version, for example to use in the toolcache. Semantic
* versions are normalized with `semver.clean`, which drops a leading `v` and any build metadata.
* Anything else, such as a date, becomes a prerelease of `0.0.0`. Throws if the result isn't a
* valid semantic version.
*/
export function convertToSemVer(version: string, logger: Logger): string {
if (!semver.valid(version)) {
logger.debug(
@@ -374,6 +428,26 @@ async function resolveDefaultCliVersion(
* Determines where the CodeQL CLI we want to use comes from. This can be from a local file,
* the Actions toolcache, or a download.
*
* We handle the `tools` input in this order:
*
* - A local path is extracted without using the toolcache.
* - A release URL selects a bundle from that release, and takes precedence over the `force_nightly`
* feature flag. Only stable releases in the repositories we download the default bundles from use
* the toolcache, since other releases may contain a different build than the cached bundle for
* their version. Bundle URLs keep using the toolcache for the version in their tag, for
* compatibility.
* - `nightly` or `nightly-latest`, or the `force_nightly` feature flag in a dynamic workflow,
* selects a bundle from the latest nightly release. We then continue with that bundle's URL.
* - `linked`, or its old name `latest`, selects the version shipped with the Action.
* - `toolcache` selects the latest version in the toolcache, falling back to the default version
* outside dynamic workflows or if there isn't one.
* - Any other value is the URL of a bundle.
* - Without a `tools` input, we use the default version.
*
* For other inputs apart from a local path, we look for the resolved version in the toolcache
* before downloading. A cached version takes precedence even if the job could use a per-language
* bundle.
*
* @param toolsInput The argument provided for the `tools` input, if any.
* @param defaultCliVersion The default CLI version that's linked to the CodeQL Action.
* @param rawLanguages Raw set of languages.
@@ -421,6 +495,11 @@ export async function getCodeQLSource(
};
}
const requestedRelease =
toolsInput === undefined
? undefined
: parseCodeQLReleaseUrl(toolsInput, apiDetails);
/** Requested CLI version number, for example 2.12.6. */
let cliVersion: string | undefined;
/** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */
@@ -432,10 +511,17 @@ export async function getCodeQLSource(
*/
let url: string | undefined;
let bundle: CodeQLBundle | undefined;
/** The release requested by URL, which we select the bundle from. */
let release: CodeQLRelease | undefined;
/** The page of a requested release whose bundles we don't cache. */
let customReleaseURL: string | undefined;
// We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the
// `tools` input cannot be adjusted to explicitly request it.
const canForceNightlyWithFF = isDynamicWorkflow() || util.isInTestMode();
// `tools` input cannot be adjusted to explicitly request it. An explicitly requested release
// takes precedence.
const canForceNightlyWithFF =
requestedRelease === undefined &&
(isDynamicWorkflow() || util.isInTestMode());
const forceNightlyValueFF = await features.getValue(Feature.ForceNightly);
const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
@@ -479,6 +565,7 @@ export async function getCodeQLSource(
{ env: getEnv(), features, logger },
rawLanguages,
variant,
tarSupportsZstd,
);
toolsInput = bundle.url;
}
@@ -555,8 +642,24 @@ export async function getCodeQLSource(
cliVersion = version.cliVersion;
tagName = version.tagName;
}
} else if (requestedRelease !== undefined) {
release = await getRequestedRelease(
{ apiClient: api.getApiClient() },
requestedRelease,
);
tagName = requestedRelease.tagName;
cliVersion = getReleaseCliVersion(
tagName,
release.assetNames ?? [],
logger,
);
if (!isCacheableRelease(requestedRelease, cliVersion, apiDetails, logger)) {
customReleaseURL = release.url;
}
} else if (toolsInput !== undefined) {
// If a tools URL was provided, then use that.
// Any other value is a bundle URL, including one we selected from the latest nightly above.
// We use the version in its tag, if any, for the toolcache, so we assume that bundles with the
// same version are the same build, whichever repository they're in.
tagName = tryGetTagNameFromUrl(toolsInput, logger);
url = toolsInput;
@@ -580,7 +683,8 @@ export async function getCodeQLSource(
}
const bundleVersion =
tagName !== undefined
// Custom releases aren't cached, and their tags needn't contain a bundle version.
tagName !== undefined && customReleaseURL === undefined
? tryGetBundleVersionFromTagName(tagName, logger)
: undefined;
const resolvedVersion =
@@ -597,13 +701,138 @@ export async function getCodeQLSource(
`URL: ${url ?? "unspecified"}.`,
);
// A custom release may contain a different build than the bundle with the same version.
const codeqlFolder =
customReleaseURL === undefined
? await findCodeQLInToolcache(
cliVersion,
tagName,
humanReadableVersion,
logger,
)
: undefined;
if (codeqlFolder) {
if (cliVersion) {
logger.info(
`Using CodeQL CLI version ${cliVersion} from toolcache at ${codeqlFolder}`,
);
} else {
logger.info(`Using CodeQL CLI from toolcache at ${codeqlFolder}`);
}
return {
codeqlFolder,
sourceType: "toolcache",
toolsVersion: cliVersion ?? humanReadableVersion,
};
}
// If we don't find the requested version on Enterprise, we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (
variant === util.GitHubVariant.GHES &&
!forceShippedTools &&
!toolsInput
) {
const result = await findOverridingToolsInCache(
humanReadableVersion,
logger,
);
if (result !== undefined) {
return result;
}
}
let compressionMethod: tar.CompressionMethod;
let perLanguageBundleFallback: true | undefined;
if (!url) {
const selectionOptions: BundleSelectionOptions = {
rawLanguages,
cliVersion,
platform: getBundlePlatform(),
variant,
tarSupportsZstd,
};
let selection: BundleSelection;
if (release !== undefined) {
selection = await selectBundle(
{ env: getEnv(), features, logger },
release,
selectionOptions,
);
} else {
if (tagName === undefined) {
throw new Error(
"Could not determine a release tag for the requested CodeQL bundle.",
);
}
selection = await selectDefaultBundle(
{ env: getEnv(), features, logger },
tagName,
apiDetails,
selectionOptions,
);
}
({ bundle, compressionMethod, perLanguageBundleFallback } = selection);
url = bundle.url;
} else {
const method = tar.inferCompressionMethod(url);
if (method === undefined) {
throw new util.ConfigurationError(
`Could not infer compression method from URL ${url}. Please specify a URL ` +
"ending in '.tar.gz' or '.tar.zst'.",
);
}
compressionMethod = method;
// Keep the bundle we selected from the latest nightly, which records the combined bundle to
// fall back to. Otherwise, classify the explicit URL, which gets no fallback.
bundle ??= getCodeQLBundleFromUrl(url);
if (bundle.kind === "per-language") {
logger.info(
`${url} appears to be a CodeQL bundle that contains only ${bundle.language}.`,
);
}
}
if (cliVersion) {
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url} .`);
} else {
logger.info(`Using CodeQL CLI sourced from ${url} .`);
}
return {
bundle,
bundleVersion,
cliVersion,
compressionMethod,
...(customReleaseURL !== undefined ? { customReleaseURL } : {}),
...(perLanguageBundleFallback ? { perLanguageBundleFallback } : {}),
sourceType: "download",
toolsVersion: resolvedVersion ?? "unknown",
};
}
/**
* Looks for the requested version of the CodeQL tools in the toolcache, allowing for the different
* version numbers that toolcaches may use for the same bundle. We try the exact CLI version, then a
* single `x.y.z-*` entry for that version, then `0.0.0-<bundle version>`. The `x.y.z-*` entry can
* be any prerelease of the CLI version.
*/
async function findCodeQLInToolcache(
cliVersion: string | undefined,
tagName: string | undefined,
humanReadableVersion: string,
logger: Logger,
): Promise<string | undefined> {
let codeqlFolder: string | undefined;
if (cliVersion) {
// If we find the specified CLI version, we always use that.
codeqlFolder = toolcache.find("CodeQL", cliVersion);
// Fall back to matching `x.y.z-<tagName>`.
// Fall back to a single `x.y.z-*` entry, since older toolcaches store bundles as
// `x.y.z-<bundle version>`.
if (!codeqlFolder) {
logger.debug(
"Didn't find a version of the CodeQL tools in the toolcache with a version number " +
@@ -615,8 +844,6 @@ export async function getCodeQLSource(
allVersions,
)}.`,
);
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is
// the form `x.y.z-<tagName>`, then use it.
const candidateVersions = allVersions.filter((version) =>
version.startsWith(`${cliVersion}-`),
);
@@ -667,123 +894,7 @@ export async function getCodeQLSource(
`Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.`,
);
}
if (codeqlFolder) {
if (cliVersion) {
logger.info(
`Using CodeQL CLI version ${cliVersion} from toolcache at ${codeqlFolder}`,
);
} else {
logger.info(`Using CodeQL CLI from toolcache at ${codeqlFolder}`);
}
return {
codeqlFolder,
sourceType: "toolcache",
toolsVersion: cliVersion ?? humanReadableVersion,
};
}
// If we don't find the requested version on Enterprise, we may allow a
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (
variant === util.GitHubVariant.GHES &&
!forceShippedTools &&
!toolsInput
) {
const result = await findOverridingToolsInCache(
humanReadableVersion,
logger,
);
if (result !== undefined) {
return result;
}
}
let compressionMethod: tar.CompressionMethod;
if (!url) {
const bundleTagName = tagName;
if (bundleTagName === undefined) {
throw new Error(
"Could not determine a release tag for the requested CodeQL bundle.",
);
}
compressionMethod =
cliVersion !== undefined &&
(await useZstdBundle(cliVersion, tarSupportsZstd))
? "zstd"
: "gzip";
const platform = getBundlePlatform();
const perLanguageBundleLanguage = await getPerLanguageBundleLanguage(
{ env: getEnv(), features, logger },
{
rawLanguages,
cliVersion,
compressionMethod,
platform,
variant,
},
);
// Resolves the combined or per-language bundle URL for the requested release.
const resolveBundleURL = (language?: BuiltInLanguage) =>
getCodeQLBundleDownloadURL(
bundleTagName,
apiDetails,
getCodeQLBundleName(compressionMethod, platform, language),
logger,
);
const combinedBundleURL = await resolveBundleURL();
if (perLanguageBundleLanguage !== undefined) {
logger.info(
`Selected the per-language CodeQL bundle for '${perLanguageBundleLanguage}'.`,
);
url = await resolveBundleURL(perLanguageBundleLanguage);
bundle = {
kind: "per-language",
url,
language: perLanguageBundleLanguage,
combinedBundleURL,
};
} else {
url = combinedBundleURL;
bundle = { kind: "combined", url };
}
} else {
const method = tar.inferCompressionMethod(url);
if (method === undefined) {
throw new util.ConfigurationError(
`Could not infer compression method from URL ${url}. Please specify a URL ` +
"ending in '.tar.gz' or '.tar.zst'.",
);
}
compressionMethod = method;
bundle ??= getCodeQLBundleFromUrl(url);
if (bundle.kind === "per-language") {
logger.info(
`${url} appears to be a CodeQL bundle that contains only ${bundle.language}.`,
);
}
}
if (cliVersion) {
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url} .`);
} else {
logger.info(`Using CodeQL CLI sourced from ${url} .`);
}
return {
bundle,
bundleVersion,
cliVersion,
compressionMethod,
sourceType: "download",
toolsVersion: resolvedVersion ?? "unknown",
};
return codeqlFolder;
}
/**
@@ -868,17 +979,32 @@ export const downloadCodeQL = async function (
...statusReport,
perLanguage: { tools_bundle_language: bundle.language },
}
: statusReport,
: source.perLanguageBundleFallback
? {
...statusReport,
perLanguage: { tools_per_language_bundle_fallback: true },
}
: statusReport,
};
};
/**
* Returns the canonical toolcache directory, or the reason the bundle cannot be cached.
*
* The toolcache is keyed by version, so we don't cache bundles that would give a later request for
* the same version the wrong tools: per-language bundles, which lack the other languages, and
* custom releases, which may be a different build.
*/
function getToolcacheDestination(
{ logger }: ActionState<["Logger"]>,
source: CodeQLDownloadSource,
): util.Result<string, string> {
if (source.customReleaseURL !== undefined) {
return new util.Failure(
`Not caching the CodeQL tools from ${source.customReleaseURL}, since we only cache stable ` +
"releases in the CodeQL Action repositories.",
);
}
if (source.bundle.kind !== "combined") {
return new util.Failure(
"Not caching the CodeQL tools because they came from a bundle that contains only a " +
@@ -969,11 +1095,11 @@ function getCanonicalToolcacheVersion(
bundleVersion: string,
logger: Logger,
): string {
// If the CLI version is a pre-release or contains build metadata, then cache the
// bundle as `0.0.0-<bundleVersion>` to avoid the bundle being interpreted as containing a stable
// CLI release. In principle, it should be enough to just check that the CLI version isn't a
// pre-release, but the version numbers of CodeQL nightlies have the format `x.y.z+<timestamp>`,
// and we don't want these nightlies to override stable CLI versions in the toolcache.
// If the CLI version is unknown, as for nightlies, which are tagged by date, or isn't a plain
// `x.y.z`, for example a prerelease, cache the bundle under its bundle version, such as
// `0.0.0-<date>` or `x.y.z-rc.1`, so that it isn't cached as a stable release. However,
// `convertToSemVer` drops build metadata, so a bundle URL tagged `codeql-bundle-vX.Y.Z+<build>`
// is still cached as `X.Y.Z`.
if (!cliVersion?.match(/^[0-9]+\.[0-9]+\.[0-9]+$/)) {
return convertToSemVer(bundleVersion, logger);
}
@@ -1113,12 +1239,13 @@ export async function downloadCodeQLBundle(
) {
throw e;
}
logPerLanguageBundleFallback(action, bundle.language, bundle.url);
logMissingPerLanguageBundle(action, bundle.language, bundle.url);
const result = await downloadCodeQL(
{
...source,
bundle: { kind: "combined", url: bundle.combinedBundleURL },
perLanguageBundleFallback: true,
},
apiDetails,
tarVersion,
@@ -1130,24 +1257,11 @@ export async function downloadCodeQLBundle(
statusReport: {
...result.statusReport,
totalDurationMs: util.durationMsSince(startTime),
perLanguage: { tools_per_language_bundle_fallback: true },
},
};
}
}
async function useZstdBundle(
cliVersion: string,
tarSupportsZstd: boolean,
): Promise<boolean> {
return (
// In testing, gzip performs better than zstd on Windows.
process.platform !== "win32" &&
tarSupportsZstd &&
semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE)
);
}
function getTempExtractionDir(tempDir: string) {
return path.join(tempDir, uuidV4());
}
@@ -1160,27 +1274,9 @@ async function getLatestNightlyBundle(
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
rawLanguages: string[] | undefined,
variant: util.GitHubVariant,
tarSupportsZstd: boolean,
): Promise<CodeQLBundle> {
const { logger } = action;
const zstdAvailability = await tar.isZstdAvailable(logger);
// The nightly is guaranteed to have a zstd bundle
const compressionMethod = (await useZstdBundle(
CODEQL_VERSION_ZSTD_BUNDLE,
zstdAvailability.available,
))
? "zstd"
: "gzip";
const platform = getBundlePlatform();
const language = await getPerLanguageBundleLanguage(action, {
rawLanguages,
cliVersion: undefined,
compressionMethod,
platform,
variant,
isLatestNightly: true,
});
let tagName: string;
try {
// Since nightlies are prereleases, we can't just download the latest release
// on the repository. So instead we need to find the latest pre-release
@@ -1196,26 +1292,31 @@ async function getLatestNightlyBundle(
if (!latestRelease) {
throw new Error("Could not find the latest nightly release.");
}
const assetUrl = (name: string) =>
`https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`;
const url = assetUrl(
getCodeQLBundleName(compressionMethod, platform, language),
);
return language === undefined
? { kind: "combined", url }
: {
kind: "per-language",
url,
language,
combinedBundleURL: assetUrl(
getCodeQLBundleName(compressionMethod, platform),
),
};
tagName = latestRelease.tag_name;
} catch (e) {
throw new Error(
`Failed to retrieve the latest nightly release: ${util.wrapError(e)}`,
);
}
const { bundle } = await selectBundle(
action,
getPublicRelease({
serverURL: util.GITHUB_DOTCOM_URL,
owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER,
repo: CODEQL_NIGHTLIES_REPOSITORY_NAME,
tagName,
}),
{
rawLanguages,
cliVersion: undefined,
platform: getBundlePlatform(),
variant,
tarSupportsZstd,
isLatestNightly: true,
},
);
return bundle;
}
/**