mirror of
https://github.com/github/codeql-action.git
synced 2026-11-04 20:20:23 +00:00
Compare commits
17 Commits
sam-robson
...
henrymerce
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9bd4b5ea8f | ||
|
|
c1e8d8d12c | ||
|
|
17f14786a5 | ||
|
|
0e574bc821 | ||
|
|
4cea925541 | ||
|
|
e8f5d72ffe | ||
|
|
4a4922e538 | ||
|
|
78a5dc251b | ||
|
|
db99327672 | ||
|
|
7f964a9e6e | ||
|
|
027ab3179c | ||
|
|
caa2cae2fb | ||
|
|
ffc453273e | ||
|
|
303b251f60 | ||
|
|
f03d2dfd0b | ||
|
|
6393cd425b | ||
|
|
6ec2b18519 |
@@ -4,7 +4,7 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
|
||||
|
||||
## [UNRELEASED]
|
||||
|
||||
No user facing changes.
|
||||
- The `tools` input to the `init` and `setup-codeql` actions now accepts GitHub release URLs. The Action selects a compatible bundle from the specified release, including a per-language bundle when eligible. [#4179](https://github.com/github/codeql-action/pull/4179)
|
||||
|
||||
## 4.38.2 - 24 Sept 2026
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ inputs:
|
||||
|
||||
- A local path to a CodeQL Bundle tarball, or
|
||||
- The URL of a CodeQL Bundle tarball GitHub release asset, or
|
||||
- A release URL from GitHub.com or the current GitHub instance, such as
|
||||
https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.27.0.
|
||||
- A special value `linked` which uses the version of the CodeQL tools
|
||||
that the Action has been bundled with.
|
||||
- A special value `nightly` which uses the latest nightly version of the
|
||||
|
||||
857
lib/entry-points.js
generated
857
lib/entry-points.js
generated
File diff suppressed because it is too large
Load Diff
@@ -10,6 +10,8 @@ inputs:
|
||||
|
||||
- A local path to a CodeQL Bundle tarball, or
|
||||
- The URL of a CodeQL Bundle tarball GitHub release asset, or
|
||||
- A release URL from GitHub.com or the current GitHub instance, such as
|
||||
https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.27.0.
|
||||
- A special value `linked` which uses the version of the CodeQL tools
|
||||
that the Action has been bundled with.
|
||||
- A special value `nightly` which uses the latest nightly version of the
|
||||
|
||||
@@ -27,6 +27,13 @@ export interface CodeQLDownloadSource {
|
||||
cliVersion?: string;
|
||||
/** Resolved version for telemetry, independent of whether the bundle can be cached. */
|
||||
toolsVersion: string;
|
||||
/** The release lacks the eligible per-language bundle, so we selected the combined bundle. */
|
||||
perLanguageBundleFallback?: true;
|
||||
/**
|
||||
* The page of a requested release that may contain a different build than the bundle we cache for
|
||||
* its version, so we don't cache it.
|
||||
*/
|
||||
customReleaseURL?: string;
|
||||
}
|
||||
|
||||
/** Returns the exact bundle asset name for a platform and optional language. */
|
||||
|
||||
354
src/codeql-release.test.ts
Normal file
354
src/codeql-release.test.ts
Normal file
@@ -0,0 +1,354 @@
|
||||
import * as github from "@actions/github";
|
||||
import test from "ava";
|
||||
|
||||
import {
|
||||
BundleSelectionOptions,
|
||||
getPublicRelease,
|
||||
getRelease,
|
||||
getReleaseCliVersion,
|
||||
getRequestedRelease,
|
||||
parseCodeQLReleaseUrl,
|
||||
selectBundle,
|
||||
} from "./codeql-release";
|
||||
import { ActionsEnvVars } from "./environment";
|
||||
import { Feature } from "./feature-flags";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { BundlePlatform } from "./platform";
|
||||
import {
|
||||
createFeatures,
|
||||
getRecordingLogger,
|
||||
getTestEnv,
|
||||
initAllState,
|
||||
LoggedMessage,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
} from "./testing-utils";
|
||||
import { ConfigurationError, GitHubVariant } from "./util";
|
||||
|
||||
const TAG = "codeql-bundle-v2.27.1";
|
||||
const REFERENCE = {
|
||||
serverURL: "https://github.com",
|
||||
owner: "octo",
|
||||
repo: "tools",
|
||||
tagName: TAG,
|
||||
};
|
||||
const RELEASE_PAGE = `https://github.com/octo/tools/releases/tag/${TAG}`;
|
||||
const API_URL = "https://api.github.com";
|
||||
const COMBINED = "codeql-bundle-linux64.tar.zst";
|
||||
const JAVA = "codeql-bundle-java-linux64.tar.zst";
|
||||
|
||||
const OPTIONS: BundleSelectionOptions = {
|
||||
rawLanguages: ["java-kotlin"],
|
||||
cliVersion: "2.27.1",
|
||||
platform: BundlePlatform.Linux64,
|
||||
variant: GitHubVariant.DOTCOM,
|
||||
tarSupportsZstd: true,
|
||||
};
|
||||
|
||||
/** Serves the release tagged `TAG` with the given assets from a stubbed API. */
|
||||
function releaseFixture({ assetNames = [COMBINED, JAVA], status = 200 } = {}) {
|
||||
const apiBase = `${API_URL}/repos/octo/tools/releases`;
|
||||
const releaseAPIURL = `${apiBase}/tags/${TAG}`;
|
||||
const assets = assetNames.map((name, index) => ({
|
||||
name,
|
||||
url: `${apiBase}/assets/${1000 + index}`,
|
||||
}));
|
||||
const requests: string[] = [];
|
||||
const messages: LoggedMessage[] = [];
|
||||
const state = initAllState({
|
||||
env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted" }),
|
||||
logger: getRecordingLogger(messages, { logToConsole: false }),
|
||||
features: createFeatures([Feature.PerLanguageBundles]),
|
||||
apiClient: github.getOctokit("123", {
|
||||
baseUrl: API_URL,
|
||||
request: {
|
||||
fetch: async (url) => {
|
||||
requests.push(String(url));
|
||||
if (String(url) !== releaseAPIURL) {
|
||||
throw new Error(`Unexpected API request: ${url}`);
|
||||
}
|
||||
return new Response(JSON.stringify({ tag_name: TAG, assets }), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
return {
|
||||
assets,
|
||||
messages,
|
||||
releaseAPIURL,
|
||||
requests,
|
||||
state,
|
||||
select: async (options: Partial<BundleSelectionOptions> = {}) =>
|
||||
selectBundle(state, await getRelease(state, REFERENCE), {
|
||||
...OPTIONS,
|
||||
...options,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
test("selectBundle selects an eligible per-language bundle with a single release lookup", async (t) => {
|
||||
const fixture = releaseFixture();
|
||||
t.deepEqual(await fixture.select(), {
|
||||
bundle: {
|
||||
kind: "per-language",
|
||||
url: fixture.assets[1].url,
|
||||
language: BuiltInLanguage.java,
|
||||
combinedBundleURL: fixture.assets[0].url,
|
||||
},
|
||||
compressionMethod: "zstd",
|
||||
});
|
||||
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
|
||||
});
|
||||
|
||||
test("selectBundle falls back to the combined bundle from the same release", async (t) => {
|
||||
const fixture = releaseFixture({ assetNames: [COMBINED] });
|
||||
t.deepEqual(await fixture.select(), {
|
||||
bundle: { kind: "combined", url: fixture.assets[0].url },
|
||||
compressionMethod: "zstd",
|
||||
perLanguageBundleFallback: true,
|
||||
});
|
||||
t.true(
|
||||
fixture.messages.some(
|
||||
(message) =>
|
||||
message.type === "warning" &&
|
||||
typeof message.message === "string" &&
|
||||
message.message.includes(`'java' at ${RELEASE_PAGE}`),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test("selectBundle selects the combined bundle for jobs that aren't eligible for a per-language bundle", async (t) => {
|
||||
const fixture = releaseFixture();
|
||||
t.deepEqual(await fixture.select({ rawLanguages: ["java", "python"] }), {
|
||||
bundle: { kind: "combined", url: fixture.assets[0].url },
|
||||
compressionMethod: "zstd",
|
||||
});
|
||||
});
|
||||
|
||||
test("selectBundle uses the other compression method when the preferred one is missing", async (t) => {
|
||||
const gzipOnly = releaseFixture({
|
||||
assetNames: ["codeql-bundle-linux64.tar.gz", JAVA],
|
||||
});
|
||||
t.deepEqual(await gzipOnly.select(), {
|
||||
bundle: { kind: "combined", url: gzipOnly.assets[0].url },
|
||||
compressionMethod: "gzip",
|
||||
});
|
||||
|
||||
for (const [platform, options] of [
|
||||
[BundlePlatform.Win64, {}],
|
||||
[BundlePlatform.Linux64, { cliVersion: "2.18.4" }],
|
||||
[BundlePlatform.Linux64, { cliVersion: undefined }],
|
||||
[BundlePlatform.Linux64, { tarSupportsZstd: false }],
|
||||
] as const) {
|
||||
const selection = await releaseFixture({
|
||||
assetNames: [
|
||||
`codeql-bundle-${platform}.tar.zst`,
|
||||
`codeql-bundle-${platform}.tar.gz`,
|
||||
],
|
||||
}).select({ platform, ...options });
|
||||
t.is(
|
||||
selection.compressionMethod,
|
||||
"gzip",
|
||||
`${platform} ${JSON.stringify(options)}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("selectBundle requires a combined bundle that we can extract", async (t) => {
|
||||
await t.throwsAsync(releaseFixture({ assetNames: [JAVA] }).select(), {
|
||||
instanceOf: ConfigurationError,
|
||||
message: `No compatible CodeQL bundle was found in release ${RELEASE_PAGE}. Expected codeql-bundle-linux64.tar.zst or codeql-bundle-linux64.tar.gz.`,
|
||||
});
|
||||
await t.throwsAsync(
|
||||
releaseFixture({ assetNames: [COMBINED] }).select({
|
||||
tarSupportsZstd: false,
|
||||
}),
|
||||
{ instanceOf: ConfigurationError, message: /Expected [^ ]+\.tar\.gz\.$/ },
|
||||
);
|
||||
});
|
||||
|
||||
test("getRelease propagates API errors", async (t) => {
|
||||
const fixture = releaseFixture({ status: 404 });
|
||||
t.like(await t.throwsAsync(fixture.select()), { status: 404 });
|
||||
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
|
||||
});
|
||||
|
||||
test("getRequestedRelease names a requested release that can't be found", async (t) => {
|
||||
const requested = { ...REFERENCE, isCurrentInstance: true };
|
||||
await t.throwsAsync(
|
||||
getRequestedRelease(releaseFixture({ status: 404 }).state, requested),
|
||||
{
|
||||
instanceOf: ConfigurationError,
|
||||
message: `Could not find the CodeQL release ${RELEASE_PAGE}. Check that it exists and that the token has access to it.`,
|
||||
},
|
||||
);
|
||||
t.like(
|
||||
await t.throwsAsync(
|
||||
getRequestedRelease(releaseFixture({ status: 500 }).state, requested),
|
||||
),
|
||||
{ status: 500 },
|
||||
);
|
||||
});
|
||||
|
||||
test("getRequestedRelease refers to a release on another instance by URL", async (t) => {
|
||||
const fixture = releaseFixture();
|
||||
const release = await getRequestedRelease(fixture.state, {
|
||||
...REFERENCE,
|
||||
isCurrentInstance: false,
|
||||
});
|
||||
t.is(
|
||||
release.getAssetURL(COMBINED),
|
||||
`https://github.com/octo/tools/releases/download/${TAG}/${COMBINED}`,
|
||||
);
|
||||
t.is(release.assetNames, undefined);
|
||||
t.deepEqual(fixture.requests, []);
|
||||
});
|
||||
|
||||
test("getPublicRelease constructs download URLs without looking up the release", async (t) => {
|
||||
const fixture = releaseFixture();
|
||||
const release = getPublicRelease({ ...REFERENCE, tagName: "nightly/v1+2" });
|
||||
const baseURL =
|
||||
"https://github.com/octo/tools/releases/download/nightly/v1%2B2";
|
||||
t.is(
|
||||
release.url,
|
||||
"https://github.com/octo/tools/releases/tag/nightly/v1%2B2",
|
||||
);
|
||||
t.deepEqual(
|
||||
await selectBundle(fixture.state, release, {
|
||||
...OPTIONS,
|
||||
cliVersion: undefined,
|
||||
isLatestNightly: true,
|
||||
}),
|
||||
{
|
||||
bundle: {
|
||||
kind: "per-language",
|
||||
url: `${baseURL}/${JAVA}`,
|
||||
language: BuiltInLanguage.java,
|
||||
combinedBundleURL: `${baseURL}/${COMBINED}`,
|
||||
},
|
||||
compressionMethod: "zstd",
|
||||
},
|
||||
);
|
||||
t.deepEqual(fixture.requests, []);
|
||||
});
|
||||
|
||||
test("parseCodeQLReleaseUrl accepts web and legacy links and decodes tags once", (t) => {
|
||||
for (const [suffix, tagName] of [
|
||||
[`tag/${TAG}`, TAG],
|
||||
[TAG, TAG],
|
||||
["codeql-bundle-20230120", "codeql-bundle-20230120"],
|
||||
["codeql-bundle-v2.27.1-rc.1", "codeql-bundle-v2.27.1-rc.1"],
|
||||
["tag/run-123", "run-123"],
|
||||
["tag/build/123", "build/123"],
|
||||
["tag/build%2F123%2Brc%231", "build/123+rc#1"],
|
||||
["tag/build%252F123", "build%2F123"],
|
||||
[`tag/${TAG}/?expanded=true#assets`, TAG],
|
||||
]) {
|
||||
t.deepEqual(
|
||||
parseCodeQLReleaseUrl(
|
||||
`https://github.com/octo/tools/releases/${suffix}`,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
),
|
||||
{ ...REFERENCE, isCurrentInstance: true, tagName },
|
||||
);
|
||||
}
|
||||
t.throws(
|
||||
() =>
|
||||
parseCodeQLReleaseUrl(
|
||||
"https://github.com/octo/tools/releases/tag/%zz",
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
),
|
||||
{ instanceOf: ConfigurationError, message: /Invalid URL encoding/ },
|
||||
);
|
||||
});
|
||||
|
||||
test("parseCodeQLReleaseUrl matches the current instance and GitHub.com by origin", (t) => {
|
||||
for (const [url, origin, serverURL] of [
|
||||
["https://github.example.test", "https://github.com", "https://github.com"],
|
||||
[
|
||||
"https://github.example.test/",
|
||||
"https://github.example.test",
|
||||
"https://github.example.test",
|
||||
],
|
||||
[
|
||||
"https://GitHub.Example.test",
|
||||
"https://github.example.test",
|
||||
"https://github.example.test",
|
||||
],
|
||||
[
|
||||
"https://github.example.test:443",
|
||||
"https://GITHUB.example.test:443",
|
||||
"https://github.example.test",
|
||||
],
|
||||
]) {
|
||||
t.deepEqual(
|
||||
parseCodeQLReleaseUrl(`${origin}/octo/tools/releases/tag/${TAG}`, {
|
||||
auth: "token",
|
||||
url,
|
||||
apiURL: undefined,
|
||||
}),
|
||||
{
|
||||
...REFERENCE,
|
||||
serverURL,
|
||||
isCurrentInstance: serverURL !== "https://github.com",
|
||||
},
|
||||
`${url} ${origin}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("parseCodeQLReleaseUrl excludes archives, REST references and untrusted URLs", (t) => {
|
||||
for (const input of [
|
||||
"/tmp/codeql-bundle.tar.zst",
|
||||
"nightly",
|
||||
`https://github.com/octo/tools/releases/download/${TAG}/${JAVA}`,
|
||||
"https://api.github.com/repos/octo/tools/releases/assets/123",
|
||||
"https://api.github.com/repos/octo/tools/releases/123",
|
||||
`https://api.github.com/repos/octo/tools/releases/tags/${TAG}`,
|
||||
"https://github.com/octo/tools/releases/latest",
|
||||
"https://github.com/octo/tools/releases/tag/",
|
||||
"https://github.com/octo/tools/releases/run-123",
|
||||
`https://github.com.example.test/octo/tools/releases/tag/${TAG}`,
|
||||
`https://github.com:8443/octo/tools/releases/tag/${TAG}`,
|
||||
`https://github.com@example.test/octo/tools/releases/tag/${TAG}`,
|
||||
`https://user@github.com/octo/tools/releases/tag/${TAG}`,
|
||||
`http://github.com/octo/tools/releases/tag/${TAG}`,
|
||||
]) {
|
||||
t.is(
|
||||
parseCodeQLReleaseUrl(input, SAMPLE_DOTCOM_API_DETAILS),
|
||||
undefined,
|
||||
input,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("getRelease returns the names of the release's assets", async (t) => {
|
||||
const assetNames = [COMBINED, "cli-version-2.27.2.txt"];
|
||||
const fixture = releaseFixture({ assetNames });
|
||||
const release = await getRelease(fixture.state, REFERENCE);
|
||||
t.deepEqual(release.assetNames, assetNames);
|
||||
});
|
||||
|
||||
test("getReleaseCliVersion prefers an unambiguous marker asset to the tag", (t) => {
|
||||
for (const [tagName, markers, cliVersion] of [
|
||||
[TAG, [], "2.27.1"],
|
||||
[TAG, ["invalid"], "2.27.1"],
|
||||
[TAG, ["2.27.2"], "2.27.2"],
|
||||
[TAG, ["2.27.1", "2.28.0"], undefined],
|
||||
["codeql-bundle-20260101", [], undefined],
|
||||
["run-123", [], undefined],
|
||||
["run-123", ["2.27.2+202601011200"], "2.27.2+202601011200"],
|
||||
] as const) {
|
||||
t.is(
|
||||
getReleaseCliVersion(
|
||||
tagName,
|
||||
[COMBINED, ...markers.map((version) => `cli-version-${version}.txt`)],
|
||||
getRecordingLogger([], { logToConsole: false }),
|
||||
),
|
||||
cliVersion,
|
||||
`${tagName} ${markers.join(",")}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
327
src/codeql-release.ts
Normal file
327
src/codeql-release.ts
Normal file
@@ -0,0 +1,327 @@
|
||||
import * as semver from "semver";
|
||||
|
||||
import { ActionState } from "./action-common";
|
||||
import type { GitHubApiDetails } from "./api-client";
|
||||
import { CodeQLBundle, getCodeQLBundleName } from "./codeql-bundle";
|
||||
import { CODEQL_VERSION_ZSTD_BUNDLE } from "./feature-flags";
|
||||
import { Logger } from "./logging";
|
||||
import {
|
||||
getPerLanguageBundleLanguage,
|
||||
logMissingPerLanguageBundle,
|
||||
} from "./per-language-bundles";
|
||||
import { BundlePlatform } from "./platform";
|
||||
import type { CompressionMethod } from "./tar";
|
||||
import {
|
||||
asHTTPError,
|
||||
ConfigurationError,
|
||||
GITHUB_DOTCOM_URL,
|
||||
GitHubVariant,
|
||||
} from "./util";
|
||||
|
||||
/** Identifies a release on a GitHub instance. */
|
||||
export interface CodeQLReleaseReference {
|
||||
serverURL: string;
|
||||
owner: string;
|
||||
repo: string;
|
||||
tagName: string;
|
||||
}
|
||||
|
||||
/** A GitHub release that contains CodeQL bundles. */
|
||||
export interface CodeQLRelease {
|
||||
/** The release's web page, for messages. */
|
||||
url: string;
|
||||
/** Returns the download URL for an asset, or `undefined` if the release doesn't have it. */
|
||||
getAssetURL(name: string): string | undefined;
|
||||
/** Names of the release's assets, if we looked up the release. */
|
||||
assetNames?: string[];
|
||||
}
|
||||
|
||||
/** A release requested by URL, on this GitHub instance or on GitHub.com. */
|
||||
export interface RequestedRelease extends CodeQLReleaseReference {
|
||||
isCurrentInstance: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognizes release pages, including legacy bundle links, but never asset URLs.
|
||||
*
|
||||
* We only accept https URLs without credentials, on this GitHub instance, whose API we can use, or
|
||||
* on GitHub.com, whose public releases we can download without credentials. The tag is
|
||||
* percent-decoded once, and may contain `/`.
|
||||
*/
|
||||
export function parseCodeQLReleaseUrl(
|
||||
input: string,
|
||||
apiDetails: GitHubApiDetails,
|
||||
): RequestedRelease | undefined {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(input);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
if (url.protocol !== "https:" || url.username || url.password) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// GitHub instances are served from the root of their origin.
|
||||
const isCurrentInstance = url.origin === new URL(apiDetails.url).origin;
|
||||
if (!isCurrentInstance && url.origin !== new URL(GITHUB_DOTCOM_URL).origin) {
|
||||
return undefined;
|
||||
}
|
||||
// Older links to bundle releases omit "tag/", which GitHub still supports. We only accept this
|
||||
// form for bundle tags, since other names can clash with routes like `releases/latest`.
|
||||
const match = url.pathname
|
||||
.replace(/\/$/, "")
|
||||
.match(
|
||||
/^\/([\w.-]+)\/([\w.-]+)\/releases\/(?:tag\/(.+)|(codeql-bundle-[^/]+))$/,
|
||||
);
|
||||
if (match === null) {
|
||||
return undefined;
|
||||
}
|
||||
let tagName: string;
|
||||
try {
|
||||
tagName = decodeURIComponent(match[3] ?? match[4]);
|
||||
} catch {
|
||||
throw new ConfigurationError(
|
||||
"Invalid URL encoding in the CodeQL release tag.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
serverURL: url.origin,
|
||||
isCurrentInstance,
|
||||
owner: match[1],
|
||||
repo: match[2],
|
||||
tagName,
|
||||
};
|
||||
}
|
||||
|
||||
function parseCliVersion(value: string): string | undefined {
|
||||
const parsed = semver.parse(value);
|
||||
if (parsed === null) {
|
||||
return undefined;
|
||||
}
|
||||
// semver's normalized version omits build metadata, which distinguishes nightly builds.
|
||||
return (
|
||||
parsed.version + (parsed.build.length ? `+${parsed.build.join(".")}` : "")
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the CLI version from the release's `cli-version-<version>.txt` asset if it has one, and
|
||||
* otherwise the version in the tag. Returns `undefined` if the release has conflicting markers.
|
||||
*/
|
||||
export function getReleaseCliVersion(
|
||||
tagName: string,
|
||||
assetNames: string[],
|
||||
logger: Logger,
|
||||
): string | undefined {
|
||||
const versions = new Set<string>();
|
||||
for (const name of assetNames) {
|
||||
const match = name.match(/^cli-version-(.+)\.txt$/);
|
||||
if (match === null) {
|
||||
continue;
|
||||
}
|
||||
const version = parseCliVersion(match[1]);
|
||||
if (version !== undefined) {
|
||||
versions.add(version);
|
||||
} else {
|
||||
logger.debug(`Ignoring invalid CLI version marker ${name}.`);
|
||||
}
|
||||
}
|
||||
if (versions.size > 1) {
|
||||
logger.warning(
|
||||
`Release ${tagName} has conflicting CLI version markers. Using a combined CodeQL bundle.`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
if (versions.size === 1) {
|
||||
return versions.values().next().value;
|
||||
}
|
||||
return parseCliVersion(tagName.replace(/^codeql-bundle-/, ""));
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a tag for use in a URL path. Slashes stay as path separators, as in GitHub's release URLs
|
||||
* for tags like `build/123`.
|
||||
*/
|
||||
function encodeTag(tagName: string): string {
|
||||
return tagName.split("/").map(encodeURIComponent).join("/");
|
||||
}
|
||||
|
||||
function getReleasePageURL(reference: CodeQLReleaseReference): string {
|
||||
const { serverURL, owner, repo, tagName } = reference;
|
||||
return `${serverURL}/${owner}/${repo}/releases/tag/${encodeTag(tagName)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks up a release on the current GitHub instance, which works for private repositories.
|
||||
*
|
||||
* The API client determines which instance we query, so `reference.serverURL` is only used for the
|
||||
* release page URL in messages. The asset URLs are REST API endpoints, which accept the token and
|
||||
* return the file when requested with `Accept: application/octet-stream`, unlike browser download
|
||||
* URLs.
|
||||
*/
|
||||
export async function getRelease(
|
||||
{ apiClient }: ActionState<["Api"]>,
|
||||
reference: CodeQLReleaseReference,
|
||||
): Promise<CodeQLRelease> {
|
||||
const { owner, repo, tagName } = reference;
|
||||
const { data: release } = await apiClient.rest.repos.getReleaseByTag({
|
||||
owner,
|
||||
repo,
|
||||
tag: tagName,
|
||||
});
|
||||
return {
|
||||
url: getReleasePageURL(reference),
|
||||
getAssetURL: (name) =>
|
||||
release.assets.find((asset) => asset.name === name)?.url,
|
||||
assetNames: release.assets.map((asset) => asset.name),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Refers to a public release without looking it up. Every asset gets a download URL, so a missing
|
||||
* asset shows up as a failed download.
|
||||
*/
|
||||
export function getPublicRelease(
|
||||
reference: CodeQLReleaseReference,
|
||||
): CodeQLRelease {
|
||||
const { serverURL, owner, repo, tagName } = reference;
|
||||
return {
|
||||
url: getReleasePageURL(reference),
|
||||
getAssetURL: (name) =>
|
||||
`${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** Gets a release requested by URL, looking it up if it's on the current GitHub instance. */
|
||||
export async function getRequestedRelease(
|
||||
action: ActionState<["Api"]>,
|
||||
requested: RequestedRelease,
|
||||
): Promise<CodeQLRelease> {
|
||||
if (!requested.isCurrentInstance) {
|
||||
// We can't send the token to another instance, so only public releases are usable there, and we
|
||||
// download them by URL rather than querying that instance's API. On the current instance we use
|
||||
// the API instead, because release download URLs don't accept tokens and so would fail for
|
||||
// private and internal repositories. Without the release's asset list, we assume the preferred
|
||||
// compression is available and take the CLI version from the tag.
|
||||
return getPublicRelease(requested);
|
||||
}
|
||||
try {
|
||||
return await getRelease(action, requested);
|
||||
} catch (e) {
|
||||
if (asHTTPError(e)?.status === 404) {
|
||||
throw new ConfigurationError(
|
||||
`Could not find the CodeQL release ${getReleasePageURL(requested)}. Check that it ` +
|
||||
"exists and that the token has access to it.",
|
||||
);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/** Describes the job and runner that we are selecting a bundle for. */
|
||||
export interface BundleSelectionOptions {
|
||||
/** Explicit `languages` input, which determines whether a per-language bundle is eligible. */
|
||||
rawLanguages: string[] | undefined;
|
||||
/** The CLI version in the release, if known. */
|
||||
cliVersion: string | undefined;
|
||||
platform: BundlePlatform | undefined;
|
||||
variant: GitHubVariant;
|
||||
tarSupportsZstd: boolean;
|
||||
/** Whether the release is the latest nightly, whose CLI version we don't know yet. */
|
||||
isLatestNightly?: boolean;
|
||||
}
|
||||
|
||||
/** A bundle selected from a release. */
|
||||
export interface BundleSelection {
|
||||
bundle: CodeQLBundle;
|
||||
compressionMethod: CompressionMethod;
|
||||
/** The release lacks the eligible per-language bundle, so we selected the combined bundle. */
|
||||
perLanguageBundleFallback?: true;
|
||||
}
|
||||
|
||||
/** Returns the compression methods that we can extract, most preferred first. */
|
||||
function getCompressionMethods({
|
||||
cliVersion,
|
||||
isLatestNightly,
|
||||
platform,
|
||||
tarSupportsZstd,
|
||||
}: BundleSelectionOptions): CompressionMethod[] {
|
||||
if (!tarSupportsZstd) {
|
||||
return ["gzip"];
|
||||
}
|
||||
const preferZstd =
|
||||
// In testing, gzip performs better than zstd on Windows.
|
||||
platform !== BundlePlatform.Win64 &&
|
||||
// Standard bundles have zstd archives from this version, and so does the latest nightly. For a
|
||||
// release we looked up, gzip comes next if it lacks the zstd archive.
|
||||
(isLatestNightly ||
|
||||
(cliVersion !== undefined &&
|
||||
semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE)));
|
||||
return preferZstd ? ["zstd", "gzip"] : ["gzip", "zstd"];
|
||||
}
|
||||
|
||||
/**
|
||||
* Selects a per-language bundle if the job is eligible for one, and otherwise the combined bundle.
|
||||
* We only select a per-language bundle from a release that also has a combined bundle with the same
|
||||
* compression, so that we can fall back to it.
|
||||
*
|
||||
* If we looked up the release, we can tell that an eligible per-language bundle is missing, so we
|
||||
* warn and select the combined bundle straight away. A public release gives every asset a URL, so a
|
||||
* missing per-language bundle only shows up as a 404 when downloading, which is when we fall back.
|
||||
*
|
||||
* Throws a `ConfigurationError` if the release has no combined bundle that we can extract.
|
||||
*/
|
||||
export async function selectBundle(
|
||||
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
|
||||
release: CodeQLRelease,
|
||||
options: BundleSelectionOptions,
|
||||
): Promise<BundleSelection> {
|
||||
const { logger } = action;
|
||||
const compressionMethods = getCompressionMethods(options);
|
||||
for (const compressionMethod of compressionMethods) {
|
||||
const combinedBundleName = getCodeQLBundleName(
|
||||
compressionMethod,
|
||||
options.platform,
|
||||
);
|
||||
const combinedBundleURL = release.getAssetURL(combinedBundleName);
|
||||
if (combinedBundleURL === undefined) {
|
||||
continue;
|
||||
}
|
||||
const combined: BundleSelection = {
|
||||
bundle: { kind: "combined", url: combinedBundleURL },
|
||||
compressionMethod,
|
||||
};
|
||||
|
||||
const language = await getPerLanguageBundleLanguage(action, {
|
||||
...options,
|
||||
compressionMethod,
|
||||
});
|
||||
if (language === undefined) {
|
||||
logger.info(
|
||||
`Selected CodeQL bundle ${combinedBundleName} from ${release.url}.`,
|
||||
);
|
||||
return combined;
|
||||
}
|
||||
const name = getCodeQLBundleName(
|
||||
compressionMethod,
|
||||
options.platform,
|
||||
language,
|
||||
);
|
||||
const url = release.getAssetURL(name);
|
||||
if (url === undefined) {
|
||||
logMissingPerLanguageBundle(action, language, release.url);
|
||||
return { ...combined, perLanguageBundleFallback: true };
|
||||
}
|
||||
logger.info(`Selected CodeQL bundle ${name} from ${release.url}.`);
|
||||
return {
|
||||
bundle: { kind: "per-language", url, language, combinedBundleURL },
|
||||
compressionMethod,
|
||||
};
|
||||
}
|
||||
throw new ConfigurationError(
|
||||
`No compatible CodeQL bundle was found in release ${release.url}. Expected ${compressionMethods
|
||||
.map((method) => getCodeQLBundleName(method, options.platform))
|
||||
.join(" or ")}.`,
|
||||
);
|
||||
}
|
||||
@@ -132,15 +132,15 @@ export async function getPerLanguageBundleLanguage(
|
||||
return language;
|
||||
}
|
||||
|
||||
/** Explains why an eligible per-language bundle is being replaced by a combined bundle. */
|
||||
export function logPerLanguageBundleFallback(
|
||||
/** Logs that the selected per-language bundle is missing, so we're using the combined bundle. */
|
||||
export function logMissingPerLanguageBundle(
|
||||
{ logger }: ActionState<["Logger"]>,
|
||||
language: BuiltInLanguage,
|
||||
location: string,
|
||||
): void {
|
||||
logger.warning(
|
||||
`No per-language CodeQL bundle for '${language}' was found at ${location}, so ` +
|
||||
"falling back to the bundle that contains all languages. This analysis will still " +
|
||||
"produce correct results, but will take longer to set up.",
|
||||
`Expected a per-language CodeQL bundle for '${language}' at ${location}, but it wasn't ` +
|
||||
"found, so using the bundle that contains all languages instead. This analysis will " +
|
||||
"still produce correct results, but will take longer to set up.",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
} from "./testing-utils";
|
||||
import * as toolsDownload from "./tools-download";
|
||||
import {
|
||||
ConfigurationError,
|
||||
getErrorMessage,
|
||||
GitHubVariant,
|
||||
HTTPError,
|
||||
@@ -428,7 +429,7 @@ test.serial(
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
false,
|
||||
true, // tarSupportsZstd
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
@@ -498,7 +499,7 @@ test.serial(
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
false,
|
||||
true, // tarSupportsZstd
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
@@ -1214,9 +1215,72 @@ test.serial(
|
||||
},
|
||||
);
|
||||
|
||||
for (const fallback of [false, true]) {
|
||||
for (const status of [200, 404]) {
|
||||
test.serial(
|
||||
`setupCodeQLBundle retains the selected release identity for an opaque asset URL${fallback ? " with fallback" : ""}`,
|
||||
`getCodeQLSource falls back to the public release when the Action's release ${status === 200 ? "lacks a compatible bundle" : "can't be found"}`,
|
||||
async (t) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
|
||||
const tag = PER_LANGUAGE_CLI_VERSION.enabledVersions[0].tagName;
|
||||
const fetchRelease = sinon
|
||||
.stub<Parameters<typeof fetch>, ReturnType<typeof fetch>>()
|
||||
.callsFake(
|
||||
async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
assets: [
|
||||
{
|
||||
name: "codeql-bundle-java-linux64.tar.zst",
|
||||
url: "https://api.github.com/repos/codeql-testing/action-fork/releases/assets/1",
|
||||
},
|
||||
],
|
||||
}),
|
||||
{ status, headers: { "content-type": "application/json" } },
|
||||
),
|
||||
);
|
||||
const client = github.getOctokit("123", {
|
||||
request: { fetch: fetchRelease },
|
||||
});
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir, {
|
||||
GITHUB_ACTION_REPOSITORY: "codeql-testing/action-fork",
|
||||
});
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
undefined,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
undefined, // rawLanguages
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(fetchRelease.calledOnce);
|
||||
t.like(source, {
|
||||
sourceType: "download",
|
||||
bundle: {
|
||||
kind: "combined",
|
||||
url: `https://github.com/github/codeql-action/releases/download/${tag}/codeql-bundle-linux64.tar.zst`,
|
||||
},
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
for (const scenario of ["per-language", "fallback", "missing"] as const) {
|
||||
const suffix = {
|
||||
"per-language": "",
|
||||
fallback: " with fallback",
|
||||
missing: " when the release lacks the per-language bundle",
|
||||
}[scenario];
|
||||
test.serial(
|
||||
`setupCodeQLBundle retains the selected release identity for an opaque asset URL${suffix}`,
|
||||
async (t) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
@@ -1230,22 +1294,22 @@ for (const fallback of [false, true]) {
|
||||
const assetURL =
|
||||
"https://api.github.com/repos/codeql-testing/action-fork/releases/assets/123";
|
||||
const combinedURL = `${assetURL}4`;
|
||||
const assets = [
|
||||
{ name: "codeql-bundle-linux64.tar.zst", url: combinedURL },
|
||||
];
|
||||
if (scenario !== "missing") {
|
||||
assets.push({
|
||||
name: "codeql-bundle-java-linux64.tar.zst",
|
||||
url: assetURL,
|
||||
});
|
||||
}
|
||||
const fetchRelease = sinon
|
||||
.stub<Parameters<typeof fetch>, ReturnType<typeof fetch>>()
|
||||
.callsFake(
|
||||
async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
assets: [
|
||||
{ name: "codeql-bundle-java-linux64.tar.zst", url: assetURL },
|
||||
{
|
||||
name: "codeql-bundle-linux64.tar.zst",
|
||||
url: combinedURL,
|
||||
},
|
||||
],
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
),
|
||||
new Response(JSON.stringify({ assets }), {
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
const client = github.getOctokit("123", {
|
||||
request: { fetch: fetchRelease },
|
||||
@@ -1253,7 +1317,7 @@ for (const fallback of [false, true]) {
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
const authorizationSpy = sinon.spy(api, "getAuthorizationHeaderFor");
|
||||
const extractStub = stubDownloadAndExtract();
|
||||
if (fallback) {
|
||||
if (scenario === "fallback") {
|
||||
extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404));
|
||||
}
|
||||
|
||||
@@ -1273,47 +1337,407 @@ for (const fallback of [false, true]) {
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(fetchRelease.calledTwice);
|
||||
const usesPerLanguageBundle = scenario === "per-language";
|
||||
t.true(fetchRelease.calledOnce);
|
||||
t.is(
|
||||
fetchRelease.firstCall.args[0],
|
||||
`https://api.github.com/repos/codeql-testing/action-fork/releases/tags/${tag}`,
|
||||
);
|
||||
t.is(extractStub.callCount, fallback ? 2 : 1);
|
||||
t.is(extractStub.firstCall.args[0], assetURL);
|
||||
t.is(extractStub.lastCall.args[0], fallback ? combinedURL : assetURL);
|
||||
t.is(authorizationSpy.callCount, extractStub.callCount);
|
||||
t.is(authorizationSpy.firstCall.args[2], assetURL);
|
||||
t.is(extractStub.callCount, scenario === "fallback" ? 2 : 1);
|
||||
t.is(
|
||||
authorizationSpy.lastCall.args[2],
|
||||
fallback ? combinedURL : assetURL,
|
||||
extractStub.firstCall.args[0],
|
||||
scenario === "missing" ? combinedURL : assetURL,
|
||||
);
|
||||
t.is(
|
||||
extractStub.lastCall.args[0],
|
||||
usesPerLanguageBundle ? assetURL : combinedURL,
|
||||
);
|
||||
t.deepEqual(
|
||||
authorizationSpy.getCalls().map((call) => call.args[2]),
|
||||
extractStub.getCalls().map((call) => call.args[0]),
|
||||
);
|
||||
t.is(extractStub.lastCall.args[3], "token token");
|
||||
t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION);
|
||||
t.is(
|
||||
result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language,
|
||||
fallback ? undefined : BuiltInLanguage.java,
|
||||
usesPerLanguageBundle ? BuiltInLanguage.java : undefined,
|
||||
);
|
||||
t.is(
|
||||
result.toolsDownloadStatusReport?.perLanguage
|
||||
?.tools_per_language_bundle_fallback,
|
||||
fallback ? true : undefined,
|
||||
usesPerLanguageBundle ? undefined : true,
|
||||
);
|
||||
if (fallback) {
|
||||
if (usesPerLanguageBundle) {
|
||||
t.is(path.dirname(result.codeqlFolder), tmpDir);
|
||||
t.deepEqual(toolcache.findAllVersions("CodeQL"), []);
|
||||
t.false(fs.existsSync(`${result.codeqlFolder}.complete`));
|
||||
} else {
|
||||
t.is(
|
||||
result.codeqlFolder,
|
||||
toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION),
|
||||
);
|
||||
t.true(fs.existsSync(`${result.codeqlFolder}.complete`));
|
||||
} else {
|
||||
t.is(path.dirname(result.codeqlFolder), tmpDir);
|
||||
t.deepEqual(toolcache.findAllVersions("CodeQL"), []);
|
||||
t.false(fs.existsSync(`${result.codeqlFolder}.complete`));
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const GHES_API_DETAILS = {
|
||||
auth: "enterprise-token",
|
||||
url: "https://github.example.test",
|
||||
apiURL: "https://github.example.test/api/v3",
|
||||
};
|
||||
|
||||
/** Models a hosted Linux runner with zstd and a release in `repository` on the instance `apiDetails` describes. */
|
||||
function stubRequestedRelease({
|
||||
apiDetails = SAMPLE_DOTCOM_API_DETAILS,
|
||||
repository = "octo/tools",
|
||||
tagName = `codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`,
|
||||
assetNames = [
|
||||
"codeql-bundle-linux64.tar.zst",
|
||||
"codeql-bundle-java-linux64.tar.zst",
|
||||
],
|
||||
markers = [] as string[],
|
||||
status = 200,
|
||||
} = {}) {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
const apiBase = `${apiDetails.apiURL}/repos/${repository}/releases`;
|
||||
const assets = [
|
||||
...assetNames,
|
||||
...markers.map((version) => `cli-version-${version}.txt`),
|
||||
].map((name, index) => ({ name, url: `${apiBase}/assets/${1000 + index}` }));
|
||||
const requests: string[] = [];
|
||||
const client = github.getOctokit("123", {
|
||||
baseUrl: apiDetails.apiURL,
|
||||
request: {
|
||||
fetch: async (input) => {
|
||||
const url = String(input);
|
||||
requests.push(url);
|
||||
if (url !== `${apiBase}/tags/${tagName}`) {
|
||||
throw new Error(`Unexpected API request: ${url}`);
|
||||
}
|
||||
return new Response(JSON.stringify({ tag_name: tagName, assets }), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
},
|
||||
},
|
||||
});
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
return {
|
||||
assets,
|
||||
requests,
|
||||
releaseURL: `${apiDetails.url}/${repository}/releases/tag/${tagName}`,
|
||||
};
|
||||
}
|
||||
|
||||
for (const scenario of ["combined", "per-language", "fallback"] as const) {
|
||||
test.serial(
|
||||
`setupCodeQLBundle downloads a ${scenario} bundle from a custom release without using the toolcache`,
|
||||
async (t) => {
|
||||
const fixture = stubRequestedRelease();
|
||||
const extract = stubDownloadAndExtract();
|
||||
if (scenario === "fallback") {
|
||||
extract.onFirstCall().rejects(new HTTPError("Not Found", 404));
|
||||
}
|
||||
const find = sinon.spy(toolcache, "find");
|
||||
sinon.stub(actionsUtil, "isDynamicWorkflow").returns(true);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
createToolcacheEntry(
|
||||
tmpDir,
|
||||
"CodeQL",
|
||||
MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
|
||||
);
|
||||
const result = await setupCodeql.setupCodeQLBundle(
|
||||
fixture.releaseURL,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
scenario === "combined" ? undefined : ["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
// The requested release takes precedence over forcing the nightly.
|
||||
createFeatures([Feature.PerLanguageBundles, Feature.ForceNightly]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(fixture.requests.length, 1);
|
||||
t.true(find.notCalled);
|
||||
t.is(result.toolsSource, setupCodeql.ToolsSource.Download);
|
||||
t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION);
|
||||
t.is(extract.callCount, scenario === "fallback" ? 2 : 1);
|
||||
t.is(
|
||||
extract.lastCall.args[0],
|
||||
fixture.assets[scenario === "per-language" ? 1 : 0].url,
|
||||
);
|
||||
t.is(extract.lastCall.args[3], "token token");
|
||||
t.is(path.dirname(result.codeqlFolder), tmpDir);
|
||||
t.false(fs.existsSync(`${result.codeqlFolder}.complete`));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test.serial(
|
||||
"setupCodeQLBundle downloads the gzip bundle from a requested release that only has gzip bundles",
|
||||
async (t) => {
|
||||
const fixture = stubRequestedRelease({
|
||||
assetNames: ["codeql-bundle-linux64.tar.gz"],
|
||||
});
|
||||
const extract = stubDownloadAndExtract();
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const result = await setupCodeql.setupCodeQLBundle(
|
||||
fixture.releaseURL,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(extract.calledOnce);
|
||||
t.is(extract.firstCall.args[0], fixture.assets[0].url);
|
||||
t.is(extract.firstCall.args[1], "gzip");
|
||||
t.is(result.toolsSource, setupCodeql.ToolsSource.Download);
|
||||
t.is(result.toolsDownloadStatusReport?.perLanguage, undefined);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
for (const [apiDetails, variant] of [
|
||||
[SAMPLE_DOTCOM_API_DETAILS, GitHubVariant.DOTCOM],
|
||||
[GHES_API_DETAILS, GitHubVariant.GHES],
|
||||
] as const) {
|
||||
test.serial(
|
||||
`setupCodeQLBundle caches a stable release in the CodeQL Action repository on ${variant}`,
|
||||
async (t) => {
|
||||
const fixture = stubRequestedRelease({
|
||||
apiDetails,
|
||||
repository: "github/codeql-action",
|
||||
});
|
||||
const extract = stubDownloadAndExtract();
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const setup = () =>
|
||||
setupCodeql.setupCodeQLBundle(
|
||||
fixture.releaseURL,
|
||||
apiDetails,
|
||||
tmpDir,
|
||||
variant,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
undefined, // rawLanguages
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
const downloaded = await setup();
|
||||
t.deepEqual(fixture.requests, [
|
||||
`${apiDetails.apiURL}/repos/github/codeql-action/releases/tags/codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`,
|
||||
]);
|
||||
t.is(downloaded.toolsSource, setupCodeql.ToolsSource.Download);
|
||||
t.is(extract.firstCall.args[0], fixture.assets[0].url);
|
||||
t.is(extract.firstCall.args[3], `token ${apiDetails.auth}`);
|
||||
t.is(
|
||||
downloaded.codeqlFolder,
|
||||
toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION),
|
||||
);
|
||||
|
||||
const cached = await setup();
|
||||
t.is(cached.toolsSource, setupCodeql.ToolsSource.Toolcache);
|
||||
t.is(cached.codeqlFolder, downloaded.codeqlFolder);
|
||||
t.true(extract.calledOnce);
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
for (const { repository, tagName, markers } of [
|
||||
{
|
||||
repository: "github/codeql-action",
|
||||
tagName: "codeql-bundle-v2.27.1-rc.1",
|
||||
markers: [],
|
||||
},
|
||||
{
|
||||
repository: "github/codeql-action",
|
||||
tagName: "codeql-bundle-v2.27.1",
|
||||
markers: ["2.27.1+202609241200"],
|
||||
},
|
||||
{
|
||||
repository: "github/codeql-action",
|
||||
tagName: "codeql-bundle-20260924",
|
||||
markers: ["2.27.1+202609241200"],
|
||||
},
|
||||
{
|
||||
repository: "github/codeql-action",
|
||||
tagName: "codeql-bundle-acme-2.27.1",
|
||||
markers: ["2.27.1"],
|
||||
},
|
||||
{
|
||||
repository: "octo/tools",
|
||||
tagName: "codeql-bundle-feature_branch",
|
||||
markers: [],
|
||||
},
|
||||
]) {
|
||||
const description =
|
||||
markers.length === 0
|
||||
? tagName
|
||||
: `${tagName} with marker ${markers.join(", ")}`;
|
||||
test.serial(
|
||||
`setupCodeQLBundle doesn't share the toolcache with ${description} in ${repository}`,
|
||||
async (t) => {
|
||||
const fixture = stubRequestedRelease({ repository, tagName, markers });
|
||||
const extract = stubDownloadAndExtract();
|
||||
const find = sinon.spy(toolcache, "find");
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
createToolcacheEntry(
|
||||
tmpDir,
|
||||
"CodeQL",
|
||||
MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
|
||||
);
|
||||
const result = await setupCodeql.setupCodeQLBundle(
|
||||
fixture.releaseURL,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
undefined, // rawLanguages
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(find.notCalled);
|
||||
t.is(result.toolsSource, setupCodeql.ToolsSource.Download);
|
||||
t.is(extract.firstCall.args[0], fixture.assets[0].url);
|
||||
t.is(path.dirname(result.codeqlFolder), tmpDir);
|
||||
t.false(fs.existsSync(`${result.codeqlFolder}.complete`));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
for (const repository of ["github/codeql-action", "octo/tools"]) {
|
||||
test.serial(
|
||||
`setupCodeQLBundle downloads a GitHub.com release in ${repository} from GHES by URL without credentials`,
|
||||
async (t) => {
|
||||
const fixture = stubRequestedRelease({ repository });
|
||||
const extract = stubDownloadAndExtract();
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const result = await setupCodeql.setupCodeQLBundle(
|
||||
fixture.releaseURL,
|
||||
GHES_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.GHES,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.deepEqual(fixture.requests, []);
|
||||
t.true(extract.calledOnce);
|
||||
t.is(
|
||||
extract.firstCall.args[0],
|
||||
`https://github.com/${repository}/releases/download/codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}/codeql-bundle-linux64.tar.zst`,
|
||||
);
|
||||
t.is(extract.firstCall.args[3], undefined);
|
||||
t.is(
|
||||
fs.existsSync(`${result.codeqlFolder}.complete`),
|
||||
repository === "github/codeql-action",
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test.serial(
|
||||
"setupCodeQLBundle doesn't substitute another release for a requested release that can't be found",
|
||||
async (t) => {
|
||||
const fixture = stubRequestedRelease({ status: 404 });
|
||||
const extract = stubDownloadAndExtract();
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
await t.throwsAsync(
|
||||
setupCodeql.setupCodeQLBundle(
|
||||
fixture.releaseURL,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
undefined, // rawLanguages
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
),
|
||||
{
|
||||
instanceOf: ConfigurationError,
|
||||
message: `Could not find the CodeQL release ${fixture.releaseURL}. Check that it exists and that the token has access to it.`,
|
||||
},
|
||||
);
|
||||
t.is(fixture.requests.length, 1);
|
||||
t.true(extract.notCalled);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"setupCodeQLBundle uses the CLI version marker of a requested release",
|
||||
async (t) => {
|
||||
const fixture = stubRequestedRelease({
|
||||
tagName: "run-123",
|
||||
markers: [MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION],
|
||||
});
|
||||
const extract = stubDownloadAndExtract();
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const result = await setupCodeql.setupCodeQLBundle(
|
||||
fixture.releaseURL,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
// The tag doesn't give a version, so the job can only use the per-language bundle because of
|
||||
// the marker.
|
||||
t.is(extract.firstCall.args[0], fixture.assets[1].url);
|
||||
t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION);
|
||||
t.is(
|
||||
result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language,
|
||||
BuiltInLanguage.java,
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
for (const bundle of ["per-language", "combined", "fallback"] as const) {
|
||||
test.serial(
|
||||
`setupCodeQLBundle preserves the nightly version for a ${bundle} download`,
|
||||
@@ -1420,9 +1844,10 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) {
|
||||
t.deepEqual(downloadSpy.secondCall.args[0], {
|
||||
...source,
|
||||
bundle: { kind: "combined", url: combinedURL },
|
||||
perLanguageBundleFallback: true,
|
||||
});
|
||||
checkExpectedLogMessages(t, loggedMessages, [
|
||||
`No per-language CodeQL bundle for 'javascript' was found at ${perLanguageURL}`,
|
||||
`Expected a per-language CodeQL bundle for 'javascript' at ${perLanguageURL}`,
|
||||
]);
|
||||
}
|
||||
if (bundle === "per-language") {
|
||||
|
||||
@@ -21,8 +21,19 @@ import {
|
||||
CodeQLBundle,
|
||||
CodeQLDownloadSource,
|
||||
getCodeQLBundleFromUrl,
|
||||
getCodeQLBundleName,
|
||||
} from "./codeql-bundle";
|
||||
import {
|
||||
BundleSelection,
|
||||
BundleSelectionOptions,
|
||||
CodeQLRelease,
|
||||
CodeQLReleaseReference,
|
||||
getPublicRelease,
|
||||
getRelease,
|
||||
getReleaseCliVersion,
|
||||
getRequestedRelease,
|
||||
parseCodeQLReleaseUrl,
|
||||
selectBundle,
|
||||
} from "./codeql-release";
|
||||
import * as defaults from "./defaults.json";
|
||||
import {
|
||||
addNoLanguageDiagnostic,
|
||||
@@ -31,19 +42,14 @@ import {
|
||||
} from "./diagnostics";
|
||||
import { EnvVar, getEnv } from "./environment";
|
||||
import {
|
||||
CODEQL_VERSION_ZSTD_BUNDLE,
|
||||
CodeQLDefaultVersionInfo,
|
||||
CodeQLVersionInfo,
|
||||
Feature,
|
||||
FeatureEnablement,
|
||||
} from "./feature-flags";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching";
|
||||
import {
|
||||
getPerLanguageBundleLanguage,
|
||||
logPerLanguageBundleFallback,
|
||||
} from "./per-language-bundles";
|
||||
import { logMissingPerLanguageBundle } from "./per-language-bundles";
|
||||
import { getBundlePlatform } from "./platform";
|
||||
import * as tar from "./tar";
|
||||
import {
|
||||
@@ -88,14 +94,13 @@ export function getCodeQLActionRepository(logger: Logger): string {
|
||||
return util.getRequiredEnvParam("GITHUB_ACTION_REPOSITORY");
|
||||
}
|
||||
|
||||
async function getCodeQLBundleDownloadURL(
|
||||
tagName: string,
|
||||
/** Returns the repositories that release the default bundles, most preferred first. */
|
||||
function getDefaultBundleSources(
|
||||
apiDetails: api.GitHubApiDetails,
|
||||
codeQLBundleName: string,
|
||||
logger: Logger,
|
||||
): Promise<string> {
|
||||
): Array<[serverURL: string, repository: string]> {
|
||||
const codeQLActionRepository = getCodeQLActionRepository(logger);
|
||||
const potentialDownloadSources = [
|
||||
const potentialDownloadSources: Array<[string, string]> = [
|
||||
// This GitHub instance, and this Action.
|
||||
[apiDetails.url, codeQLActionRepository],
|
||||
// This GitHub instance, and the canonical Action.
|
||||
@@ -105,42 +110,85 @@ async function getCodeQLBundleDownloadURL(
|
||||
];
|
||||
// We now filter out any duplicates.
|
||||
// Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
|
||||
const uniqueDownloadSources = potentialDownloadSources.filter(
|
||||
(source, index, self) => {
|
||||
return !self.slice(0, index).some((other) => deepEqual(source, other));
|
||||
},
|
||||
return potentialDownloadSources.filter((source, index, self) => {
|
||||
return !self.slice(0, index).some((other) => deepEqual(source, other));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a release requested by URL contains the same build as the toolcache entry for its CLI
|
||||
* version. The toolcache is keyed by version alone, so we only assume this for stable releases
|
||||
* tagged `codeql-bundle-v<version>` in the repositories we download the default bundles from.
|
||||
*/
|
||||
function isCacheableRelease(
|
||||
reference: CodeQLReleaseReference,
|
||||
cliVersion: string | undefined,
|
||||
apiDetails: api.GitHubApiDetails,
|
||||
logger: Logger,
|
||||
): boolean {
|
||||
if (
|
||||
cliVersion === undefined ||
|
||||
!/^\d+\.\d+\.\d+$/.test(cliVersion) ||
|
||||
reference.tagName !== `codeql-bundle-v${cliVersion}`
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const repository = `${reference.owner}/${reference.repo}`.toLowerCase();
|
||||
return getDefaultBundleSources(apiDetails, logger).some(
|
||||
([serverURL, sourceRepository]) =>
|
||||
new URL(serverURL).origin === new URL(reference.serverURL).origin &&
|
||||
sourceRepository.toLowerCase() === repository,
|
||||
);
|
||||
for (const downloadSource of uniqueDownloadSources) {
|
||||
const [apiURL, repository] = downloadSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Selects a bundle from the first release tagged `tagName` that has a compatible bundle, trying the
|
||||
* Action repositories on this GitHub instance before the canonical Action on GitHub.com. If we
|
||||
* can't look up a release, or it has no compatible bundle, we move on to the next repository. We
|
||||
* assume that the public release on GitHub.com has every bundle.
|
||||
*/
|
||||
async function selectDefaultBundle(
|
||||
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
|
||||
tagName: string,
|
||||
apiDetails: api.GitHubApiDetails,
|
||||
options: BundleSelectionOptions,
|
||||
): Promise<BundleSelection> {
|
||||
const { logger } = action;
|
||||
for (const [serverURL, repository] of getDefaultBundleSources(
|
||||
apiDetails,
|
||||
logger,
|
||||
)) {
|
||||
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
|
||||
if (
|
||||
apiURL === util.GITHUB_DOTCOM_URL &&
|
||||
serverURL === util.GITHUB_DOTCOM_URL &&
|
||||
repository === CODEQL_DEFAULT_ACTION_REPOSITORY
|
||||
) {
|
||||
break;
|
||||
}
|
||||
const [repositoryOwner, repositoryName] = repository.split("/");
|
||||
const [owner, repo] = repository.split("/");
|
||||
try {
|
||||
const release = await api.getApiClient().rest.repos.getReleaseByTag({
|
||||
owner: repositoryOwner,
|
||||
repo: repositoryName,
|
||||
tag: tagName,
|
||||
});
|
||||
for (const asset of release.data.assets) {
|
||||
if (asset.name === codeQLBundleName) {
|
||||
logger.info(
|
||||
`Found CodeQL bundle ${codeQLBundleName} in ${repository} on ${apiURL} with URL ${asset.url}.`,
|
||||
);
|
||||
return asset.url;
|
||||
}
|
||||
}
|
||||
const release = await getRelease(
|
||||
{ apiClient: api.getApiClient() },
|
||||
{ serverURL, owner, repo, tagName },
|
||||
);
|
||||
return await selectBundle(action, release, options);
|
||||
} catch (e) {
|
||||
logger.info(
|
||||
`Looked for CodeQL bundle ${codeQLBundleName} in ${repository} on ${apiURL} but got error ${e}.`,
|
||||
`Looked for CodeQL bundles in release ${tagName} of ${repository} on ${serverURL} but got error ${e}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
|
||||
const [owner, repo] = CODEQL_DEFAULT_ACTION_REPOSITORY.split("/");
|
||||
return selectBundle(
|
||||
action,
|
||||
getPublicRelease({
|
||||
serverURL: util.GITHUB_DOTCOM_URL,
|
||||
owner,
|
||||
repo,
|
||||
tagName,
|
||||
}),
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
function tryGetBundleVersionFromTagName(
|
||||
@@ -182,6 +230,12 @@ export function tryGetTagNameFromUrl(
|
||||
return match[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a bundle version to a semantic version, for example to use in the toolcache. Semantic
|
||||
* versions are normalized with `semver.clean`, which drops a leading `v` and any build metadata.
|
||||
* Anything else, such as a date, becomes a prerelease of `0.0.0`. Throws if the result isn't a
|
||||
* valid semantic version.
|
||||
*/
|
||||
export function convertToSemVer(version: string, logger: Logger): string {
|
||||
if (!semver.valid(version)) {
|
||||
logger.debug(
|
||||
@@ -374,6 +428,26 @@ async function resolveDefaultCliVersion(
|
||||
* Determines where the CodeQL CLI we want to use comes from. This can be from a local file,
|
||||
* the Actions toolcache, or a download.
|
||||
*
|
||||
* We handle the `tools` input in this order:
|
||||
*
|
||||
* - A local path is extracted without using the toolcache.
|
||||
* - A release URL selects a bundle from that release, and takes precedence over the `force_nightly`
|
||||
* feature flag. Only stable releases in the repositories we download the default bundles from use
|
||||
* the toolcache, since other releases may contain a different build than the cached bundle for
|
||||
* their version. Bundle URLs keep using the toolcache for the version in their tag, for
|
||||
* compatibility.
|
||||
* - `nightly` or `nightly-latest`, or the `force_nightly` feature flag in a dynamic workflow,
|
||||
* selects a bundle from the latest nightly release. We then continue with that bundle's URL.
|
||||
* - `linked`, or its old name `latest`, selects the version shipped with the Action.
|
||||
* - `toolcache` selects the latest version in the toolcache, falling back to the default version
|
||||
* outside dynamic workflows or if there isn't one.
|
||||
* - Any other value is the URL of a bundle.
|
||||
* - Without a `tools` input, we use the default version.
|
||||
*
|
||||
* For other inputs apart from a local path, we look for the resolved version in the toolcache
|
||||
* before downloading. A cached version takes precedence even if the job could use a per-language
|
||||
* bundle.
|
||||
*
|
||||
* @param toolsInput The argument provided for the `tools` input, if any.
|
||||
* @param defaultCliVersion The default CLI version that's linked to the CodeQL Action.
|
||||
* @param rawLanguages Raw set of languages.
|
||||
@@ -421,6 +495,11 @@ export async function getCodeQLSource(
|
||||
};
|
||||
}
|
||||
|
||||
const requestedRelease =
|
||||
toolsInput === undefined
|
||||
? undefined
|
||||
: parseCodeQLReleaseUrl(toolsInput, apiDetails);
|
||||
|
||||
/** Requested CLI version number, for example 2.12.6. */
|
||||
let cliVersion: string | undefined;
|
||||
/** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */
|
||||
@@ -432,10 +511,17 @@ export async function getCodeQLSource(
|
||||
*/
|
||||
let url: string | undefined;
|
||||
let bundle: CodeQLBundle | undefined;
|
||||
/** The release requested by URL, which we select the bundle from. */
|
||||
let release: CodeQLRelease | undefined;
|
||||
/** The page of a requested release whose bundles we don't cache. */
|
||||
let customReleaseURL: string | undefined;
|
||||
|
||||
// We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the
|
||||
// `tools` input cannot be adjusted to explicitly request it.
|
||||
const canForceNightlyWithFF = isDynamicWorkflow() || util.isInTestMode();
|
||||
// `tools` input cannot be adjusted to explicitly request it. An explicitly requested release
|
||||
// takes precedence.
|
||||
const canForceNightlyWithFF =
|
||||
requestedRelease === undefined &&
|
||||
(isDynamicWorkflow() || util.isInTestMode());
|
||||
const forceNightlyValueFF = await features.getValue(Feature.ForceNightly);
|
||||
const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
|
||||
|
||||
@@ -479,6 +565,7 @@ export async function getCodeQLSource(
|
||||
{ env: getEnv(), features, logger },
|
||||
rawLanguages,
|
||||
variant,
|
||||
tarSupportsZstd,
|
||||
);
|
||||
toolsInput = bundle.url;
|
||||
}
|
||||
@@ -555,8 +642,24 @@ export async function getCodeQLSource(
|
||||
cliVersion = version.cliVersion;
|
||||
tagName = version.tagName;
|
||||
}
|
||||
} else if (requestedRelease !== undefined) {
|
||||
release = await getRequestedRelease(
|
||||
{ apiClient: api.getApiClient() },
|
||||
requestedRelease,
|
||||
);
|
||||
tagName = requestedRelease.tagName;
|
||||
cliVersion = getReleaseCliVersion(
|
||||
tagName,
|
||||
release.assetNames ?? [],
|
||||
logger,
|
||||
);
|
||||
if (!isCacheableRelease(requestedRelease, cliVersion, apiDetails, logger)) {
|
||||
customReleaseURL = release.url;
|
||||
}
|
||||
} else if (toolsInput !== undefined) {
|
||||
// If a tools URL was provided, then use that.
|
||||
// Any other value is a bundle URL, including one we selected from the latest nightly above.
|
||||
// We use the version in its tag, if any, for the toolcache, so we assume that bundles with the
|
||||
// same version are the same build, whichever repository they're in.
|
||||
tagName = tryGetTagNameFromUrl(toolsInput, logger);
|
||||
url = toolsInput;
|
||||
|
||||
@@ -580,7 +683,8 @@ export async function getCodeQLSource(
|
||||
}
|
||||
|
||||
const bundleVersion =
|
||||
tagName !== undefined
|
||||
// Custom releases aren't cached, and their tags needn't contain a bundle version.
|
||||
tagName !== undefined && customReleaseURL === undefined
|
||||
? tryGetBundleVersionFromTagName(tagName, logger)
|
||||
: undefined;
|
||||
const resolvedVersion =
|
||||
@@ -597,13 +701,138 @@ export async function getCodeQLSource(
|
||||
`URL: ${url ?? "unspecified"}.`,
|
||||
);
|
||||
|
||||
// A custom release may contain a different build than the bundle with the same version.
|
||||
const codeqlFolder =
|
||||
customReleaseURL === undefined
|
||||
? await findCodeQLInToolcache(
|
||||
cliVersion,
|
||||
tagName,
|
||||
humanReadableVersion,
|
||||
logger,
|
||||
)
|
||||
: undefined;
|
||||
if (codeqlFolder) {
|
||||
if (cliVersion) {
|
||||
logger.info(
|
||||
`Using CodeQL CLI version ${cliVersion} from toolcache at ${codeqlFolder}`,
|
||||
);
|
||||
} else {
|
||||
logger.info(`Using CodeQL CLI from toolcache at ${codeqlFolder}`);
|
||||
}
|
||||
return {
|
||||
codeqlFolder,
|
||||
sourceType: "toolcache",
|
||||
toolsVersion: cliVersion ?? humanReadableVersion,
|
||||
};
|
||||
}
|
||||
|
||||
// If we don't find the requested version on Enterprise, we may allow a
|
||||
// different version to save download time if the version hasn't been
|
||||
// specified explicitly (in which case we always honor it).
|
||||
if (
|
||||
variant === util.GitHubVariant.GHES &&
|
||||
!forceShippedTools &&
|
||||
!toolsInput
|
||||
) {
|
||||
const result = await findOverridingToolsInCache(
|
||||
humanReadableVersion,
|
||||
logger,
|
||||
);
|
||||
if (result !== undefined) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let compressionMethod: tar.CompressionMethod;
|
||||
let perLanguageBundleFallback: true | undefined;
|
||||
|
||||
if (!url) {
|
||||
const selectionOptions: BundleSelectionOptions = {
|
||||
rawLanguages,
|
||||
cliVersion,
|
||||
platform: getBundlePlatform(),
|
||||
variant,
|
||||
tarSupportsZstd,
|
||||
};
|
||||
let selection: BundleSelection;
|
||||
if (release !== undefined) {
|
||||
selection = await selectBundle(
|
||||
{ env: getEnv(), features, logger },
|
||||
release,
|
||||
selectionOptions,
|
||||
);
|
||||
} else {
|
||||
if (tagName === undefined) {
|
||||
throw new Error(
|
||||
"Could not determine a release tag for the requested CodeQL bundle.",
|
||||
);
|
||||
}
|
||||
selection = await selectDefaultBundle(
|
||||
{ env: getEnv(), features, logger },
|
||||
tagName,
|
||||
apiDetails,
|
||||
selectionOptions,
|
||||
);
|
||||
}
|
||||
({ bundle, compressionMethod, perLanguageBundleFallback } = selection);
|
||||
url = bundle.url;
|
||||
} else {
|
||||
const method = tar.inferCompressionMethod(url);
|
||||
if (method === undefined) {
|
||||
throw new util.ConfigurationError(
|
||||
`Could not infer compression method from URL ${url}. Please specify a URL ` +
|
||||
"ending in '.tar.gz' or '.tar.zst'.",
|
||||
);
|
||||
}
|
||||
compressionMethod = method;
|
||||
|
||||
// Keep the bundle we selected from the latest nightly, which records the combined bundle to
|
||||
// fall back to. Otherwise, classify the explicit URL, which gets no fallback.
|
||||
bundle ??= getCodeQLBundleFromUrl(url);
|
||||
if (bundle.kind === "per-language") {
|
||||
logger.info(
|
||||
`${url} appears to be a CodeQL bundle that contains only ${bundle.language}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (cliVersion) {
|
||||
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url} .`);
|
||||
} else {
|
||||
logger.info(`Using CodeQL CLI sourced from ${url} .`);
|
||||
}
|
||||
return {
|
||||
bundle,
|
||||
bundleVersion,
|
||||
cliVersion,
|
||||
compressionMethod,
|
||||
...(customReleaseURL !== undefined ? { customReleaseURL } : {}),
|
||||
...(perLanguageBundleFallback ? { perLanguageBundleFallback } : {}),
|
||||
sourceType: "download",
|
||||
toolsVersion: resolvedVersion ?? "unknown",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks for the requested version of the CodeQL tools in the toolcache, allowing for the different
|
||||
* version numbers that toolcaches may use for the same bundle. We try the exact CLI version, then a
|
||||
* single `x.y.z-*` entry for that version, then `0.0.0-<bundle version>`. The `x.y.z-*` entry can
|
||||
* be any prerelease of the CLI version.
|
||||
*/
|
||||
async function findCodeQLInToolcache(
|
||||
cliVersion: string | undefined,
|
||||
tagName: string | undefined,
|
||||
humanReadableVersion: string,
|
||||
logger: Logger,
|
||||
): Promise<string | undefined> {
|
||||
let codeqlFolder: string | undefined;
|
||||
|
||||
if (cliVersion) {
|
||||
// If we find the specified CLI version, we always use that.
|
||||
codeqlFolder = toolcache.find("CodeQL", cliVersion);
|
||||
|
||||
// Fall back to matching `x.y.z-<tagName>`.
|
||||
// Fall back to a single `x.y.z-*` entry, since older toolcaches store bundles as
|
||||
// `x.y.z-<bundle version>`.
|
||||
if (!codeqlFolder) {
|
||||
logger.debug(
|
||||
"Didn't find a version of the CodeQL tools in the toolcache with a version number " +
|
||||
@@ -615,8 +844,6 @@ export async function getCodeQLSource(
|
||||
allVersions,
|
||||
)}.`,
|
||||
);
|
||||
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is
|
||||
// the form `x.y.z-<tagName>`, then use it.
|
||||
const candidateVersions = allVersions.filter((version) =>
|
||||
version.startsWith(`${cliVersion}-`),
|
||||
);
|
||||
@@ -667,123 +894,7 @@ export async function getCodeQLSource(
|
||||
`Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (codeqlFolder) {
|
||||
if (cliVersion) {
|
||||
logger.info(
|
||||
`Using CodeQL CLI version ${cliVersion} from toolcache at ${codeqlFolder}`,
|
||||
);
|
||||
} else {
|
||||
logger.info(`Using CodeQL CLI from toolcache at ${codeqlFolder}`);
|
||||
}
|
||||
return {
|
||||
codeqlFolder,
|
||||
sourceType: "toolcache",
|
||||
toolsVersion: cliVersion ?? humanReadableVersion,
|
||||
};
|
||||
}
|
||||
|
||||
// If we don't find the requested version on Enterprise, we may allow a
|
||||
// different version to save download time if the version hasn't been
|
||||
// specified explicitly (in which case we always honor it).
|
||||
if (
|
||||
variant === util.GitHubVariant.GHES &&
|
||||
!forceShippedTools &&
|
||||
!toolsInput
|
||||
) {
|
||||
const result = await findOverridingToolsInCache(
|
||||
humanReadableVersion,
|
||||
logger,
|
||||
);
|
||||
if (result !== undefined) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let compressionMethod: tar.CompressionMethod;
|
||||
|
||||
if (!url) {
|
||||
const bundleTagName = tagName;
|
||||
if (bundleTagName === undefined) {
|
||||
throw new Error(
|
||||
"Could not determine a release tag for the requested CodeQL bundle.",
|
||||
);
|
||||
}
|
||||
|
||||
compressionMethod =
|
||||
cliVersion !== undefined &&
|
||||
(await useZstdBundle(cliVersion, tarSupportsZstd))
|
||||
? "zstd"
|
||||
: "gzip";
|
||||
|
||||
const platform = getBundlePlatform();
|
||||
const perLanguageBundleLanguage = await getPerLanguageBundleLanguage(
|
||||
{ env: getEnv(), features, logger },
|
||||
{
|
||||
rawLanguages,
|
||||
cliVersion,
|
||||
compressionMethod,
|
||||
platform,
|
||||
variant,
|
||||
},
|
||||
);
|
||||
|
||||
// Resolves the combined or per-language bundle URL for the requested release.
|
||||
const resolveBundleURL = (language?: BuiltInLanguage) =>
|
||||
getCodeQLBundleDownloadURL(
|
||||
bundleTagName,
|
||||
apiDetails,
|
||||
getCodeQLBundleName(compressionMethod, platform, language),
|
||||
logger,
|
||||
);
|
||||
|
||||
const combinedBundleURL = await resolveBundleURL();
|
||||
if (perLanguageBundleLanguage !== undefined) {
|
||||
logger.info(
|
||||
`Selected the per-language CodeQL bundle for '${perLanguageBundleLanguage}'.`,
|
||||
);
|
||||
url = await resolveBundleURL(perLanguageBundleLanguage);
|
||||
bundle = {
|
||||
kind: "per-language",
|
||||
url,
|
||||
language: perLanguageBundleLanguage,
|
||||
combinedBundleURL,
|
||||
};
|
||||
} else {
|
||||
url = combinedBundleURL;
|
||||
bundle = { kind: "combined", url };
|
||||
}
|
||||
} else {
|
||||
const method = tar.inferCompressionMethod(url);
|
||||
if (method === undefined) {
|
||||
throw new util.ConfigurationError(
|
||||
`Could not infer compression method from URL ${url}. Please specify a URL ` +
|
||||
"ending in '.tar.gz' or '.tar.zst'.",
|
||||
);
|
||||
}
|
||||
compressionMethod = method;
|
||||
|
||||
bundle ??= getCodeQLBundleFromUrl(url);
|
||||
if (bundle.kind === "per-language") {
|
||||
logger.info(
|
||||
`${url} appears to be a CodeQL bundle that contains only ${bundle.language}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (cliVersion) {
|
||||
logger.info(`Using CodeQL CLI version ${cliVersion} sourced from ${url} .`);
|
||||
} else {
|
||||
logger.info(`Using CodeQL CLI sourced from ${url} .`);
|
||||
}
|
||||
return {
|
||||
bundle,
|
||||
bundleVersion,
|
||||
cliVersion,
|
||||
compressionMethod,
|
||||
sourceType: "download",
|
||||
toolsVersion: resolvedVersion ?? "unknown",
|
||||
};
|
||||
return codeqlFolder;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -868,17 +979,32 @@ export const downloadCodeQL = async function (
|
||||
...statusReport,
|
||||
perLanguage: { tools_bundle_language: bundle.language },
|
||||
}
|
||||
: statusReport,
|
||||
: source.perLanguageBundleFallback
|
||||
? {
|
||||
...statusReport,
|
||||
perLanguage: { tools_per_language_bundle_fallback: true },
|
||||
}
|
||||
: statusReport,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the canonical toolcache directory, or the reason the bundle cannot be cached.
|
||||
*
|
||||
* The toolcache is keyed by version, so we don't cache bundles that would give a later request for
|
||||
* the same version the wrong tools: per-language bundles, which lack the other languages, and
|
||||
* custom releases, which may be a different build.
|
||||
*/
|
||||
function getToolcacheDestination(
|
||||
{ logger }: ActionState<["Logger"]>,
|
||||
source: CodeQLDownloadSource,
|
||||
): util.Result<string, string> {
|
||||
if (source.customReleaseURL !== undefined) {
|
||||
return new util.Failure(
|
||||
`Not caching the CodeQL tools from ${source.customReleaseURL}, since we only cache stable ` +
|
||||
"releases in the CodeQL Action repositories.",
|
||||
);
|
||||
}
|
||||
if (source.bundle.kind !== "combined") {
|
||||
return new util.Failure(
|
||||
"Not caching the CodeQL tools because they came from a bundle that contains only a " +
|
||||
@@ -969,11 +1095,11 @@ function getCanonicalToolcacheVersion(
|
||||
bundleVersion: string,
|
||||
logger: Logger,
|
||||
): string {
|
||||
// If the CLI version is a pre-release or contains build metadata, then cache the
|
||||
// bundle as `0.0.0-<bundleVersion>` to avoid the bundle being interpreted as containing a stable
|
||||
// CLI release. In principle, it should be enough to just check that the CLI version isn't a
|
||||
// pre-release, but the version numbers of CodeQL nightlies have the format `x.y.z+<timestamp>`,
|
||||
// and we don't want these nightlies to override stable CLI versions in the toolcache.
|
||||
// If the CLI version is unknown, as for nightlies, which are tagged by date, or isn't a plain
|
||||
// `x.y.z`, for example a prerelease, cache the bundle under its bundle version, such as
|
||||
// `0.0.0-<date>` or `x.y.z-rc.1`, so that it isn't cached as a stable release. However,
|
||||
// `convertToSemVer` drops build metadata, so a bundle URL tagged `codeql-bundle-vX.Y.Z+<build>`
|
||||
// is still cached as `X.Y.Z`.
|
||||
if (!cliVersion?.match(/^[0-9]+\.[0-9]+\.[0-9]+$/)) {
|
||||
return convertToSemVer(bundleVersion, logger);
|
||||
}
|
||||
@@ -1113,12 +1239,13 @@ export async function downloadCodeQLBundle(
|
||||
) {
|
||||
throw e;
|
||||
}
|
||||
logPerLanguageBundleFallback(action, bundle.language, bundle.url);
|
||||
logMissingPerLanguageBundle(action, bundle.language, bundle.url);
|
||||
|
||||
const result = await downloadCodeQL(
|
||||
{
|
||||
...source,
|
||||
bundle: { kind: "combined", url: bundle.combinedBundleURL },
|
||||
perLanguageBundleFallback: true,
|
||||
},
|
||||
apiDetails,
|
||||
tarVersion,
|
||||
@@ -1130,24 +1257,11 @@ export async function downloadCodeQLBundle(
|
||||
statusReport: {
|
||||
...result.statusReport,
|
||||
totalDurationMs: util.durationMsSince(startTime),
|
||||
perLanguage: { tools_per_language_bundle_fallback: true },
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function useZstdBundle(
|
||||
cliVersion: string,
|
||||
tarSupportsZstd: boolean,
|
||||
): Promise<boolean> {
|
||||
return (
|
||||
// In testing, gzip performs better than zstd on Windows.
|
||||
process.platform !== "win32" &&
|
||||
tarSupportsZstd &&
|
||||
semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE)
|
||||
);
|
||||
}
|
||||
|
||||
function getTempExtractionDir(tempDir: string) {
|
||||
return path.join(tempDir, uuidV4());
|
||||
}
|
||||
@@ -1160,27 +1274,9 @@ async function getLatestNightlyBundle(
|
||||
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
|
||||
rawLanguages: string[] | undefined,
|
||||
variant: util.GitHubVariant,
|
||||
tarSupportsZstd: boolean,
|
||||
): Promise<CodeQLBundle> {
|
||||
const { logger } = action;
|
||||
const zstdAvailability = await tar.isZstdAvailable(logger);
|
||||
// The nightly is guaranteed to have a zstd bundle
|
||||
const compressionMethod = (await useZstdBundle(
|
||||
CODEQL_VERSION_ZSTD_BUNDLE,
|
||||
zstdAvailability.available,
|
||||
))
|
||||
? "zstd"
|
||||
: "gzip";
|
||||
|
||||
const platform = getBundlePlatform();
|
||||
const language = await getPerLanguageBundleLanguage(action, {
|
||||
rawLanguages,
|
||||
cliVersion: undefined,
|
||||
compressionMethod,
|
||||
platform,
|
||||
variant,
|
||||
isLatestNightly: true,
|
||||
});
|
||||
|
||||
let tagName: string;
|
||||
try {
|
||||
// Since nightlies are prereleases, we can't just download the latest release
|
||||
// on the repository. So instead we need to find the latest pre-release
|
||||
@@ -1196,26 +1292,31 @@ async function getLatestNightlyBundle(
|
||||
if (!latestRelease) {
|
||||
throw new Error("Could not find the latest nightly release.");
|
||||
}
|
||||
const assetUrl = (name: string) =>
|
||||
`https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`;
|
||||
const url = assetUrl(
|
||||
getCodeQLBundleName(compressionMethod, platform, language),
|
||||
);
|
||||
return language === undefined
|
||||
? { kind: "combined", url }
|
||||
: {
|
||||
kind: "per-language",
|
||||
url,
|
||||
language,
|
||||
combinedBundleURL: assetUrl(
|
||||
getCodeQLBundleName(compressionMethod, platform),
|
||||
),
|
||||
};
|
||||
tagName = latestRelease.tag_name;
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Failed to retrieve the latest nightly release: ${util.wrapError(e)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const { bundle } = await selectBundle(
|
||||
action,
|
||||
getPublicRelease({
|
||||
serverURL: util.GITHUB_DOTCOM_URL,
|
||||
owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER,
|
||||
repo: CODEQL_NIGHTLIES_REPOSITORY_NAME,
|
||||
tagName,
|
||||
}),
|
||||
{
|
||||
rawLanguages,
|
||||
cliVersion: undefined,
|
||||
platform: getBundlePlatform(),
|
||||
variant,
|
||||
tarSupportsZstd,
|
||||
isLatestNightly: true,
|
||||
},
|
||||
);
|
||||
return bundle;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user