mirror of
https://github.com/github/codeql-action.git
synced 2026-10-03 09:14:58 +00:00
Add helpers for selecting a CodeQL bundle from a release
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Henry Mercer
200
src/codeql-release.test.ts
Normal file
200
src/codeql-release.test.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
import * as github from "@actions/github";
|
||||
import test from "ava";
|
||||
|
||||
import {
|
||||
BundleSelectionOptions,
|
||||
getPublicRelease,
|
||||
getRelease,
|
||||
selectBundle,
|
||||
} from "./codeql-release";
|
||||
import { ActionsEnvVars } from "./environment";
|
||||
import { Feature } from "./feature-flags";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { BundlePlatform } from "./platform";
|
||||
import {
|
||||
createFeatures,
|
||||
getRecordingLogger,
|
||||
getTestEnv,
|
||||
initAllState,
|
||||
LoggedMessage,
|
||||
} from "./testing-utils";
|
||||
import { ConfigurationError, GitHubVariant } from "./util";
|
||||
|
||||
const TAG = "codeql-bundle-v2.27.1";
|
||||
const REFERENCE = {
|
||||
serverURL: "https://github.com",
|
||||
owner: "octo",
|
||||
repo: "tools",
|
||||
tagName: TAG,
|
||||
};
|
||||
const RELEASE_PAGE = `https://github.com/octo/tools/releases/tag/${TAG}`;
|
||||
const API_URL = "https://api.github.com";
|
||||
const COMBINED = "codeql-bundle-linux64.tar.zst";
|
||||
const JAVA = "codeql-bundle-java-linux64.tar.zst";
|
||||
|
||||
const OPTIONS: BundleSelectionOptions = {
|
||||
rawLanguages: ["java-kotlin"],
|
||||
cliVersion: "2.27.1",
|
||||
platform: BundlePlatform.Linux64,
|
||||
variant: GitHubVariant.DOTCOM,
|
||||
tarSupportsZstd: true,
|
||||
};
|
||||
|
||||
/** Serves the release tagged `TAG` with the given assets from a stubbed API. */
|
||||
function releaseFixture({ assetNames = [COMBINED, JAVA], status = 200 } = {}) {
|
||||
const apiBase = `${API_URL}/repos/octo/tools/releases`;
|
||||
const releaseAPIURL = `${apiBase}/tags/${TAG}`;
|
||||
const assets = assetNames.map((name, index) => ({
|
||||
name,
|
||||
url: `${apiBase}/assets/${1000 + index}`,
|
||||
}));
|
||||
const requests: string[] = [];
|
||||
const messages: LoggedMessage[] = [];
|
||||
const state = initAllState({
|
||||
env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted" }),
|
||||
logger: getRecordingLogger(messages, { logToConsole: false }),
|
||||
features: createFeatures([Feature.PerLanguageBundles]),
|
||||
apiClient: github.getOctokit("123", {
|
||||
baseUrl: API_URL,
|
||||
request: {
|
||||
fetch: async (url) => {
|
||||
requests.push(String(url));
|
||||
if (String(url) !== releaseAPIURL) {
|
||||
throw new Error(`Unexpected API request: ${url}`);
|
||||
}
|
||||
return new Response(JSON.stringify({ tag_name: TAG, assets }), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
return {
|
||||
assets,
|
||||
messages,
|
||||
releaseAPIURL,
|
||||
requests,
|
||||
state,
|
||||
select: async (options: Partial<BundleSelectionOptions> = {}) =>
|
||||
selectBundle(state, await getRelease(state, REFERENCE), {
|
||||
...OPTIONS,
|
||||
...options,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
test("selectBundle selects an eligible per-language bundle with a single release lookup", async (t) => {
|
||||
const fixture = releaseFixture();
|
||||
t.deepEqual(await fixture.select(), {
|
||||
bundle: {
|
||||
kind: "per-language",
|
||||
url: fixture.assets[1].url,
|
||||
language: BuiltInLanguage.java,
|
||||
combinedBundleURL: fixture.assets[0].url,
|
||||
},
|
||||
compressionMethod: "zstd",
|
||||
});
|
||||
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
|
||||
});
|
||||
|
||||
test("selectBundle falls back to the combined bundle from the same release", async (t) => {
|
||||
const fixture = releaseFixture({ assetNames: [COMBINED] });
|
||||
t.deepEqual(await fixture.select(), {
|
||||
bundle: { kind: "combined", url: fixture.assets[0].url },
|
||||
compressionMethod: "zstd",
|
||||
perLanguageBundleFallback: true,
|
||||
});
|
||||
t.true(
|
||||
fixture.messages.some(
|
||||
(message) =>
|
||||
message.type === "warning" &&
|
||||
typeof message.message === "string" &&
|
||||
message.message.includes(`'java' at ${RELEASE_PAGE}`),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test("selectBundle selects the combined bundle for jobs that aren't eligible for a per-language bundle", async (t) => {
|
||||
const fixture = releaseFixture();
|
||||
t.deepEqual(await fixture.select({ rawLanguages: ["java", "python"] }), {
|
||||
bundle: { kind: "combined", url: fixture.assets[0].url },
|
||||
compressionMethod: "zstd",
|
||||
});
|
||||
});
|
||||
|
||||
test("selectBundle uses the other compression method when the preferred one is missing", async (t) => {
|
||||
const gzipOnly = releaseFixture({
|
||||
assetNames: ["codeql-bundle-linux64.tar.gz", JAVA],
|
||||
});
|
||||
t.deepEqual(await gzipOnly.select(), {
|
||||
bundle: { kind: "combined", url: gzipOnly.assets[0].url },
|
||||
compressionMethod: "gzip",
|
||||
});
|
||||
|
||||
for (const [platform, options] of [
|
||||
[BundlePlatform.Win64, {}],
|
||||
[BundlePlatform.Linux64, { cliVersion: "2.18.4" }],
|
||||
[BundlePlatform.Linux64, { cliVersion: undefined }],
|
||||
[BundlePlatform.Linux64, { tarSupportsZstd: false }],
|
||||
] as const) {
|
||||
const selection = await releaseFixture({
|
||||
assetNames: [
|
||||
`codeql-bundle-${platform}.tar.zst`,
|
||||
`codeql-bundle-${platform}.tar.gz`,
|
||||
],
|
||||
}).select({ platform, ...options });
|
||||
t.is(
|
||||
selection.compressionMethod,
|
||||
"gzip",
|
||||
`${platform} ${JSON.stringify(options)}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("selectBundle requires a combined bundle that we can extract", async (t) => {
|
||||
await t.throwsAsync(releaseFixture({ assetNames: [JAVA] }).select(), {
|
||||
instanceOf: ConfigurationError,
|
||||
message: `No compatible CodeQL bundle was found in release ${RELEASE_PAGE}. Expected codeql-bundle-linux64.tar.zst or codeql-bundle-linux64.tar.gz.`,
|
||||
});
|
||||
await t.throwsAsync(
|
||||
releaseFixture({ assetNames: [COMBINED] }).select({
|
||||
tarSupportsZstd: false,
|
||||
}),
|
||||
{ instanceOf: ConfigurationError, message: /Expected [^ ]+\.tar\.gz\.$/ },
|
||||
);
|
||||
});
|
||||
|
||||
test("getRelease propagates API errors", async (t) => {
|
||||
const fixture = releaseFixture({ status: 404 });
|
||||
t.like(await t.throwsAsync(fixture.select()), { status: 404 });
|
||||
t.deepEqual(fixture.requests, [fixture.releaseAPIURL]);
|
||||
});
|
||||
|
||||
test("getPublicRelease constructs download URLs without looking up the release", async (t) => {
|
||||
const fixture = releaseFixture();
|
||||
const release = getPublicRelease({ ...REFERENCE, tagName: "nightly/v1+2" });
|
||||
const baseURL =
|
||||
"https://github.com/octo/tools/releases/download/nightly/v1%2B2";
|
||||
t.is(
|
||||
release.url,
|
||||
"https://github.com/octo/tools/releases/tag/nightly/v1%2B2",
|
||||
);
|
||||
t.deepEqual(
|
||||
await selectBundle(fixture.state, release, {
|
||||
...OPTIONS,
|
||||
cliVersion: undefined,
|
||||
isLatestNightly: true,
|
||||
}),
|
||||
{
|
||||
bundle: {
|
||||
kind: "per-language",
|
||||
url: `${baseURL}/${JAVA}`,
|
||||
language: BuiltInLanguage.java,
|
||||
combinedBundleURL: `${baseURL}/${COMBINED}`,
|
||||
},
|
||||
compressionMethod: "zstd",
|
||||
},
|
||||
);
|
||||
t.deepEqual(fixture.requests, []);
|
||||
});
|
||||
188
src/codeql-release.ts
Normal file
188
src/codeql-release.ts
Normal file
@@ -0,0 +1,188 @@
|
||||
import * as semver from "semver";
|
||||
|
||||
import { ActionState } from "./action-common";
|
||||
import { CodeQLBundle, getCodeQLBundleName } from "./codeql-bundle";
|
||||
import { CODEQL_VERSION_ZSTD_BUNDLE } from "./feature-flags";
|
||||
import {
|
||||
getPerLanguageBundleLanguage,
|
||||
logMissingPerLanguageBundle,
|
||||
} from "./per-language-bundles";
|
||||
import { BundlePlatform } from "./platform";
|
||||
import type { CompressionMethod } from "./tar";
|
||||
import { ConfigurationError, GitHubVariant } from "./util";
|
||||
|
||||
/** Identifies a release on a GitHub instance. */
|
||||
export interface CodeQLReleaseReference {
|
||||
serverURL: string;
|
||||
owner: string;
|
||||
repo: string;
|
||||
tagName: string;
|
||||
}
|
||||
|
||||
/** A GitHub release that contains CodeQL bundles. */
|
||||
export interface CodeQLRelease {
|
||||
/** The release's web page, for messages. */
|
||||
url: string;
|
||||
/** Returns the download URL for an asset, or `undefined` if the release doesn't have it. */
|
||||
getAssetURL(name: string): string | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes a tag for use in a URL path. Slashes stay as path separators, as in GitHub's release URLs
|
||||
* for tags like `build/123`.
|
||||
*/
|
||||
function encodeTag(tagName: string): string {
|
||||
return tagName.split("/").map(encodeURIComponent).join("/");
|
||||
}
|
||||
|
||||
function getReleasePageURL(reference: CodeQLReleaseReference): string {
|
||||
const { serverURL, owner, repo, tagName } = reference;
|
||||
return `${serverURL}/${owner}/${repo}/releases/tag/${encodeTag(tagName)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks up a release on the current GitHub instance, which works for private repositories.
|
||||
*
|
||||
* The API client determines which instance we query, so `reference.serverURL` is only used for the
|
||||
* release page URL in messages. The asset URLs are REST API endpoints, which accept the token and
|
||||
* return the file when requested with `Accept: application/octet-stream`, unlike browser download
|
||||
* URLs.
|
||||
*/
|
||||
export async function getRelease(
|
||||
{ apiClient }: ActionState<["Api"]>,
|
||||
reference: CodeQLReleaseReference,
|
||||
): Promise<CodeQLRelease> {
|
||||
const { owner, repo, tagName } = reference;
|
||||
const { data: release } = await apiClient.rest.repos.getReleaseByTag({
|
||||
owner,
|
||||
repo,
|
||||
tag: tagName,
|
||||
});
|
||||
return {
|
||||
url: getReleasePageURL(reference),
|
||||
getAssetURL: (name) =>
|
||||
release.assets.find((asset) => asset.name === name)?.url,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Refers to a public release without looking it up. Every asset gets a download URL, so a missing
|
||||
* asset shows up as a failed download.
|
||||
*/
|
||||
export function getPublicRelease(
|
||||
reference: CodeQLReleaseReference,
|
||||
): CodeQLRelease {
|
||||
const { serverURL, owner, repo, tagName } = reference;
|
||||
return {
|
||||
url: getReleasePageURL(reference),
|
||||
getAssetURL: (name) =>
|
||||
`${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** Describes the job and runner that we are selecting a bundle for. */
|
||||
export interface BundleSelectionOptions {
|
||||
/** Explicit `languages` input, which determines whether a per-language bundle is eligible. */
|
||||
rawLanguages: string[] | undefined;
|
||||
/** The CLI version in the release, if known. */
|
||||
cliVersion: string | undefined;
|
||||
platform: BundlePlatform | undefined;
|
||||
variant: GitHubVariant;
|
||||
tarSupportsZstd: boolean;
|
||||
/** Whether the release is the latest nightly, whose CLI version we don't know yet. */
|
||||
isLatestNightly?: boolean;
|
||||
}
|
||||
|
||||
/** A bundle selected from a release. */
|
||||
export interface BundleSelection {
|
||||
bundle: CodeQLBundle;
|
||||
compressionMethod: CompressionMethod;
|
||||
/** The release lacks the eligible per-language bundle, so we selected the combined bundle. */
|
||||
perLanguageBundleFallback?: true;
|
||||
}
|
||||
|
||||
/** Returns the compression methods that we can extract, most preferred first. */
|
||||
function getCompressionMethods({
|
||||
cliVersion,
|
||||
isLatestNightly,
|
||||
platform,
|
||||
tarSupportsZstd,
|
||||
}: BundleSelectionOptions): CompressionMethod[] {
|
||||
if (!tarSupportsZstd) {
|
||||
return ["gzip"];
|
||||
}
|
||||
const preferZstd =
|
||||
// In testing, gzip performs better than zstd on Windows.
|
||||
platform !== BundlePlatform.Win64 &&
|
||||
// Standard bundles have zstd archives from this version, and so does the latest nightly. For a
|
||||
// release we looked up, gzip comes next if it lacks the zstd archive.
|
||||
(isLatestNightly ||
|
||||
(cliVersion !== undefined &&
|
||||
semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE)));
|
||||
return preferZstd ? ["zstd", "gzip"] : ["gzip", "zstd"];
|
||||
}
|
||||
|
||||
/**
|
||||
* Selects a per-language bundle if the job is eligible for one, and otherwise the combined bundle.
|
||||
* We only select a per-language bundle from a release that also has a combined bundle with the same
|
||||
* compression, so that we can fall back to it.
|
||||
*
|
||||
* If we looked up the release, we can tell that an eligible per-language bundle is missing, so we
|
||||
* warn and select the combined bundle straight away. A public release gives every asset a URL, so a
|
||||
* missing per-language bundle only shows up as a 404 when downloading, which is when we fall back.
|
||||
*
|
||||
* Throws a `ConfigurationError` if the release has no combined bundle that we can extract.
|
||||
*/
|
||||
export async function selectBundle(
|
||||
action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>,
|
||||
release: CodeQLRelease,
|
||||
options: BundleSelectionOptions,
|
||||
): Promise<BundleSelection> {
|
||||
const { logger } = action;
|
||||
const compressionMethods = getCompressionMethods(options);
|
||||
for (const compressionMethod of compressionMethods) {
|
||||
const combinedBundleName = getCodeQLBundleName(
|
||||
compressionMethod,
|
||||
options.platform,
|
||||
);
|
||||
const combinedBundleURL = release.getAssetURL(combinedBundleName);
|
||||
if (combinedBundleURL === undefined) {
|
||||
continue;
|
||||
}
|
||||
const combined: BundleSelection = {
|
||||
bundle: { kind: "combined", url: combinedBundleURL },
|
||||
compressionMethod,
|
||||
};
|
||||
|
||||
const language = await getPerLanguageBundleLanguage(action, {
|
||||
...options,
|
||||
compressionMethod,
|
||||
});
|
||||
if (language === undefined) {
|
||||
logger.info(
|
||||
`Selected CodeQL bundle ${combinedBundleName} from ${release.url}.`,
|
||||
);
|
||||
return combined;
|
||||
}
|
||||
const name = getCodeQLBundleName(
|
||||
compressionMethod,
|
||||
options.platform,
|
||||
language,
|
||||
);
|
||||
const url = release.getAssetURL(name);
|
||||
if (url === undefined) {
|
||||
logMissingPerLanguageBundle(action, language, release.url);
|
||||
return { ...combined, perLanguageBundleFallback: true };
|
||||
}
|
||||
logger.info(`Selected CodeQL bundle ${name} from ${release.url}.`);
|
||||
return {
|
||||
bundle: { kind: "per-language", url, language, combinedBundleURL },
|
||||
compressionMethod,
|
||||
};
|
||||
}
|
||||
throw new ConfigurationError(
|
||||
`No compatible CodeQL bundle was found in release ${release.url}. Expected ${compressionMethods
|
||||
.map((method) => getCodeQLBundleName(method, options.platform))
|
||||
.join(" or ")}.`,
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user