diff --git a/src/codeql-release.test.ts b/src/codeql-release.test.ts new file mode 100644 index 000000000..ce29a4bf1 --- /dev/null +++ b/src/codeql-release.test.ts @@ -0,0 +1,200 @@ +import * as github from "@actions/github"; +import test from "ava"; + +import { + BundleSelectionOptions, + getPublicRelease, + getRelease, + selectBundle, +} from "./codeql-release"; +import { ActionsEnvVars } from "./environment"; +import { Feature } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; +import { BundlePlatform } from "./platform"; +import { + createFeatures, + getRecordingLogger, + getTestEnv, + initAllState, + LoggedMessage, +} from "./testing-utils"; +import { ConfigurationError, GitHubVariant } from "./util"; + +const TAG = "codeql-bundle-v2.27.1"; +const REFERENCE = { + serverURL: "https://github.com", + owner: "octo", + repo: "tools", + tagName: TAG, +}; +const RELEASE_PAGE = `https://github.com/octo/tools/releases/tag/${TAG}`; +const API_URL = "https://api.github.com"; +const COMBINED = "codeql-bundle-linux64.tar.zst"; +const JAVA = "codeql-bundle-java-linux64.tar.zst"; + +const OPTIONS: BundleSelectionOptions = { + rawLanguages: ["java-kotlin"], + cliVersion: "2.27.1", + platform: BundlePlatform.Linux64, + variant: GitHubVariant.DOTCOM, + tarSupportsZstd: true, +}; + +/** Serves the release tagged `TAG` with the given assets from a stubbed API. */ +function releaseFixture({ assetNames = [COMBINED, JAVA], status = 200 } = {}) { + const apiBase = `${API_URL}/repos/octo/tools/releases`; + const releaseAPIURL = `${apiBase}/tags/${TAG}`; + const assets = assetNames.map((name, index) => ({ + name, + url: `${apiBase}/assets/${1000 + index}`, + })); + const requests: string[] = []; + const messages: LoggedMessage[] = []; + const state = initAllState({ + env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted" }), + logger: getRecordingLogger(messages, { logToConsole: false }), + features: createFeatures([Feature.PerLanguageBundles]), + apiClient: github.getOctokit("123", { + baseUrl: API_URL, + request: { + fetch: async (url) => { + requests.push(String(url)); + if (String(url) !== releaseAPIURL) { + throw new Error(`Unexpected API request: ${url}`); + } + return new Response(JSON.stringify({ tag_name: TAG, assets }), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }, + }), + }); + return { + assets, + messages, + releaseAPIURL, + requests, + state, + select: async (options: Partial = {}) => + selectBundle(state, await getRelease(state, REFERENCE), { + ...OPTIONS, + ...options, + }), + }; +} + +test("selectBundle selects an eligible per-language bundle with a single release lookup", async (t) => { + const fixture = releaseFixture(); + t.deepEqual(await fixture.select(), { + bundle: { + kind: "per-language", + url: fixture.assets[1].url, + language: BuiltInLanguage.java, + combinedBundleURL: fixture.assets[0].url, + }, + compressionMethod: "zstd", + }); + t.deepEqual(fixture.requests, [fixture.releaseAPIURL]); +}); + +test("selectBundle falls back to the combined bundle from the same release", async (t) => { + const fixture = releaseFixture({ assetNames: [COMBINED] }); + t.deepEqual(await fixture.select(), { + bundle: { kind: "combined", url: fixture.assets[0].url }, + compressionMethod: "zstd", + perLanguageBundleFallback: true, + }); + t.true( + fixture.messages.some( + (message) => + message.type === "warning" && + typeof message.message === "string" && + message.message.includes(`'java' at ${RELEASE_PAGE}`), + ), + ); +}); + +test("selectBundle selects the combined bundle for jobs that aren't eligible for a per-language bundle", async (t) => { + const fixture = releaseFixture(); + t.deepEqual(await fixture.select({ rawLanguages: ["java", "python"] }), { + bundle: { kind: "combined", url: fixture.assets[0].url }, + compressionMethod: "zstd", + }); +}); + +test("selectBundle uses the other compression method when the preferred one is missing", async (t) => { + const gzipOnly = releaseFixture({ + assetNames: ["codeql-bundle-linux64.tar.gz", JAVA], + }); + t.deepEqual(await gzipOnly.select(), { + bundle: { kind: "combined", url: gzipOnly.assets[0].url }, + compressionMethod: "gzip", + }); + + for (const [platform, options] of [ + [BundlePlatform.Win64, {}], + [BundlePlatform.Linux64, { cliVersion: "2.18.4" }], + [BundlePlatform.Linux64, { cliVersion: undefined }], + [BundlePlatform.Linux64, { tarSupportsZstd: false }], + ] as const) { + const selection = await releaseFixture({ + assetNames: [ + `codeql-bundle-${platform}.tar.zst`, + `codeql-bundle-${platform}.tar.gz`, + ], + }).select({ platform, ...options }); + t.is( + selection.compressionMethod, + "gzip", + `${platform} ${JSON.stringify(options)}`, + ); + } +}); + +test("selectBundle requires a combined bundle that we can extract", async (t) => { + await t.throwsAsync(releaseFixture({ assetNames: [JAVA] }).select(), { + instanceOf: ConfigurationError, + message: `No compatible CodeQL bundle was found in release ${RELEASE_PAGE}. Expected codeql-bundle-linux64.tar.zst or codeql-bundle-linux64.tar.gz.`, + }); + await t.throwsAsync( + releaseFixture({ assetNames: [COMBINED] }).select({ + tarSupportsZstd: false, + }), + { instanceOf: ConfigurationError, message: /Expected [^ ]+\.tar\.gz\.$/ }, + ); +}); + +test("getRelease propagates API errors", async (t) => { + const fixture = releaseFixture({ status: 404 }); + t.like(await t.throwsAsync(fixture.select()), { status: 404 }); + t.deepEqual(fixture.requests, [fixture.releaseAPIURL]); +}); + +test("getPublicRelease constructs download URLs without looking up the release", async (t) => { + const fixture = releaseFixture(); + const release = getPublicRelease({ ...REFERENCE, tagName: "nightly/v1+2" }); + const baseURL = + "https://github.com/octo/tools/releases/download/nightly/v1%2B2"; + t.is( + release.url, + "https://github.com/octo/tools/releases/tag/nightly/v1%2B2", + ); + t.deepEqual( + await selectBundle(fixture.state, release, { + ...OPTIONS, + cliVersion: undefined, + isLatestNightly: true, + }), + { + bundle: { + kind: "per-language", + url: `${baseURL}/${JAVA}`, + language: BuiltInLanguage.java, + combinedBundleURL: `${baseURL}/${COMBINED}`, + }, + compressionMethod: "zstd", + }, + ); + t.deepEqual(fixture.requests, []); +}); diff --git a/src/codeql-release.ts b/src/codeql-release.ts new file mode 100644 index 000000000..23f18503d --- /dev/null +++ b/src/codeql-release.ts @@ -0,0 +1,188 @@ +import * as semver from "semver"; + +import { ActionState } from "./action-common"; +import { CodeQLBundle, getCodeQLBundleName } from "./codeql-bundle"; +import { CODEQL_VERSION_ZSTD_BUNDLE } from "./feature-flags"; +import { + getPerLanguageBundleLanguage, + logMissingPerLanguageBundle, +} from "./per-language-bundles"; +import { BundlePlatform } from "./platform"; +import type { CompressionMethod } from "./tar"; +import { ConfigurationError, GitHubVariant } from "./util"; + +/** Identifies a release on a GitHub instance. */ +export interface CodeQLReleaseReference { + serverURL: string; + owner: string; + repo: string; + tagName: string; +} + +/** A GitHub release that contains CodeQL bundles. */ +export interface CodeQLRelease { + /** The release's web page, for messages. */ + url: string; + /** Returns the download URL for an asset, or `undefined` if the release doesn't have it. */ + getAssetURL(name: string): string | undefined; +} + +/** + * Encodes a tag for use in a URL path. Slashes stay as path separators, as in GitHub's release URLs + * for tags like `build/123`. + */ +function encodeTag(tagName: string): string { + return tagName.split("/").map(encodeURIComponent).join("/"); +} + +function getReleasePageURL(reference: CodeQLReleaseReference): string { + const { serverURL, owner, repo, tagName } = reference; + return `${serverURL}/${owner}/${repo}/releases/tag/${encodeTag(tagName)}`; +} + +/** + * Looks up a release on the current GitHub instance, which works for private repositories. + * + * The API client determines which instance we query, so `reference.serverURL` is only used for the + * release page URL in messages. The asset URLs are REST API endpoints, which accept the token and + * return the file when requested with `Accept: application/octet-stream`, unlike browser download + * URLs. + */ +export async function getRelease( + { apiClient }: ActionState<["Api"]>, + reference: CodeQLReleaseReference, +): Promise { + const { owner, repo, tagName } = reference; + const { data: release } = await apiClient.rest.repos.getReleaseByTag({ + owner, + repo, + tag: tagName, + }); + return { + url: getReleasePageURL(reference), + getAssetURL: (name) => + release.assets.find((asset) => asset.name === name)?.url, + }; +} + +/** + * Refers to a public release without looking it up. Every asset gets a download URL, so a missing + * asset shows up as a failed download. + */ +export function getPublicRelease( + reference: CodeQLReleaseReference, +): CodeQLRelease { + const { serverURL, owner, repo, tagName } = reference; + return { + url: getReleasePageURL(reference), + getAssetURL: (name) => + `${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}`, + }; +} + +/** Describes the job and runner that we are selecting a bundle for. */ +export interface BundleSelectionOptions { + /** Explicit `languages` input, which determines whether a per-language bundle is eligible. */ + rawLanguages: string[] | undefined; + /** The CLI version in the release, if known. */ + cliVersion: string | undefined; + platform: BundlePlatform | undefined; + variant: GitHubVariant; + tarSupportsZstd: boolean; + /** Whether the release is the latest nightly, whose CLI version we don't know yet. */ + isLatestNightly?: boolean; +} + +/** A bundle selected from a release. */ +export interface BundleSelection { + bundle: CodeQLBundle; + compressionMethod: CompressionMethod; + /** The release lacks the eligible per-language bundle, so we selected the combined bundle. */ + perLanguageBundleFallback?: true; +} + +/** Returns the compression methods that we can extract, most preferred first. */ +function getCompressionMethods({ + cliVersion, + isLatestNightly, + platform, + tarSupportsZstd, +}: BundleSelectionOptions): CompressionMethod[] { + if (!tarSupportsZstd) { + return ["gzip"]; + } + const preferZstd = + // In testing, gzip performs better than zstd on Windows. + platform !== BundlePlatform.Win64 && + // Standard bundles have zstd archives from this version, and so does the latest nightly. For a + // release we looked up, gzip comes next if it lacks the zstd archive. + (isLatestNightly || + (cliVersion !== undefined && + semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE))); + return preferZstd ? ["zstd", "gzip"] : ["gzip", "zstd"]; +} + +/** + * Selects a per-language bundle if the job is eligible for one, and otherwise the combined bundle. + * We only select a per-language bundle from a release that also has a combined bundle with the same + * compression, so that we can fall back to it. + * + * If we looked up the release, we can tell that an eligible per-language bundle is missing, so we + * warn and select the combined bundle straight away. A public release gives every asset a URL, so a + * missing per-language bundle only shows up as a 404 when downloading, which is when we fall back. + * + * Throws a `ConfigurationError` if the release has no combined bundle that we can extract. + */ +export async function selectBundle( + action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, + release: CodeQLRelease, + options: BundleSelectionOptions, +): Promise { + const { logger } = action; + const compressionMethods = getCompressionMethods(options); + for (const compressionMethod of compressionMethods) { + const combinedBundleName = getCodeQLBundleName( + compressionMethod, + options.platform, + ); + const combinedBundleURL = release.getAssetURL(combinedBundleName); + if (combinedBundleURL === undefined) { + continue; + } + const combined: BundleSelection = { + bundle: { kind: "combined", url: combinedBundleURL }, + compressionMethod, + }; + + const language = await getPerLanguageBundleLanguage(action, { + ...options, + compressionMethod, + }); + if (language === undefined) { + logger.info( + `Selected CodeQL bundle ${combinedBundleName} from ${release.url}.`, + ); + return combined; + } + const name = getCodeQLBundleName( + compressionMethod, + options.platform, + language, + ); + const url = release.getAssetURL(name); + if (url === undefined) { + logMissingPerLanguageBundle(action, language, release.url); + return { ...combined, perLanguageBundleFallback: true }; + } + logger.info(`Selected CodeQL bundle ${name} from ${release.url}.`); + return { + bundle: { kind: "per-language", url, language, combinedBundleURL }, + compressionMethod, + }; + } + throw new ConfigurationError( + `No compatible CodeQL bundle was found in release ${release.url}. Expected ${compressionMethods + .map((method) => getCodeQLBundleName(method, options.platform)) + .join(" or ")}.`, + ); +}