mirror of
https://github.com/github/codeql-action.git
synced 2026-10-03 17:41:28 +00:00
Use a per-language bundle from a nightly when one is asked for
Nightlies are the first bundles to contain per-language bundles, so allow a workflow that explicitly asks for a nightly to use one. Their tags record the date they were built rather than a version, so there is no minimum version to check against; a nightly is always at least as new as the first release to publish per-language bundles. Only do this when a nightly was asked for explicitly. Nightlies can also be forced for analyses that did not ask for one, and those should keep getting the bundle that contains every language while the feature is still being tested. Fall back to the combined bundle from the same nightly if it turns out not to contain the language, since we chose the bundle ourselves rather than being asked for it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d4c7b873-8278-4a36-b67c-6cc5ea3b2316
This commit is contained in:
Henry Mercer
@@ -1,18 +1,19 @@
|
||||
# Validates the per-language CodeQL bundles produced by a pre-release build. This is not generated
|
||||
# from `pr-checks`, since each language needs its own platform, build steps and set of expected
|
||||
# Validates the per-language CodeQL bundles in the latest nightly. This is not generated from
|
||||
# `pr-checks`, since each language needs its own platform, build steps and set of expected
|
||||
# extractors, whereas the generated checks share one set of steps across a matrix that only varies
|
||||
# the operating system and CodeQL version.
|
||||
#
|
||||
# TODO(per-language-bundles): The bundles below are from a pre-release, so this needs revisiting once
|
||||
# they ship in a release: either point `BUNDLE_TAG` at a released bundle and run this on a schedule
|
||||
# rather than on every push, or drop it if the checks that exercise the download path are enough.
|
||||
# TODO(per-language-bundles): This needs revisiting once per-language bundles ship in a release:
|
||||
# either run it on a schedule rather than on every push, or drop it if the checks that exercise the
|
||||
# download path are enough by then.
|
||||
name: Per-language bundle validation
|
||||
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GO111MODULE: auto
|
||||
# The pre-release whose per-language bundles we are validating.
|
||||
BUNDLE_TAG: codeql-bundle-20260908
|
||||
# Nightlies are the only bundles that currently contain per-language bundles, and the Action only
|
||||
# considers one for a nightly that was asked for explicitly, as this workflow does below.
|
||||
CODEQL_ACTION_PER_LANGUAGE_BUNDLES: true
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -37,42 +38,32 @@ jobs:
|
||||
matrix:
|
||||
include:
|
||||
- language: actions
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
# The Actions QL pack depends on the JavaScript one, which is the only dependency of its
|
||||
# kind, so the Actions bundle has to carry the JavaScript extractor as well.
|
||||
expected-extractors: actions javascript
|
||||
- language: cpp
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
build-mode: manual
|
||||
build-command: gcc -o main main.c
|
||||
- language: csharp
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
build-mode: none
|
||||
- language: go
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
build-mode: autobuild
|
||||
- language: java
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
build-mode: none
|
||||
- language: javascript
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
- language: python
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
- language: ruby
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
- language: rust
|
||||
platform: linux64
|
||||
os: ubuntu-latest
|
||||
- language: swift
|
||||
platform: osx64
|
||||
# Swift autobuild is too slow to finish on a standard macOS runner, which is why the
|
||||
# generated Swift checks use a larger one as well.
|
||||
os: macos-latest-xlarge
|
||||
@@ -91,7 +82,7 @@ jobs:
|
||||
id: prepare-test
|
||||
uses: ./.github/actions/prepare-test
|
||||
with:
|
||||
# Unused: we pin `tools` to a specific per-language bundle below.
|
||||
# Unused: we ask for a nightly below.
|
||||
version: linked
|
||||
use-all-platform-bundle: 'false'
|
||||
setup-kotlin: 'false'
|
||||
@@ -100,7 +91,9 @@ jobs:
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix['build-mode'] }}
|
||||
tools: https://github.com/dsp-testing/henrymercer-codeql-cli-binaries/releases/download/${{ env.BUNDLE_TAG }}/codeql-bundle-${{ matrix.language }}-${{ matrix.platform }}.tar.zst
|
||||
# The Action picks the per-language bundle from the latest nightly itself, so this also
|
||||
# exercises the code that decides which bundle to download.
|
||||
tools: nightly
|
||||
- name: Check that the bundle contains only the expected extractors
|
||||
env:
|
||||
CODEQL_PATH: ${{ steps.init.outputs.codeql-path }}
|
||||
@@ -135,17 +128,16 @@ jobs:
|
||||
CODEQL_PATH: ${{ steps.init.outputs.codeql-path }}
|
||||
run: |
|
||||
# A bundle that is missing most of its extractors must never be left in the toolcache,
|
||||
# where a later job analyzing a different language could pick it up.
|
||||
# where a later job analyzing a different language could pick it up. The runner image
|
||||
# ships with its own CodeQL in the toolcache, so check where this bundle was extracted to
|
||||
# rather than whether the toolcache contains CodeQL at all.
|
||||
echo "CodeQL is at $CODEQL_PATH"
|
||||
if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then
|
||||
echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH."
|
||||
exit 1
|
||||
fi
|
||||
# The runner image ships with its own CodeQL in the toolcache, so look for the version we
|
||||
# downloaded rather than for CodeQL in general.
|
||||
cached_version="$RUNNER_TOOL_CACHE/CodeQL/0.0.0-${BUNDLE_TAG#codeql-bundle-}"
|
||||
if [ -d "$cached_version" ]; then
|
||||
echo "::error::The per-language bundle was added to the toolcache at $cached_version."
|
||||
if [[ "$CODEQL_PATH" != "$RUNNER_TEMP"/* ]]; then
|
||||
echo "::error::Expected the per-language bundle to be extracted under $RUNNER_TEMP, but found it at $CODEQL_PATH."
|
||||
exit 1
|
||||
fi
|
||||
- name: Build code
|
||||
|
||||
47
lib/entry-points.js
generated
47
lib/entry-points.js
generated
@@ -151538,7 +151538,14 @@ var PER_LANGUAGE_BUNDLE_PLATFORMS = {
|
||||
["swift" /* swift */]: "osx64"
|
||||
};
|
||||
async function getPerLanguageBundleLanguage(options, features, logger) {
|
||||
const { rawLanguages, cliVersion: cliVersion2, compressionMethod, platform: platform2, variant } = options;
|
||||
const {
|
||||
rawLanguages,
|
||||
cliVersion: cliVersion2,
|
||||
compressionMethod,
|
||||
platform: platform2,
|
||||
variant,
|
||||
isNightly
|
||||
} = options;
|
||||
const explain = (reason) => {
|
||||
logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`);
|
||||
return void 0;
|
||||
@@ -151561,6 +151568,7 @@ async function getPerLanguageBundleLanguage(options, features, logger) {
|
||||
if (!isGitHubHostedRunner()) {
|
||||
return explain("the job is not running on a GitHub-hosted runner");
|
||||
}
|
||||
if (!isNightly) {
|
||||
if (cliVersion2 === void 0) {
|
||||
return explain("the CLI version of the bundle is unknown");
|
||||
}
|
||||
@@ -151569,6 +151577,7 @@ async function getPerLanguageBundleLanguage(options, features, logger) {
|
||||
`CodeQL ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version that publishes per-language bundles`
|
||||
);
|
||||
}
|
||||
}
|
||||
const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language];
|
||||
if (supportedPlatform === void 0) {
|
||||
return explain(`no per-language bundle is published for ${language}`);
|
||||
@@ -152226,6 +152235,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
const forceNightlyValueFF = await features.getValue("force_nightly" /* ForceNightly */);
|
||||
const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
|
||||
const nightlyRequestedByToolsInput = toolsInput !== void 0 && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
|
||||
let nightlyCombinedBundleURL;
|
||||
if (forceNightly || nightlyRequestedByToolsInput) {
|
||||
if (forceNightly) {
|
||||
logger.info(
|
||||
@@ -152252,7 +152262,15 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
`Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`
|
||||
);
|
||||
}
|
||||
toolsInput = await getNightlyToolsUrl(logger);
|
||||
const nightly = await getNightlyToolsUrl(
|
||||
rawLanguages,
|
||||
variant,
|
||||
nightlyRequestedByToolsInput,
|
||||
features,
|
||||
logger
|
||||
);
|
||||
toolsInput = nightly.url;
|
||||
nightlyCombinedBundleURL = nightly.combinedBundleURL;
|
||||
}
|
||||
const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput);
|
||||
if (forceShippedTools) {
|
||||
@@ -152448,7 +152466,10 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
logger.info(
|
||||
`${url2} appears to be a CodeQL bundle that contains only ${language}.`
|
||||
);
|
||||
perLanguageBundle = { language };
|
||||
perLanguageBundle = {
|
||||
language,
|
||||
combinedBundleURL: nightlyCombinedBundleURL
|
||||
};
|
||||
}
|
||||
}
|
||||
if (cliVersion2) {
|
||||
@@ -152692,12 +152713,24 @@ async function useZstdBundle(cliVersion2, tarSupportsZstd) {
|
||||
function getTempExtractionDir(tempDir) {
|
||||
return path13.join(tempDir, v4_default());
|
||||
}
|
||||
async function getNightlyToolsUrl(logger) {
|
||||
async function getNightlyToolsUrl(rawLanguages, variant, requestedExplicitly, features, logger) {
|
||||
const zstdAvailability = await isZstdAvailable(logger);
|
||||
const compressionMethod = await useZstdBundle(
|
||||
CODEQL_VERSION_ZSTD_BUNDLE,
|
||||
zstdAvailability.available
|
||||
) ? "zstd" : "gzip";
|
||||
const language = requestedExplicitly ? await getPerLanguageBundleLanguage(
|
||||
{
|
||||
rawLanguages,
|
||||
cliVersion: void 0,
|
||||
compressionMethod,
|
||||
platform: getBundlePlatform(),
|
||||
variant,
|
||||
isNightly: true
|
||||
},
|
||||
features,
|
||||
logger
|
||||
) : void 0;
|
||||
try {
|
||||
const release2 = await getApiClient().rest.repos.listReleases({
|
||||
owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER,
|
||||
@@ -152710,7 +152743,11 @@ async function getNightlyToolsUrl(logger) {
|
||||
if (!latestRelease) {
|
||||
throw new Error("Could not find the latest nightly release.");
|
||||
}
|
||||
return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`;
|
||||
const assetUrl = (name) => `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`;
|
||||
return {
|
||||
url: assetUrl(getCodeQLBundleName(compressionMethod, language)),
|
||||
combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod))
|
||||
};
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Failed to retrieve the latest nightly release: ${wrapError(e)}`
|
||||
|
||||
@@ -11,9 +11,13 @@ import { GitHubVariant } from "./util";
|
||||
* First version of the CodeQL CLI whose releases include per-language bundles.
|
||||
*
|
||||
* TODO(per-language-bundles): This is a sentinel value that no release can ever satisfy, so
|
||||
* per-language bundles are disabled no matter what the feature flag says. This MUST be updated to
|
||||
* the first CLI version that actually publishes per-language bundles before the feature can be
|
||||
* rolled out.
|
||||
* per-language bundles are disabled for releases no matter what the feature flag says. This MUST be
|
||||
* updated to the first CLI version that actually publishes per-language bundles before the feature
|
||||
* can be rolled out.
|
||||
*
|
||||
* Nightlies are exempt, since their tags record the date they were built rather than a version we
|
||||
* could compare. A workflow that explicitly asks for a nightly can therefore use a per-language
|
||||
* bundle while this is still a sentinel, which is how the feature is tested before rollout.
|
||||
*/
|
||||
export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "99.99.99";
|
||||
|
||||
@@ -82,7 +86,11 @@ export interface PerLanguageBundleOptions {
|
||||
* download.
|
||||
*/
|
||||
rawLanguages: string[] | undefined;
|
||||
/** The CLI version of the bundle we are about to download, if known. */
|
||||
/**
|
||||
* The CLI version of the bundle we are about to download, if known.
|
||||
*
|
||||
* Not consulted for nightlies, whose version cannot be determined from their tag.
|
||||
*/
|
||||
cliVersion: string | undefined;
|
||||
/** The compression method of the bundle we are about to download. */
|
||||
compressionMethod: tar.CompressionMethod;
|
||||
@@ -90,6 +98,8 @@ export interface PerLanguageBundleOptions {
|
||||
platform: string | undefined;
|
||||
/** The GitHub product we are running against. */
|
||||
variant: GitHubVariant;
|
||||
/** Whether the bundle we are about to download is a nightly build. */
|
||||
isNightly?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -107,8 +117,14 @@ export async function getPerLanguageBundleLanguage(
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
): Promise<BuiltInLanguage | undefined> {
|
||||
const { rawLanguages, cliVersion, compressionMethod, platform, variant } =
|
||||
options;
|
||||
const {
|
||||
rawLanguages,
|
||||
cliVersion,
|
||||
compressionMethod,
|
||||
platform,
|
||||
variant,
|
||||
isNightly,
|
||||
} = options;
|
||||
|
||||
const explain = (reason: string) => {
|
||||
logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`);
|
||||
@@ -153,6 +169,10 @@ export async function getPerLanguageBundleLanguage(
|
||||
return explain("the job is not running on a GitHub-hosted runner");
|
||||
}
|
||||
|
||||
// A nightly is built from the newest source we have, so it is always at least as new as the
|
||||
// first release to publish per-language bundles. Its tag records the date it was built rather
|
||||
// than a version we could compare, so there is nothing to check here.
|
||||
if (!isNightly) {
|
||||
if (cliVersion === undefined) {
|
||||
return explain("the CLI version of the bundle is unknown");
|
||||
}
|
||||
@@ -163,6 +183,7 @@ export async function getPerLanguageBundleLanguage(
|
||||
"first version that publishes per-language bundles",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language];
|
||||
if (supportedPlatform === undefined) {
|
||||
|
||||
@@ -480,6 +480,177 @@ test.serial(
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource downloads a per-language nightly bundle when eligible",
|
||||
async (t) => {
|
||||
const expectedTag = "codeql-bundle-30260213";
|
||||
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
|
||||
const client = github.getOctokit("123");
|
||||
const listReleases = sinon.stub(client.rest.repos, "listReleases");
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
listReleases.resolves({
|
||||
data: [{ tag_name: expectedTag }],
|
||||
} as any);
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
"nightly",
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
// Default setup passes the combined language name, which needs normalizing.
|
||||
["java-kotlin"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
createFeatures([Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(source.sourceType, "download");
|
||||
if (source.sourceType === "download") {
|
||||
// A nightly is always newer than the first release to publish per-language bundles, so it
|
||||
// is eligible even though its tag contains no version to compare.
|
||||
t.true(
|
||||
source.codeqlURL.endsWith(
|
||||
`/${expectedTag}/codeql-bundle-java-linux64.tar.zst`,
|
||||
),
|
||||
`Unexpected URL ${source.codeqlURL}`,
|
||||
);
|
||||
t.is(source.perLanguageBundle?.language, BuiltInLanguage.java);
|
||||
// We chose this bundle, so an older nightly without it should fall back rather than fail.
|
||||
t.true(
|
||||
source.perLanguageBundle?.combinedBundleURL?.endsWith(
|
||||
`/${expectedTag}/codeql-bundle-linux64.tar.zst`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource downloads the combined nightly bundle when not eligible",
|
||||
async (t) => {
|
||||
const expectedTag = "codeql-bundle-30260213";
|
||||
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
|
||||
const client = github.getOctokit("123");
|
||||
const listReleases = sinon.stub(client.rest.repos, "listReleases");
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
listReleases.resolves({
|
||||
data: [{ tag_name: expectedTag }],
|
||||
} as any);
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
for (const { languages, features } of [
|
||||
// The feature is disabled.
|
||||
{ languages: ["java"], features: createFeatures([]) },
|
||||
// More than one language is being analyzed.
|
||||
{
|
||||
languages: ["java", "python"],
|
||||
features: createFeatures([Feature.PerLanguageBundles]),
|
||||
},
|
||||
]) {
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
"nightly",
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
languages,
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
features,
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(source.sourceType, "download");
|
||||
if (source.sourceType === "download") {
|
||||
t.true(
|
||||
source.codeqlURL.endsWith(
|
||||
`/${expectedTag}/codeql-bundle-linux64.tar.zst`,
|
||||
),
|
||||
`Unexpected URL ${source.codeqlURL}`,
|
||||
);
|
||||
t.is(source.perLanguageBundle, undefined);
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource does not use a per-language bundle for a nightly that was not asked for",
|
||||
async (t) => {
|
||||
const expectedTag = "codeql-bundle-30260213";
|
||||
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
|
||||
const client = github.getOctokit("123");
|
||||
const listReleases = sinon.stub(client.rest.repos, "listReleases");
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
listReleases.resolves({
|
||||
data: [{ tag_name: expectedTag }],
|
||||
} as any);
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
// `dynamic` is the event name that Code Scanning default setup uses, which is the case we
|
||||
// most need to keep the per-language bundle feature away from while it is being tested.
|
||||
setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" });
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
// No `tools` input: the nightly is forced by a feature flag instead.
|
||||
undefined,
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
createFeatures([Feature.ForceNightly, Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(source.sourceType, "download");
|
||||
if (source.sourceType === "download") {
|
||||
// Analyses that did not ask for a nightly should not have the bundle they get changed by
|
||||
// the per-language bundle feature.
|
||||
t.true(
|
||||
source.codeqlURL.endsWith(
|
||||
`/${expectedTag}/codeql-bundle-linux64.tar.zst`,
|
||||
),
|
||||
`Unexpected URL ${source.codeqlURL}`,
|
||||
);
|
||||
t.is(source.perLanguageBundle, undefined);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource correctly returns latest version from toolcache when tools == toolcache",
|
||||
async (t) => {
|
||||
|
||||
@@ -507,6 +507,14 @@ export async function getCodeQLSource(
|
||||
toolsInput !== undefined &&
|
||||
CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
|
||||
|
||||
/**
|
||||
* The combined bundle from the nightly release we are using, if any.
|
||||
*
|
||||
* Only set when we chose the nightly ourselves, so that we can fall back to it if the nightly
|
||||
* turns out not to have a bundle for the language we asked for.
|
||||
*/
|
||||
let nightlyCombinedBundleURL: string | undefined;
|
||||
|
||||
if (forceNightly || nightlyRequestedByToolsInput) {
|
||||
if (forceNightly) {
|
||||
logger.info(
|
||||
@@ -536,7 +544,15 @@ export async function getCodeQLSource(
|
||||
`Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`,
|
||||
);
|
||||
}
|
||||
toolsInput = await getNightlyToolsUrl(logger);
|
||||
const nightly = await getNightlyToolsUrl(
|
||||
rawLanguages,
|
||||
variant,
|
||||
nightlyRequestedByToolsInput,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
toolsInput = nightly.url;
|
||||
nightlyCombinedBundleURL = nightly.combinedBundleURL;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -809,14 +825,20 @@ export async function getCodeQLSource(
|
||||
}
|
||||
compressionMethod = method;
|
||||
|
||||
// The bundle was requested explicitly rather than chosen by us, but we still need to know
|
||||
// whether it contains a single language so that we do not add it to the toolcache.
|
||||
// We chose this bundle ourselves if it is a nightly, and otherwise it was requested explicitly.
|
||||
// Either way we need to know whether it contains a single language, so that we do not add it to
|
||||
// the toolcache.
|
||||
const language = tryGetBundleLanguageFromUrl(url);
|
||||
if (language !== undefined) {
|
||||
logger.info(
|
||||
`${url} appears to be a CodeQL bundle that contains only ${language}.`,
|
||||
);
|
||||
perLanguageBundle = { language };
|
||||
// We only have somewhere to fall back to if we chose this bundle; when it was requested
|
||||
// explicitly we should honor the request rather than substituting a different bundle.
|
||||
perLanguageBundle = {
|
||||
language,
|
||||
combinedBundleURL: nightlyCombinedBundleURL,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1232,7 +1254,17 @@ function getTempExtractionDir(tempDir: string) {
|
||||
/**
|
||||
* Get the URL of the latest nightly CodeQL bundle.
|
||||
*/
|
||||
async function getNightlyToolsUrl(logger: Logger) {
|
||||
/**
|
||||
* Get the URL of the latest nightly CodeQL bundle, and of the combined bundle from the same
|
||||
* nightly release to fall back to if that bundle does not exist.
|
||||
*/
|
||||
async function getNightlyToolsUrl(
|
||||
rawLanguages: string[] | undefined,
|
||||
variant: util.GitHubVariant,
|
||||
requestedExplicitly: boolean,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
): Promise<{ url: string; combinedBundleURL: string }> {
|
||||
const zstdAvailability = await tar.isZstdAvailable(logger);
|
||||
// The nightly is guaranteed to have a zstd bundle
|
||||
const compressionMethod = (await useZstdBundle(
|
||||
@@ -1242,6 +1274,24 @@ async function getNightlyToolsUrl(logger: Logger) {
|
||||
? "zstd"
|
||||
: "gzip";
|
||||
|
||||
// We only consider a per-language bundle when a nightly was asked for explicitly. Nightlies can
|
||||
// also be forced for analyses that did not ask for one, and those should keep getting the bundle
|
||||
// that contains every language.
|
||||
const language = requestedExplicitly
|
||||
? await getPerLanguageBundleLanguage(
|
||||
{
|
||||
rawLanguages,
|
||||
cliVersion: undefined,
|
||||
compressionMethod,
|
||||
platform: getBundlePlatform(),
|
||||
variant,
|
||||
isNightly: true,
|
||||
},
|
||||
features,
|
||||
logger,
|
||||
)
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
// Since nightlies are prereleases, we can't just download the latest release
|
||||
// on the repository. So instead we need to find the latest pre-release
|
||||
@@ -1257,7 +1307,12 @@ async function getNightlyToolsUrl(logger: Logger) {
|
||||
if (!latestRelease) {
|
||||
throw new Error("Could not find the latest nightly release.");
|
||||
}
|
||||
return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`;
|
||||
const assetUrl = (name: string) =>
|
||||
`https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`;
|
||||
return {
|
||||
url: assetUrl(getCodeQLBundleName(compressionMethod, language)),
|
||||
combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod)),
|
||||
};
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Failed to retrieve the latest nightly release: ${util.wrapError(e)}`,
|
||||
|
||||
Reference in New Issue
Block a user