mirror of
https://github.com/github/codeql-action.git
synced 2026-11-04 20:20:23 +00:00
Compare commits
13 Commits
codeql-bun
...
henrymerce
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f8e73499b4 | ||
|
|
73491bfa12 | ||
|
|
4e422c4b8e | ||
|
|
b40dfaea3a | ||
|
|
e9f64a6f54 | ||
|
|
4519edc232 | ||
|
|
cc807b741b | ||
|
|
a1e61b1512 | ||
|
|
8847921389 | ||
|
|
b877f27829 | ||
|
|
a113a2b6d2 | ||
|
|
aa3f24b2eb | ||
|
|
b84b1e3854 |
164
.github/workflows/per-language-bundle-validation.yml
vendored
Normal file
164
.github/workflows/per-language-bundle-validation.yml
vendored
Normal file
@@ -0,0 +1,164 @@
|
||||
# Validates the per-language CodeQL bundles in the latest nightly. This is not generated from
|
||||
# `pr-checks`, since each language needs its own platform, build steps and set of expected
|
||||
# extractors, whereas the generated checks share one set of steps across a matrix that only varies
|
||||
# the operating system and CodeQL version.
|
||||
#
|
||||
# TODO(per-language-bundles): This needs revisiting once per-language bundles ship in a release:
|
||||
# either run it on a schedule rather than on every push, or drop it if the checks that exercise the
|
||||
# download path are enough by then.
|
||||
name: Per-language bundle validation
|
||||
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GO111MODULE: auto
|
||||
# Nightlies are the only bundles that currently contain per-language bundles, and the Action only
|
||||
# considers one for a nightly that was asked for explicitly, as this workflow does below.
|
||||
CODEQL_ACTION_PER_LANGUAGE_BUNDLES: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- henrymercer/per-language-bundles
|
||||
workflow_dispatch: {}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
concurrency:
|
||||
cancel-in-progress: true
|
||||
group: per-language-bundle-validation-${{ github.ref }}
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: ${{ matrix.language }}
|
||||
if: github.triggering_actor != 'dependabot[bot]'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- language: actions
|
||||
os: ubuntu-latest
|
||||
# The Actions QL pack depends on the JavaScript one, which is the only dependency of its
|
||||
# kind, so the Actions bundle has to carry the JavaScript extractor as well.
|
||||
expected-extractors: actions javascript
|
||||
- language: cpp
|
||||
os: ubuntu-latest
|
||||
build-mode: manual
|
||||
build-command: gcc -o main main.c
|
||||
- language: csharp
|
||||
os: ubuntu-latest
|
||||
build-mode: none
|
||||
- language: go
|
||||
os: ubuntu-latest
|
||||
build-mode: autobuild
|
||||
- language: java
|
||||
os: ubuntu-latest
|
||||
build-mode: none
|
||||
- language: javascript
|
||||
os: ubuntu-latest
|
||||
- language: python
|
||||
os: ubuntu-latest
|
||||
- language: ruby
|
||||
os: ubuntu-latest
|
||||
- language: rust
|
||||
os: ubuntu-latest
|
||||
- language: swift
|
||||
# Swift autobuild is too slow to finish on a standard macOS runner, which is why the
|
||||
# generated Swift checks use a larger one as well.
|
||||
os: macos-latest-xlarge
|
||||
build-mode: autobuild
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: read
|
||||
timeout-minutes: 45
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
CODEQL_ACTION_TEST_MODE: true
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Prepare test
|
||||
id: prepare-test
|
||||
uses: ./.github/actions/prepare-test
|
||||
with:
|
||||
# Unused: we ask for a nightly below.
|
||||
version: linked
|
||||
use-all-platform-bundle: 'false'
|
||||
setup-kotlin: 'false'
|
||||
- uses: ./../action/init
|
||||
id: init
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix['build-mode'] }}
|
||||
# The Action picks the per-language bundle from the latest nightly itself, so this also
|
||||
# exercises the code that decides which bundle to download.
|
||||
tools: nightly
|
||||
- name: Check that the bundle contains only the expected extractors
|
||||
env:
|
||||
CODEQL_PATH: ${{ steps.init.outputs.codeql-path }}
|
||||
LANGUAGE: ${{ matrix.language }}
|
||||
EXPECTED_EXTRACTORS: ${{ matrix['expected-extractors'] || matrix.language }}
|
||||
run: |
|
||||
extractors="$("$CODEQL_PATH" resolve languages --format=json | jq -r 'keys[]')"
|
||||
echo "Extractors in the bundle:"
|
||||
echo "$extractors"
|
||||
echo "Expected: $EXPECTED_EXTRACTORS"
|
||||
|
||||
for expected in $EXPECTED_EXTRACTORS; do
|
||||
if ! echo "$extractors" | grep -qx "$expected"; then
|
||||
echo "::error::The ${LANGUAGE} bundle does not contain the ${expected} extractor."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# If the bundle contained extractors beyond those the language needs, then it would not
|
||||
# have been trimmed, and this job would be silently validating the combined bundle.
|
||||
for other in actions cpp csharp go java javascript python ruby rust swift; do
|
||||
if echo "$EXPECTED_EXTRACTORS" | grep -qw "$other"; then
|
||||
continue
|
||||
fi
|
||||
if echo "$extractors" | grep -qx "$other"; then
|
||||
echo "::error::The ${LANGUAGE} bundle also contains the ${other} extractor, so it is not trimmed."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
- name: Check that the bundle was not added to the toolcache
|
||||
env:
|
||||
CODEQL_PATH: ${{ steps.init.outputs.codeql-path }}
|
||||
run: |
|
||||
# A bundle that is missing most of its extractors must never be left in the toolcache,
|
||||
# where a later job analyzing a different language could pick it up. The runner image
|
||||
# ships with its own CodeQL in the toolcache, so check where this bundle was extracted to
|
||||
# rather than whether the toolcache contains CodeQL at all.
|
||||
echo "CodeQL is at $CODEQL_PATH"
|
||||
if [[ "$CODEQL_PATH" == "$RUNNER_TOOL_CACHE"/* ]]; then
|
||||
echo "::error::The per-language bundle was added to the toolcache at $CODEQL_PATH."
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$CODEQL_PATH" != "$RUNNER_TEMP"/* ]]; then
|
||||
echo "::error::Expected the per-language bundle to be extracted under $RUNNER_TEMP, but found it at $CODEQL_PATH."
|
||||
exit 1
|
||||
fi
|
||||
- name: Build code
|
||||
if: matrix['build-command']
|
||||
run: ${{ matrix['build-command'] }}
|
||||
- uses: ./../action/autobuild
|
||||
if: matrix['build-mode'] == 'autobuild'
|
||||
timeout-minutes: 30
|
||||
- uses: ./../action/analyze
|
||||
id: analysis
|
||||
with:
|
||||
upload-database: false
|
||||
- name: Check that a database was created for the language
|
||||
env:
|
||||
DB_LOCATIONS: ${{ steps.analysis.outputs.db-locations }}
|
||||
LANGUAGE: ${{ matrix.language }}
|
||||
run: |
|
||||
database="$(echo "$DB_LOCATIONS" | jq -r --arg lang "$LANGUAGE" '.[$lang] // empty')"
|
||||
if [ -z "$database" ] || [ ! -d "$database" ]; then
|
||||
echo "::error::No CodeQL database was created for ${LANGUAGE}."
|
||||
echo "Databases: $DB_LOCATIONS"
|
||||
exit 1
|
||||
fi
|
||||
echo "Created a ${LANGUAGE} database at ${database}."
|
||||
387
lib/entry-points.js
generated
387
lib/entry-points.js
generated
@@ -27238,8 +27238,8 @@ var require_gte = __commonJS({
|
||||
"node_modules/semver/functions/gte.js"(exports2, module2) {
|
||||
"use strict";
|
||||
var compare3 = require_compare();
|
||||
var gte7 = (a, b, loose) => compare3(a, b, loose) >= 0;
|
||||
module2.exports = gte7;
|
||||
var gte8 = (a, b, loose) => compare3(a, b, loose) >= 0;
|
||||
module2.exports = gte8;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -27260,7 +27260,7 @@ var require_cmp = __commonJS({
|
||||
var eq = require_eq();
|
||||
var neq = require_neq();
|
||||
var gt = require_gt();
|
||||
var gte7 = require_gte();
|
||||
var gte8 = require_gte();
|
||||
var lt2 = require_lt();
|
||||
var lte2 = require_lte();
|
||||
var cmp = (a, op, b, loose) => {
|
||||
@@ -27290,7 +27290,7 @@ var require_cmp = __commonJS({
|
||||
case ">":
|
||||
return gt(a, b, loose);
|
||||
case ">=":
|
||||
return gte7(a, b, loose);
|
||||
return gte8(a, b, loose);
|
||||
case "<":
|
||||
return lt2(a, b, loose);
|
||||
case "<=":
|
||||
@@ -28098,7 +28098,7 @@ var require_outside = __commonJS({
|
||||
var gt = require_gt();
|
||||
var lt2 = require_lt();
|
||||
var lte2 = require_lte();
|
||||
var gte7 = require_gte();
|
||||
var gte8 = require_gte();
|
||||
var outside = (version, range2, hilo, options) => {
|
||||
version = new SemVer(version, options);
|
||||
range2 = new Range2(range2, options);
|
||||
@@ -28113,7 +28113,7 @@ var require_outside = __commonJS({
|
||||
break;
|
||||
case "<":
|
||||
gtfn = lt2;
|
||||
ltefn = gte7;
|
||||
ltefn = gte8;
|
||||
ltfn = gt;
|
||||
comp = "<";
|
||||
ecomp = "<=";
|
||||
@@ -28428,7 +28428,7 @@ var require_semver2 = __commonJS({
|
||||
var lt2 = require_lt();
|
||||
var eq = require_eq();
|
||||
var neq = require_neq();
|
||||
var gte7 = require_gte();
|
||||
var gte8 = require_gte();
|
||||
var lte2 = require_lte();
|
||||
var cmp = require_cmp();
|
||||
var coerce3 = require_coerce();
|
||||
@@ -28467,7 +28467,7 @@ var require_semver2 = __commonJS({
|
||||
lt: lt2,
|
||||
eq,
|
||||
neq,
|
||||
gte: gte7,
|
||||
gte: gte8,
|
||||
lte: lte2,
|
||||
cmp,
|
||||
coerce: coerce3,
|
||||
@@ -31743,7 +31743,7 @@ var require_brace_expansion = __commonJS({
|
||||
function lte2(i, y) {
|
||||
return i <= y;
|
||||
}
|
||||
function gte7(i, y) {
|
||||
function gte8(i, y) {
|
||||
return i >= y;
|
||||
}
|
||||
function combine2(acc, base, pre, values, max, maxLength, dropEmpties, outBase) {
|
||||
@@ -31776,7 +31776,7 @@ var require_brace_expansion = __commonJS({
|
||||
var reverse = y < x;
|
||||
if (reverse) {
|
||||
incr *= -1;
|
||||
test = gte7;
|
||||
test = gte8;
|
||||
}
|
||||
var pad = n.some(isPadded2);
|
||||
var length = 0;
|
||||
@@ -33923,8 +33923,8 @@ var require_semver3 = __commonJS({
|
||||
function neq(a, b, loose) {
|
||||
return compare3(a, b, loose) !== 0;
|
||||
}
|
||||
exports2.gte = gte7;
|
||||
function gte7(a, b, loose) {
|
||||
exports2.gte = gte8;
|
||||
function gte8(a, b, loose) {
|
||||
return compare3(a, b, loose) >= 0;
|
||||
}
|
||||
exports2.lte = lte2;
|
||||
@@ -33955,7 +33955,7 @@ var require_semver3 = __commonJS({
|
||||
case ">":
|
||||
return gt(a, b, loose);
|
||||
case ">=":
|
||||
return gte7(a, b, loose);
|
||||
return gte8(a, b, loose);
|
||||
case "<":
|
||||
return lt2(a, b, loose);
|
||||
case "<=":
|
||||
@@ -34500,7 +34500,7 @@ var require_semver3 = __commonJS({
|
||||
break;
|
||||
case "<":
|
||||
gtfn = lt2;
|
||||
ltefn = gte7;
|
||||
ltefn = gte8;
|
||||
ltfn = gt;
|
||||
comp = "<";
|
||||
ecomp = "<=";
|
||||
@@ -34721,7 +34721,7 @@ var require_cacheUtils = __commonJS({
|
||||
var crypto3 = __importStar2(require("crypto"));
|
||||
var fs32 = __importStar2(require("fs"));
|
||||
var path30 = __importStar2(require("path"));
|
||||
var semver11 = __importStar2(require_semver3());
|
||||
var semver12 = __importStar2(require_semver3());
|
||||
var util3 = __importStar2(require("util"));
|
||||
var constants_1 = require_constants7();
|
||||
var versionSalt = "1.0";
|
||||
@@ -34814,7 +34814,7 @@ var require_cacheUtils = __commonJS({
|
||||
function getCompressionMethod() {
|
||||
return __awaiter2(this, void 0, void 0, function* () {
|
||||
const versionOutput = yield getVersion("zstd", ["--quiet"]);
|
||||
const version = semver11.clean(versionOutput);
|
||||
const version = semver12.clean(versionOutput);
|
||||
core32.debug(`zstd version: ${version}`);
|
||||
if (versionOutput === "") {
|
||||
return constants_1.CompressionMethod.Gzip;
|
||||
@@ -82423,7 +82423,7 @@ var require_manifest = __commonJS({
|
||||
exports2._findMatch = _findMatch;
|
||||
exports2._getOsVersion = _getOsVersion;
|
||||
exports2._readLinuxVersionFile = _readLinuxVersionFile;
|
||||
var semver11 = __importStar2(require_semver2());
|
||||
var semver12 = __importStar2(require_semver2());
|
||||
var core_1 = require_core();
|
||||
var os7 = require("os");
|
||||
var cp = require("child_process");
|
||||
@@ -82437,7 +82437,7 @@ var require_manifest = __commonJS({
|
||||
for (const candidate of candidates) {
|
||||
const version = candidate.version;
|
||||
(0, core_1.debug)(`check ${version} satisfies ${versionSpec}`);
|
||||
if (semver11.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) {
|
||||
if (semver12.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) {
|
||||
file = candidate.files.find((item) => {
|
||||
(0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`);
|
||||
let chk = item.arch === archFilter && item.platform === platFilter;
|
||||
@@ -82446,7 +82446,7 @@ var require_manifest = __commonJS({
|
||||
if (osVersion === item.platform_version) {
|
||||
chk = true;
|
||||
} else {
|
||||
chk = semver11.satisfies(osVersion, item.platform_version);
|
||||
chk = semver12.satisfies(osVersion, item.platform_version);
|
||||
}
|
||||
}
|
||||
return chk;
|
||||
@@ -82706,7 +82706,7 @@ var require_tool_cache = __commonJS({
|
||||
var os7 = __importStar2(require("os"));
|
||||
var path30 = __importStar2(require("path"));
|
||||
var httpm = __importStar2(require_lib());
|
||||
var semver11 = __importStar2(require_semver2());
|
||||
var semver12 = __importStar2(require_semver2());
|
||||
var stream2 = __importStar2(require("stream"));
|
||||
var util3 = __importStar2(require("util"));
|
||||
var assert_1 = require("assert");
|
||||
@@ -82979,7 +82979,7 @@ var require_tool_cache = __commonJS({
|
||||
}
|
||||
function cacheDir2(sourceDir, tool, version, arch2) {
|
||||
return __awaiter2(this, void 0, void 0, function* () {
|
||||
version = semver11.clean(version) || version;
|
||||
version = semver12.clean(version) || version;
|
||||
arch2 = arch2 || os7.arch();
|
||||
core32.debug(`Caching tool ${tool} ${version} ${arch2}`);
|
||||
core32.debug(`source dir: ${sourceDir}`);
|
||||
@@ -82997,7 +82997,7 @@ var require_tool_cache = __commonJS({
|
||||
}
|
||||
function cacheFile(sourceFile, targetFile, tool, version, arch2) {
|
||||
return __awaiter2(this, void 0, void 0, function* () {
|
||||
version = semver11.clean(version) || version;
|
||||
version = semver12.clean(version) || version;
|
||||
arch2 = arch2 || os7.arch();
|
||||
core32.debug(`Caching tool ${tool} ${version} ${arch2}`);
|
||||
core32.debug(`source file: ${sourceFile}`);
|
||||
@@ -83027,7 +83027,7 @@ var require_tool_cache = __commonJS({
|
||||
}
|
||||
let toolPath = "";
|
||||
if (versionSpec) {
|
||||
versionSpec = semver11.clean(versionSpec) || "";
|
||||
versionSpec = semver12.clean(versionSpec) || "";
|
||||
const cachePath = path30.join(_getCacheDirectory(), toolName, versionSpec, arch2);
|
||||
core32.debug(`checking cache: ${cachePath}`);
|
||||
if (fs32.existsSync(cachePath) && fs32.existsSync(`${cachePath}.complete`)) {
|
||||
@@ -83107,7 +83107,7 @@ var require_tool_cache = __commonJS({
|
||||
}
|
||||
function _createToolPath(tool, version, arch2) {
|
||||
return __awaiter2(this, void 0, void 0, function* () {
|
||||
const folderPath = path30.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || "");
|
||||
const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || "");
|
||||
core32.debug(`destination ${folderPath}`);
|
||||
const markerPath = `${folderPath}.complete`;
|
||||
yield io9.rmRF(folderPath);
|
||||
@@ -83117,15 +83117,15 @@ var require_tool_cache = __commonJS({
|
||||
});
|
||||
}
|
||||
function _completeToolPath(tool, version, arch2) {
|
||||
const folderPath = path30.join(_getCacheDirectory(), tool, semver11.clean(version) || version, arch2 || "");
|
||||
const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || "");
|
||||
const markerPath = `${folderPath}.complete`;
|
||||
fs32.writeFileSync(markerPath, "");
|
||||
core32.debug("finished caching tool");
|
||||
}
|
||||
function isExplicitVersion(versionSpec) {
|
||||
const c = semver11.clean(versionSpec) || "";
|
||||
const c = semver12.clean(versionSpec) || "";
|
||||
core32.debug(`isExplicit: ${c}`);
|
||||
const valid4 = semver11.valid(c) != null;
|
||||
const valid4 = semver12.valid(c) != null;
|
||||
core32.debug(`explicit? ${valid4}`);
|
||||
return valid4;
|
||||
}
|
||||
@@ -83133,14 +83133,14 @@ var require_tool_cache = __commonJS({
|
||||
let version = "";
|
||||
core32.debug(`evaluating ${versions.length} versions`);
|
||||
versions = versions.sort((a, b) => {
|
||||
if (semver11.gt(a, b)) {
|
||||
if (semver12.gt(a, b)) {
|
||||
return 1;
|
||||
}
|
||||
return -1;
|
||||
});
|
||||
for (let i = versions.length - 1; i >= 0; i--) {
|
||||
const potential = versions[i];
|
||||
const satisfied = semver11.satisfies(potential, versionSpec);
|
||||
const satisfied = semver12.satisfies(potential, versionSpec);
|
||||
if (satisfied) {
|
||||
version = potential;
|
||||
break;
|
||||
@@ -89617,7 +89617,7 @@ var require_brace_expansion2 = __commonJS({
|
||||
function lte2(i, y) {
|
||||
return i <= y;
|
||||
}
|
||||
function gte7(i, y) {
|
||||
function gte8(i, y) {
|
||||
return i >= y;
|
||||
}
|
||||
function combine2(acc, pre, values, max, maxLength, dropEmpties) {
|
||||
@@ -89649,7 +89649,7 @@ var require_brace_expansion2 = __commonJS({
|
||||
var reverse = y < x;
|
||||
if (reverse) {
|
||||
incr *= -1;
|
||||
test = gte7;
|
||||
test = gte8;
|
||||
}
|
||||
var pad = n.some(isPadded2);
|
||||
var length = 0;
|
||||
@@ -148105,6 +148105,11 @@ var featureConfig = {
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS",
|
||||
minimumVersion: void 0
|
||||
},
|
||||
["per_language_bundles" /* PerLanguageBundles */]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES",
|
||||
minimumVersion: void 0
|
||||
},
|
||||
["qa_telemetry_enabled" /* QaTelemetryEnabled */]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_QA_TELEMETRY",
|
||||
@@ -151206,7 +151211,7 @@ var path13 = __toESM(require("path"));
|
||||
var core12 = __toESM(require_core());
|
||||
var toolcache3 = __toESM(require_tool_cache());
|
||||
var import_fast_deep_equal = __toESM(require_fast_deep_equal());
|
||||
var semver9 = __toESM(require_semver2());
|
||||
var semver10 = __toESM(require_semver2());
|
||||
|
||||
// src/overlay/caching.ts
|
||||
var fs11 = __toESM(require("fs"));
|
||||
@@ -151506,6 +151511,88 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) {
|
||||
return versions;
|
||||
}
|
||||
|
||||
// src/per-language-bundles.ts
|
||||
var semver7 = __toESM(require_semver2());
|
||||
var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "99.99.99";
|
||||
var PER_LANGUAGE_BUNDLE_NAME = /^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/;
|
||||
function tryGetBundleLanguageFromUrl(url2) {
|
||||
let assetName;
|
||||
try {
|
||||
assetName = new URL(url2).pathname.split("/").pop() ?? "";
|
||||
} catch {
|
||||
return void 0;
|
||||
}
|
||||
const match2 = assetName.match(PER_LANGUAGE_BUNDLE_NAME);
|
||||
return match2 ? parseBuiltInLanguage(match2[1]) : void 0;
|
||||
}
|
||||
var PER_LANGUAGE_BUNDLE_PLATFORMS = {
|
||||
["actions" /* actions */]: "linux64",
|
||||
["cpp" /* cpp */]: "linux64",
|
||||
["csharp" /* csharp */]: "linux64",
|
||||
["go" /* go */]: "linux64",
|
||||
["java" /* java */]: "linux64",
|
||||
["javascript" /* javascript */]: "linux64",
|
||||
["python" /* python */]: "linux64",
|
||||
["ruby" /* ruby */]: "linux64",
|
||||
["rust" /* rust */]: "linux64",
|
||||
["swift" /* swift */]: "osx64"
|
||||
};
|
||||
async function getPerLanguageBundleLanguage(options, features, logger) {
|
||||
const {
|
||||
rawLanguages,
|
||||
cliVersion: cliVersion2,
|
||||
compressionMethod,
|
||||
platform: platform2,
|
||||
variant,
|
||||
isNightly
|
||||
} = options;
|
||||
const explain = (reason) => {
|
||||
logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`);
|
||||
return void 0;
|
||||
};
|
||||
if (rawLanguages?.length !== 1) {
|
||||
return explain(
|
||||
`exactly one language must be requested via the 'languages' input, but ${rawLanguages?.length ?? 0} were`
|
||||
);
|
||||
}
|
||||
const language = parseBuiltInLanguage(rawLanguages[0]);
|
||||
if (language === void 0) {
|
||||
return explain(`'${rawLanguages[0]}' is not a known CodeQL language`);
|
||||
}
|
||||
if (compressionMethod !== "zstd") {
|
||||
return explain(`the bundle would be downloaded as ${compressionMethod}`);
|
||||
}
|
||||
if (variant !== "GitHub.com" /* DOTCOM */) {
|
||||
return explain(`we are running against ${variant}`);
|
||||
}
|
||||
if (!isGitHubHostedRunner()) {
|
||||
return explain("the job is not running on a GitHub-hosted runner");
|
||||
}
|
||||
if (!isNightly) {
|
||||
if (cliVersion2 === void 0) {
|
||||
return explain("the CLI version of the bundle is unknown");
|
||||
}
|
||||
if (!semver7.gte(cliVersion2, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) {
|
||||
return explain(
|
||||
`CodeQL ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version that publishes per-language bundles`
|
||||
);
|
||||
}
|
||||
}
|
||||
const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language];
|
||||
if (supportedPlatform === void 0) {
|
||||
return explain(`no per-language bundle is published for ${language}`);
|
||||
}
|
||||
if (supportedPlatform !== platform2) {
|
||||
return explain(
|
||||
`the ${language} bundle is only published for ${supportedPlatform}, but this job is running on ${platform2 ?? "an unknown platform"}`
|
||||
);
|
||||
}
|
||||
if (!await features.getValue("per_language_bundles" /* PerLanguageBundles */)) {
|
||||
return explain(`the ${"per_language_bundles" /* PerLanguageBundles */} feature is disabled`);
|
||||
}
|
||||
return language;
|
||||
}
|
||||
|
||||
// src/tar.ts
|
||||
var import_child_process = require("child_process");
|
||||
var fs12 = __toESM(require("fs"));
|
||||
@@ -151513,7 +151600,7 @@ var stream = __toESM(require("stream"));
|
||||
var import_toolrunner = __toESM(require_toolrunner());
|
||||
var io4 = __toESM(require_io());
|
||||
var toolcache = __toESM(require_tool_cache());
|
||||
var semver7 = __toESM(require_semver2());
|
||||
var semver8 = __toESM(require_semver2());
|
||||
var MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3";
|
||||
var MIN_REQUIRED_GNU_TAR_VERSION = "1.31";
|
||||
async function getTarVersion() {
|
||||
@@ -151555,9 +151642,9 @@ async function isZstdAvailable(logger) {
|
||||
case "gnu":
|
||||
return {
|
||||
available: foundZstdBinary && // GNU tar only uses major and minor version numbers
|
||||
semver7.gte(
|
||||
semver7.coerce(version),
|
||||
semver7.coerce(MIN_REQUIRED_GNU_TAR_VERSION)
|
||||
semver8.gte(
|
||||
semver8.coerce(version),
|
||||
semver8.coerce(MIN_REQUIRED_GNU_TAR_VERSION)
|
||||
),
|
||||
foundZstdBinary,
|
||||
version: tarVersion
|
||||
@@ -151566,7 +151653,7 @@ async function isZstdAvailable(logger) {
|
||||
return {
|
||||
available: foundZstdBinary && // Do a loose comparison since these version numbers don't contain
|
||||
// a patch version number.
|
||||
semver7.gte(version, MIN_REQUIRED_BSD_TAR_VERSION),
|
||||
semver8.gte(version, MIN_REQUIRED_BSD_TAR_VERSION),
|
||||
foundZstdBinary,
|
||||
version: tarVersion
|
||||
};
|
||||
@@ -151675,7 +151762,7 @@ var core11 = __toESM(require_core());
|
||||
var import_http_client = __toESM(require_lib());
|
||||
var toolcache2 = __toESM(require_tool_cache());
|
||||
var import_follow_redirects = __toESM(require_follow_redirects());
|
||||
var semver8 = __toESM(require_semver2());
|
||||
var semver9 = __toESM(require_semver2());
|
||||
var STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024;
|
||||
var STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1e3;
|
||||
var TOOLCACHE_TOOL_NAME = "CodeQL";
|
||||
@@ -151704,11 +151791,14 @@ async function downloadAndExtract(codeqlURL, compressionMethod, dest, authorizat
|
||||
return { totalDurationMs };
|
||||
}
|
||||
} catch (e) {
|
||||
await cleanUpPath(dest, "CodeQL bundle", logger);
|
||||
if (asHTTPError(e)?.status === 404) {
|
||||
throw e;
|
||||
}
|
||||
core11.warning(
|
||||
`Failed to download and extract CodeQL bundle using streaming with error: ${getErrorMessage(e)}`
|
||||
);
|
||||
core11.warning(`Falling back to downloading the bundle before extracting.`);
|
||||
await cleanUpPath(dest, "CodeQL bundle", logger);
|
||||
}
|
||||
const toolsDownloadStart = import_perf_hooks2.performance.now();
|
||||
const archivedBundlePath = await toolcache2.downloadTool(
|
||||
@@ -151780,8 +151870,9 @@ async function downloadAndExtractZstdWithStreaming(codeqlURL, dest, authorizatio
|
||||
});
|
||||
if (response.statusCode !== 200) {
|
||||
response.resume();
|
||||
throw new Error(
|
||||
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`
|
||||
throw new HTTPError(
|
||||
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
|
||||
response.statusCode ?? 0
|
||||
);
|
||||
}
|
||||
await extractTarZst(response, dest, tarVersion, logger);
|
||||
@@ -151793,7 +151884,7 @@ function getToolcacheToolDirectory(env) {
|
||||
);
|
||||
}
|
||||
function getToolcacheVersionDirectoryName(version) {
|
||||
return semver8.clean(version) || version;
|
||||
return semver9.clean(version) || version;
|
||||
}
|
||||
function getToolcacheDirectory(version) {
|
||||
return path12.join(
|
||||
@@ -151905,18 +151996,27 @@ function getCodeQLBundleExtension(compressionMethod) {
|
||||
assertNever(compressionMethod);
|
||||
}
|
||||
}
|
||||
function getCodeQLBundleName(compressionMethod) {
|
||||
function getBundlePlatform() {
|
||||
switch (process.platform) {
|
||||
case "win32":
|
||||
return "win64";
|
||||
case "linux":
|
||||
return process.arch === "arm64" ? "linux-arm64" : "linux64";
|
||||
case "darwin":
|
||||
return "osx64";
|
||||
default:
|
||||
return void 0;
|
||||
}
|
||||
}
|
||||
function getCodeQLBundleName(compressionMethod, language) {
|
||||
const extension = getCodeQLBundleExtension(compressionMethod);
|
||||
let platform2;
|
||||
if (process.platform === "win32") {
|
||||
platform2 = "win64";
|
||||
} else if (process.platform === "linux") {
|
||||
platform2 = process.arch === "arm64" ? "linux-arm64" : "linux64";
|
||||
} else if (process.platform === "darwin") {
|
||||
platform2 = "osx64";
|
||||
} else {
|
||||
const platform2 = getBundlePlatform();
|
||||
if (platform2 === void 0) {
|
||||
return `codeql-bundle${extension}`;
|
||||
}
|
||||
if (language !== void 0) {
|
||||
return `codeql-bundle-${language}-${platform2}${extension}`;
|
||||
}
|
||||
return `codeql-bundle-${platform2}${extension}`;
|
||||
}
|
||||
function getCodeQLActionRepository(logger) {
|
||||
@@ -151928,7 +152028,7 @@ function getCodeQLActionRepository(logger) {
|
||||
}
|
||||
return getRequiredEnvParam("GITHUB_ACTION_REPOSITORY");
|
||||
}
|
||||
async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod, logger) {
|
||||
async function getCodeQLBundleDownloadURL(tagName, apiDetails, codeQLBundleName, logger) {
|
||||
const codeQLActionRepository = getCodeQLActionRepository(logger);
|
||||
const potentialDownloadSources = [
|
||||
// This GitHub instance, and this Action.
|
||||
@@ -151943,7 +152043,6 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, compressionMethod
|
||||
return !self2.slice(0, index2).some((other) => (0, import_fast_deep_equal.default)(source, other));
|
||||
}
|
||||
);
|
||||
const codeQLBundleName = getCodeQLBundleName(compressionMethod);
|
||||
for (const downloadSource of uniqueDownloadSources) {
|
||||
const [apiURL, repository] = downloadSource;
|
||||
if (apiURL === GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
|
||||
@@ -151998,13 +152097,13 @@ function tryGetTagNameFromUrl(url2, logger) {
|
||||
return match2[1];
|
||||
}
|
||||
function convertToSemVer(version, logger) {
|
||||
if (!semver9.valid(version)) {
|
||||
if (!semver10.valid(version)) {
|
||||
logger.debug(
|
||||
`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`
|
||||
);
|
||||
version = `0.0.0-${version}`;
|
||||
}
|
||||
const s = semver9.clean(version);
|
||||
const s = semver10.clean(version);
|
||||
if (!s) {
|
||||
throw new Error(`Bundle version ${version} is not in SemVer format.`);
|
||||
}
|
||||
@@ -152136,6 +152235,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
const forceNightlyValueFF = await features.getValue("force_nightly" /* ForceNightly */);
|
||||
const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
|
||||
const nightlyRequestedByToolsInput = toolsInput !== void 0 && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
|
||||
let nightlyCombinedBundleURL;
|
||||
if (forceNightly || nightlyRequestedByToolsInput) {
|
||||
if (forceNightly) {
|
||||
logger.info(
|
||||
@@ -152162,7 +152262,15 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
`Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`
|
||||
);
|
||||
}
|
||||
toolsInput = await getNightlyToolsUrl(logger);
|
||||
const nightly = await getNightlyToolsUrl(
|
||||
rawLanguages,
|
||||
variant,
|
||||
nightlyRequestedByToolsInput,
|
||||
features,
|
||||
logger
|
||||
);
|
||||
toolsInput = nightly.url;
|
||||
nightlyCombinedBundleURL = nightly.combinedBundleURL;
|
||||
}
|
||||
const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput);
|
||||
if (forceShippedTools) {
|
||||
@@ -152213,7 +152321,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
url2 = toolsInput;
|
||||
if (tagName) {
|
||||
const bundleVersion3 = tryGetBundleVersionFromTagName(tagName, logger);
|
||||
if (bundleVersion3 && semver9.valid(bundleVersion3)) {
|
||||
if (bundleVersion3 && semver10.valid(bundleVersion3)) {
|
||||
cliVersion2 = convertToSemVer(bundleVersion3, logger);
|
||||
}
|
||||
}
|
||||
@@ -152313,14 +152421,38 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
}
|
||||
}
|
||||
let compressionMethod;
|
||||
let perLanguageBundle;
|
||||
if (!url2) {
|
||||
compressionMethod = cliVersion2 !== void 0 && await useZstdBundle(cliVersion2, tarSupportsZstd) ? "zstd" : "gzip";
|
||||
url2 = await getCodeQLBundleDownloadURL(
|
||||
tagName,
|
||||
apiDetails,
|
||||
compressionMethod,
|
||||
const perLanguageBundleLanguage = await getPerLanguageBundleLanguage(
|
||||
{
|
||||
rawLanguages,
|
||||
cliVersion: cliVersion2,
|
||||
compressionMethod,
|
||||
platform: getBundlePlatform(),
|
||||
variant
|
||||
},
|
||||
features,
|
||||
logger
|
||||
);
|
||||
const resolveBundleURL = (language) => getCodeQLBundleDownloadURL(
|
||||
tagName,
|
||||
apiDetails,
|
||||
getCodeQLBundleName(compressionMethod, language),
|
||||
logger
|
||||
);
|
||||
if (perLanguageBundleLanguage !== void 0) {
|
||||
logger.info(
|
||||
`Downloading the ${perLanguageBundleLanguage} CodeQL bundle, since ${perLanguageBundleLanguage} is the only language being analyzed.`
|
||||
);
|
||||
url2 = await resolveBundleURL(perLanguageBundleLanguage);
|
||||
perLanguageBundle = {
|
||||
language: perLanguageBundleLanguage,
|
||||
combinedBundleURL: await resolveBundleURL()
|
||||
};
|
||||
} else {
|
||||
url2 = await resolveBundleURL();
|
||||
}
|
||||
} else {
|
||||
const method = inferCompressionMethod(url2);
|
||||
if (method === void 0) {
|
||||
@@ -152329,6 +152461,16 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
);
|
||||
}
|
||||
compressionMethod = method;
|
||||
const language = tryGetBundleLanguageFromUrl(url2);
|
||||
if (language !== void 0) {
|
||||
logger.info(
|
||||
`${url2} appears to be a CodeQL bundle that contains only ${language}.`
|
||||
);
|
||||
perLanguageBundle = {
|
||||
language,
|
||||
combinedBundleURL: nightlyCombinedBundleURL
|
||||
};
|
||||
}
|
||||
}
|
||||
if (cliVersion2) {
|
||||
logger.info(`Using CodeQL CLI version ${cliVersion2} sourced from ${url2} .`);
|
||||
@@ -152340,6 +152482,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO
|
||||
cliVersion: cliVersion2,
|
||||
codeqlURL: url2,
|
||||
compressionMethod,
|
||||
perLanguageBundle,
|
||||
sourceType: "download",
|
||||
toolsVersion: cliVersion2 ?? humanReadableVersion
|
||||
};
|
||||
@@ -152355,7 +152498,7 @@ async function tryGetFallbackToolcacheVersion(cliVersion2, tagName, logger) {
|
||||
);
|
||||
return fallbackVersion;
|
||||
}
|
||||
var downloadCodeQL = async function(codeqlURL, compressionMethod, maybeBundleVersion, maybeCliVersion, apiDetails, tarVersion, tempDir, features, logger) {
|
||||
var downloadCodeQL = async function(codeqlURL, compressionMethod, maybeBundleVersion, maybeCliVersion, isPerLanguageBundle, apiDetails, tarVersion, tempDir, features, logger) {
|
||||
const parsedCodeQLURL = new URL(codeqlURL);
|
||||
const searchParams = new URLSearchParams(parsedCodeQLURL.search);
|
||||
const headers = {
|
||||
@@ -152371,11 +152514,7 @@ var downloadCodeQL = async function(codeqlURL, compressionMethod, maybeBundleVer
|
||||
codeqlURL
|
||||
);
|
||||
}
|
||||
const toolcacheInfo = getToolcacheDestinationInfo(
|
||||
maybeBundleVersion,
|
||||
maybeCliVersion,
|
||||
logger
|
||||
);
|
||||
const toolcacheInfo = isPerLanguageBundle ? void 0 : getToolcacheDestinationInfo(maybeBundleVersion, maybeCliVersion, logger);
|
||||
const extractedBundlePath = toolcacheInfo?.path ?? getTempExtractionDir(tempDir);
|
||||
await tryDeleteToolcacheBundles({ env: getEnv(), features, logger });
|
||||
const statusReport = await downloadAndExtract(
|
||||
@@ -152389,7 +152528,7 @@ var downloadCodeQL = async function(codeqlURL, compressionMethod, maybeBundleVer
|
||||
);
|
||||
if (!toolcacheInfo) {
|
||||
logger.debug(
|
||||
`Could not cache CodeQL tools because we could not determine the bundle version from the URL ${codeqlURL}.`
|
||||
isPerLanguageBundle ? "Not caching the CodeQL tools because they came from a bundle that contains only a single language." : `Could not cache CodeQL tools because we could not determine the bundle version from the URL ${codeqlURL}.`
|
||||
);
|
||||
return {
|
||||
codeqlFolder: extractedBundlePath,
|
||||
@@ -152488,11 +152627,8 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau
|
||||
toolsSource = "TOOLCACHE" /* Toolcache */;
|
||||
break;
|
||||
case "download": {
|
||||
const result = await downloadCodeQL(
|
||||
source.codeqlURL,
|
||||
source.compressionMethod,
|
||||
source.bundleVersion,
|
||||
source.cliVersion,
|
||||
const result = await downloadCodeQLBundle(
|
||||
source,
|
||||
apiDetails,
|
||||
zstdAvailability.version,
|
||||
tempDir,
|
||||
@@ -152516,21 +152652,85 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau
|
||||
toolsVersion
|
||||
};
|
||||
}
|
||||
async function downloadCodeQLBundle(source, apiDetails, tarVersion, tempDir, features, logger) {
|
||||
const { perLanguageBundle } = source;
|
||||
if (perLanguageBundle !== void 0) {
|
||||
try {
|
||||
const result2 = await downloadCodeQL(
|
||||
source.codeqlURL,
|
||||
source.compressionMethod,
|
||||
source.bundleVersion,
|
||||
source.cliVersion,
|
||||
true,
|
||||
apiDetails,
|
||||
tarVersion,
|
||||
tempDir,
|
||||
features,
|
||||
logger
|
||||
);
|
||||
return {
|
||||
...result2,
|
||||
statusReport: {
|
||||
...result2.statusReport,
|
||||
bundleLanguage: perLanguageBundle.language
|
||||
}
|
||||
};
|
||||
} catch (e) {
|
||||
if (perLanguageBundle.combinedBundleURL === void 0 || asHTTPError(e)?.status !== 404) {
|
||||
throw e;
|
||||
}
|
||||
logger.warning(
|
||||
`No ${perLanguageBundle.language} CodeQL bundle was found at ${source.codeqlURL}, so falling back to the bundle that contains all languages. This analysis will still produce correct results, but will take longer to set up.`
|
||||
);
|
||||
}
|
||||
}
|
||||
const result = await downloadCodeQL(
|
||||
perLanguageBundle?.combinedBundleURL ?? source.codeqlURL,
|
||||
source.compressionMethod,
|
||||
source.bundleVersion,
|
||||
source.cliVersion,
|
||||
false,
|
||||
apiDetails,
|
||||
tarVersion,
|
||||
tempDir,
|
||||
features,
|
||||
logger
|
||||
);
|
||||
return perLanguageBundle === void 0 ? result : {
|
||||
...result,
|
||||
statusReport: {
|
||||
...result.statusReport,
|
||||
perLanguageBundleFallback: true
|
||||
}
|
||||
};
|
||||
}
|
||||
async function useZstdBundle(cliVersion2, tarSupportsZstd) {
|
||||
return (
|
||||
// In testing, gzip performs better than zstd on Windows.
|
||||
process.platform !== "win32" && tarSupportsZstd && semver9.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE)
|
||||
process.platform !== "win32" && tarSupportsZstd && semver10.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE)
|
||||
);
|
||||
}
|
||||
function getTempExtractionDir(tempDir) {
|
||||
return path13.join(tempDir, v4_default());
|
||||
}
|
||||
async function getNightlyToolsUrl(logger) {
|
||||
async function getNightlyToolsUrl(rawLanguages, variant, requestedExplicitly, features, logger) {
|
||||
const zstdAvailability = await isZstdAvailable(logger);
|
||||
const compressionMethod = await useZstdBundle(
|
||||
CODEQL_VERSION_ZSTD_BUNDLE,
|
||||
zstdAvailability.available
|
||||
) ? "zstd" : "gzip";
|
||||
const language = requestedExplicitly ? await getPerLanguageBundleLanguage(
|
||||
{
|
||||
rawLanguages,
|
||||
cliVersion: void 0,
|
||||
compressionMethod,
|
||||
platform: getBundlePlatform(),
|
||||
variant,
|
||||
isNightly: true
|
||||
},
|
||||
features,
|
||||
logger
|
||||
) : void 0;
|
||||
try {
|
||||
const release2 = await getApiClient().rest.repos.listReleases({
|
||||
owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER,
|
||||
@@ -152543,7 +152743,11 @@ async function getNightlyToolsUrl(logger) {
|
||||
if (!latestRelease) {
|
||||
throw new Error("Could not find the latest nightly release.");
|
||||
}
|
||||
return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`;
|
||||
const assetUrl = (name) => `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`;
|
||||
return {
|
||||
url: assetUrl(getCodeQLBundleName(compressionMethod, language)),
|
||||
combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod))
|
||||
};
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Failed to retrieve the latest nightly release: ${wrapError(e)}`
|
||||
@@ -152551,7 +152755,7 @@ async function getNightlyToolsUrl(logger) {
|
||||
}
|
||||
}
|
||||
function getLatestToolcacheVersion(logger) {
|
||||
const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver9.compare(b, a));
|
||||
const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver10.compare(b, a));
|
||||
logger.debug(
|
||||
`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(
|
||||
allVersions
|
||||
@@ -155659,6 +155863,9 @@ async function combineSarifFilesUsingCLI(sarifFiles, gitHubVersion, features, lo
|
||||
tempDir,
|
||||
gitHubVersion.type,
|
||||
codeQLDefaultVersionInfo,
|
||||
// Leaving the languages unset is load-bearing as well as accurate: it is what stops us from
|
||||
// downloading a bundle that contains only one language, which would not have the extractors
|
||||
// that a later step might need. Do not start threading languages through here.
|
||||
void 0,
|
||||
// rawLanguages: upload-lib does not run analysis
|
||||
false,
|
||||
@@ -156735,7 +156942,7 @@ function isPadded(el) {
|
||||
function lte(i, y) {
|
||||
return i <= y;
|
||||
}
|
||||
function gte6(i, y) {
|
||||
function gte7(i, y) {
|
||||
return i >= y;
|
||||
}
|
||||
function combine(acc, pre, values, max, maxLength, dropEmpties) {
|
||||
@@ -156770,7 +156977,7 @@ function expandSequence(body, isAlphaSequence, max, maxLength) {
|
||||
const reverse = y < x;
|
||||
if (reverse) {
|
||||
incr *= -1;
|
||||
test = gte6;
|
||||
test = gte7;
|
||||
}
|
||||
const pad = n.some(isPadded);
|
||||
let length = 0;
|
||||
@@ -158667,7 +158874,7 @@ var import_async = __toESM(require_async(), 1);
|
||||
var import_path7 = require("path");
|
||||
|
||||
// node_modules/archiver/lib/error.js
|
||||
var import_util34 = __toESM(require("util"), 1);
|
||||
var import_util35 = __toESM(require("util"), 1);
|
||||
var ERROR_CODES = {
|
||||
ABORTED: "archive was aborted",
|
||||
DIRECTORYDIRPATHREQUIRED: "diretory dirpath argument must be a non-empty string value",
|
||||
@@ -158692,7 +158899,7 @@ function ArchiverError(code, data) {
|
||||
this.code = code;
|
||||
this.data = data;
|
||||
}
|
||||
import_util34.default.inherits(ArchiverError, Error);
|
||||
import_util35.default.inherits(ArchiverError, Error);
|
||||
|
||||
// node_modules/archiver/lib/core.js
|
||||
var import_readable_stream2 = __toESM(require_ours(), 1);
|
||||
@@ -161624,7 +161831,7 @@ var fs29 = __toESM(require("fs"));
|
||||
var path25 = __toESM(require("path"));
|
||||
var core22 = __toESM(require_core());
|
||||
var io7 = __toESM(require_io());
|
||||
var semver10 = __toESM(require_semver2());
|
||||
var semver11 = __toESM(require_semver2());
|
||||
|
||||
// src/config/inputs.ts
|
||||
async function getToolsInput(action, repositoryProperties) {
|
||||
@@ -161985,6 +162192,12 @@ async function sendCompletedStatusReport2(startedAt, config, configFile, toolsIn
|
||||
if (toolsDownloadStatusReport?.totalDurationMs !== void 0) {
|
||||
initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.bundleLanguage !== void 0) {
|
||||
initToolsDownloadFields.tools_bundle_language = toolsDownloadStatusReport.bundleLanguage;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.perLanguageBundleFallback !== void 0) {
|
||||
initToolsDownloadFields.tools_per_language_bundle_fallback = toolsDownloadStatusReport.perLanguageBundleFallback;
|
||||
}
|
||||
if (toolsFeatureFlagsValid !== void 0) {
|
||||
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
|
||||
}
|
||||
@@ -162104,12 +162317,12 @@ async function run3(actionState) {
|
||||
const experimental = "2.19.3";
|
||||
const publicPreview = "2.22.1";
|
||||
const actualVer = (await codeql.getVersion()).version;
|
||||
if (semver10.lt(actualVer, experimental)) {
|
||||
if (semver11.lt(actualVer, experimental)) {
|
||||
throw new ConfigurationError(
|
||||
`Rust analysis is supported by CodeQL CLI version ${experimental} or higher, but found version ${actualVer}`
|
||||
);
|
||||
}
|
||||
if (semver10.lt(actualVer, publicPreview)) {
|
||||
if (semver11.lt(actualVer, publicPreview)) {
|
||||
core22.exportVariable("CODEQL_ENABLE_EXPERIMENTAL_FEATURES" /* EXPERIMENTAL_FEATURES */, "true");
|
||||
logger.info("Experimental Rust analysis enabled");
|
||||
}
|
||||
@@ -163033,6 +163246,12 @@ async function sendCompletedStatusReport3(startedAt, toolsInput, toolsDownloadSt
|
||||
if (toolsDownloadStatusReport?.totalDurationMs !== void 0) {
|
||||
initToolsDownloadFields.tools_total_duration_ms = toolsDownloadStatusReport.totalDurationMs;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.bundleLanguage !== void 0) {
|
||||
initToolsDownloadFields.tools_bundle_language = toolsDownloadStatusReport.bundleLanguage;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.perLanguageBundleFallback !== void 0) {
|
||||
initToolsDownloadFields.tools_per_language_bundle_fallback = toolsDownloadStatusReport.perLanguageBundleFallback;
|
||||
}
|
||||
if (toolsFeatureFlagsValid !== void 0) {
|
||||
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
|
||||
}
|
||||
|
||||
@@ -164,6 +164,11 @@ export enum Feature {
|
||||
OverlayAnalysisStatusCheck = "overlay_analysis_status_check",
|
||||
/** Controls whether overlay build failures on the default branch are stored in the Actions cache. */
|
||||
OverlayAnalysisStatusSave = "overlay_analysis_status_save",
|
||||
/**
|
||||
* Controls whether we may download a bundle containing only the single language being analysed,
|
||||
* rather than the combined bundle that contains every language.
|
||||
*/
|
||||
PerLanguageBundles = "per_language_bundles",
|
||||
QaTelemetryEnabled = "qa_telemetry_enabled",
|
||||
/** Routes (some) API requests through the registry proxy. */
|
||||
ProxyApiRequests = "proxy_api_requests",
|
||||
@@ -434,6 +439,11 @@ export const featureConfig = {
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.PerLanguageBundles]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_PER_LANGUAGE_BUNDLES",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.QaTelemetryEnabled]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_QA_TELEMETRY",
|
||||
|
||||
@@ -182,6 +182,14 @@ async function sendCompletedStatusReport(
|
||||
initToolsDownloadFields.tools_total_duration_ms =
|
||||
toolsDownloadStatusReport.totalDurationMs;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.bundleLanguage !== undefined) {
|
||||
initToolsDownloadFields.tools_bundle_language =
|
||||
toolsDownloadStatusReport.bundleLanguage;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) {
|
||||
initToolsDownloadFields.tools_per_language_bundle_fallback =
|
||||
toolsDownloadStatusReport.perLanguageBundleFallback;
|
||||
}
|
||||
if (toolsFeatureFlagsValid !== undefined) {
|
||||
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
|
||||
}
|
||||
|
||||
195
src/per-language-bundles.test.ts
Normal file
195
src/per-language-bundles.test.ts
Normal file
@@ -0,0 +1,195 @@
|
||||
import test from "ava";
|
||||
|
||||
import { ActionsEnvVars } from "./environment";
|
||||
import { Feature } from "./feature-flags";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import {
|
||||
getPerLanguageBundleLanguage,
|
||||
MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
|
||||
PerLanguageBundleOptions,
|
||||
tryGetBundleLanguageFromUrl,
|
||||
} from "./per-language-bundles";
|
||||
import { createFeatures, setupTests } from "./testing-utils";
|
||||
import { GitHubVariant } from "./util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
/** Options for which we would use a per-language bundle. */
|
||||
const ELIGIBLE_OPTIONS: PerLanguageBundleOptions = {
|
||||
rawLanguages: ["java"],
|
||||
// Any version at least as new as the minimum will do.
|
||||
cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
|
||||
compressionMethod: "zstd",
|
||||
platform: "linux64",
|
||||
variant: GitHubVariant.DOTCOM,
|
||||
};
|
||||
|
||||
async function checkEligibility(
|
||||
overrides: Partial<PerLanguageBundleOptions>,
|
||||
enabledFeatures: Feature[] = [Feature.PerLanguageBundles],
|
||||
) {
|
||||
return getPerLanguageBundleLanguage(
|
||||
{ ...ELIGIBLE_OPTIONS, ...overrides },
|
||||
createFeatures(enabledFeatures),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
}
|
||||
|
||||
test.beforeEach(() => {
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
});
|
||||
|
||||
test.serial(
|
||||
"uses a per-language bundle when everything lines up",
|
||||
async (t) => {
|
||||
t.is(await checkEligibility({}), BuiltInLanguage.java);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial("resolves the language names used by default setup", async (t) => {
|
||||
// Code Scanning default setup passes the combined language names, which do not match the
|
||||
// extractor names that the bundles are published under.
|
||||
for (const [rawLanguage, expected] of [
|
||||
["java-kotlin", BuiltInLanguage.java],
|
||||
["kotlin", BuiltInLanguage.java],
|
||||
["javascript-typescript", BuiltInLanguage.javascript],
|
||||
["typescript", BuiltInLanguage.javascript],
|
||||
["c-cpp", BuiltInLanguage.cpp],
|
||||
["c", BuiltInLanguage.cpp],
|
||||
["c++", BuiltInLanguage.cpp],
|
||||
["c#", BuiltInLanguage.csharp],
|
||||
["actions", BuiltInLanguage.actions],
|
||||
["go", BuiltInLanguage.go],
|
||||
["python", BuiltInLanguage.python],
|
||||
["ruby", BuiltInLanguage.ruby],
|
||||
["rust", BuiltInLanguage.rust],
|
||||
] as const) {
|
||||
t.is(
|
||||
await checkEligibility({ rawLanguages: [rawLanguage] }),
|
||||
expected,
|
||||
`Expected '${rawLanguage}' to resolve to '${expected}'`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test.serial("uses the macOS bundle for Swift", async (t) => {
|
||||
t.is(
|
||||
await checkEligibility({ rawLanguages: ["swift"], platform: "osx64" }),
|
||||
BuiltInLanguage.swift,
|
||||
);
|
||||
// Swift is only published for macOS.
|
||||
t.is(
|
||||
await checkEligibility({ rawLanguages: ["swift"], platform: "linux64" }),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
test.serial("only publishes non-Swift languages for Linux", async (t) => {
|
||||
t.is(await checkEligibility({ platform: "osx64" }), undefined);
|
||||
t.is(await checkEligibility({ platform: "win64" }), undefined);
|
||||
// We do not publish per-language bundles for Linux Arm64 either.
|
||||
t.is(await checkEligibility({ platform: "linux-arm64" }), undefined);
|
||||
t.is(await checkEligibility({ platform: undefined }), undefined);
|
||||
});
|
||||
|
||||
test.serial("requires exactly one language", async (t) => {
|
||||
t.is(await checkEligibility({ rawLanguages: undefined }), undefined);
|
||||
t.is(await checkEligibility({ rawLanguages: [] }), undefined);
|
||||
t.is(await checkEligibility({ rawLanguages: ["java", "python"] }), undefined);
|
||||
});
|
||||
|
||||
test.serial("requires a language that CodeQL knows about", async (t) => {
|
||||
t.is(await checkEligibility({ rawLanguages: ["cobol"] }), undefined);
|
||||
});
|
||||
|
||||
test.serial("requires a zstd bundle", async (t) => {
|
||||
t.is(await checkEligibility({ compressionMethod: "gzip" }), undefined);
|
||||
});
|
||||
|
||||
test.serial("requires GitHub.com", async (t) => {
|
||||
// Other products resolve the combined bundle against their own instance, so asking for a
|
||||
// per-language bundle they do not mirror would move the download off that instance.
|
||||
for (const variant of [GitHubVariant.GHES, GitHubVariant.GHEC_DR]) {
|
||||
t.is(await checkEligibility({ variant }), undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test.serial("requires a GitHub-hosted runner", async (t) => {
|
||||
// A self-hosted runner may have a toolcache that persists between jobs, which is worth more than
|
||||
// a smaller download.
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "self-hosted";
|
||||
t.is(await checkEligibility({}), undefined);
|
||||
|
||||
// Self-hosted runners are routinely configured to look like hosted ones, for example by mounting
|
||||
// a persistent volume at `/opt/hostedtoolcache`, so we require the service to tell us explicitly.
|
||||
delete process.env[ActionsEnvVars.RUNNER_ENVIRONMENT];
|
||||
process.env["RUNNER_TOOL_CACHE"] = "/opt/hostedtoolcache";
|
||||
t.is(await checkEligibility({}), undefined);
|
||||
});
|
||||
|
||||
test.serial("requires a new enough CLI version", async (t) => {
|
||||
t.is(await checkEligibility({ cliVersion: undefined }), undefined);
|
||||
t.is(
|
||||
await checkEligibility({
|
||||
cliVersion: decrementPatchVersion(MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION),
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
test.serial("requires the feature flag", async (t) => {
|
||||
t.is(await checkEligibility({}, []), undefined);
|
||||
});
|
||||
|
||||
test.serial("recognizes a per-language bundle from its URL", (t) => {
|
||||
const url = (name: string) =>
|
||||
`https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`;
|
||||
|
||||
t.is(
|
||||
tryGetBundleLanguageFromUrl(url("codeql-bundle-java-linux64.tar.zst")),
|
||||
BuiltInLanguage.java,
|
||||
);
|
||||
t.is(
|
||||
tryGetBundleLanguageFromUrl(url("codeql-bundle-swift-osx64.tar.zst")),
|
||||
BuiltInLanguage.swift,
|
||||
);
|
||||
// We do not publish these, but should still recognize them if we ever do.
|
||||
t.is(
|
||||
tryGetBundleLanguageFromUrl(url("codeql-bundle-csharp-win64.tar.gz")),
|
||||
BuiltInLanguage.csharp,
|
||||
);
|
||||
// A token in the URL must not prevent us from recognizing the bundle.
|
||||
t.is(
|
||||
tryGetBundleLanguageFromUrl(
|
||||
`${url("codeql-bundle-ruby-linux64.tar.zst")}?token=secret`,
|
||||
),
|
||||
BuiltInLanguage.ruby,
|
||||
);
|
||||
});
|
||||
|
||||
test.serial("does not mistake other bundles for per-language ones", (t) => {
|
||||
const url = (name: string) =>
|
||||
`https://github.com/github/codeql-action/releases/download/codeql-bundle-v1.2.3/${name}`;
|
||||
|
||||
for (const name of [
|
||||
"codeql-bundle-linux64.tar.zst",
|
||||
"codeql-bundle-osx64.tar.gz",
|
||||
"codeql-bundle-win64.tar.zst",
|
||||
// The all-platform bundle.
|
||||
"codeql-bundle.tar.gz",
|
||||
// A platform we do not publish per-language bundles for, whose name also contains a hyphen.
|
||||
"codeql-bundle-linux-arm64.tar.zst",
|
||||
// Not a language we know about.
|
||||
"codeql-bundle-cobol-linux64.tar.zst",
|
||||
]) {
|
||||
t.is(tryGetBundleLanguageFromUrl(url(name)), undefined, name);
|
||||
}
|
||||
|
||||
t.is(tryGetBundleLanguageFromUrl("not a url"), undefined);
|
||||
});
|
||||
|
||||
function decrementPatchVersion(version: string): string {
|
||||
const [major, minor, patch] = version.split(".").map(Number);
|
||||
return `${major}.${minor}.${patch - 1}`;
|
||||
}
|
||||
204
src/per-language-bundles.ts
Normal file
204
src/per-language-bundles.ts
Normal file
@@ -0,0 +1,204 @@
|
||||
import * as semver from "semver";
|
||||
|
||||
import { isGitHubHostedRunner } from "./actions-util";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import { BuiltInLanguage, parseBuiltInLanguage } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import * as tar from "./tar";
|
||||
import { GitHubVariant } from "./util";
|
||||
|
||||
/**
|
||||
* First version of the CodeQL CLI whose releases include per-language bundles.
|
||||
*
|
||||
* TODO(per-language-bundles): This is a sentinel value that no release can ever satisfy, so
|
||||
* per-language bundles are disabled for releases no matter what the feature flag says. This MUST be
|
||||
* updated to the first CLI version that actually publishes per-language bundles before the feature
|
||||
* can be rolled out.
|
||||
*
|
||||
* Nightlies are exempt, since their tags record the date they were built rather than a version we
|
||||
* could compare. A workflow that explicitly asks for a nightly can therefore use a per-language
|
||||
* bundle while this is still a sentinel, which is how the feature is tested before rollout.
|
||||
*/
|
||||
export const MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "99.99.99";
|
||||
|
||||
/**
|
||||
* Matches the name of a bundle that contains a single language, capturing that language.
|
||||
*
|
||||
* The language comes before the platform so that the name of a per-language bundle never has the
|
||||
* name of the combined bundle for the same platform as a prefix.
|
||||
*/
|
||||
const PER_LANGUAGE_BUNDLE_NAME =
|
||||
/^codeql-bundle-(.+)-(?:linux64|osx64|win64)\.tar\.(?:gz|zst)$/;
|
||||
|
||||
/**
|
||||
* Determines whether a URL points at a bundle that contains only a single language.
|
||||
*
|
||||
* We need to recognise these even when we did not choose to download one ourselves, since a bundle
|
||||
* that is missing most of its extractors must not be added to the toolcache no matter how we came
|
||||
* to be downloading it.
|
||||
*
|
||||
* @returns The language that the bundle contains, or `undefined` if the URL does not point at a
|
||||
* bundle for a single language.
|
||||
*/
|
||||
export function tryGetBundleLanguageFromUrl(
|
||||
url: string,
|
||||
): BuiltInLanguage | undefined {
|
||||
let assetName: string;
|
||||
try {
|
||||
assetName = new URL(url).pathname.split("/").pop() ?? "";
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const match = assetName.match(PER_LANGUAGE_BUNDLE_NAME);
|
||||
return match ? parseBuiltInLanguage(match[1]) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The platform of the per-language bundle we use for each language.
|
||||
*
|
||||
* We publish at most one per-language bundle per language, so a language is only eligible when the
|
||||
* job is running on the corresponding platform. Languages that are absent from this record have no
|
||||
* per-language bundle at all.
|
||||
*/
|
||||
const PER_LANGUAGE_BUNDLE_PLATFORMS: Readonly<
|
||||
Partial<Record<BuiltInLanguage, string>>
|
||||
> = {
|
||||
[BuiltInLanguage.actions]: "linux64",
|
||||
[BuiltInLanguage.cpp]: "linux64",
|
||||
[BuiltInLanguage.csharp]: "linux64",
|
||||
[BuiltInLanguage.go]: "linux64",
|
||||
[BuiltInLanguage.java]: "linux64",
|
||||
[BuiltInLanguage.javascript]: "linux64",
|
||||
[BuiltInLanguage.python]: "linux64",
|
||||
[BuiltInLanguage.ruby]: "linux64",
|
||||
[BuiltInLanguage.rust]: "linux64",
|
||||
[BuiltInLanguage.swift]: "osx64",
|
||||
};
|
||||
|
||||
/** Inputs that determine whether we may download a per-language bundle. */
|
||||
export interface PerLanguageBundleOptions {
|
||||
/**
|
||||
* The languages that were requested for analysis.
|
||||
*
|
||||
* These must have been set explicitly via the `languages` input rather than autodetected, since
|
||||
* autodetection needs a CodeQL instance that we do not have while we are choosing which bundle to
|
||||
* download.
|
||||
*/
|
||||
rawLanguages: string[] | undefined;
|
||||
/**
|
||||
* The CLI version of the bundle we are about to download, if known.
|
||||
*
|
||||
* Not consulted for nightlies, whose version cannot be determined from their tag.
|
||||
*/
|
||||
cliVersion: string | undefined;
|
||||
/** The compression method of the bundle we are about to download. */
|
||||
compressionMethod: tar.CompressionMethod;
|
||||
/** The platform component of the bundle name, for example `linux64`. */
|
||||
platform: string | undefined;
|
||||
/** The GitHub product we are running against. */
|
||||
variant: GitHubVariant;
|
||||
/** Whether the bundle we are about to download is a nightly build. */
|
||||
isNightly?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether we should download a bundle containing only the language being analysed
|
||||
* instead of the combined bundle that contains every language.
|
||||
*
|
||||
* Per-language bundles are substantially smaller than the combined bundle, so using one saves both
|
||||
* download time and disk space on the runner.
|
||||
*
|
||||
* @returns The language whose per-language bundle we should download, or `undefined` if we should
|
||||
* download the combined bundle.
|
||||
*/
|
||||
export async function getPerLanguageBundleLanguage(
|
||||
options: PerLanguageBundleOptions,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
): Promise<BuiltInLanguage | undefined> {
|
||||
const {
|
||||
rawLanguages,
|
||||
cliVersion,
|
||||
compressionMethod,
|
||||
platform,
|
||||
variant,
|
||||
isNightly,
|
||||
} = options;
|
||||
|
||||
const explain = (reason: string) => {
|
||||
logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`);
|
||||
return undefined;
|
||||
};
|
||||
|
||||
if (rawLanguages?.length !== 1) {
|
||||
return explain(
|
||||
`exactly one language must be requested via the 'languages' input, but ${
|
||||
rawLanguages?.length ?? 0
|
||||
} were`,
|
||||
);
|
||||
}
|
||||
|
||||
const language = parseBuiltInLanguage(rawLanguages[0]);
|
||||
if (language === undefined) {
|
||||
return explain(`'${rawLanguages[0]}' is not a known CodeQL language`);
|
||||
}
|
||||
|
||||
if (compressionMethod !== "zstd") {
|
||||
// Per-language bundles are only published as zstd archives.
|
||||
return explain(`the bundle would be downloaded as ${compressionMethod}`);
|
||||
}
|
||||
|
||||
if (variant !== GitHubVariant.DOTCOM) {
|
||||
// Other products resolve bundles against their own instance first, and only fall back to
|
||||
// github.com when the instance does not have the asset. Since they are unlikely to mirror
|
||||
// per-language bundles, asking for one would quietly move the download off the instance.
|
||||
//
|
||||
// We cannot simply let that fail and fall back: an instance whose runners cannot reach
|
||||
// github.com produces a connection error rather than a 404, which is not something we fall back
|
||||
// from, so the job would fail outright where today it succeeds.
|
||||
return explain(`we are running against ${variant}`);
|
||||
}
|
||||
|
||||
if (!isGitHubHostedRunner()) {
|
||||
// We never add per-language bundles to the toolcache, since a bundle for one language must not
|
||||
// be reused for another. That trade-off only pays off on GitHub-hosted runners, which are
|
||||
// discarded after the job, so their toolcache could not have been reused anyway. A self-hosted
|
||||
// runner may well have a toolcache that persists between jobs, and populating it is worth more
|
||||
// than a smaller download.
|
||||
return explain("the job is not running on a GitHub-hosted runner");
|
||||
}
|
||||
|
||||
// A nightly is built from the newest source we have, so it is always at least as new as the
|
||||
// first release to publish per-language bundles. Its tag records the date it was built rather
|
||||
// than a version we could compare, so there is nothing to check here.
|
||||
if (!isNightly) {
|
||||
if (cliVersion === undefined) {
|
||||
return explain("the CLI version of the bundle is unknown");
|
||||
}
|
||||
|
||||
if (!semver.gte(cliVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) {
|
||||
return explain(
|
||||
`CodeQL ${cliVersion} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the ` +
|
||||
"first version that publishes per-language bundles",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const supportedPlatform = PER_LANGUAGE_BUNDLE_PLATFORMS[language];
|
||||
if (supportedPlatform === undefined) {
|
||||
return explain(`no per-language bundle is published for ${language}`);
|
||||
}
|
||||
if (supportedPlatform !== platform) {
|
||||
return explain(
|
||||
`the ${language} bundle is only published for ${supportedPlatform}, but this job is ` +
|
||||
`running on ${platform ?? "an unknown platform"}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!(await features.getValue(Feature.PerLanguageBundles))) {
|
||||
return explain(`the ${Feature.PerLanguageBundles} feature is disabled`);
|
||||
}
|
||||
|
||||
return language;
|
||||
}
|
||||
@@ -93,6 +93,14 @@ async function sendCompletedStatusReport(
|
||||
initToolsDownloadFields.tools_total_duration_ms =
|
||||
toolsDownloadStatusReport.totalDurationMs;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.bundleLanguage !== undefined) {
|
||||
initToolsDownloadFields.tools_bundle_language =
|
||||
toolsDownloadStatusReport.bundleLanguage;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.perLanguageBundleFallback !== undefined) {
|
||||
initToolsDownloadFields.tools_per_language_bundle_fallback =
|
||||
toolsDownloadStatusReport.perLanguageBundleFallback;
|
||||
}
|
||||
if (toolsFeatureFlagsValid !== undefined) {
|
||||
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
|
||||
}
|
||||
|
||||
@@ -12,8 +12,10 @@ import * as api from "./api-client";
|
||||
import * as diagnostics from "./diagnostics";
|
||||
import { ActionsEnvVars, EnvVar, ReadOnlyEnv } from "./environment";
|
||||
import { Feature } from "./feature-flags";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { getCacheRestoreKeyPrefix } from "./overlay/caching";
|
||||
import { MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION } from "./per-language-bundles";
|
||||
import * as setupCodeql from "./setup-codeql";
|
||||
import * as tar from "./tar";
|
||||
import {
|
||||
@@ -31,12 +33,14 @@ import {
|
||||
setupTests,
|
||||
} from "./testing-utils";
|
||||
import * as toolsDownload from "./tools-download";
|
||||
import { getToolcacheDirectory } from "./tools-download";
|
||||
import {
|
||||
getErrorMessage,
|
||||
GitHubVariant,
|
||||
initializeEnvironment,
|
||||
withTmpDir,
|
||||
} from "./util";
|
||||
import * as util from "./util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
@@ -359,6 +363,7 @@ test.serial(
|
||||
|
||||
// Ensure that we consistently select "zstd" for the test.
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
@@ -394,6 +399,7 @@ test.serial(
|
||||
cliVersion: undefined,
|
||||
codeqlURL: expectedURL,
|
||||
compressionMethod: "zstd",
|
||||
perLanguageBundle: undefined,
|
||||
sourceType: "download",
|
||||
toolsVersion: expectedVersion,
|
||||
} satisfies setupCodeql.CodeQLToolsSource);
|
||||
@@ -421,6 +427,7 @@ test.serial(
|
||||
|
||||
// Ensure that we consistently select "zstd" for the test.
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
@@ -457,6 +464,7 @@ test.serial(
|
||||
cliVersion: undefined,
|
||||
codeqlURL: expectedURL,
|
||||
compressionMethod: "zstd",
|
||||
perLanguageBundle: undefined,
|
||||
sourceType: "download",
|
||||
toolsVersion: expectedVersion,
|
||||
} satisfies setupCodeql.CodeQLToolsSource);
|
||||
@@ -472,6 +480,177 @@ test.serial(
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource downloads a per-language nightly bundle when eligible",
|
||||
async (t) => {
|
||||
const expectedTag = "codeql-bundle-30260213";
|
||||
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
|
||||
const client = github.getOctokit("123");
|
||||
const listReleases = sinon.stub(client.rest.repos, "listReleases");
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
listReleases.resolves({
|
||||
data: [{ tag_name: expectedTag }],
|
||||
} as any);
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
"nightly",
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
// Default setup passes the combined language name, which needs normalizing.
|
||||
["java-kotlin"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
createFeatures([Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(source.sourceType, "download");
|
||||
if (source.sourceType === "download") {
|
||||
// A nightly is always newer than the first release to publish per-language bundles, so it
|
||||
// is eligible even though its tag contains no version to compare.
|
||||
t.true(
|
||||
source.codeqlURL.endsWith(
|
||||
`/${expectedTag}/codeql-bundle-java-linux64.tar.zst`,
|
||||
),
|
||||
`Unexpected URL ${source.codeqlURL}`,
|
||||
);
|
||||
t.is(source.perLanguageBundle?.language, BuiltInLanguage.java);
|
||||
// We chose this bundle, so an older nightly without it should fall back rather than fail.
|
||||
t.true(
|
||||
source.perLanguageBundle?.combinedBundleURL?.endsWith(
|
||||
`/${expectedTag}/codeql-bundle-linux64.tar.zst`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource downloads the combined nightly bundle when not eligible",
|
||||
async (t) => {
|
||||
const expectedTag = "codeql-bundle-30260213";
|
||||
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
|
||||
const client = github.getOctokit("123");
|
||||
const listReleases = sinon.stub(client.rest.repos, "listReleases");
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
listReleases.resolves({
|
||||
data: [{ tag_name: expectedTag }],
|
||||
} as any);
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
for (const { languages, features } of [
|
||||
// The feature is disabled.
|
||||
{ languages: ["java"], features: createFeatures([]) },
|
||||
// More than one language is being analyzed.
|
||||
{
|
||||
languages: ["java", "python"],
|
||||
features: createFeatures([Feature.PerLanguageBundles]),
|
||||
},
|
||||
]) {
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
"nightly",
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
languages,
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
features,
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(source.sourceType, "download");
|
||||
if (source.sourceType === "download") {
|
||||
t.true(
|
||||
source.codeqlURL.endsWith(
|
||||
`/${expectedTag}/codeql-bundle-linux64.tar.zst`,
|
||||
),
|
||||
`Unexpected URL ${source.codeqlURL}`,
|
||||
);
|
||||
t.is(source.perLanguageBundle, undefined);
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource does not use a per-language bundle for a nightly that was not asked for",
|
||||
async (t) => {
|
||||
const expectedTag = "codeql-bundle-30260213";
|
||||
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
|
||||
const client = github.getOctokit("123");
|
||||
const listReleases = sinon.stub(client.rest.repos, "listReleases");
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
listReleases.resolves({
|
||||
data: [{ tag_name: expectedTag }],
|
||||
} as any);
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
// `dynamic` is the event name that Code Scanning default setup uses, which is the case we
|
||||
// most need to keep the per-language bundle feature away from while it is being tested.
|
||||
setupActionsVars(tmpDir, tmpDir, { GITHUB_EVENT_NAME: "dynamic" });
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
// No `tools` input: the nightly is forced by a feature flag instead.
|
||||
undefined,
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
createFeatures([Feature.ForceNightly, Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(source.sourceType, "download");
|
||||
if (source.sourceType === "download") {
|
||||
// Analyses that did not ask for a nightly should not have the bundle they get changed by
|
||||
// the per-language bundle feature.
|
||||
t.true(
|
||||
source.codeqlURL.endsWith(
|
||||
`/${expectedTag}/codeql-bundle-linux64.tar.zst`,
|
||||
),
|
||||
`Unexpected URL ${source.codeqlURL}`,
|
||||
);
|
||||
t.is(source.perLanguageBundle, undefined);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource correctly returns latest version from toolcache when tools == toolcache",
|
||||
async (t) => {
|
||||
@@ -756,6 +935,345 @@ test.serial(
|
||||
},
|
||||
);
|
||||
|
||||
const PER_LANGUAGE_CLI_VERSION = {
|
||||
enabledVersions: [
|
||||
{
|
||||
cliVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
|
||||
tagName: `codeql-bundle-v${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads the `isPerLanguageBundle` argument of a call to `downloadCodeQL`, which controls whether
|
||||
* the bundle may be added to the toolcache.
|
||||
*/
|
||||
function isPerLanguageBundleArg(
|
||||
call: sinon.SinonSpyCall<
|
||||
Parameters<typeof setupCodeql.downloadCodeQL>,
|
||||
unknown
|
||||
>,
|
||||
): boolean {
|
||||
return call.args[4];
|
||||
}
|
||||
|
||||
test.serial("getCodeQLBundleName names the per-language bundle", (t) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
t.is(
|
||||
setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.java),
|
||||
"codeql-bundle-java-linux64.tar.zst",
|
||||
);
|
||||
t.is(
|
||||
setupCodeql.getCodeQLBundleName("zstd"),
|
||||
"codeql-bundle-linux64.tar.zst",
|
||||
);
|
||||
});
|
||||
|
||||
test.serial("getCodeQLBundleName names the Swift bundle for macOS", (t) => {
|
||||
sinon.stub(process, "platform").value("darwin");
|
||||
t.is(
|
||||
setupCodeql.getCodeQLBundleName("zstd", BuiltInLanguage.swift),
|
||||
"codeql-bundle-swift-osx64.tar.zst",
|
||||
);
|
||||
});
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource downloads the per-language bundle for a single explicit language",
|
||||
async (t) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
undefined,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
// Default setup passes the combined language name, which needs normalizing.
|
||||
["java-kotlin"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
createFeatures([Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(source.sourceType, "download");
|
||||
if (source.sourceType === "download") {
|
||||
t.true(
|
||||
source.codeqlURL.endsWith("/codeql-bundle-java-linux64.tar.zst"),
|
||||
`Unexpected URL ${source.codeqlURL}`,
|
||||
);
|
||||
t.is(source.perLanguageBundle?.language, BuiltInLanguage.java);
|
||||
t.true(
|
||||
source.perLanguageBundle?.combinedBundleURL?.endsWith(
|
||||
"/codeql-bundle-linux64.tar.zst",
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQLSource downloads the combined bundle when the feature is disabled",
|
||||
async (t) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
undefined,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
true, // tarSupportsZstd
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(source.sourceType, "download");
|
||||
if (source.sourceType === "download") {
|
||||
t.true(source.codeqlURL.endsWith("/codeql-bundle-linux64.tar.zst"));
|
||||
t.is(source.perLanguageBundle, undefined);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* Stubs out the download of the CodeQL bundle, creating the destination directory as the real
|
||||
* implementation does.
|
||||
*/
|
||||
function stubDownloadAndExtract() {
|
||||
return sinon
|
||||
.stub(toolsDownload, "downloadAndExtract")
|
||||
.callsFake(async (_url, _compressionMethod, dest) => {
|
||||
fs.mkdirSync(dest, { recursive: true });
|
||||
return { totalDurationMs: 100 };
|
||||
});
|
||||
}
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL does not add a per-language bundle to the toolcache",
|
||||
async (t) => {
|
||||
const extractStub = stubDownloadAndExtract();
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
|
||||
const { codeqlFolder } = await setupCodeql.downloadCodeQL(
|
||||
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-java-linux64.tar.zst",
|
||||
"zstd",
|
||||
"v9.9.9",
|
||||
"9.9.9",
|
||||
true, // isPerLanguageBundle
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
undefined,
|
||||
tmpDir,
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
// Even though we know the bundle version, the bundle must be extracted somewhere that a
|
||||
// later job analyzing a different language cannot pick it up from.
|
||||
t.is(codeqlFolder, extractStub.firstCall.args[2]);
|
||||
t.false(codeqlFolder.startsWith(getToolcacheDirectory("9.9.9")));
|
||||
t.true(codeqlFolder.startsWith(tmpDir));
|
||||
t.false(fs.existsSync(`${codeqlFolder}.complete`));
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL adds the combined bundle to the toolcache",
|
||||
async (t) => {
|
||||
const extractStub = stubDownloadAndExtract();
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
|
||||
const { codeqlFolder } = await setupCodeql.downloadCodeQL(
|
||||
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-linux64.tar.zst",
|
||||
"zstd",
|
||||
"v9.9.9",
|
||||
"9.9.9",
|
||||
false, // isPerLanguageBundle
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
undefined,
|
||||
tmpDir,
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(codeqlFolder, extractStub.firstCall.args[2]);
|
||||
t.is(codeqlFolder, getToolcacheDirectory("9.9.9"));
|
||||
t.true(fs.existsSync(`${codeqlFolder}.complete`));
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"setupCodeQLBundle asks for the per-language bundle to be kept out of the toolcache",
|
||||
async (t) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
const downloadStub = sinon.stub(setupCodeql, "downloadCodeQL").resolves({
|
||||
codeqlFolder: "codeql",
|
||||
statusReport: { totalDurationMs: 100 },
|
||||
toolsVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
|
||||
});
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const result = await setupCodeql.setupCodeQLBundle(
|
||||
undefined,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([Feature.PerLanguageBundles]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(downloadStub.calledOnce);
|
||||
t.true(isPerLanguageBundleArg(downloadStub.firstCall));
|
||||
t.is(
|
||||
result.toolsDownloadStatusReport?.bundleLanguage,
|
||||
BuiltInLanguage.java,
|
||||
);
|
||||
t.is(
|
||||
result.toolsDownloadStatusReport?.perLanguageBundleFallback,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"setupCodeQLBundle falls back to the combined bundle if the per-language bundle is missing",
|
||||
async (t) => {
|
||||
const loggedMessages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(process, "arch").value("x64");
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
const downloadStub = sinon.stub(setupCodeql, "downloadCodeQL");
|
||||
downloadStub
|
||||
.onFirstCall()
|
||||
.rejects(new util.HTTPError("Not Found", 404))
|
||||
.onSecondCall()
|
||||
.resolves({
|
||||
codeqlFolder: "codeql",
|
||||
statusReport: { totalDurationMs: 100 },
|
||||
toolsVersion: MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION,
|
||||
});
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const result = await setupCodeql.setupCodeQLBundle(
|
||||
undefined,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
PER_LANGUAGE_CLI_VERSION,
|
||||
["java"],
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([Feature.PerLanguageBundles]),
|
||||
logger,
|
||||
);
|
||||
|
||||
t.true(downloadStub.calledTwice);
|
||||
t.true(
|
||||
downloadStub.secondCall.args[0].endsWith(
|
||||
"/codeql-bundle-linux64.tar.zst",
|
||||
),
|
||||
);
|
||||
// The combined bundle may be added to the toolcache.
|
||||
t.false(isPerLanguageBundleArg(downloadStub.secondCall));
|
||||
t.is(result.toolsDownloadStatusReport?.perLanguageBundleFallback, true);
|
||||
t.is(result.toolsDownloadStatusReport?.bundleLanguage, undefined);
|
||||
checkExpectedLogMessages(t, loggedMessages, [
|
||||
"No java CodeQL bundle was found at",
|
||||
]);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"setupCodeQLBundle keeps an explicitly requested per-language bundle out of the toolcache",
|
||||
async (t) => {
|
||||
const downloadStub = sinon.stub(setupCodeql, "downloadCodeQL").resolves({
|
||||
codeqlFolder: "codeql",
|
||||
statusReport: { totalDurationMs: 100 },
|
||||
toolsVersion: "9.9.9",
|
||||
});
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const result = await setupCodeql.setupCodeQLBundle(
|
||||
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst",
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
undefined, // rawLanguages
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
// Even though we did not choose this bundle, it is still missing most of its extractors.
|
||||
t.true(downloadStub.calledOnce);
|
||||
t.true(isPerLanguageBundleArg(downloadStub.firstCall));
|
||||
t.is(
|
||||
result.toolsDownloadStatusReport?.bundleLanguage,
|
||||
BuiltInLanguage.ruby,
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"setupCodeQLBundle does not substitute a bundle for an explicitly requested one that is missing",
|
||||
async (t) => {
|
||||
const downloadStub = sinon
|
||||
.stub(setupCodeql, "downloadCodeQL")
|
||||
.rejects(new util.HTTPError("Not Found", 404));
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
await t.throwsAsync(
|
||||
setupCodeql.setupCodeQLBundle(
|
||||
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v9.9.9/codeql-bundle-ruby-linux64.tar.zst",
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
undefined, // rawLanguages
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
),
|
||||
);
|
||||
|
||||
// Falling back would silently ignore the bundle that was asked for.
|
||||
t.true(downloadStub.calledOnce);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getEnabledVersionsWithOverlayBaseDatabases returns flag-enabled versions present in cache, sorted desc",
|
||||
async (t) => {
|
||||
@@ -993,6 +1511,7 @@ async function runDownloadCodeQL(
|
||||
"gzip",
|
||||
bundleVersion,
|
||||
CLEANUP_CLI_VERSION,
|
||||
false, // isPerLanguageBundle
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
undefined, // tarVersion
|
||||
toolcacheRoot, // tempDir
|
||||
|
||||
@@ -30,8 +30,13 @@ import {
|
||||
Feature,
|
||||
FeatureEnablement,
|
||||
} from "./feature-flags";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching";
|
||||
import {
|
||||
getPerLanguageBundleLanguage,
|
||||
tryGetBundleLanguageFromUrl,
|
||||
} from "./per-language-bundles";
|
||||
import * as tar from "./tar";
|
||||
import {
|
||||
deleteToolcacheBundles,
|
||||
@@ -72,21 +77,40 @@ function getCodeQLBundleExtension(
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the platform component of the CodeQL bundle name for the current platform. */
|
||||
export function getBundlePlatform(): string | undefined {
|
||||
switch (process.platform) {
|
||||
case "win32":
|
||||
return "win64";
|
||||
case "linux":
|
||||
return process.arch === "arm64" ? "linux-arm64" : "linux64";
|
||||
case "darwin":
|
||||
return "osx64";
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the name of the CodeQL bundle asset to download.
|
||||
*
|
||||
* @param compressionMethod The compression method of the bundle.
|
||||
* @param language If provided, the name of the bundle that contains only this language, rather than
|
||||
* the name of the combined bundle that contains every language.
|
||||
*/
|
||||
export function getCodeQLBundleName(
|
||||
compressionMethod: tar.CompressionMethod,
|
||||
language?: BuiltInLanguage,
|
||||
): string {
|
||||
const extension = getCodeQLBundleExtension(compressionMethod);
|
||||
const platform = getBundlePlatform();
|
||||
|
||||
let platform: string;
|
||||
if (process.platform === "win32") {
|
||||
platform = "win64";
|
||||
} else if (process.platform === "linux") {
|
||||
platform = process.arch === "arm64" ? "linux-arm64" : "linux64";
|
||||
} else if (process.platform === "darwin") {
|
||||
platform = "osx64";
|
||||
} else {
|
||||
if (platform === undefined) {
|
||||
return `codeql-bundle${extension}`;
|
||||
}
|
||||
if (language !== undefined) {
|
||||
return `codeql-bundle-${language}-${platform}${extension}`;
|
||||
}
|
||||
return `codeql-bundle-${platform}${extension}`;
|
||||
}
|
||||
|
||||
@@ -107,7 +131,7 @@ export function getCodeQLActionRepository(logger: Logger): string {
|
||||
async function getCodeQLBundleDownloadURL(
|
||||
tagName: string,
|
||||
apiDetails: api.GitHubApiDetails,
|
||||
compressionMethod: tar.CompressionMethod,
|
||||
codeQLBundleName: string,
|
||||
logger: Logger,
|
||||
): Promise<string> {
|
||||
const codeQLActionRepository = getCodeQLActionRepository(logger);
|
||||
@@ -126,7 +150,6 @@ async function getCodeQLBundleDownloadURL(
|
||||
return !self.slice(0, index).some((other) => deepEqual(source, other));
|
||||
},
|
||||
);
|
||||
const codeQLBundleName = getCodeQLBundleName(compressionMethod);
|
||||
for (const downloadSource of uniqueDownloadSources) {
|
||||
const [apiURL, repository] = downloadSource;
|
||||
// If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
|
||||
@@ -236,6 +259,19 @@ export type CodeQLToolsSource =
|
||||
cliVersion?: string;
|
||||
compressionMethod: tar.CompressionMethod;
|
||||
codeqlURL: string;
|
||||
/**
|
||||
* Set when `codeqlURL` points at a bundle that contains only a single language. Records which
|
||||
* language that is, and where to find the combined bundle if the per-language bundle turns
|
||||
* out not to exist.
|
||||
*/
|
||||
perLanguageBundle?: {
|
||||
language: BuiltInLanguage;
|
||||
/**
|
||||
* Absent when the bundle was requested explicitly rather than chosen by us, since in that
|
||||
* case we should honor the request rather than substituting a different bundle.
|
||||
*/
|
||||
combinedBundleURL?: string;
|
||||
};
|
||||
sourceType: "download";
|
||||
/** Human-readable description of the source of the tools for telemetry purposes. */
|
||||
toolsVersion: string;
|
||||
@@ -471,6 +507,14 @@ export async function getCodeQLSource(
|
||||
toolsInput !== undefined &&
|
||||
CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
|
||||
|
||||
/**
|
||||
* The combined bundle from the nightly release we are using, if any.
|
||||
*
|
||||
* Only set when we chose the nightly ourselves, so that we can fall back to it if the nightly
|
||||
* turns out not to have a bundle for the language we asked for.
|
||||
*/
|
||||
let nightlyCombinedBundleURL: string | undefined;
|
||||
|
||||
if (forceNightly || nightlyRequestedByToolsInput) {
|
||||
if (forceNightly) {
|
||||
logger.info(
|
||||
@@ -500,7 +544,15 @@ export async function getCodeQLSource(
|
||||
`Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`,
|
||||
);
|
||||
}
|
||||
toolsInput = await getNightlyToolsUrl(logger);
|
||||
const nightly = await getNightlyToolsUrl(
|
||||
rawLanguages,
|
||||
variant,
|
||||
nightlyRequestedByToolsInput,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
toolsInput = nightly.url;
|
||||
nightlyCombinedBundleURL = nightly.combinedBundleURL;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -719,6 +771,9 @@ export async function getCodeQLSource(
|
||||
}
|
||||
|
||||
let compressionMethod: tar.CompressionMethod;
|
||||
let perLanguageBundle:
|
||||
| { language: BuiltInLanguage; combinedBundleURL?: string }
|
||||
| undefined;
|
||||
|
||||
if (!url) {
|
||||
compressionMethod =
|
||||
@@ -727,12 +782,39 @@ export async function getCodeQLSource(
|
||||
? "zstd"
|
||||
: "gzip";
|
||||
|
||||
url = await getCodeQLBundleDownloadURL(
|
||||
tagName!,
|
||||
apiDetails,
|
||||
compressionMethod,
|
||||
const perLanguageBundleLanguage = await getPerLanguageBundleLanguage(
|
||||
{
|
||||
rawLanguages,
|
||||
cliVersion,
|
||||
compressionMethod,
|
||||
platform: getBundlePlatform(),
|
||||
variant,
|
||||
},
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
|
||||
const resolveBundleURL = (language?: BuiltInLanguage) =>
|
||||
getCodeQLBundleDownloadURL(
|
||||
tagName!,
|
||||
apiDetails,
|
||||
getCodeQLBundleName(compressionMethod, language),
|
||||
logger,
|
||||
);
|
||||
|
||||
if (perLanguageBundleLanguage !== undefined) {
|
||||
logger.info(
|
||||
`Downloading the ${perLanguageBundleLanguage} CodeQL bundle, since ${perLanguageBundleLanguage} ` +
|
||||
"is the only language being analyzed.",
|
||||
);
|
||||
url = await resolveBundleURL(perLanguageBundleLanguage);
|
||||
perLanguageBundle = {
|
||||
language: perLanguageBundleLanguage,
|
||||
combinedBundleURL: await resolveBundleURL(),
|
||||
};
|
||||
} else {
|
||||
url = await resolveBundleURL();
|
||||
}
|
||||
} else {
|
||||
const method = tar.inferCompressionMethod(url);
|
||||
if (method === undefined) {
|
||||
@@ -742,6 +824,22 @@ export async function getCodeQLSource(
|
||||
);
|
||||
}
|
||||
compressionMethod = method;
|
||||
|
||||
// We chose this bundle ourselves if it is a nightly, and otherwise it was requested explicitly.
|
||||
// Either way we need to know whether it contains a single language, so that we do not add it to
|
||||
// the toolcache.
|
||||
const language = tryGetBundleLanguageFromUrl(url);
|
||||
if (language !== undefined) {
|
||||
logger.info(
|
||||
`${url} appears to be a CodeQL bundle that contains only ${language}.`,
|
||||
);
|
||||
// We only have somewhere to fall back to if we chose this bundle; when it was requested
|
||||
// explicitly we should honor the request rather than substituting a different bundle.
|
||||
perLanguageBundle = {
|
||||
language,
|
||||
combinedBundleURL: nightlyCombinedBundleURL,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (cliVersion) {
|
||||
@@ -754,6 +852,7 @@ export async function getCodeQLSource(
|
||||
cliVersion,
|
||||
codeqlURL: url,
|
||||
compressionMethod,
|
||||
perLanguageBundle,
|
||||
sourceType: "download",
|
||||
toolsVersion: cliVersion ?? humanReadableVersion,
|
||||
};
|
||||
@@ -787,6 +886,7 @@ export const downloadCodeQL = async function (
|
||||
compressionMethod: tar.CompressionMethod,
|
||||
maybeBundleVersion: string | undefined,
|
||||
maybeCliVersion: string | undefined,
|
||||
isPerLanguageBundle: boolean,
|
||||
apiDetails: api.GitHubApiDetails,
|
||||
tarVersion: tar.TarVersion | undefined,
|
||||
tempDir: string,
|
||||
@@ -815,11 +915,11 @@ export const downloadCodeQL = async function (
|
||||
);
|
||||
}
|
||||
|
||||
const toolcacheInfo = getToolcacheDestinationInfo(
|
||||
maybeBundleVersion,
|
||||
maybeCliVersion,
|
||||
logger,
|
||||
);
|
||||
// We deliberately never populate the toolcache with a per-language bundle, since a subsequent job
|
||||
// that analyzes a different language would find it there and be missing its extractor.
|
||||
const toolcacheInfo = isPerLanguageBundle
|
||||
? undefined
|
||||
: getToolcacheDestinationInfo(maybeBundleVersion, maybeCliVersion, logger);
|
||||
|
||||
const extractedBundlePath =
|
||||
toolcacheInfo?.path ?? getTempExtractionDir(tempDir);
|
||||
@@ -838,8 +938,11 @@ export const downloadCodeQL = async function (
|
||||
|
||||
if (!toolcacheInfo) {
|
||||
logger.debug(
|
||||
"Could not cache CodeQL tools because we could not determine the bundle version from the " +
|
||||
`URL ${codeqlURL}.`,
|
||||
isPerLanguageBundle
|
||||
? "Not caching the CodeQL tools because they came from a bundle that contains only a " +
|
||||
"single language."
|
||||
: "Could not cache CodeQL tools because we could not determine the bundle version from the " +
|
||||
`URL ${codeqlURL}.`,
|
||||
);
|
||||
return {
|
||||
codeqlFolder: extractedBundlePath,
|
||||
@@ -1021,11 +1124,8 @@ export async function setupCodeQLBundle(
|
||||
toolsSource = ToolsSource.Toolcache;
|
||||
break;
|
||||
case "download": {
|
||||
const result = await downloadCodeQL(
|
||||
source.codeqlURL,
|
||||
source.compressionMethod,
|
||||
source.bundleVersion,
|
||||
source.cliVersion,
|
||||
const result = await downloadCodeQLBundle(
|
||||
source,
|
||||
apiDetails,
|
||||
zstdAvailability.version,
|
||||
tempDir,
|
||||
@@ -1054,6 +1154,87 @@ export async function setupCodeQLBundle(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads the CodeQL bundle described by `source`.
|
||||
*
|
||||
* If `source` refers to a bundle for a single language and that bundle turns out not to exist, this
|
||||
* falls back to downloading the combined bundle.
|
||||
*/
|
||||
async function downloadCodeQLBundle(
|
||||
source: Extract<CodeQLToolsSource, { sourceType: "download" }>,
|
||||
apiDetails: api.GitHubApiDetails,
|
||||
tarVersion: tar.TarVersion | undefined,
|
||||
tempDir: string,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
): Promise<{
|
||||
codeqlFolder: string;
|
||||
statusReport: ToolsDownloadStatusReport;
|
||||
toolsVersion: string;
|
||||
}> {
|
||||
const { perLanguageBundle } = source;
|
||||
|
||||
if (perLanguageBundle !== undefined) {
|
||||
try {
|
||||
const result = await downloadCodeQL(
|
||||
source.codeqlURL,
|
||||
source.compressionMethod,
|
||||
source.bundleVersion,
|
||||
source.cliVersion,
|
||||
true,
|
||||
apiDetails,
|
||||
tarVersion,
|
||||
tempDir,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
return {
|
||||
...result,
|
||||
statusReport: {
|
||||
...result.statusReport,
|
||||
bundleLanguage: perLanguageBundle.language,
|
||||
},
|
||||
};
|
||||
} catch (e) {
|
||||
// Without a combined bundle to fall back to, the bundle was requested explicitly, so there is
|
||||
// nothing we can substitute for it.
|
||||
if (
|
||||
perLanguageBundle.combinedBundleURL === undefined ||
|
||||
util.asHTTPError(e)?.status !== 404
|
||||
) {
|
||||
throw e;
|
||||
}
|
||||
logger.warning(
|
||||
`No ${perLanguageBundle.language} CodeQL bundle was found at ${source.codeqlURL}, so ` +
|
||||
"falling back to the bundle that contains all languages. This analysis will still " +
|
||||
"produce correct results, but will take longer to set up.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const result = await downloadCodeQL(
|
||||
perLanguageBundle?.combinedBundleURL ?? source.codeqlURL,
|
||||
source.compressionMethod,
|
||||
source.bundleVersion,
|
||||
source.cliVersion,
|
||||
false,
|
||||
apiDetails,
|
||||
tarVersion,
|
||||
tempDir,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
return perLanguageBundle === undefined
|
||||
? result
|
||||
: {
|
||||
...result,
|
||||
statusReport: {
|
||||
...result.statusReport,
|
||||
perLanguageBundleFallback: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function useZstdBundle(
|
||||
cliVersion: string,
|
||||
tarSupportsZstd: boolean,
|
||||
@@ -1073,7 +1254,17 @@ function getTempExtractionDir(tempDir: string) {
|
||||
/**
|
||||
* Get the URL of the latest nightly CodeQL bundle.
|
||||
*/
|
||||
async function getNightlyToolsUrl(logger: Logger) {
|
||||
/**
|
||||
* Get the URL of the latest nightly CodeQL bundle, and of the combined bundle from the same
|
||||
* nightly release to fall back to if that bundle does not exist.
|
||||
*/
|
||||
async function getNightlyToolsUrl(
|
||||
rawLanguages: string[] | undefined,
|
||||
variant: util.GitHubVariant,
|
||||
requestedExplicitly: boolean,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
): Promise<{ url: string; combinedBundleURL: string }> {
|
||||
const zstdAvailability = await tar.isZstdAvailable(logger);
|
||||
// The nightly is guaranteed to have a zstd bundle
|
||||
const compressionMethod = (await useZstdBundle(
|
||||
@@ -1083,6 +1274,24 @@ async function getNightlyToolsUrl(logger: Logger) {
|
||||
? "zstd"
|
||||
: "gzip";
|
||||
|
||||
// We only consider a per-language bundle when a nightly was asked for explicitly. Nightlies can
|
||||
// also be forced for analyses that did not ask for one, and those should keep getting the bundle
|
||||
// that contains every language.
|
||||
const language = requestedExplicitly
|
||||
? await getPerLanguageBundleLanguage(
|
||||
{
|
||||
rawLanguages,
|
||||
cliVersion: undefined,
|
||||
compressionMethod,
|
||||
platform: getBundlePlatform(),
|
||||
variant,
|
||||
isNightly: true,
|
||||
},
|
||||
features,
|
||||
logger,
|
||||
)
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
// Since nightlies are prereleases, we can't just download the latest release
|
||||
// on the repository. So instead we need to find the latest pre-release
|
||||
@@ -1098,7 +1307,12 @@ async function getNightlyToolsUrl(logger: Logger) {
|
||||
if (!latestRelease) {
|
||||
throw new Error("Could not find the latest nightly release.");
|
||||
}
|
||||
return `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${getCodeQLBundleName(compressionMethod)}`;
|
||||
const assetUrl = (name: string) =>
|
||||
`https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`;
|
||||
return {
|
||||
url: assetUrl(getCodeQLBundleName(compressionMethod, language)),
|
||||
combinedBundleURL: assetUrl(getCodeQLBundleName(compressionMethod)),
|
||||
};
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Failed to retrieve the latest nightly release: ${util.wrapError(e)}`,
|
||||
|
||||
@@ -645,6 +645,13 @@ export interface InitToolsDownloadFields {
|
||||
* Whether the relevant tools dotcom feature flags have been misconfigured.
|
||||
* Only populated if we attempt to determine the default version based on the dotcom feature flags. */
|
||||
tools_feature_flags_valid?: boolean;
|
||||
/** The language of the single-language bundle that was downloaded, if any. */
|
||||
tools_bundle_language?: string;
|
||||
/**
|
||||
* Whether we tried to download a single-language bundle, but it did not exist and we fell back to
|
||||
* the combined bundle.
|
||||
*/
|
||||
tools_per_language_bundle_fallback?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -10,7 +10,7 @@ import { getRunnerLogger } from "./logging";
|
||||
import * as tar from "./tar";
|
||||
import { setupTests } from "./testing-utils";
|
||||
import { downloadAndExtract } from "./tools-download";
|
||||
import { withTmpDir } from "./util";
|
||||
import { HTTPError, withTmpDir } from "./util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
@@ -78,6 +78,74 @@ test.serial(
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadAndExtract rethrows a 404 rather than retrying the download",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
const downloadTool = sinon.stub(toolcache, "downloadTool");
|
||||
const extractTarZst = sinon.stub(tar, "extractTarZst").resolves();
|
||||
const request = nock("https://example.com")
|
||||
.get("/codeql-bundle.tar.zst")
|
||||
.reply(404);
|
||||
|
||||
const error = await t.throwsAsync(
|
||||
downloadAndExtract(
|
||||
"https://example.com/codeql-bundle.tar.zst",
|
||||
"zstd",
|
||||
path.join(tmpDir, "codeql"),
|
||||
undefined,
|
||||
{},
|
||||
{ type: "gnu", version: "1.34" },
|
||||
getRunnerLogger(true),
|
||||
),
|
||||
{ instanceOf: HTTPError },
|
||||
);
|
||||
|
||||
// Callers rely on the status to decide whether to fall back to a different bundle.
|
||||
t.is(error?.status, 404);
|
||||
t.true(request.isDone());
|
||||
t.false(extractTarZst.called);
|
||||
// Downloading the missing asset a second time would just fail again.
|
||||
t.false(downloadTool.called);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadAndExtract falls back to downloading before extracting on a server error",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
const archivePath = path.join(tmpDir, "codeql-bundle.tar.zst");
|
||||
const destination = path.join(tmpDir, "codeql");
|
||||
const downloadTool = sinon
|
||||
.stub(toolcache, "downloadTool")
|
||||
.resolves(archivePath);
|
||||
const extract = sinon.stub(tar, "extract").resolves(destination);
|
||||
const request = nock("https://example.com")
|
||||
.get("/codeql-bundle.tar.zst")
|
||||
.reply(500);
|
||||
|
||||
const statusReport = await downloadAndExtract(
|
||||
"https://example.com/codeql-bundle.tar.zst",
|
||||
"zstd",
|
||||
destination,
|
||||
undefined,
|
||||
{},
|
||||
{ type: "gnu", version: "1.34" },
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.assert(Number.isInteger(statusReport.downloadDurationMs));
|
||||
t.true(request.isDone());
|
||||
// A server error may be transient, so it is worth retrying via `downloadTool`.
|
||||
t.true(downloadTool.calledOnce);
|
||||
t.true(extract.calledOnce);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadAndExtract reports only the total duration when streaming extraction",
|
||||
async (t) => {
|
||||
|
||||
@@ -14,7 +14,13 @@ import { ActionState } from "./action-common";
|
||||
import { ActionsEnvVars, getEnv, ReadOnlyEnv } from "./environment";
|
||||
import { formatDuration, Logger } from "./logging";
|
||||
import * as tar from "./tar";
|
||||
import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
|
||||
import {
|
||||
asHTTPError,
|
||||
cleanUpPath,
|
||||
getErrorMessage,
|
||||
getRequiredEnvParam,
|
||||
HTTPError,
|
||||
} from "./util";
|
||||
|
||||
/**
|
||||
* High watermark to use when streaming the download and extraction of the CodeQL tools.
|
||||
@@ -48,6 +54,13 @@ export type ToolsDownloadStatusReport = {
|
||||
* spent on a streaming attempt that failed and fell back to downloading before extracting.
|
||||
*/
|
||||
totalDurationMs: number;
|
||||
/** The language of the single-language bundle that was downloaded, if any. */
|
||||
bundleLanguage?: string;
|
||||
/**
|
||||
* Whether we tried to download a single-language bundle, but it did not exist and we fell back to
|
||||
* the combined bundle.
|
||||
*/
|
||||
perLanguageBundleFallback?: boolean;
|
||||
};
|
||||
|
||||
export async function downloadAndExtract(
|
||||
@@ -88,14 +101,21 @@ export async function downloadAndExtract(
|
||||
return { totalDurationMs };
|
||||
}
|
||||
} catch (e) {
|
||||
// If we failed during processing, we want to clean up the destination directory
|
||||
// before we either try again or give up.
|
||||
await cleanUpPath(dest, "CodeQL bundle", logger);
|
||||
|
||||
// Retrying a 404 is pointless: the asset does not exist, so downloading it a different way
|
||||
// will fail in the same way. Surface it so that callers can react to it, for example by
|
||||
// falling back to a different bundle.
|
||||
if (asHTTPError(e)?.status === 404) {
|
||||
throw e;
|
||||
}
|
||||
|
||||
core.warning(
|
||||
`Failed to download and extract CodeQL bundle using streaming with error: ${getErrorMessage(e)}`,
|
||||
);
|
||||
core.warning(`Falling back to downloading the bundle before extracting.`);
|
||||
|
||||
// If we failed during processing, we want to clean up the destination directory
|
||||
// before we try again.
|
||||
await cleanUpPath(dest, "CodeQL bundle", logger);
|
||||
}
|
||||
|
||||
const toolsDownloadStart = performance.now();
|
||||
@@ -191,8 +211,9 @@ async function downloadAndExtractZstdWithStreaming(
|
||||
if (response.statusCode !== 200) {
|
||||
// Discard the response body so that the connection can be released.
|
||||
response.resume();
|
||||
throw new Error(
|
||||
throw new HTTPError(
|
||||
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
|
||||
response.statusCode ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -165,6 +165,9 @@ async function combineSarifFilesUsingCLI(
|
||||
tempDir,
|
||||
gitHubVersion.type,
|
||||
codeQLDefaultVersionInfo,
|
||||
// Leaving the languages unset is load-bearing as well as accurate: it is what stops us from
|
||||
// downloading a bundle that contains only one language, which would not have the extractors
|
||||
// that a later step might need. Do not start threading languages through here.
|
||||
undefined, // rawLanguages: upload-lib does not run analysis
|
||||
false, // useOverlayAwareDefaultCliVersion: upload-lib does not run analysis
|
||||
features,
|
||||
|
||||
Reference in New Issue
Block a user