github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-10-30 01:27:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Stream the download and extraction of zstd bundles

Browse Source 
Behind a feature flag
This commit is contained in:
Henry Mercer
2024-10-10 18:23:06 +01:00
parent cd83b08c78
commit ddead5420c
22 changed files with 371 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/codeql.test.ts
View File

@@ -30,6 +30,7 @@ import {
makeVersionInfo,
createTestConfig,
} from "./testing-utils";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import * as util from "./util";
import { initializeEnvironment } from "./util";
@@ -138,9 +139,6 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, `0.0.0-${version}`);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
}
t.is(toolcache.findAllVersions("CodeQL").length, 2);
@@ -170,9 +168,9 @@ test("caches semantically versioned bundles using their semantic version number"
t.assert(toolcache.find("CodeQL", `2.14.0`));
t.is(result.toolsVersion, `2.14.0`);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
@@ -203,9 +201,9 @@ test("downloads an explicitly requested bundle even if a different version is ca
t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610");
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
});
});
@@ -288,7 +286,9 @@ for (const toolcacheVersion of [
);
t.is(result.toolsVersion, SAMPLE_DEFAULT_CLI_VERSION.cliVersion);
t.is(result.toolsSource, ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
});
},
);
@@ -320,7 +320,9 @@ test(`uses a cached bundle when no tools input is given on GHES`, async (t) => {
);
t.deepEqual(result.toolsVersion, "0.0.0-20200601");
t.is(result.toolsSource, ToolsSource.Toolcache);
t.is(result.toolsDownloadStatusReport?.combinedDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.downloadDurationMs, undefined);
t.is(result.toolsDownloadStatusReport?.extractionDurationMs, undefined);
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
@@ -356,9 +358,9 @@ test(`downloads bundle if only an unpinned version is cached on GHES`, async (t)
);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
@@ -391,9 +393,9 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
);
t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
@@ -430,9 +432,9 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
t.is(result.toolsVersion, "0.0.0-20230203");
t.is(result.toolsSource, ToolsSource.Download);
t.true(
Number.isInteger(result.toolsDownloadStatusReport?.downloadDurationMs),
);
if (result.toolsDownloadStatusReport) {
assertDurationsInteger(t, result.toolsDownloadStatusReport);
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
@@ -442,6 +444,17 @@ test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t)
});
});
function assertDurationsInteger(
t: ExecutionContext<unknown>,
statusReport: ToolsDownloadStatusReport,
) {
t.assert(Number.isInteger(statusReport?.combinedDurationMs));
if (statusReport.downloadDurationMs !== undefined) {
t.assert(Number.isInteger(statusReport?.downloadDurationMs));
t.assert(Number.isInteger(statusReport?.extractionDurationMs));
}
}
test("getExtraOptions works for explicit paths", (t) => {
t.deepEqual(codeql.getExtraOptions({}, ["foo"], []), []);

3
src/codeql.ts
View File

@@ -28,6 +28,7 @@ import { Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { shouldEnableIndirectTracing } from "./tracer-config";
import * as util from "./util";
@@ -356,7 +357,7 @@ export async function setupCodeQL(
checkVersion: boolean,
): Promise<{
codeql: CodeQL;
toolsDownloadStatusReport?: setupCodeql.ToolsDownloadStatusReport;
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: setupCodeql.ToolsSource;
toolsVersion: string;
zstdAvailability: ZstdAvailability;

6
src/feature-flags.ts
View File

@@ -54,6 +54,7 @@ export enum Feature {
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
QaTelemetryEnabled = "qa_telemetry_enabled",
ZstdBundle = "zstd_bundle",
ZstdBundleStreamingExtraction = "zstd_bundle_streaming_extraction",
}
export const featureConfig: Record<
@@ -139,6 +140,11 @@ export const featureConfig: Record<
// version check separately.
minimumVersion: undefined,
},
[Feature.ZstdBundleStreamingExtraction]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ZSTD_BUNDLE_STREAMING_EXTRACTION",
minimumVersion: undefined,
},
};
/**

5
src/init-action.ts
View File

@@ -34,7 +34,7 @@ import {
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
import { ToolsSource } from "./setup-codeql";
import {
ActionName,
StatusReportBase,
@@ -43,6 +43,7 @@ import {
sendStatusReport,
} from "./status-report";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import { getTotalCacheSize } from "./trap-caching";
import {
@@ -153,7 +154,7 @@ async function sendCompletedStatusReport(
const initToolsDownloadFields: InitToolsDownloadFields = {};
if (toolsDownloadStatusReport !== undefined) {
if (toolsDownloadStatusReport?.downloadDurationMs !== undefined) {
initToolsDownloadFields.tools_download_duration_ms =
toolsDownloadStatusReport.downloadDurationMs;
}

3
src/init.ts
View File

@@ -11,8 +11,9 @@ import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { Language, isScannedLanguage } from "./languages";
import { Logger } from "./logging";
import { ToolsDownloadStatusReport, ToolsSource } from "./setup-codeql";
import { ToolsSource } from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";

2
src/setup-codeql.test.ts
View File

@@ -161,6 +161,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to use
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,
@@ -210,6 +211,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
codeqlFolder: "codeql",
statusReport: {
combinedDurationMs: 500,
compressionMethod: "gzip",
downloadDurationMs: 200,
extractionDurationMs: 300,

83
src/setup-codeql.ts
View File

@@ -1,18 +1,13 @@
import * as fs from "fs";
import { OutgoingHttpHeaders } from "http";
import * as path from "path";
import { performance } from "perf_hooks";
import * as toolcache from "@actions/tool-cache";
import { default as deepEqual } from "fast-deep-equal";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { CommandInvocationError, isRunningLocalAction } from "./actions-util";
import * as api from "./api-client";
// Note: defaults.json is referenced from the CodeQL Action sync tool and the Actions runner image
// creation scripts. Ensure that any changes to the format of this file are compatible with both of
// these dependents.
import * as defaults from "./defaults.json";
import {
CODEQL_VERSION_ZSTD_BUNDLE,
@@ -22,6 +17,10 @@ import {
} from "./feature-flags";
import { Logger } from "./logging";
import * as tar from "./tar";
import {
downloadAndExtract,
ToolsDownloadStatusReport,
} from "./tools-download";
import * as util from "./util";
import { cleanUpGlob, isGoodVersion } from "./util";
@@ -481,14 +480,6 @@ export async function tryGetFallbackToolcacheVersion(
return fallbackVersion;
}
export interface ToolsDownloadStatusReport {
compressionMethod: tar.CompressionMethod;
downloadDurationMs: number;
extractionDurationMs: number;
toolsUrl: string;
zstdFailureReason?: string;
}
// Exported using `export const` for testing purposes. Specifically, we want to
// be able to stub this function and have other functions in this file use that stub.
export const downloadCodeQL = async function (
@@ -498,6 +489,7 @@ export const downloadCodeQL = async function (
apiDetails: api.GitHubApiDetails,
tarVersion: tar.TarVersion | undefined,
tempDir: string,
features: FeatureEnablement,
logger: Logger,
): Promise<{
codeqlFolder: string;
@@ -529,45 +521,15 @@ export const downloadCodeQL = async function (
`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
);
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
const dest = path.join(tempDir, uuidV4());
const finalHeaders = Object.assign(
{ "User-Agent": "CodeQL Action" },
headers,
);
const toolsDownloadStart = performance.now();
const archivedBundlePath = await toolcache.downloadTool(
const { extractedBundlePath, statusReport } = await downloadAndExtract(
codeqlURL,
dest,
authorization,
finalHeaders,
{ "User-Agent": "CodeQL Action", ...headers },
tarVersion,
tempDir,
features,
logger,
);
const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);
logger.debug(
`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`,
);
let extractedBundlePath: string;
let extractionDurationMs: number;
try {
logger.debug("Extracting CodeQL bundle.");
const extractionStart = performance.now();
extractedBundlePath = await tar.extract(
archivedBundlePath,
compressionMethod,
tarVersion,
logger,
);
extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.debug(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
);
} finally {
await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
}
const bundleVersion =
maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
@@ -579,12 +541,7 @@ export const downloadCodeQL = async function (
);
return {
codeqlFolder: extractedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? "unknown",
};
}
@@ -612,12 +569,7 @@ export const downloadCodeQL = async function (
return {
codeqlFolder: toolcachedBundlePath,
statusReport: {
compressionMethod,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
statusReport,
toolsVersion: maybeCliVersion ?? toolcacheVersion,
};
};
@@ -789,6 +741,7 @@ async function setupCodeQLBundleWithCompressionMethod(
apiDetails,
zstdAvailability.version,
tempDir,
features,
logger,
);
toolsVersion = result.toolsVersion;
@@ -809,14 +762,6 @@ async function setupCodeQLBundleWithCompressionMethod(
};
}
function sanitizeUrlForStatusReport(url: string): string {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
(repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
)
? url
: "sanitized-value";
}
async function useZstdBundle(
cliVersion: string,
features: FeatureEnablement,

167
src/tools-download.ts Normal file
View File

@@ -0,0 +1,167 @@
import * as http from "http";
import { OutgoingHttpHeaders } from "http";
import * as path from "path";
import { performance } from "perf_hooks";
import * as toolcache from "@actions/tool-cache";
import { v4 as uuidV4 } from "uuid";
import { Feature, FeatureEnablement } from "./feature-flags";
import { Logger } from "./logging";
import * as tar from "./tar";
import { cleanUpGlob } from "./util";
/**
* Timing information for the download and extraction of the CodeQL tools when
* we fully download the bundle before extracting.
*/
type DownloadFirstToolsDownloadDurations = {
combinedDurationMs: number;
downloadDurationMs: number;
extractionDurationMs: number;
};
/**
* Timing information for the download and extraction of the CodeQL tools when
* we stream the download and extraction of the bundle.
*/
type StreamedToolsDownloadDurations = {
combinedDurationMs: number;
downloadDurationMs: undefined;
extractionDurationMs: undefined;
};
type ToolsDownloadDurations =
| DownloadFirstToolsDownloadDurations
| StreamedToolsDownloadDurations;
export type ToolsDownloadStatusReport = {
compressionMethod: tar.CompressionMethod;
toolsUrl: string;
zstdFailureReason?: string;
} & ToolsDownloadDurations;
export async function downloadAndExtract(
codeqlURL: string,
authorization: string | undefined,
headers: OutgoingHttpHeaders,
tarVersion: tar.TarVersion | undefined,
tempDir: string,
features: FeatureEnablement,
logger: Logger,
): Promise<{
extractedBundlePath: string;
statusReport: ToolsDownloadStatusReport;
}> {
const compressionMethod = tar.inferCompressionMethod(codeqlURL);
if (
compressionMethod === "zstd" &&
(await features.getValue(Feature.ZstdBundleStreamingExtraction))
) {
const toolsInstallStart = performance.now();
const extractedBundlePath = await downloadAndExtractZstdWithStreaming(
codeqlURL,
authorization,
headers,
tarVersion!,
logger,
);
const combinedDurationMs = Math.round(
performance.now() - toolsInstallStart,
);
logger.info(
`Finished downloading and extracting CodeQL bundle to ${extractedBundlePath} (${combinedDurationMs} ms).`,
);
return {
extractedBundlePath,
statusReport: {
combinedDurationMs,
compressionMethod,
downloadDurationMs: undefined,
extractionDurationMs: undefined,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
};
}
const dest = path.join(tempDir, uuidV4());
const toolsDownloadStart = performance.now();
const archivedBundlePath = await toolcache.downloadTool(
codeqlURL,
dest,
authorization,
headers,
);
const downloadDurationMs = Math.round(performance.now() - toolsDownloadStart);
logger.info(
`Finished downloading CodeQL bundle to ${archivedBundlePath} (${downloadDurationMs} ms).`,
);
let extractedBundlePath: string;
let extractionDurationMs: number;
try {
logger.info("Extracting CodeQL bundle.");
const extractionStart = performance.now();
extractedBundlePath = await tar.extract(
archivedBundlePath,
compressionMethod,
tarVersion,
logger,
);
extractionDurationMs = Math.round(performance.now() - extractionStart);
logger.info(
`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
);
} finally {
await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
}
return {
extractedBundlePath,
statusReport: {
compressionMethod,
combinedDurationMs: downloadDurationMs + extractionDurationMs,
downloadDurationMs,
extractionDurationMs,
toolsUrl: sanitizeUrlForStatusReport(codeqlURL),
},
};
}
async function downloadAndExtractZstdWithStreaming(
codeqlURL: string,
authorization: string | undefined,
headers: OutgoingHttpHeaders,
tarVersion: tar.TarVersion,
logger: Logger,
): Promise<string> {
headers = Object.assign(
{ "User-Agent": "CodeQL Action", authorization },
headers,
);
const response = await new Promise<http.IncomingMessage>((resolve) =>
http.get(codeqlURL, { headers }, (r) => resolve(r)),
);
if (response.statusCode !== 200) {
throw new Error(
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
);
}
return await tar.extractTarZst(response, tarVersion, logger);
}
function sanitizeUrlForStatusReport(url: string): string {
return ["github/codeql-action", "dsp-testing/codeql-cli-nightlies"].some(
(repo) => url.startsWith(`https://github.com/${repo}/releases/download/`),
)
? url
: "sanitized-value";
}