github/codeql-action
1
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-10-31 01:49:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into mbg/ignore-generated

Browse Source
This commit is contained in:
Michael B. Gale
2026-01-19 13:09:15 +00:00
parent b4db38273c c99e493099
commit 1782089bde
146 changed files with 253426 additions and 203535 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/actions-util.ts
View File

@@ -80,7 +80,7 @@ export function isRunningLocalAction(): boolean {
*
* This can be used to get the Action's name or tell if we're running a local Action.
*/
export function getRelativeScriptPath(): string {
function getRelativeScriptPath(): string {
const runnerTemp = getRequiredEnvParam("RUNNER_TEMP");
const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
return path.relative(actionsDirectory, __filename);

2
src/analyses.ts
View File

@@ -98,7 +98,7 @@ export async function getAnalysisKinds(
export const codeQualityQueries: string[] = ["code-quality"];
// Enumerates API endpoints that accept SARIF files.
export enum SARIF_UPLOAD_ENDPOINT {
enum SARIF_UPLOAD_ENDPOINT {
CODE_SCANNING = "PUT /repos/:owner/:repo/code-scanning/analysis",
CODE_QUALITY = "PUT /repos/:owner/:repo/code-quality/analysis",
}

21
src/analyze-action-env.test.ts
View File

@@ -74,11 +74,20 @@ test("analyze action with RAM & threads from environment variables", async (t) =
// wait for the action promise to complete before starting verification.
await analyzeAction.runPromise;
t.assert(runFinalizeStub.calledOnce);
t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=4992");
t.assert(runQueriesStub.calledOnce);
t.deepEqual(runQueriesStub.firstCall.args[2], "--threads=-1");
t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=4992");
t.assert(
runFinalizeStub.calledOnceWith(
sinon.match.any,
sinon.match.any,
"--threads=-1",
"--ram=4992",
),
);
t.assert(
runQueriesStub.calledOnceWith(
sinon.match.any,
"--ram=4992",
"--threads=-1",
),
);
});
});

21
src/analyze-action-input.test.ts
View File

@@ -72,11 +72,20 @@ test("analyze action with RAM & threads from action inputs", async (t) => {
// wait for the action promise to complete before starting verification.
await analyzeAction.runPromise;
t.assert(runFinalizeStub.calledOnce);
t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=3012");
t.assert(runQueriesStub.calledOnce);
t.deepEqual(runQueriesStub.firstCall.args[2], "--threads=-1");
t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=3012");
t.assert(
runFinalizeStub.calledOnceWith(
sinon.match.any,
sinon.match.any,
"--threads=-1",
"--ram=3012",
),
);
t.assert(
runQueriesStub.calledOnceWith(
sinon.match.any,
"--ram=3012",
"--threads=-1",
),
);
});
});

28
src/analyze-action-post.ts
View File

@@ -12,7 +12,10 @@ import { getGitHubVersion } from "./api-client";
import { getCodeQL } from "./codeql";
import { getConfig } from "./config-utils";
import * as debugArtifacts from "./debug-artifacts";
import { getJavaTempDependencyDir } from "./dependency-caching";
import {
getCsharpTempDependencyDir,
getJavaTempDependencyDir,
} from "./dependency-caching";
import { EnvVar } from "./environment";
import { getActionsLogger } from "./logging";
import { checkGitHubVersionInRange, getErrorMessage } from "./util";
@@ -42,17 +45,22 @@ async function runWrapper() {
}
}
// If we analysed Java in build-mode: none, we may have downloaded dependencies
// If we analysed Java or C# in build-mode: none, we may have downloaded dependencies
// to the temp directory. Clean these up so they don't persist unnecessarily
// long on self-hosted runners.
const javaTempDependencyDir = getJavaTempDependencyDir();
if (fs.existsSync(javaTempDependencyDir)) {
try {
fs.rmSync(javaTempDependencyDir, { recursive: true });
} catch (error) {
logger.info(
`Failed to remove temporary Java dependencies directory: ${getErrorMessage(error)}`,
);
const tempDependencyDirs = [
getJavaTempDependencyDir(),
getCsharpTempDependencyDir(),
];
for (const tempDependencyDir of tempDependencyDirs) {
if (fs.existsSync(tempDependencyDir)) {
try {
fs.rmSync(tempDependencyDir, { recursive: true });
} catch (error) {
logger.info(
`Failed to remove temporary dependencies directory: ${getErrorMessage(error)}`,
);
}
}
}
} catch (error) {

81
src/analyze-action.ts
View File

@@ -19,20 +19,18 @@ import { getApiDetails, getGitHubVersion } from "./api-client";
import { runAutobuild } from "./autobuild";
import { getTotalCacheSize, shouldStoreCache } from "./caching-utils";
import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import {
Config,
getConfig,
isCodeQualityEnabled,
isCodeScanningEnabled,
} from "./config-utils";
import { cleanupAndUploadDatabases } from "./database-upload";
cleanupAndUploadDatabases,
DatabaseUploadResult,
} from "./database-upload";
import {
DependencyCacheUploadStatusReport,
uploadDependencyCaches,
} from "./dependency-caching";
import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import { Features } from "./feature-flags";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
@@ -59,15 +57,13 @@ interface AnalysisStatusReport
extends uploadLib.UploadStatusReport,
QueriesStatusReport {}
interface DependencyCachingUploadStatusReport {
dependency_caching_upload_results?: DependencyCacheUploadStatusReport;
}
interface FinishStatusReport
extends StatusReportBase,
DatabaseCreationTimings,
AnalysisStatusReport,
DependencyCachingUploadStatusReport {}
AnalysisStatusReport {
dependency_caching_upload_results?: DependencyCacheUploadStatusReport;
database_upload_results: DatabaseUploadResult[];
}
interface FinishWithTrapUploadStatusReport extends FinishStatusReport {
/** Size of TRAP caches that we uploaded, in bytes. */
@@ -86,6 +82,7 @@ async function sendStatusReport(
didUploadTrapCaches: boolean,
trapCacheCleanup: TrapCacheCleanupStatusReport | undefined,
dependencyCacheResults: DependencyCacheUploadStatusReport | undefined,
databaseUploadResults: DatabaseUploadResult[],
logger: Logger,
) {
const status = getActionsStatus(error, stats?.analyze_failure_language);
@@ -106,6 +103,7 @@ async function sendStatusReport(
...(dbCreationTimings || {}),
...(trapCacheCleanup || {}),
dependency_caching_upload_results: dependencyCacheResults,
database_upload_results: databaseUploadResults,
};
if (config && didUploadTrapCaches) {
const trapCacheUploadStatusReport: FinishWithTrapUploadStatusReport = {
@@ -223,6 +221,7 @@ async function run() {
let dbCreationTimings: DatabaseCreationTimings | undefined = undefined;
let didUploadTrapCaches = false;
let dependencyCacheResults: DependencyCacheUploadStatusReport | undefined;
let databaseUploadResults: DatabaseUploadResult[] = [];
util.initializeEnvironment(actionsUtil.getActionVersion());
// Make inputs accessible in the `post` step, details at
@@ -315,6 +314,7 @@ async function run() {
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await runFinalize(
features,
outputDir,
threads,
memory,
@@ -357,46 +357,15 @@ async function run() {
const checkoutPath = actionsUtil.getRequiredInput("checkout_path");
const category = actionsUtil.getOptionalInput("category");
if (await features.getValue(Feature.AnalyzeUseNewUpload)) {
uploadResults = await postProcessAndUploadSarif(
logger,
features,
uploadKind,
checkoutPath,
outputDir,
category,
actionsUtil.getOptionalInput("post-processed-sarif-path"),
);
} else if (uploadKind === "always") {
uploadResults = {};
if (isCodeScanningEnabled(config)) {
uploadResults[analyses.AnalysisKind.CodeScanning] =
await uploadLib.uploadFiles(
outputDir,
checkoutPath,
category,
features,
logger,
analyses.CodeScanning,
);
}
if (isCodeQualityEnabled(config)) {
uploadResults[analyses.AnalysisKind.CodeQuality] =
await uploadLib.uploadFiles(
outputDir,
checkoutPath,
category,
features,
logger,
analyses.CodeQuality,
);
}
} else {
uploadResults = {};
logger.info("Not uploading results");
}
uploadResults = await postProcessAndUploadSarif(
logger,
features,
uploadKind,
checkoutPath,
outputDir,
category,
actionsUtil.getOptionalInput("post-processed-sarif-path"),
);
// Set the SARIF id outputs only if we have results for them, to avoid
// having keys with empty values in the action output.
@@ -424,7 +393,7 @@ async function run() {
// Possibly upload the database bundles for remote queries.
// Note: Take care with the ordering of this call since databases may be cleaned up
// at the `overlay` or `clear` level.
await cleanupAndUploadDatabases(
databaseUploadResults = await cleanupAndUploadDatabases(
repositoryNwo,
codeql,
config,
@@ -496,6 +465,7 @@ async function run() {
didUploadTrapCaches,
trapCacheCleanupTelemetry,
dependencyCacheResults,
databaseUploadResults,
logger,
);
return;
@@ -518,6 +488,7 @@ async function run() {
didUploadTrapCaches,
trapCacheCleanupTelemetry,
dependencyCacheResults,
databaseUploadResults,
logger,
);
} else if (runStats !== undefined) {
@@ -531,6 +502,7 @@ async function run() {
didUploadTrapCaches,
trapCacheCleanupTelemetry,
dependencyCacheResults,
databaseUploadResults,
logger,
);
} else {
@@ -544,6 +516,7 @@ async function run() {
didUploadTrapCaches,
trapCacheCleanupTelemetry,
dependencyCacheResults,
databaseUploadResults,
logger,
);
}

22
src/analyze.ts
View File

@@ -10,7 +10,10 @@ import * as analyses from "./analyses";
import { setupCppAutobuild } from "./autobuild";
import { type CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { getJavaTempDependencyDir } from "./dependency-caching";
import {
getCsharpTempDependencyDir,
getJavaTempDependencyDir,
} from "./dependency-caching";
import { addDiagnostic, makeDiagnostic } from "./diagnostics";
import {
DiffThunkRange,
@@ -98,6 +101,7 @@ async function setupPythonExtractor(logger: Logger) {
export async function runExtraction(
codeql: CodeQL,
features: FeatureEnablement,
config: configUtils.Config,
logger: Logger,
) {
@@ -122,7 +126,7 @@ export async function runExtraction(
await setupCppAutobuild(codeql, logger);
}
// The Java `build-mode: none` extractor places dependencies (.jar files) in the
// The Java and C# `build-mode: none` extractors place dependencies in the
// database scratch directory by default. For dependency caching purposes, we want
// a stable path that caches can be restored into and that we can cache at the
// end of the workflow (i.e. that does not get removed when the scratch directory is).
@@ -133,6 +137,15 @@ export async function runExtraction(
process.env["CODEQL_EXTRACTOR_JAVA_OPTION_BUILDLESS_DEPENDENCY_DIR"] =
getJavaTempDependencyDir();
}
if (
language === KnownLanguage.csharp &&
config.buildMode === BuildMode.None &&
(await features.getValue(Feature.CsharpCacheBuildModeNone))
) {
process.env[
"CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR"
] = getCsharpTempDependencyDir();
}
await codeql.extractUsingBuildMode(config, language);
} else {
@@ -177,13 +190,14 @@ export function dbIsFinalized(
async function finalizeDatabaseCreation(
codeql: CodeQL,
features: FeatureEnablement,
config: configUtils.Config,
threadsFlag: string,
memoryFlag: string,
logger: Logger,
): Promise<DatabaseCreationTimings> {
const extractionStart = performance.now();
await runExtraction(codeql, config, logger);
await runExtraction(codeql, features, config, logger);
const extractionTime = performance.now() - extractionStart;
const trapImportStart = performance.now();
@@ -597,6 +611,7 @@ export async function runQueries(
}
export async function runFinalize(
features: FeatureEnablement,
outputDir: string,
threadsFlag: string,
memoryFlag: string,
@@ -615,6 +630,7 @@ export async function runFinalize(
const timings = await finalizeDatabaseCreation(
codeql,
features,
config,
threadsFlag,
memoryFlag,

4
src/api-client.test.ts
View File

@@ -95,14 +95,14 @@ test("getGitHubVersion for different domain", async (t) => {
t.deepEqual({ type: util.GitHubVariant.DOTCOM }, v3);
});
test("getGitHubVersion for GHE_DOTCOM", async (t) => {
test("getGitHubVersion for GHEC-DR", async (t) => {
mockGetMetaVersionHeader("ghe.com");
const gheDotcom = await api.getGitHubVersionFromApi(api.getApiClient(), {
auth: "",
url: "https://foo.ghe.com",
apiURL: undefined,
});
t.deepEqual({ type: util.GitHubVariant.GHE_DOTCOM }, gheDotcom);
t.deepEqual({ type: util.GitHubVariant.GHEC_DR }, gheDotcom);
});
test("wrapApiConfigurationError correctly wraps specific configuration errors", (t) => {

7
src/api-client.ts
View File

@@ -18,11 +18,6 @@ import {
const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
export enum DisallowedAPIVersionReason {
ACTION_TOO_OLD,
ACTION_TOO_NEW,
}
export type GitHubApiCombinedDetails = GitHubApiDetails &
GitHubApiExternalRepoDetails;
@@ -130,7 +125,7 @@ export async function getGitHubVersionFromApi(
}
if (response.headers[GITHUB_ENTERPRISE_VERSION_HEADER] === "ghe.com") {
return { type: GitHubVariant.GHE_DOTCOM };
return { type: GitHubVariant.GHEC_DR };
}
const version = response.headers[GITHUB_ENTERPRISE_VERSION_HEADER] as string;

2
src/api-compatibility.json
View File

@@ -1 +1 @@
{"maximumVersion": "3.19", "minimumVersion": "3.14"}
{"maximumVersion": "3.20", "minimumVersion": "3.14"}

98
src/artifact-scanner.test.ts Normal file
View File

@@ -0,0 +1,98 @@
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import test from "ava";
import { scanArtifactsForTokens } from "./artifact-scanner";
import { getRunnerLogger } from "./logging";
import { getRecordingLogger, LoggedMessage } from "./testing-utils";
test("scanArtifactsForTokens detects GitHub tokens in files", async (t) => {
const logger = getRunnerLogger(true);
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "scanner-test-"));
try {
// Create a test file with a fake GitHub token
const testFile = path.join(tempDir, "test.txt");
fs.writeFileSync(
testFile,
"This is a test file with token ghp_1234567890123456789012345678901234AB",
);
const error = await t.throwsAsync(
async () => await scanArtifactsForTokens([testFile], logger),
);
t.regex(
error?.message || "",
/Found 1 potential GitHub token.*Personal Access Token/,
);
t.regex(error?.message || "", /test\.txt/);
} finally {
// Clean up
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test("scanArtifactsForTokens handles files without tokens", async (t) => {
const logger = getRunnerLogger(true);
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "scanner-test-"));
try {
// Create a test file without tokens
const testFile = path.join(tempDir, "test.txt");
fs.writeFileSync(
testFile,
"This is a test file without any sensitive data",
);
await t.notThrowsAsync(
async () => await scanArtifactsForTokens([testFile], logger),
);
} finally {
// Clean up
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
if (os.platform() !== "win32") {
test("scanArtifactsForTokens finds token in debug artifacts", async (t) => {
t.timeout(15000); // 15 seconds
const messages: LoggedMessage[] = [];
const logger = getRecordingLogger(messages, { logToConsole: false });
// The zip here is a regression test based on
// https://github.com/github/codeql-action/security/advisories/GHSA-vqf5-2xx6-9wfm
const testZip = path.join(
__dirname,
"..",
"src",
"testdata",
"debug-artifacts-with-fake-token.zip",
);
// This zip file contains a nested structure with a fake token in:
// my-db-java-partial.zip/trap/java/invocations/kotlin.9017231652989744319.trap
const error = await t.throwsAsync(
async () => await scanArtifactsForTokens([testZip], logger),
);
t.regex(
error?.message || "",
/Found.*potential GitHub token/,
"Should detect token in nested zip",
);
t.regex(
error?.message || "",
/kotlin\.9017231652989744319\.trap/,
"Should report the .trap file containing the token",
);
const logOutput = messages.map((msg) => msg.message).join("\n");
t.regex(
logOutput,
/^Extracting gz file: .*\.gz$/m,
"Logs should show that .gz files were extracted",
);
});
}

379
src/artifact-scanner.ts Normal file
View File

@@ -0,0 +1,379 @@
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import * as exec from "@actions/exec";
import { Logger } from "./logging";
import { getErrorMessage } from "./util";
/**
* GitHub token patterns to scan for.
* These patterns match various GitHub token formats.
*/
const GITHUB_TOKEN_PATTERNS = [
{
name: "Personal Access Token",
pattern: /\bghp_[a-zA-Z0-9]{36}\b/g,
},
{
name: "OAuth Access Token",
pattern: /\bgho_[a-zA-Z0-9]{36}\b/g,
},
{
name: "User-to-Server Token",
pattern: /\bghu_[a-zA-Z0-9]{36}\b/g,
},
{
name: "Server-to-Server Token",
pattern: /\bghs_[a-zA-Z0-9]{36}\b/g,
},
{
name: "Refresh Token",
pattern: /\bghr_[a-zA-Z0-9]{36}\b/g,
},
{
name: "App Installation Access Token",
pattern: /\bghs_[a-zA-Z0-9]{255}\b/g,
},
];
interface TokenFinding {
tokenType: string;
filePath: string;
}
interface ScanResult {
scannedFiles: number;
findings: TokenFinding[];
}
/**
* Scans a file for GitHub tokens.
*
* @param filePath Path to the file to scan
* @param relativePath Relative path for display purposes
* @param logger Logger instance
* @returns Array of token findings in the file
*/
function scanFileForTokens(
filePath: string,
relativePath: string,
logger: Logger,
): TokenFinding[] {
const findings: TokenFinding[] = [];
try {
const content = fs.readFileSync(filePath, "utf8");
for (const { name, pattern } of GITHUB_TOKEN_PATTERNS) {
const matches = content.match(pattern);
if (matches) {
for (let i = 0; i < matches.length; i++) {
findings.push({ tokenType: name, filePath: relativePath });
}
logger.debug(`Found ${matches.length} ${name}(s) in ${relativePath}`);
}
}
return findings;
} catch (e) {
// If we can't read the file as text, it's likely binary or inaccessible
logger.debug(
`Could not scan file ${filePath} for tokens: ${getErrorMessage(e)}`,
);
return [];
}
}
/**
* Recursively extracts and scans archive files (.zip, .gz, .tar.gz).
*
* @param archivePath Path to the archive file
* @param relativeArchivePath Relative path of the archive for display
* @param extractDir Directory to extract to
* @param logger Logger instance
* @param depth Current recursion depth (to prevent infinite loops)
* @returns Scan results
*/
async function scanArchiveFile(
archivePath: string,
relativeArchivePath: string,
extractDir: string,
logger: Logger,
depth: number = 0,
): Promise<ScanResult> {
const MAX_DEPTH = 10; // Prevent infinite recursion
if (depth > MAX_DEPTH) {
throw new Error(
`Maximum archive extraction depth (${MAX_DEPTH}) reached for ${archivePath}`,
);
}
const result: ScanResult = {
scannedFiles: 0,
findings: [],
};
try {
const tempExtractDir = fs.mkdtempSync(
path.join(extractDir, `extract-${depth}-`),
);
// Determine archive type and extract accordingly
const fileName = path.basename(archivePath).toLowerCase();
if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
// Extract tar.gz files
logger.debug(`Extracting tar.gz file: ${archivePath}`);
await exec.exec("tar", ["-xzf", archivePath, "-C", tempExtractDir], {
silent: true,
});
} else if (fileName.endsWith(".tar.zst")) {
// Extract tar.zst files
logger.debug(`Extracting tar.zst file: ${archivePath}`);
await exec.exec(
"tar",
["--zstd", "-xf", archivePath, "-C", tempExtractDir],
{
silent: true,
},
);
} else if (fileName.endsWith(".zst")) {
// Extract .zst files (single file compression)
logger.debug(`Extracting zst file: ${archivePath}`);
const outputFile = path.join(
tempExtractDir,
path.basename(archivePath, ".zst"),
);
await exec.exec("zstd", ["-d", archivePath, "-o", outputFile], {
silent: true,
});
} else if (fileName.endsWith(".gz")) {
// Extract .gz files (single file compression)
logger.debug(`Extracting gz file: ${archivePath}`);
const outputFile = path.join(
tempExtractDir,
path.basename(archivePath, ".gz"),
);
await exec.exec("gunzip", ["-c", archivePath], {
outStream: fs.createWriteStream(outputFile),
silent: true,
});
} else if (fileName.endsWith(".zip")) {
// Extract zip files
logger.debug(`Extracting zip file: ${archivePath}`);
await exec.exec(
"unzip",
["-q", "-o", archivePath, "-d", tempExtractDir],
{
silent: true,
},
);
}
// Scan the extracted contents
const scanResult = await scanDirectory(
tempExtractDir,
relativeArchivePath,
logger,
depth + 1,
);
result.scannedFiles += scanResult.scannedFiles;
result.findings.push(...scanResult.findings);
// Clean up extracted files
fs.rmSync(tempExtractDir, { recursive: true, force: true });
} catch (e) {
logger.debug(
`Could not extract or scan archive file ${archivePath}: ${getErrorMessage(e)}`,
);
}
return result;
}
/**
* Scans a single file, including recursive archive extraction if applicable.
*
* @param fullPath Full path to the file
* @param relativePath Relative path for display
* @param extractDir Directory to use for extraction (for archive files)
* @param logger Logger instance
* @param depth Current recursion depth
* @returns Scan results
*/
async function scanFile(
fullPath: string,
relativePath: string,
extractDir: string,
logger: Logger,
depth: number = 0,
): Promise<ScanResult> {
const result: ScanResult = {
scannedFiles: 1,
findings: [],
};
// Check if it's an archive file and recursively scan it
const fileName = path.basename(fullPath).toLowerCase();
const isArchive =
fileName.endsWith(".zip") ||
fileName.endsWith(".tar.gz") ||
fileName.endsWith(".tgz") ||
fileName.endsWith(".tar.zst") ||
fileName.endsWith(".zst") ||
fileName.endsWith(".gz");
if (isArchive) {
const archiveResult = await scanArchiveFile(
fullPath,
relativePath,
extractDir,
logger,
depth,
);
result.scannedFiles += archiveResult.scannedFiles;
result.findings.push(...archiveResult.findings);
}
// Scan the file itself for tokens (unless it's a pure binary archive format)
const fileFindings = scanFileForTokens(fullPath, relativePath, logger);
result.findings.push(...fileFindings);
return result;
}
/**
* Recursively scans a directory for GitHub tokens.
*
* @param dirPath Directory path to scan
* @param baseRelativePath Base relative path for computing display paths
* @param logger Logger instance
* @param depth Current recursion depth
* @returns Scan results
*/
async function scanDirectory(
dirPath: string,
baseRelativePath: string,
logger: Logger,
depth: number = 0,
): Promise<ScanResult> {
const result: ScanResult = {
scannedFiles: 0,
findings: [],
};
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name);
const relativePath = path.join(baseRelativePath, entry.name);
if (entry.isDirectory()) {
const subResult = await scanDirectory(
fullPath,
relativePath,
logger,
depth,
);
result.scannedFiles += subResult.scannedFiles;
result.findings.push(...subResult.findings);
} else if (entry.isFile()) {
const fileResult = await scanFile(
fullPath,
relativePath,
path.dirname(fullPath),
logger,
depth,
);
result.scannedFiles += fileResult.scannedFiles;
result.findings.push(...fileResult.findings);
}
}
return result;
}
/**
* Scans a list of files and directories for GitHub tokens.
* Recursively extracts and scans archive files (.zip, .gz, .tar.gz).
*
* @param filesToScan List of file paths to scan
* @param logger Logger instance
* @returns Scan results
*/
export async function scanArtifactsForTokens(
filesToScan: string[],
logger: Logger,
): Promise<void> {
logger.info(
"Starting best-effort check for potential GitHub tokens in debug artifacts (for testing purposes only)...",
);
const result: ScanResult = {
scannedFiles: 0,
findings: [],
};
// Create a temporary directory for extraction
const tempScanDir = fs.mkdtempSync(path.join(os.tmpdir(), "artifact-scan-"));
try {
for (const filePath of filesToScan) {
const stats = fs.statSync(filePath);
const fileName = path.basename(filePath);
if (stats.isDirectory()) {
const dirResult = await scanDirectory(filePath, fileName, logger);
result.scannedFiles += dirResult.scannedFiles;
result.findings.push(...dirResult.findings);
} else if (stats.isFile()) {
const fileResult = await scanFile(
filePath,
fileName,
tempScanDir,
logger,
);
result.scannedFiles += fileResult.scannedFiles;
result.findings.push(...fileResult.findings);
}
}
// Compute statistics from findings
const tokenTypesCounts = new Map<string, number>();
const filesWithTokens = new Set<string>();
for (const finding of result.findings) {
tokenTypesCounts.set(
finding.tokenType,
(tokenTypesCounts.get(finding.tokenType) || 0) + 1,
);
filesWithTokens.add(finding.filePath);
}
const tokenTypesSummary = Array.from(tokenTypesCounts.entries())
.map(([type, count]) => `${count} ${type}${count > 1 ? "s" : ""}`)
.join(", ");
const baseSummary = `scanned ${result.scannedFiles} files, found ${result.findings.length} potential token(s) in ${filesWithTokens.size} file(s)`;
const summaryWithTypes = tokenTypesSummary
? `${baseSummary} (${tokenTypesSummary})`
: baseSummary;
logger.info(`Artifact check complete: ${summaryWithTypes}`);
if (result.findings.length > 0) {
const fileList = Array.from(filesWithTokens).join(", ");
throw new Error(
`Found ${result.findings.length} potential GitHub token(s) (${tokenTypesSummary}) in debug artifacts at: ${fileList}. This is a best-effort check for testing purposes only.`,
);
}
} finally {
// Clean up temporary directory
try {
fs.rmSync(tempScanDir, { recursive: true, force: true });
} catch (e) {
logger.debug(
`Could not clean up temporary scan directory: ${getErrorMessage(e)}`,
);
}
}
}

5
src/cli-errors.ts
View File

@@ -159,10 +159,7 @@ type CliErrorConfiguration = {
* All of our caught CLI error messages that we handle specially: ie. if we
* would like to categorize an error as a configuration error or not.
*/
export const cliErrorsConfig: Record<
CliConfigErrorCategory,
CliErrorConfiguration
> = {
const cliErrorsConfig: Record<CliConfigErrorCategory, CliErrorConfiguration> = {
[CliConfigErrorCategory.AutobuildError]: {
cliErrorMessageCandidates: [
new RegExp("We were unable to automatically build your code"),

30
src/codeql.ts
View File

@@ -206,6 +206,7 @@ export interface CodeQL {
* Run 'codeql resolve queries --format=startingpacks'.
*/
resolveQueriesStartingPacks(queries: string[]): Promise<string[]>;
resolveDatabase(databasePath: string): Promise<ResolveDatabaseOutput>;
/**
* Run 'codeql github merge-results'.
*/
@@ -230,6 +231,10 @@ export interface VersionInfo {
overlayVersion?: number;
}
export interface ResolveDatabaseOutput {
overlayBaseSpecifier?: string;
}
export interface ResolveLanguagesOutput {
[language: string]: [string];
}
@@ -493,6 +498,7 @@ export function createStubCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
partialCodeql,
"resolveQueriesStartingPacks",
),
resolveDatabase: resolveFunction(partialCodeql, "resolveDatabase"),
mergeResults: resolveFunction(partialCodeql, "mergeResults"),
};
}
@@ -516,7 +522,7 @@ export async function getCodeQLForTesting(
* version requirement. Must be set to true outside tests.
* @returns A new CodeQL object
*/
export async function getCodeQLForCmd(
async function getCodeQLForCmd(
cmd: string,
checkVersion: boolean,
): Promise<CodeQL> {
@@ -1003,6 +1009,26 @@ export async function getCodeQLForCmd(
);
}
},
async resolveDatabase(
databasePath: string,
): Promise<ResolveDatabaseOutput> {
const codeqlArgs = [
"resolve",
"database",
databasePath,
"--format=json",
...getExtraOptionsFromEnv(["resolve", "database"]),
];
const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
try {
return JSON.parse(output) as ResolveDatabaseOutput;
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve database --format=json: ${e}`,
);
}
},
async mergeResults(
sarifFiles: string[],
outputFile: string,
@@ -1225,7 +1251,7 @@ export async function getTrapCachingExtractorConfigArgsForLang(
*
* This will not exist if the configuration is being parsed in the Action.
*/
export function getGeneratedCodeScanningConfigPath(config: Config): string {
function getGeneratedCodeScanningConfigPath(config: Config): string {
return path.resolve(config.tempDir, "user-config.yaml");
}

157
src/config-utils.test.ts
View File

@@ -15,6 +15,7 @@ import * as configUtils from "./config-utils";
import * as errorMessages from "./error-messages";
import { Feature } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { GitVersionInfo } from "./git-utils";
import { KnownLanguage, Language } from "./languages";
import { getRunnerLogger } from "./logging";
import {
@@ -59,6 +60,7 @@ function createTestInitConfigInputs(
dbLocation: undefined,
configInput: undefined,
buildModeInput: undefined,
ramInput: undefined,
trapCachingEnabled: false,
dependencyCachingEnabled: CachingKind.None,
debugMode: false,
@@ -977,8 +979,10 @@ interface OverlayDatabaseModeTestSetup {
languages: Language[];
codeqlVersion: string;
gitRoot: string | undefined;
gitVersion: GitVersionInfo | undefined;
codeScanningConfig: configUtils.UserConfig;
diskUsage: DiskUsage | undefined;
memoryFlagValue: number;
}
const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
@@ -990,11 +994,16 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
languages: [KnownLanguage.javascript],
codeqlVersion: CODEQL_OVERLAY_MINIMUM_VERSION,
gitRoot: "/some/git/root",
gitVersion: new GitVersionInfo(
gitUtils.GIT_MINIMUM_VERSION_FOR_OVERLAY,
gitUtils.GIT_MINIMUM_VERSION_FOR_OVERLAY,
),
codeScanningConfig: {},
diskUsage: {
numAvailableBytes: 50_000_000_000,
numTotalBytes: 100_000_000_000,
},
memoryFlagValue: 6920,
};
const getOverlayDatabaseModeMacro = test.macro({
@@ -1037,6 +1046,8 @@ const getOverlayDatabaseModeMacro = test.macro({
.stub(actionsUtil, "isAnalyzingPullRequest")
.returns(setup.isPullRequest);
sinon.stub(util, "getCodeQLMemoryLimit").returns(setup.memoryFlagValue);
// Set up CodeQL mock
const codeql = mockCodeQLVersion(setup.codeqlVersion);
@@ -1063,7 +1074,9 @@ const getOverlayDatabaseModeMacro = test.macro({
setup.languages,
tempDir, // sourceRoot
setup.buildMode,
undefined,
setup.codeScanningConfig,
setup.gitVersion,
logger,
);
@@ -1225,6 +1238,65 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"Overlay-base database on default branch if runner disk space is too low and skip resource checks flag is enabled",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisSkipResourceChecks,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch if memory flag is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isDefaultBranch: true,
memoryFlagValue: 3072,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"Overlay-base database on default branch if memory flag is too low and skip resource checks flag is enabled",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisSkipResourceChecks,
],
isDefaultBranch: true,
memoryFlagValue: 3072,
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
@@ -1416,6 +1488,28 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"Overlay analysis on PR if runner disk space is too low and skip resource checks flag is enabled",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisSkipResourceChecks,
],
isPullRequest: true,
diskUsage: {
numAvailableBytes: 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
useOverlayDatabaseCaching: true,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if we can't determine runner disk space",
@@ -1434,6 +1528,43 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR if memory flag is too low",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
],
isPullRequest: true,
memoryFlagValue: 3072,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"Overlay analysis on PR if memory flag is too low and skip resource checks flag is enabled",
{
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisSkipResourceChecks,
],
isPullRequest: true,
memoryFlagValue: 3072,
},
{
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
useOverlayDatabaseCaching: true,
},
);
test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with disable-default-queries",
@@ -1649,6 +1780,32 @@ test(
},
);
test(
getOverlayDatabaseModeMacro,
"Fallback due to old git version",
{
overlayDatabaseEnvVar: "overlay",
gitVersion: new GitVersionInfo("2.30.0", "2.30.0"), // Version below required 2.38.0
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
test(
getOverlayDatabaseModeMacro,
"Fallback when git version cannot be determined",
{
overlayDatabaseEnvVar: "overlay",
gitVersion: undefined,
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
useOverlayDatabaseCaching: false,
},
);
// Exercise language-specific overlay analysis features code paths
for (const language in KnownLanguage) {
test(

163
src/config-utils.ts
View File

@@ -26,13 +26,18 @@ import {
parseUserConfig,
UserConfig,
} from "./config/db-config";
import { addDiagnostic, makeTelemetryDiagnostic } from "./diagnostics";
import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import * as errorMessages from "./error-messages";
import { Feature, FeatureEnablement } from "./feature-flags";
import { RepositoryProperties } from "./feature-flags/properties";
import {
getGeneratedFiles,
getGitRoot,
getGitVersionOrThrow,
GIT_MINIMUM_VERSION_FOR_OVERLAY,
GitVersionInfo,
isAnalyzingDefaultBranch,
} from "./git-utils";
import { KnownLanguage, Language } from "./languages";
@@ -52,7 +57,9 @@ import {
cloneObject,
isDefined,
checkDiskUsage,
getCodeQLMemoryLimit,
getErrorMessage,
isInTestMode,
} from "./util";
export * from "./config/db-config";
@@ -68,6 +75,14 @@ const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
/**
* The minimum memory (in MB) that must be available for CodeQL to perform overlay
* analysis. If CodeQL will be given less memory than this threshold, then the
* action will not perform overlay analysis unless overlay analysis has been
* explicitly enabled via environment variable.
*/
const OVERLAY_MINIMUM_MEMORY_MB = 5 * 1024;
export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & {
// Token to use when downloading packs from this registry.
token: string;
@@ -200,7 +215,7 @@ export interface Config {
repositoryProperties: RepositoryProperties;
}
export async function getSupportedLanguageMap(
async function getSupportedLanguageMap(
codeql: CodeQL,
logger: Logger,
): Promise<Record<string, string>> {
@@ -263,7 +278,7 @@ export function hasActionsWorkflows(sourceRoot: string): boolean {
/**
* Gets the set of languages in the current repository.
*/
export async function getRawLanguagesInRepo(
async function getRawLanguagesInRepo(
repository: RepositoryNwo,
sourceRoot: string,
logger: Logger,
@@ -372,7 +387,7 @@ export function getRawLanguagesNoAutodetect(
* @returns A tuple containing a list of languages in this repository that might be
* analyzable and whether or not this list was determined automatically.
*/
export async function getRawLanguages(
async function getRawLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
sourceRoot: string,
@@ -402,6 +417,7 @@ export interface InitConfigInputs {
dbLocation: string | undefined;
configInput: string | undefined;
buildModeInput: string | undefined;
ramInput: string | undefined;
trapCachingEnabled: boolean;
dependencyCachingEnabled: string | undefined;
debugMode: boolean;
@@ -643,6 +659,42 @@ async function isOverlayAnalysisFeatureEnabled(
return true;
}
/**
* Checks if the runner supports overlay analysis based on available disk space
* and the maximum memory CodeQL will be allowed to use.
*/
async function runnerSupportsOverlayAnalysis(
ramInput: string | undefined,
logger: Logger,
): Promise<boolean> {
const diskUsage = await checkDiskUsage(logger);
if (
diskUsage === undefined ||
diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES
) {
const diskSpaceMb =
diskUsage === undefined
? 0
: Math.round(diskUsage.numAvailableBytes / 1_000_000);
logger.info(
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
`due to insufficient disk space (${diskSpaceMb} MB).`,
);
return false;
}
const memoryFlagValue = getCodeQLMemoryLimit(ramInput, logger);
if (memoryFlagValue < OVERLAY_MINIMUM_MEMORY_MB) {
logger.info(
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
`due to insufficient memory for CodeQL analysis (${memoryFlagValue} MB).`,
);
return false;
}
return true;
}
/**
* Calculate and validate the overlay database mode and caching to use.
*
@@ -670,7 +722,9 @@ export async function getOverlayDatabaseMode(
languages: Language[],
sourceRoot: string,
buildMode: BuildMode | undefined,
ramInput: string | undefined,
codeScanningConfig: UserConfig,
gitVersion: GitVersionInfo | undefined,
logger: Logger,
): Promise<{
overlayDatabaseMode: OverlayDatabaseMode;
@@ -700,37 +754,29 @@ export async function getOverlayDatabaseMode(
codeScanningConfig,
)
) {
const diskUsage = await checkDiskUsage(logger);
const performResourceChecks = !(await features.getValue(
Feature.OverlayAnalysisSkipResourceChecks,
codeql,
));
if (
diskUsage === undefined ||
diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES
performResourceChecks &&
!(await runnerSupportsOverlayAnalysis(ramInput, logger))
) {
const diskSpaceMb =
diskUsage === undefined
? 0
: Math.round(diskUsage.numAvailableBytes / 1_000_000);
overlayDatabaseMode = OverlayDatabaseMode.None;
useOverlayDatabaseCaching = false;
} else if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
`due to insufficient disk space (${diskSpaceMb} MB).`,
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
);
} else {
if (isAnalyzingPullRequest()) {
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing a pull request.",
);
} else if (await isAnalyzingDefaultBranch()) {
overlayDatabaseMode = OverlayDatabaseMode.OverlayBase;
useOverlayDatabaseCaching = true;
logger.info(
`Setting overlay database mode to ${overlayDatabaseMode} ` +
"with caching because we are analyzing the default branch.",
);
}
}
}
@@ -781,6 +827,22 @@ export async function getOverlayDatabaseMode(
);
return nonOverlayAnalysis;
}
if (gitVersion === undefined) {
logger.warning(
`Cannot build an ${overlayDatabaseMode} database because ` +
"the Git version could not be determined. " +
"Falling back to creating a normal full database instead.",
);
return nonOverlayAnalysis;
}
if (!gitVersion.isAtLeast(GIT_MINIMUM_VERSION_FOR_OVERLAY)) {
logger.warning(
`Cannot build an ${overlayDatabaseMode} database because ` +
`the installed Git version is older than ${GIT_MINIMUM_VERSION_FOR_OVERLAY}. ` +
"Falling back to creating a normal full database instead.",
);
return nonOverlayAnalysis;
}
return {
overlayDatabaseMode,
@@ -886,6 +948,24 @@ export async function initConfig(
config.computedConfig["query-filters"] = [];
}
let gitVersion: GitVersionInfo | undefined = undefined;
try {
gitVersion = await getGitVersionOrThrow();
logger.info(`Using Git version ${gitVersion.fullVersion}`);
await logGitVersionTelemetry(config, gitVersion);
} catch (e) {
logger.warning(`Could not determine Git version: ${getErrorMessage(e)}`);
// Throw the error in test mode so it's more visible, unless the environment
// variable is set to tolerate this, for example because we're running in a
// Docker container where git may not be available.
if (
isInTestMode() &&
process.env[EnvVar.TOLERATE_MISSING_GIT_VERSION] !== "true"
) {
throw e;
}
}
// The choice of overlay database mode depends on the selection of languages
// and queries, which in turn depends on the user config and the augmentation
// properties. So we need to calculate the overlay database mode after the
@@ -897,7 +977,9 @@ export async function initConfig(
config.languages,
inputs.sourceRoot,
config.buildMode,
inputs.ramInput,
config.computedConfig,
gitVersion,
logger,
);
logger.info(
@@ -1281,7 +1363,7 @@ export function isCodeQualityEnabled(config: Config): boolean {
* @returns Returns `AnalysisKind.CodeScanning` if `AnalysisKind.CodeScanning` is enabled;
* otherwise `AnalysisKind.CodeQuality`.
*/
export function getPrimaryAnalysisKind(config: Config): AnalysisKind {
function getPrimaryAnalysisKind(config: Config): AnalysisKind {
return isCodeScanningEnabled(config)
? AnalysisKind.CodeScanning
: AnalysisKind.CodeQuality;
@@ -1298,3 +1380,26 @@ export function getPrimaryAnalysisConfig(config: Config): AnalysisConfig {
? CodeScanning
: CodeQuality;
}
/** Logs the Git version as a telemetry diagnostic. */
async function logGitVersionTelemetry(
config: Config,
gitVersion: GitVersionInfo,
): Promise<void> {
if (config.languages.length > 0) {
addDiagnostic(
config,
// Arbitrarily choose the first language. We could also choose all languages, but that
// increases the risk of misinterpreting the data.
config.languages[0],
makeTelemetryDiagnostic(
"codeql-action/git-version-telemetry",
"Git version telemetry",
{
fullVersion: gitVersion.fullVersion,
truncatedVersion: gitVersion.truncatedVersion,
},
),
);
}
}

2
src/database-upload.test.ts
View File

@@ -231,7 +231,7 @@ test("Don't crash if uploading a database fails", async (t) => {
(v) =>
v.type === "warning" &&
v.message ===
"Failed to upload database for javascript: Error: some error message",
"Failed to upload database for javascript: some error message",
) !== undefined,
);
});

62
src/database-upload.ts
View File

@@ -13,6 +13,20 @@ import { RepositoryNwo } from "./repository";
import * as util from "./util";
import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
/** Information about a database upload. */
export interface DatabaseUploadResult {
/** Language of the database. */
language: string;
/** Size of the zipped database in bytes. */
zipped_upload_size_bytes?: number;
/** Whether the uploaded database is an overlay base. */
is_overlay_base?: boolean;
/** Time taken to upload database in milliseconds. */
upload_duration_ms?: number;
/** If there was an error during database upload, this is its message. */
error?: string;
}
export async function cleanupAndUploadDatabases(
repositoryNwo: RepositoryNwo,
codeql: CodeQL,
@@ -20,44 +34,46 @@ export async function cleanupAndUploadDatabases(
apiDetails: GitHubApiDetails,
features: FeatureEnablement,
logger: Logger,
): Promise<void> {
): Promise<DatabaseUploadResult[]> {
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
logger.debug("Database upload disabled in workflow. Skipping upload.");
return;
return [];
}
if (!config.analysisKinds.includes(AnalysisKind.CodeScanning)) {
logger.debug(
`Not uploading database because 'analysis-kinds: ${AnalysisKind.CodeScanning}' is not enabled.`,
);
return;
return [];
}
if (util.isInTestMode()) {
logger.debug("In test mode. Skipping database upload.");
return;
return [];
}
// Do nothing when not running against github.com
if (
config.gitHubVersion.type !== util.GitHubVariant.DOTCOM &&
config.gitHubVersion.type !== util.GitHubVariant.GHE_DOTCOM
config.gitHubVersion.type !== util.GitHubVariant.GHEC_DR
) {
logger.debug("Not running against github.com or GHEC-DR. Skipping upload.");
return;
return [];
}
if (!(await gitUtils.isAnalyzingDefaultBranch())) {
// We only want to upload a database if we are analyzing the default branch.
logger.debug("Not analyzing default branch. Skipping upload.");
return;
return [];
}
const cleanupLevel =
// If config.overlayDatabaseMode is OverlayBase, then we have overlay base databases for all languages.
const shouldUploadOverlayBase =
config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase &&
(await features.getValue(Feature.UploadOverlayDbToApi))
? CleanupLevel.Overlay
: CleanupLevel.Clear;
(await features.getValue(Feature.UploadOverlayDbToApi, codeql));
const cleanupLevel = shouldUploadOverlayBase
? CleanupLevel.Overlay
: CleanupLevel.Clear;
// Clean up the database, since intermediate results may still be written to the
// database if there is high RAM pressure.
@@ -77,19 +93,22 @@ export async function cleanupAndUploadDatabases(
uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
}
const reports: DatabaseUploadResult[] = [];
for (const language of config.languages) {
let bundledDbSize: number | undefined = undefined;
try {
// Upload the database bundle.
// Although we are uploading arbitrary file contents to the API, it's worth
// noting that it's the API's job to validate that the contents is acceptable.
// This API method is available to anyone with write access to the repo.
const bundledDb = await bundleDb(config, language, codeql, language);
const bundledDbSize = fs.statSync(bundledDb).size;
bundledDbSize = fs.statSync(bundledDb).size;
const bundledDbReadStream = fs.createReadStream(bundledDb);
const commitOid = await gitUtils.getCommitOid(
actionsUtil.getRequiredInput("checkout_path"),
);
try {
const startTime = performance.now();
await client.request(
`POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
{
@@ -107,13 +126,30 @@ export async function cleanupAndUploadDatabases(
},
},
);
const endTime = performance.now();
reports.push({
language,
zipped_upload_size_bytes: bundledDbSize,
is_overlay_base: shouldUploadOverlayBase,
upload_duration_ms: endTime - startTime,
});
logger.debug(`Successfully uploaded database for ${language}`);
} finally {
bundledDbReadStream.close();
}
} catch (e) {
// Log a warning but don't fail the workflow
logger.warning(`Failed to upload database for ${language}: ${e}`);
logger.warning(
`Failed to upload database for ${language}: ${util.getErrorMessage(e)}`,
);
reports.push({
language,
error: util.getErrorMessage(e),
...(bundledDbSize !== undefined
? { zipped_upload_size_bytes: bundledDbSize }
: {}),
});
}
}
return reports;
}

10
src/debug-artifacts.ts
View File

@@ -8,6 +8,7 @@ import archiver from "archiver";
import { getOptionalInput, getTemporaryDirectory } from "./actions-util";
import { dbIsFinalized } from "./analyze";
import { scanArtifactsForTokens } from "./artifact-scanner";
import { type CodeQL } from "./codeql";
import { Config } from "./config-utils";
import { EnvVar } from "./environment";
@@ -23,6 +24,7 @@ import {
getCodeQLDatabasePath,
getErrorMessage,
GitHubVariant,
isInTestMode,
listFolder,
} from "./util";
@@ -269,6 +271,14 @@ export async function uploadDebugArtifacts(
return "upload-not-supported";
}
// When running in test mode, perform a best effort scan of the debug artifacts. The artifact
// scanner is basic and not reliable or fast enough for production use, but it can help catch
// some issues early.
if (isInTestMode()) {
await scanArtifactsForTokens(toUpload, logger);
core.exportVariable("CODEQL_ACTION_ARTIFACT_SCAN_FINISHED", "true");
}
let suffix = "";
const matrix = getOptionalInput("matrix");
if (matrix) {

8
src/defaults.json
View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.23.5",
"cliVersion": "2.23.5",
"priorBundleVersion": "codeql-bundle-v2.23.3",
"priorCliVersion": "2.23.3"
"bundleVersion": "codeql-bundle-v2.23.9",
"cliVersion": "2.23.9",
"priorBundleVersion": "codeql-bundle-v2.23.8",
"priorCliVersion": "2.23.8"
}

75
src/dependency-caching.test.ts
View File

@@ -21,6 +21,8 @@ import {
downloadDependencyCaches,
CacheHitKind,
cacheKey,
getCsharpDependencyDirs,
getCsharpTempDependencyDir,
uploadDependencyCaches,
CacheStoreResult,
} from "./dependency-caching";
@@ -42,6 +44,28 @@ function makeAbsolutePatterns(tmpDir: string, patterns: string[]): string[] {
return patterns.map((pattern) => path.join(tmpDir, pattern));
}
test("getCsharpDependencyDirs - does not include BMN dir if FF is enabled", async (t) => {
await withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
const codeql = createStubCodeQL({});
const features = createFeatures([]);
const results = await getCsharpDependencyDirs(codeql, features);
t.false(results.includes(getCsharpTempDependencyDir()));
});
});
test("getCsharpDependencyDirs - includes BMN dir if FF is enabled", async (t) => {
await withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpCacheBuildModeNone]);
const results = await getCsharpDependencyDirs(codeql, features);
t.assert(results.includes(getCsharpTempDependencyDir()));
});
});
test("makePatternCheck - returns undefined if no patterns match", async (t) => {
await withTmpDir(async (tmpDir) => {
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
@@ -130,7 +154,7 @@ test("checkHashPatterns - logs when no patterns match", async (t) => {
const features = createFeatures([]);
const messages: LoggedMessage[] = [];
const config: CacheConfig = {
getDependencyPaths: () => [],
getDependencyPaths: async () => [],
getHashPatterns: async () => undefined,
};
@@ -159,7 +183,7 @@ test("checkHashPatterns - returns patterns when patterns match", async (t) => {
fs.writeFileSync(path.join(tmpDir, "test.java"), "");
const config: CacheConfig = {
getDependencyPaths: () => [],
getDependencyPaths: async () => [],
getHashPatterns: async () => makePatternCheck(patterns),
};
@@ -579,28 +603,6 @@ test("getFeaturePrefix - returns empty string if no features are enabled", async
}
});
test("getFeaturePrefix - Java - returns 'minify-' if JavaMinimizeDependencyJars is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.JavaMinimizeDependencyJars]);
const result = await getFeaturePrefix(codeql, features, KnownLanguage.java);
t.deepEqual(result, "minify-");
});
test("getFeaturePrefix - non-Java - returns '' if JavaMinimizeDependencyJars is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.JavaMinimizeDependencyJars]);
for (const knownLanguage of Object.values(KnownLanguage)) {
// Skip Java since we expect a result for it, which is tested in the previous test.
if (knownLanguage === KnownLanguage.java) {
continue;
}
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});
test("getFeaturePrefix - C# - returns prefix if CsharpNewCacheKey is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpNewCacheKey]);
@@ -625,3 +627,28 @@ test("getFeaturePrefix - non-C# - returns '' if CsharpNewCacheKey is enabled", a
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});
test("getFeaturePrefix - C# - returns prefix if CsharpCacheBuildModeNone is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpCacheBuildModeNone]);
const result = await getFeaturePrefix(codeql, features, KnownLanguage.csharp);
t.notDeepEqual(result, "");
t.assert(result.endsWith("-"));
// Check the length of the prefix, which should correspond to `cacheKeyHashLength` + 1 for the trailing `-`.
t.is(result.length, cacheKeyHashLength + 1);
});
test("getFeaturePrefix - non-C# - returns '' if CsharpCacheBuildModeNone is enabled", async (t) => {
const codeql = createStubCodeQL({});
const features = createFeatures([Feature.CsharpCacheBuildModeNone]);
for (const knownLanguage of Object.values(KnownLanguage)) {
// Skip C# since we expect a result for it, which is tested in the previous test.
if (knownLanguage === KnownLanguage.csharp) {
continue;
}
const result = await getFeaturePrefix(codeql, features, knownLanguage);
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
}
});

76
src/dependency-caching.ts
View File

@@ -20,7 +20,10 @@ import { getErrorMessage, getRequiredEnvParam } from "./util";
*/
export interface CacheConfig {
/** Gets the paths of directories on the runner that should be included in the cache. */
getDependencyPaths: () => string[];
getDependencyPaths: (
codeql: CodeQL,
features: FeatureEnablement,
) => Promise<string[]>;
/**
* Gets an array of glob patterns for the paths of files whose contents affect which dependencies are used
* by a project. This function also checks whether there are any matching files and returns
@@ -55,7 +58,7 @@ export function getJavaTempDependencyDir(): string {
* @returns The paths of directories on the runner that should be included in a dependency cache
* for a Java analysis.
*/
export function getJavaDependencyDirs(): string[] {
export async function getJavaDependencyDirs(): Promise<string[]> {
return [
// Maven
join(os.homedir(), ".m2", "repository"),
@@ -66,6 +69,38 @@ export function getJavaDependencyDirs(): string[] {
];
}
/**
* Returns a path to a directory intended to be used to store dependencies
* for the C# `build-mode: none` extractor.
* @returns The path to the directory that should be used by the `build-mode: none` extractor.
*/
export function getCsharpTempDependencyDir(): string {
return join(getTemporaryDirectory(), "codeql_csharp", "repository");
}
/**
* Returns an array of paths of directories on the runner that should be included in a dependency cache
* for a C# analysis.
*
* @returns The paths of directories on the runner that should be included in a dependency cache
* for a C# analysis.
*/
export async function getCsharpDependencyDirs(
codeql: CodeQL,
features: FeatureEnablement,
): Promise<string[]> {
const dirs = [
// Nuget
join(os.homedir(), ".nuget", "packages"),
];
if (await features.getValue(Feature.CsharpCacheBuildModeNone, codeql)) {
dirs.push(getCsharpTempDependencyDir());
}
return dirs;
}
/**
* Checks that there are files which match `patterns`. If there are matching files for any of the patterns,
* this function returns all `patterns`. Otherwise, `undefined` is returned.
@@ -158,11 +193,11 @@ const defaultCacheConfigs: { [language: string]: CacheConfig } = {
]),
},
csharp: {
getDependencyPaths: () => [join(os.homedir(), ".nuget", "packages")],
getDependencyPaths: getCsharpDependencyDirs,
getHashPatterns: getCsharpHashPatterns,
},
go: {
getDependencyPaths: () => [join(os.homedir(), "go", "pkg", "mod")],
getDependencyPaths: async () => [join(os.homedir(), "go", "pkg", "mod")],
getHashPatterns: async () => internal.makePatternCheck(["**/go.sum"]),
},
};
@@ -289,7 +324,7 @@ export async function downloadDependencyCaches(
const start = performance.now();
const hitKey = await actionsCache.restoreCache(
cacheConfig.getDependencyPaths(),
await cacheConfig.getDependencyPaths(codeql, features),
primaryKey,
restoreKeys,
);
@@ -408,7 +443,7 @@ export async function uploadDependencyCaches(
// with the dependency caches. For this, we could use the Cache API to check whether other workflows
// are using the quota and how full it is.
const size = await getTotalCacheSize(
cacheConfig.getDependencyPaths(),
await cacheConfig.getDependencyPaths(codeql, features),
logger,
true,
);
@@ -428,7 +463,10 @@ export async function uploadDependencyCaches(
try {
const start = performance.now();
await actionsCache.saveCache(cacheConfig.getDependencyPaths(), key);
await actionsCache.saveCache(
await cacheConfig.getDependencyPaths(codeql, features),
key,
);
const upload_duration_ms = Math.round(performance.now() - start);
status.push({
@@ -503,19 +541,9 @@ export async function getFeaturePrefix(
}
};
if (language === KnownLanguage.java) {
// To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled.
const minimizeJavaJars = await features.getValue(
Feature.JavaMinimizeDependencyJars,
codeql,
);
// To maintain backwards compatibility with this, we return "minify-" instead of a hash.
if (minimizeJavaJars) {
return "minify-";
}
} else if (language === KnownLanguage.csharp) {
if (language === KnownLanguage.csharp) {
await addFeatureIfEnabled(Feature.CsharpNewCacheKey);
await addFeatureIfEnabled(Feature.CsharpCacheBuildModeNone);
}
// If any features that affect the cache are enabled, return a feature prefix by
@@ -554,14 +582,8 @@ async function cachePrefix(
// experimental features that affect the cache contents.
const featurePrefix = await getFeaturePrefix(codeql, features, language);
// Assemble the cache key. For backwards compatibility with the JAR minification experiment's existing
// feature prefix usage, we add that feature prefix at the start. Other feature prefixes are inserted
// after the general CodeQL dependency cache prefix.
if (featurePrefix === "minify-") {
return `${featurePrefix}${prefix}-${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
} else {
return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
// Assemble the cache key.
return `${prefix}-${featurePrefix}${CODEQL_DEPENDENCY_CACHE_VERSION}-${runnerOs}-${language}-`;
}
/** Represents information about our overall cache usage for CodeQL dependency caches. */

24
src/diagnostics.ts
View File

@@ -185,3 +185,27 @@ export function flushDiagnostics(config: Config) {
// Reset the unwritten diagnostics array.
unwrittenDiagnostics = [];
}
/**
* Creates a telemetry-only diagnostic message. This is a convenience function
* for creating diagnostics that should only be sent to telemetry and not
* displayed on the status page or CLI summary table.
*
* @param id An identifier under which it makes sense to group this diagnostic message
* @param name Display name
* @param attributes Structured metadata
*/
export function makeTelemetryDiagnostic(
id: string,
name: string,
attributes: { [key: string]: any },
): DiagnosticMessage {
return makeDiagnostic(id, name, {
attributes,
visibility: {
cliSummaryTable: false,
statusPage: false,
telemetry: true,
},
});
}

20
src/environment.ts
View File

@@ -20,12 +20,6 @@ export enum EnvVar {
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
/**
* Whether to disable the SARIF post-processing in the Action that removes duplicate locations from
* notifications in the `run[].invocations[].toolExecutionNotifications` SARIF property.
*/
DISABLE_DUPLICATE_LOCATION_FIX = "CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX",
/**
* Whether the CodeQL Action is using its own deprecated and non-standard way of scanning for
* multiple languages.
@@ -56,20 +50,12 @@ export enum EnvVar {
/** Whether the error for a deprecated version of the CodeQL Action was logged. */
LOG_VERSION_DEPRECATION = "CODEQL_ACTION_DID_LOG_VERSION_DEPRECATION",
/**
* For macOS. Result of `csrutil status` to determine whether System Integrity
* Protection is enabled.
*/
IS_SIP_ENABLED = "CODEQL_ACTION_IS_SIP_ENABLED",
/** UUID representing the current job run. */
JOB_RUN_UUID = "JOB_RUN_UUID",
/** Status for the entire job, submitted to the status report in `init-post` */
JOB_STATUS = "CODEQL_ACTION_JOB_STATUS",
ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION",
/** The value of the `output` input for the analyze action. */
SARIF_RESULTS_OUTPUT_DIR = "CODEQL_ACTION_SARIF_RESULTS_OUTPUT_DIR",
@@ -143,4 +129,10 @@ export enum EnvVar {
* the workflow is valid and validation is not necessary.
*/
SKIP_WORKFLOW_VALIDATION = "CODEQL_ACTION_SKIP_WORKFLOW_VALIDATION",
/**
* Whether to tolerate failure to determine the git version (only applicable in test mode).
* Intended for use in environments where git may not be installed, such as Docker containers.
*/
TOLERATE_MISSING_GIT_VERSION = "CODEQL_ACTION_TOLERATE_MISSING_GIT_VERSION",
}

256
src/feature-flags.test.ts
View File

@@ -10,6 +10,8 @@ import {
FeatureEnablement,
Features,
FEATURE_FLAGS_FILE_NAME,
FeatureConfig,
FeatureWithoutCLI,
} from "./feature-flags";
import { getRunnerLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
@@ -46,7 +48,7 @@ test(`All features are disabled if running against GHES`, async (t) => {
for (const feature of Object.values(Feature)) {
t.deepEqual(
await features.getValue(feature, includeCodeQlIfRequired(feature)),
await getFeatureIncludingCodeQlIfRequired(features, feature),
featureConfig[feature].defaultValue,
);
}
@@ -62,22 +64,20 @@ test(`All features are disabled if running against GHES`, async (t) => {
});
});
test(`Feature flags are requested in Proxima`, async (t) => {
test(`Feature flags are requested in GHEC-DR`, async (t) => {
await withTmpDir(async (tmpDir) => {
const loggedMessages = [];
const features = setUpFeatureFlagTests(
tmpDir,
getRecordingLogger(loggedMessages),
{ type: GitHubVariant.GHE_DOTCOM },
{ type: GitHubVariant.GHEC_DR },
);
mockFeatureFlagApiEndpoint(200, initializeFeatures(true));
for (const feature of Object.values(Feature)) {
// Ensure we have gotten a response value back from the Mock API
t.assert(
await features.getValue(feature, includeCodeQlIfRequired(feature)),
);
t.assert(await getFeatureIncludingCodeQlIfRequired(features, feature));
}
// And that we haven't bailed preemptively.
@@ -104,7 +104,7 @@ test("API response missing and features use default value", async (t) => {
for (const feature of Object.values(Feature)) {
t.assert(
(await features.getValue(feature, includeCodeQlIfRequired(feature))) ===
(await getFeatureIncludingCodeQlIfRequired(features, feature)) ===
featureConfig[feature].defaultValue,
);
}
@@ -124,7 +124,7 @@ test("Features use default value if they're not returned in API response", async
for (const feature of Object.values(Feature)) {
t.assert(
(await features.getValue(feature, includeCodeQlIfRequired(feature))) ===
(await getFeatureIncludingCodeQlIfRequired(features, feature)) ===
featureConfig[feature].defaultValue,
);
}
@@ -151,7 +151,7 @@ test("Include no more than 25 features in each API request", async (t) => {
// from the API.
const feature = Object.values(Feature)[0];
await t.notThrowsAsync(async () =>
features.getValue(feature, includeCodeQlIfRequired(feature)),
getFeatureIncludingCodeQlIfRequired(features, feature),
);
});
});
@@ -165,8 +165,7 @@ test("Feature flags exception is propagated if the API request errors", async (t
const someFeature = Object.values(Feature)[0];
await t.throwsAsync(
async () =>
features.getValue(someFeature, includeCodeQlIfRequired(someFeature)),
async () => getFeatureIncludingCodeQlIfRequired(features, someFeature),
{
message:
"Encountered an error while trying to determine feature enablement: Error: some error message",
@@ -190,9 +189,9 @@ for (const feature of Object.keys(featureConfig)) {
// retrieve the values of the actual features
const actualFeatureEnablement: { [feature: string]: boolean } = {};
for (const f of Object.keys(featureConfig)) {
actualFeatureEnablement[f] = await features.getValue(
actualFeatureEnablement[f] = await getFeatureIncludingCodeQlIfRequired(
features,
f as Feature,
includeCodeQlIfRequired(f),
);
}
@@ -210,19 +209,16 @@ for (const feature of Object.keys(featureConfig)) {
// feature should be disabled initially
t.assert(
!(await features.getValue(
!(await getFeatureIncludingCodeQlIfRequired(
features,
feature as Feature,
includeCodeQlIfRequired(feature),
)),
);
// set env var to true and check that the feature is now enabled
process.env[featureConfig[feature].envVar] = "true";
t.assert(
await features.getValue(
feature as Feature,
includeCodeQlIfRequired(feature),
),
await getFeatureIncludingCodeQlIfRequired(features, feature as Feature),
);
});
});
@@ -236,18 +232,15 @@ for (const feature of Object.keys(featureConfig)) {
// feature should be enabled initially
t.assert(
await features.getValue(
feature as Feature,
includeCodeQlIfRequired(feature),
),
await getFeatureIncludingCodeQlIfRequired(features, feature as Feature),
);
// set env var to false and check that the feature is now disabled
process.env[featureConfig[feature].envVar] = "false";
t.assert(
!(await features.getValue(
!(await getFeatureIncludingCodeQlIfRequired(
features,
feature as Feature,
includeCodeQlIfRequired(feature),
)),
);
});
@@ -264,13 +257,19 @@ for (const feature of Object.keys(featureConfig)) {
const expectedFeatureEnablement = initializeFeatures(true);
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
await t.throwsAsync(async () => features.getValue(feature as Feature), {
message: `Internal error: A ${
featureConfig[feature].minimumVersion !== undefined
? "minimum version"
: "required tools feature"
} is specified for feature ${feature}, but no instance of CodeQL was provided.`,
});
// The type system should prevent this happening, but test that if we
// bypass it we get the expected error.
await t.throwsAsync(
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
async () => features.getValue(feature as any),
{
message: `Internal error: A ${
featureConfig[feature].minimumVersion !== undefined
? "minimum version"
: "required tools feature"
} is specified for feature ${feature}, but no instance of CodeQL was provided.`,
},
);
});
});
}
@@ -354,9 +353,9 @@ test("Feature flags are saved to disk", async (t) => {
);
t.true(
await features.getValue(
await getFeatureIncludingCodeQlIfRequired(
features,
Feature.QaTelemetryEnabled,
includeCodeQlIfRequired(Feature.QaTelemetryEnabled),
),
"Feature flag should be enabled initially",
);
@@ -382,9 +381,9 @@ test("Feature flags are saved to disk", async (t) => {
(features as any).gitHubFeatureFlags.cachedApiResponse = undefined;
t.false(
await features.getValue(
await getFeatureIncludingCodeQlIfRequired(
features,
Feature.QaTelemetryEnabled,
includeCodeQlIfRequired(Feature.QaTelemetryEnabled),
),
"Feature flag should be enabled after reading from cached file",
);
@@ -399,9 +398,9 @@ test("Environment variable can override feature flag cache", async (t) => {
const cachedFeatureFlags = path.join(tmpDir, FEATURE_FLAGS_FILE_NAME);
t.true(
await features.getValue(
await getFeatureIncludingCodeQlIfRequired(
features,
Feature.QaTelemetryEnabled,
includeCodeQlIfRequired(Feature.QaTelemetryEnabled),
),
"Feature flag should be enabled initially",
);
@@ -413,9 +412,9 @@ test("Environment variable can override feature flag cache", async (t) => {
process.env.CODEQL_ACTION_QA_TELEMETRY = "false";
t.false(
await features.getValue(
await getFeatureIncludingCodeQlIfRequired(
features,
Feature.QaTelemetryEnabled,
includeCodeQlIfRequired(Feature.QaTelemetryEnabled),
),
"Feature flag should be disabled after setting env var",
);
@@ -436,101 +435,83 @@ test(`selects CLI from defaults.json on GHES`, async (t) => {
});
});
test("selects CLI v2.20.1 on Dotcom when feature flags enable v2.20.0 and v2.20.1", async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_2_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_20_3_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_20_4_enabled"] = false;
expectedFeatureEnablement["default_codeql_version_2_20_5_enabled"] = false;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
for (const variant of [GitHubVariant.DOTCOM, GitHubVariant.GHEC_DR]) {
test(`selects CLI v2.20.1 on ${variant} when feature flags enable v2.20.0 and v2.20.1`, async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_2_enabled"] =
false;
expectedFeatureEnablement["default_codeql_version_2_20_3_enabled"] =
false;
expectedFeatureEnablement["default_codeql_version_2_20_4_enabled"] =
false;
expectedFeatureEnablement["default_codeql_version_2_20_5_enabled"] =
false;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
const defaultCliVersion = await features.getDefaultCliVersion(
GitHubVariant.DOTCOM,
);
t.deepEqual(defaultCliVersion, {
cliVersion: "2.20.1",
tagName: "codeql-bundle-v2.20.1",
toolsFeatureFlagsValid: true,
const defaultCliVersion = await features.getDefaultCliVersion(variant);
t.deepEqual(defaultCliVersion, {
cliVersion: "2.20.1",
tagName: "codeql-bundle-v2.20.1",
toolsFeatureFlagsValid: true,
});
});
});
});
test("includes tag name", async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
test(`selects CLI from defaults.json on ${variant} when no default version feature flags are enabled`, async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
const defaultCliVersion = await features.getDefaultCliVersion(
GitHubVariant.DOTCOM,
);
t.deepEqual(defaultCliVersion, {
cliVersion: "2.20.0",
tagName: "codeql-bundle-v2.20.0",
toolsFeatureFlagsValid: true,
const defaultCliVersion = await features.getDefaultCliVersion(variant);
t.deepEqual(defaultCliVersion, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
toolsFeatureFlagsValid: false,
});
});
});
});
test(`selects CLI from defaults.json on Dotcom when no default version feature flags are enabled`, async (t) => {
await withTmpDir(async (tmpDir) => {
const features = setUpFeatureFlagTests(tmpDir);
const expectedFeatureEnablement = initializeFeatures(true);
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
test(`ignores invalid version numbers in default version feature flags on ${variant}`, async (t) => {
await withTmpDir(async (tmpDir) => {
const loggedMessages = [];
const features = setUpFeatureFlagTests(
tmpDir,
getRecordingLogger(loggedMessages),
);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_invalid_enabled"] =
true;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
const defaultCliVersion = await features.getDefaultCliVersion(
GitHubVariant.DOTCOM,
);
t.deepEqual(defaultCliVersion, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
toolsFeatureFlagsValid: false,
const defaultCliVersion = await features.getDefaultCliVersion(variant);
t.deepEqual(defaultCliVersion, {
cliVersion: "2.20.1",
tagName: "codeql-bundle-v2.20.1",
toolsFeatureFlagsValid: true,
});
t.assert(
loggedMessages.find(
(v: LoggedMessage) =>
v.type === "warning" &&
v.message ===
"Ignoring feature flag default_codeql_version_2_20_invalid_enabled as it does not specify a valid CodeQL version.",
) !== undefined,
);
});
});
});
test("ignores invalid version numbers in default version feature flags", async (t) => {
await withTmpDir(async (tmpDir) => {
const loggedMessages = [];
const features = setUpFeatureFlagTests(
tmpDir,
getRecordingLogger(loggedMessages),
);
const expectedFeatureEnablement = initializeFeatures(true);
expectedFeatureEnablement["default_codeql_version_2_20_0_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_1_enabled"] = true;
expectedFeatureEnablement["default_codeql_version_2_20_invalid_enabled"] =
true;
mockFeatureFlagApiEndpoint(200, expectedFeatureEnablement);
const defaultCliVersion = await features.getDefaultCliVersion(
GitHubVariant.DOTCOM,
);
t.deepEqual(defaultCliVersion, {
cliVersion: "2.20.1",
tagName: "codeql-bundle-v2.20.1",
toolsFeatureFlagsValid: true,
});
t.assert(
loggedMessages.find(
(v: LoggedMessage) =>
v.type === "warning" &&
v.message ===
"Ignoring feature flag default_codeql_version_2_20_invalid_enabled as it does not specify a valid CodeQL version.",
) !== undefined,
);
});
});
}
test("legacy feature flags should end with _enabled", async (t) => {
for (const [feature, config] of Object.entries(featureConfig)) {
if (config.legacyApi) {
if ((config satisfies FeatureConfig as FeatureConfig).legacyApi) {
t.assert(
feature.endsWith("_enabled"),
`legacy feature ${feature} should end with '_enabled'`,
@@ -541,7 +522,7 @@ test("legacy feature flags should end with _enabled", async (t) => {
test("non-legacy feature flags should not end with _enabled", async (t) => {
for (const [feature, config] of Object.entries(featureConfig)) {
if (!config.legacyApi) {
if (!(config satisfies FeatureConfig as FeatureConfig).legacyApi) {
t.false(
feature.endsWith("_enabled"),
`non-legacy feature ${feature} should not end with '_enabled'`,
@@ -552,7 +533,7 @@ test("non-legacy feature flags should not end with _enabled", async (t) => {
test("non-legacy feature flags should not start with codeql_action_", async (t) => {
for (const [feature, config] of Object.entries(featureConfig)) {
if (!config.legacyApi) {
if (!(config satisfies FeatureConfig as FeatureConfig).legacyApi) {
t.false(
feature.startsWith("codeql_action_"),
`non-legacy feature ${feature} should not start with 'codeql_action_'`,
@@ -591,12 +572,25 @@ function setUpFeatureFlagTests(
* Returns an argument to pass to `getValue` that if required includes a CodeQL object meeting the
* minimum version or tool feature requirements specified by the feature.
*/
function includeCodeQlIfRequired(feature: string) {
return featureConfig[feature].minimumVersion !== undefined ||
featureConfig[feature].toolsFeature !== undefined
? mockCodeQLVersion(
"9.9.9",
Object.fromEntries(Object.values(ToolsFeature).map((v) => [v, true])),
)
: undefined;
function getFeatureIncludingCodeQlIfRequired(
features: FeatureEnablement,
feature: Feature,
) {
const config = featureConfig[
feature
] satisfies FeatureConfig as FeatureConfig;
if (
config.minimumVersion === undefined &&
config.toolsFeature === undefined
) {
return features.getValue(feature as FeatureWithoutCLI);
}
return features.getValue(
feature,
mockCodeQLVersion(
"9.9.9",
Object.fromEntries(Object.values(ToolsFeature).map((v) => [v, true])),
),
);
}

167
src/feature-flags.ts
View File

@@ -26,16 +26,8 @@ export interface CodeQLDefaultVersionInfo {
toolsFeatureFlagsValid?: boolean;
}
export interface FeatureEnablement {
/** Gets the default version of the CodeQL tools. */
getDefaultCliVersion(
variant: util.GitHubVariant,
): Promise<CodeQLDefaultVersionInfo>;
getValue(feature: Feature, codeql?: CodeQL): Promise<boolean>;
}
/**
* Feature enablement as returned by the GitHub API endpoint.
* Features as named by the GitHub API endpoint.
*
* Do not include the `codeql_action_` prefix as this is stripped by the API
* endpoint.
@@ -44,9 +36,9 @@ export interface FeatureEnablement {
*/
export enum Feature {
AllowToolcacheInput = "allow_toolcache_input",
AnalyzeUseNewUpload = "analyze_use_new_upload",
CleanupTrapCaches = "cleanup_trap_caches",
CppDependencyInstallation = "cpp_dependency_installation_enabled",
CsharpCacheBuildModeNone = "csharp_cache_bmn",
CsharpNewCacheKey = "csharp_new_cache_key",
DiffInformedQueries = "diff_informed_queries",
DisableCsharpBuildless = "disable_csharp_buildless",
@@ -54,7 +46,6 @@ export enum Feature {
DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
IgnoreGeneratedFiles = "ignore_generated_files",
JavaMinimizeDependencyJars = "java_minimize_dependency_jars",
OverlayAnalysis = "overlay_analysis",
OverlayAnalysisActions = "overlay_analysis_actions",
OverlayAnalysisCodeScanningActions = "overlay_analysis_code_scanning_actions",
@@ -75,6 +66,7 @@ export enum Feature {
OverlayAnalysisPython = "overlay_analysis_python",
OverlayAnalysisRuby = "overlay_analysis_ruby",
OverlayAnalysisRust = "overlay_analysis_rust",
OverlayAnalysisSkipResourceChecks = "overlay_analysis_skip_resource_checks",
OverlayAnalysisSwift = "overlay_analysis_swift",
PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib",
QaTelemetryEnabled = "qa_telemetry_enabled",
@@ -83,47 +75,41 @@ export enum Feature {
ValidateDbConfig = "validate_db_config",
}
export const featureConfig: Record<
Feature,
{
/**
* Default value in environments where the feature flags API is not available,
* such as GitHub Enterprise Server.
*/
defaultValue: boolean;
/**
* Environment variable for explicitly enabling or disabling the feature.
*
* This overrides enablement status from the feature flags API.
*/
envVar: string;
/**
* Whether the feature flag is part of the legacy feature flags API (defaults to false).
*
* These feature flags are included by default in the API response and do not need to be
* explicitly requested.
*/
legacyApi?: boolean;
/**
* Minimum version of the CLI, if applicable.
*
* Prefer using `ToolsFeature`s for future flags.
*/
minimumVersion: string | undefined;
/** Required tools feature, if applicable. */
toolsFeature?: ToolsFeature;
}
> = {
export type FeatureConfig = {
/**
* Default value in environments where the feature flags API is not available,
* such as GitHub Enterprise Server.
*/
defaultValue: boolean;
/**
* Environment variable for explicitly enabling or disabling the feature.
*
* This overrides enablement status from the feature flags API.
*/
envVar: string;
/**
* Whether the feature flag is part of the legacy feature flags API (defaults to false).
*
* These feature flags are included by default in the API response and do not need to be
* explicitly requested.
*/
legacyApi?: boolean;
/**
* Minimum version of the CLI, if applicable.
*
* Prefer using `ToolsFeature`s for future flags.
*/
minimumVersion: string | undefined;
/** Required tools feature, if applicable. */
toolsFeature?: ToolsFeature;
};
export const featureConfig = {
[Feature.AllowToolcacheInput]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ALLOW_TOOLCACHE_INPUT",
minimumVersion: undefined,
},
[Feature.AnalyzeUseNewUpload]: {
defaultValue: false,
envVar: "CODEQL_ACTION_ANALYZE_USE_NEW_UPLOAD",
minimumVersion: undefined,
},
[Feature.CleanupTrapCaches]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CLEANUP_TRAP_CACHES",
@@ -135,6 +121,11 @@ export const featureConfig: Record<
legacyApi: true,
minimumVersion: "2.15.0",
},
[Feature.CsharpCacheBuildModeNone]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_CACHE_BMN",
minimumVersion: undefined,
},
[Feature.CsharpNewCacheKey]: {
defaultValue: false,
envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY",
@@ -173,11 +164,6 @@ export const featureConfig: Record<
envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",
minimumVersion: undefined,
},
[Feature.JavaMinimizeDependencyJars]: {
defaultValue: false,
envVar: "CODEQL_ACTION_JAVA_MINIMIZE_DEPENDENCY_JARS",
minimumVersion: "2.23.0",
},
[Feature.OverlayAnalysis]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
@@ -278,6 +264,11 @@ export const featureConfig: Record<
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RUST",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisSkipResourceChecks]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisSwift]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SWIFT",
@@ -299,6 +290,7 @@ export const featureConfig: Record<
defaultValue: false,
envVar: "CODEQL_ACTION_UPLOAD_OVERLAY_DB_TO_API",
minimumVersion: undefined,
toolsFeature: ToolsFeature.BundleSupportsOverlay,
},
[Feature.UseRepositoryProperties]: {
defaultValue: false,
@@ -310,7 +302,29 @@ export const featureConfig: Record<
envVar: "CODEQL_ACTION_VALIDATE_DB_CONFIG",
minimumVersion: undefined,
},
};
} satisfies Record<Feature, FeatureConfig>;
/** A feature whose enablement does not depend on the version of the CodeQL CLI. */
export type FeatureWithoutCLI = {
[K in Feature]: (typeof featureConfig)[K] extends
| {
minimumVersion: string;
}
| {
toolsFeature: ToolsFeature;
}
? never
: K;
}[keyof typeof featureConfig];
export interface FeatureEnablement {
/** Gets the default version of the CodeQL tools. */
getDefaultCliVersion(
variant: util.GitHubVariant,
): Promise<CodeQLDefaultVersionInfo>;
getValue(feature: FeatureWithoutCLI): Promise<boolean>;
getValue(feature: Feature, codeql: CodeQL): Promise<boolean>;
}
/**
* A response from the GitHub API that contains feature flag enablement information for the CodeQL
@@ -363,31 +377,35 @@ export class Features implements FeatureEnablement {
* @throws if a `minimumVersion` is specified for the feature, and `codeql` is not provided.
*/
async getValue(feature: Feature, codeql?: CodeQL): Promise<boolean> {
if (!codeql && featureConfig[feature].minimumVersion) {
// Narrow the type to FeatureConfig to avoid type errors. To avoid unsafe use of `as`, we
// check that the required properties exist using `satisfies`.
const config = featureConfig[
feature
] satisfies FeatureConfig as FeatureConfig;
if (!codeql && config.minimumVersion) {
throw new Error(
`Internal error: A minimum version is specified for feature ${feature}, but no instance of CodeQL was provided.`,
);
}
if (!codeql && featureConfig[feature].toolsFeature) {
if (!codeql && config.toolsFeature) {
throw new Error(
`Internal error: A required tools feature is specified for feature ${feature}, but no instance of CodeQL was provided.`,
);
}
const envVar = (
process.env[featureConfig[feature].envVar] || ""
).toLocaleLowerCase();
const envVar = (process.env[config.envVar] || "").toLocaleLowerCase();
// Do not use this feature if user explicitly disables it via an environment variable.
if (envVar === "false") {
this.logger.debug(
`Feature ${feature} is disabled via the environment variable ${featureConfig[feature].envVar}.`,
`Feature ${feature} is disabled via the environment variable ${config.envVar}.`,
);
return false;
}
// Never use this feature if the CLI version explicitly can't support it.
const minimumVersion = featureConfig[feature].minimumVersion;
const minimumVersion = config.minimumVersion;
if (codeql && minimumVersion) {
if (!(await util.codeQlVersionAtLeast(codeql, minimumVersion))) {
this.logger.debug(
@@ -404,7 +422,7 @@ export class Features implements FeatureEnablement {
);
}
}
const toolsFeature = featureConfig[feature].toolsFeature;
const toolsFeature = config.toolsFeature;
if (codeql && toolsFeature) {
if (!(await codeql.supportsFeature(toolsFeature))) {
this.logger.debug(
@@ -424,7 +442,7 @@ export class Features implements FeatureEnablement {
// Use this feature if user explicitly enables it via an environment variable.
if (envVar === "true") {
this.logger.debug(
`Feature ${feature} is enabled via the environment variable ${featureConfig[feature].envVar}.`,
`Feature ${feature} is enabled via the environment variable ${config.envVar}.`,
);
return true;
}
@@ -440,7 +458,7 @@ export class Features implements FeatureEnablement {
return apiValue;
}
const defaultValue = featureConfig[feature].defaultValue;
const defaultValue = config.defaultValue;
this.logger.debug(
`Feature ${feature} is ${
defaultValue ? "enabled" : "disabled"
@@ -492,8 +510,8 @@ class GitHubFeatureFlags {
async getDefaultCliVersion(
variant: util.GitHubVariant,
): Promise<CodeQLDefaultVersionInfo> {
if (variant === util.GitHubVariant.DOTCOM) {
return await this.getDefaultDotcomCliVersion();
if (supportsFeatureFlags(variant)) {
return await this.getDefaultCliVersionFromFlags();
}
return {
cliVersion: defaults.cliVersion,
@@ -501,7 +519,7 @@ class GitHubFeatureFlags {
};
}
async getDefaultDotcomCliVersion(): Promise<CodeQLDefaultVersionInfo> {
async getDefaultCliVersionFromFlags(): Promise<CodeQLDefaultVersionInfo> {
const response = await this.getAllFeatures();
const enabledFeatureFlagCliVersions = Object.entries(response)
@@ -627,10 +645,7 @@ class GitHubFeatureFlags {
private async loadApiResponse(): Promise<GitHubFeatureFlagsApiResponse> {
// Do nothing when not running against github.com
if (
this.gitHubVersion.type !== util.GitHubVariant.DOTCOM &&
this.gitHubVersion.type !== util.GitHubVariant.GHE_DOTCOM
) {
if (!supportsFeatureFlags(this.gitHubVersion.type)) {
this.logger.debug(
"Not running against github.com. Disabling all toggleable features.",
);
@@ -639,7 +654,10 @@ class GitHubFeatureFlags {
}
try {
const featuresToRequest = Object.entries(featureConfig)
.filter(([, config]) => !config.legacyApi)
.filter(
([, config]) =>
!(config satisfies FeatureConfig as FeatureConfig).legacyApi,
)
.map(([f]) => f);
const FEATURES_PER_REQUEST = 25;
@@ -696,3 +714,10 @@ class GitHubFeatureFlags {
}
}
}
function supportsFeatureFlags(githubVariant: util.GitHubVariant): boolean {
return (
githubVariant === util.GitHubVariant.DOTCOM ||
githubVariant === util.GitHubVariant.GHEC_DR
);
}

140
src/git-utils.test.ts
View File

@@ -316,27 +316,23 @@ test("getFileOidsUnderPath returns correct file mapping", async (t) => {
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
);
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {
"lib/git-utils.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/git-utils.js.map": "d89514599a9a99f22b4085766d40af7b99974827",
"src/git-utils.ts": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
t.deepEqual(result, {
"lib/git-utils.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/git-utils.js.map": "d89514599a9a99f22b4085766d40af7b99974827",
"src/git-utils.ts": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
t.deepEqual(runGitCommandStub.firstCall.args, [
"/fake/path",
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
"Cannot list Git OIDs of tracked files.",
]);
} finally {
runGitCommandStub.restore();
}
t.deepEqual(runGitCommandStub.firstCall.args, [
"/fake/path",
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
"Cannot list Git OIDs of tracked files.",
]);
});
test("getFileOidsUnderPath handles quoted paths", async (t) => {
const runGitCommandStub = sinon
sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(
"30d998ded095371488be3a729eb61d86ed721a18_lib/normal-file.js\n" +
@@ -344,34 +340,24 @@ test("getFileOidsUnderPath handles quoted paths", async (t) => {
'a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_"lib/file\\twith\\ttabs.js"',
);
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {
"lib/normal-file.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/file with spaces.js": "d89514599a9a99f22b4085766d40af7b99974827",
"lib/file\twith\ttabs.js": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
} finally {
runGitCommandStub.restore();
}
t.deepEqual(result, {
"lib/normal-file.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/file with spaces.js": "d89514599a9a99f22b4085766d40af7b99974827",
"lib/file\twith\ttabs.js": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
});
test("getFileOidsUnderPath handles empty output", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves("");
sinon.stub(gitUtils as any, "runGitCommand").resolves("");
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {});
} finally {
runGitCommandStub.restore();
}
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {});
});
test("getFileOidsUnderPath throws on unexpected output format", async (t) => {
const runGitCommandStub = sinon
sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(
"30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
@@ -379,19 +365,75 @@ test("getFileOidsUnderPath throws on unexpected output format", async (t) => {
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
);
try {
await t.throwsAsync(
async () => {
await gitUtils.getFileOidsUnderPath("/fake/path");
},
{
instanceOf: Error,
message: 'Unexpected "git ls-files" output: invalid-line-format',
},
);
} finally {
runGitCommandStub.restore();
}
await t.throwsAsync(
async () => {
await gitUtils.getFileOidsUnderPath("/fake/path");
},
{
instanceOf: Error,
message: 'Unexpected "git ls-files" output: invalid-line-format',
},
);
});
test("getGitVersionOrThrow returns version for valid git output", async (t) => {
sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(`git version 2.40.0${os.EOL}`);
const version = await gitUtils.getGitVersionOrThrow();
t.is(version.truncatedVersion, "2.40.0");
t.is(version.fullVersion, "2.40.0");
});
test("getGitVersionOrThrow throws for invalid git output", async (t) => {
sinon.stub(gitUtils as any, "runGitCommand").resolves("invalid output");
await t.throwsAsync(
async () => {
await gitUtils.getGitVersionOrThrow();
},
{
instanceOf: Error,
message: "Could not parse Git version from output: invalid output",
},
);
});
test("getGitVersionOrThrow handles Windows-style git output", async (t) => {
sinon
.stub(gitUtils as any, "runGitCommand")
.resolves("git version 2.40.0.windows.1");
const version = await gitUtils.getGitVersionOrThrow();
// The truncated version should contain just the major.minor.patch portion
t.is(version.truncatedVersion, "2.40.0");
t.is(version.fullVersion, "2.40.0.windows.1");
});
test("getGitVersionOrThrow throws when git command fails", async (t) => {
sinon
.stub(gitUtils as any, "runGitCommand")
.rejects(new Error("git not found"));
await t.throwsAsync(
async () => {
await gitUtils.getGitVersionOrThrow();
},
{
instanceOf: Error,
message: "git not found",
},
);
});
test("GitVersionInfo.isAtLeast correctly compares versions", async (t) => {
const version = new gitUtils.GitVersionInfo("2.40.0", "2.40.0");
t.true(version.isAtLeast("2.38.0"));
t.true(version.isAtLeast("2.40.0"));
t.false(version.isAtLeast("2.41.0"));
t.false(version.isAtLeast("3.0.0"));
});
test("listFiles returns array of file paths", async (t) => {

108
src/git-utils.ts
View File

@@ -3,6 +3,7 @@ import * as os from "os";
import * as core from "@actions/core";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as io from "@actions/io";
import * as semver from "semver";
import {
getOptionalInput,
@@ -11,6 +12,52 @@ import {
} from "./actions-util";
import { ConfigurationError, getRequiredEnvParam } from "./util";
/**
* Minimum Git version required for overlay analysis. The `git ls-files --format`
* option, which is used by `getFileOidsUnderPath`, was introduced in Git 2.38.0.
*/
export const GIT_MINIMUM_VERSION_FOR_OVERLAY = "2.38.0";
/**
* Git version information
*
* The full version string as reported by `git --version` may not be
* semver-compatible (e.g., "2.40.0.windows.1"). This class captures both
* the full version string and a truncated semver-compatible version string
* (e.g., "2.40.0").
*/
export class GitVersionInfo {
constructor(
/** Truncated semver-compatible version */
public truncatedVersion: string,
/** Full version string as reported by `git --version` */
public fullVersion: string,
) {}
isAtLeast(minVersion: string): boolean {
return semver.gte(this.truncatedVersion, minVersion);
}
}
/**
* Gets the version of Git installed on the system and throws an error if
* the version cannot be determined.
*/
export async function getGitVersionOrThrow(): Promise<GitVersionInfo> {
const stdout = await runGitCommand(
undefined,
["--version"],
"Failed to get git version.",
);
// Git version output can vary: "git version 2.40.0" or "git version 2.40.0.windows.1"
// We capture just the major.minor.patch portion to ensure semver compatibility.
const match = stdout.trim().match(/^git version ((\d+\.\d+\.\d+).*)$/);
if (match?.[1] && match?.[2]) {
return new GitVersionInfo(match[2], match[1]);
}
throw new Error(`Could not parse Git version from output: ${stdout.trim()}`);
}
export const runGitCommand = async function (
workingDirectory: string | undefined,
args: string[],
@@ -124,67 +171,6 @@ export const determineBaseBranchHeadCommitOid = async function (
}
};
/**
* Deepen the git history of HEAD by one level. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const deepenGitHistory = async function () {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
[
"fetch",
"origin",
"HEAD",
"--no-tags",
"--no-recurse-submodules",
"--deepen=1",
],
"Cannot deepen the shallow repository.",
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Fetch the given remote branch. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const gitFetch = async function (branch: string, extraFlags: string[]) {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
["fetch", "--no-tags", ...extraFlags, "origin", `${branch}:${branch}`],
`Cannot fetch ${branch}.`,
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Repack the git repository, using with the given flags. Errors are logged.
*
* This function uses the `checkout_path` to determine the repository path and
* works only when called from `analyze` or `upload-sarif`.
*/
export const gitRepack = async function (flags: string[]) {
try {
await runGitCommand(
getOptionalInput("checkout_path"),
["repack", ...flags],
"Cannot repack the repository.",
);
} catch {
// Errors are already logged by runGitCommand()
}
};
/**
* Decode, if necessary, a file path produced by Git. See
* https://git-scm.com/docs/git-config#Documentation/git-config.txt-corequotePath

51
src/init-action.ts
View File

@@ -33,6 +33,7 @@ import {
flushDiagnostics,
logUnwrittenDiagnostics,
makeDiagnostic,
makeTelemetryDiagnostic,
} from "./diagnostics";
import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
@@ -75,7 +76,7 @@ import {
codeQlVersionAtLeast,
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
getMemoryFlagValue,
getCodeQLMemoryLimit,
getRequiredEnvParam,
getThreadsFlagValue,
initializeEnvironment,
@@ -88,6 +89,13 @@ import {
} from "./util";
import { checkWorkflow } from "./workflow";
/**
* First version of CodeQL where the Java extractor safely supports the option to minimize
* dependency jars. Note: some earlier versions of the extractor will respond to the corresponding
* option, but may rewrite jars in ways that lead to extraction errors.
*/
export const CODEQL_VERSION_JAR_MINIMIZATION = "2.23.0";
/**
* Sends a status report indicating that the `init` Action is starting.
*
@@ -324,6 +332,7 @@ async function run() {
queriesInput: getOptionalInput("queries"),
packsInput: getOptionalInput("packs"),
buildModeInput: getOptionalInput("build-mode"),
ramInput: getOptionalInput("ram"),
configFile,
dbLocation: getOptionalInput("db-location"),
configInput: getOptionalInput("config"),
@@ -417,17 +426,10 @@ async function run() {
// Arbitrarily choose the first language. We could also choose all languages, but that
// increases the risk of misinterpreting the data.
config.languages[0],
makeDiagnostic(
makeTelemetryDiagnostic(
"codeql-action/bundle-download-telemetry",
"CodeQL bundle download telemetry",
{
attributes: toolsDownloadStatusReport,
visibility: {
cliSummaryTable: false,
statusPage: false,
telemetry: true,
},
},
toolsDownloadStatusReport,
),
);
}
@@ -537,7 +539,7 @@ async function run() {
core.exportVariable(
"CODEQL_RAM",
process.env["CODEQL_RAM"] ||
getMemoryFlagValue(getOptionalInput("ram"), logger).toString(),
getCodeQLMemoryLimit(getOptionalInput("ram"), logger).toString(),
);
core.exportVariable(
"CODEQL_THREADS",
@@ -637,18 +639,20 @@ async function run() {
}
}
// If the feature flag to minimize Java dependency jars is enabled, and we are doing a Java
// `build-mode: none` analysis (i.e. the flag is relevant), then set the environment variable
// that enables the corresponding option in the Java extractor. We also only do this if
// dependency caching is enabled, since the option is intended to reduce the size of
// dependency caches, but the jar-rewriting does have a performance cost that we'd like to avoid
// when caching is not being used.
// If we are doing a Java `build-mode: none` analysis, then set the environment variable that
// enables the option in the Java extractor to minimize dependency jars. We also only do this if
// dependency caching is enabled, since the option is intended to reduce the size of dependency
// caches, but the jar-rewriting does have a performance cost that we'd like to avoid when
// caching is not being used.
// TODO: Remove this language-specific mechanism and replace it with a more general one that
// tells extractors when dependency caching is enabled, and then the Java extractor can make its
// own decision about whether to rewrite jars.
if (process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]) {
logger.debug(
`${EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS} is already set to '${process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]}', so the Action will not override it.`,
);
} else if (
(await features.getValue(Feature.JavaMinimizeDependencyJars, codeql)) &&
(await codeQlVersionAtLeast(codeql, CODEQL_VERSION_JAR_MINIMIZATION)) &&
config.dependencyCachingEnabled &&
config.buildMode === BuildMode.None &&
config.languages.includes(KnownLanguage.java)
@@ -784,17 +788,10 @@ async function recordZstdAvailability(
// Arbitrarily choose the first language. We could also choose all languages, but that
// increases the risk of misinterpreting the data.
config.languages[0],
makeDiagnostic(
makeTelemetryDiagnostic(
"codeql-action/zstd-availability",
"Zstandard availability",
{
attributes: zstdAvailability,
visibility: {
cliSummaryTable: false,
statusPage: false,
telemetry: true,
},
},
zstdAvailability,
),
);
}

44
src/overlay-database-utils.test.ts
View File

@@ -7,7 +7,9 @@ import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as apiClient from "./api-client";
import { ResolveDatabaseOutput } from "./codeql";
import * as gitUtils from "./git-utils";
import { KnownLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import {
downloadOverlayBaseDatabaseFromCache,
@@ -95,6 +97,7 @@ interface DownloadOverlayBaseDatabaseTestCase {
hasBaseDatabaseOidsFile: boolean;
tryGetFolderBytesSucceeds: boolean;
codeQLVersion: string;
resolveDatabaseOutput: ResolveDatabaseOutput | Error;
}
const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = {
@@ -105,6 +108,7 @@ const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = {
hasBaseDatabaseOidsFile: true,
tryGetFolderBytesSucceeds: true,
codeQLVersion: "2.20.5",
resolveDatabaseOutput: { overlayBaseSpecifier: "20250626:XXX" },
};
const testDownloadOverlayBaseDatabaseFromCache = test.macro({
@@ -119,9 +123,11 @@ const testDownloadOverlayBaseDatabaseFromCache = test.macro({
await fs.promises.mkdir(dbLocation, { recursive: true });
const logger = getRunnerLogger(true);
const config = createTestConfig({ dbLocation });
const testCase = { ...defaultDownloadTestCase, ...partialTestCase };
const config = createTestConfig({
dbLocation,
languages: [KnownLanguage.java],
});
config.overlayDatabaseMode = testCase.overlayDatabaseMode;
config.useOverlayDatabaseCaching = testCase.useOverlayDatabaseCaching;
@@ -163,9 +169,23 @@ const testDownloadOverlayBaseDatabaseFromCache = test.macro({
.resolves(testCase.tryGetFolderBytesSucceeds ? 1024 * 1024 : undefined);
stubs.push(tryGetFolderBytesStub);
const codeql = mockCodeQLVersion(testCase.codeQLVersion);
if (testCase.resolveDatabaseOutput instanceof Error) {
const resolveDatabaseStub = sinon
.stub(codeql, "resolveDatabase")
.rejects(testCase.resolveDatabaseOutput);
stubs.push(resolveDatabaseStub);
} else {
const resolveDatabaseStub = sinon
.stub(codeql, "resolveDatabase")
.resolves(testCase.resolveDatabaseOutput);
stubs.push(resolveDatabaseStub);
}
try {
const result = await downloadOverlayBaseDatabaseFromCache(
mockCodeQLVersion(testCase.codeQLVersion),
codeql,
config,
logger,
);
@@ -255,6 +275,24 @@ test(
false,
);
test(
testDownloadOverlayBaseDatabaseFromCache,
"returns undefined when downloaded database doesn't have an overlayBaseSpecifier",
{
resolveDatabaseOutput: {},
},
false,
);
test(
testDownloadOverlayBaseDatabaseFromCache,
"returns undefined when resolving database metadata fails",
{
resolveDatabaseOutput: new Error("Failed to resolve database metadata"),
},
false,
);
test(
testDownloadOverlayBaseDatabaseFromCache,
"returns undefined when filesystem error occurs",

37
src/overlay-database-utils.ts
View File

@@ -17,6 +17,7 @@ import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import {
CleanupLevel,
getCodeQLDatabasePath,
getErrorMessage,
isInTestMode,
tryGetFolderBytes,
@@ -29,7 +30,7 @@ export enum OverlayDatabaseMode {
None = "none",
}
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.23.5";
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.23.8";
/**
* The maximum (uncompressed) size of the overlay base database that we will
@@ -176,11 +177,12 @@ const MAX_CACHE_OPERATION_MS = 600_000;
* @param warningPrefix Prefix for the check failure warning message
* @returns True if the verification succeeded, false otherwise
*/
export function checkOverlayBaseDatabase(
async function checkOverlayBaseDatabase(
codeql: CodeQL,
config: Config,
logger: Logger,
warningPrefix: string,
): boolean {
): Promise<boolean> {
// An overlay-base database should contain the base database OIDs file.
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
if (!fs.existsSync(baseDatabaseOidsFilePath)) {
@@ -189,6 +191,29 @@ export function checkOverlayBaseDatabase(
);
return false;
}
for (const language of config.languages) {
const dbPath = getCodeQLDatabasePath(config, language);
try {
const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath);
if (
resolveDatabaseOutput === undefined ||
!("overlayBaseSpecifier" in resolveDatabaseOutput)
) {
logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`);
return false;
} else {
logger.debug(
`Overlay base specifier for ${language} overlay-base database found: ` +
`${resolveDatabaseOutput.overlayBaseSpecifier}`,
);
}
} catch (e) {
logger.warning(`${warningPrefix}: failed to resolve database: ${e}`);
return false;
}
}
return true;
}
@@ -232,7 +257,8 @@ export async function cleanupAndUploadOverlayBaseDatabaseToCache(
return false;
}
const databaseIsValid = checkOverlayBaseDatabase(
const databaseIsValid = await checkOverlayBaseDatabase(
codeql,
config,
logger,
"Abort uploading overlay-base database to cache",
@@ -415,7 +441,8 @@ export async function downloadOverlayBaseDatabaseFromCache(
return undefined;
}
const databaseIsValid = checkOverlayBaseDatabase(
const databaseIsValid = await checkOverlayBaseDatabase(
codeql,
config,
logger,
"Downloaded overlay-base database is invalid",

21
src/setup-codeql.ts
View File

@@ -34,7 +34,7 @@ export enum ToolsSource {
Download = "DOWNLOAD",
}
export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
const CODEQL_NIGHTLIES_REPOSITORY_OWNER = "dsp-testing";
const CODEQL_NIGHTLIES_REPOSITORY_NAME = "codeql-cli-nightlies";
@@ -180,17 +180,6 @@ export function tryGetTagNameFromUrl(
return match[1];
}
export function tryGetBundleVersionFromUrl(
url: string,
logger: Logger,
): string | undefined {
const tagName = tryGetTagNameFromUrl(url, logger);
if (tagName === undefined) {
return undefined;
}
return tryGetBundleVersionFromTagName(tagName, logger);
}
export function convertToSemVer(version: string, logger: Logger): string {
if (!semver.valid(version)) {
logger.debug(
@@ -522,7 +511,7 @@ export async function getCodeQLSource(
// different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it).
if (
variant !== util.GitHubVariant.DOTCOM &&
variant === util.GitHubVariant.GHES &&
!forceShippedTools &&
!toolsInput
) {
@@ -580,7 +569,7 @@ export async function getCodeQLSource(
* Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find
* the `x.y.z` version. This is to support old versions of the toolcache.
*/
export async function tryGetFallbackToolcacheVersion(
async function tryGetFallbackToolcacheVersion(
cliVersion: string | undefined,
tagName: string,
logger: Logger,
@@ -729,7 +718,7 @@ function getCanonicalToolcacheVersion(
return cliVersion;
}
export interface SetupCodeQLResult {
interface SetupCodeQLResult {
codeqlFolder: string;
toolsDownloadStatusReport?: ToolsDownloadStatusReport;
toolsSource: ToolsSource;
@@ -750,7 +739,7 @@ export async function setupCodeQLBundle(
defaultCliVersion: CodeQLDefaultVersionInfo,
features: FeatureEnablement,
logger: Logger,
) {
): Promise<SetupCodeQLResult> {
if (!(await util.isBinaryAccessible("tar", logger))) {
throw new util.ConfigurationError(
"Could not find tar in PATH, so unable to extract CodeQL bundle.",

4
src/start-proxy.ts
View File

@@ -8,7 +8,7 @@ import { ConfigurationError, getErrorMessage, isDefined } from "./util";
export const UPDATEJOB_PROXY = "update-job-proxy";
export const UPDATEJOB_PROXY_VERSION = "v2.0.20250624110901";
export const UPDATEJOB_PROXY_URL_PREFIX =
const UPDATEJOB_PROXY_URL_PREFIX =
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.22.0/";
export type Credential = {
@@ -202,7 +202,7 @@ export function getFallbackUrl(proxyPackage: string): string {
*
* @returns The response from the GitHub API.
*/
export async function getLinkedRelease() {
async function getLinkedRelease() {
return getApiClient().rest.repos.getReleaseByTag({
owner: "github",
repo: "codeql-action",

2
src/status-report.ts
View File

@@ -54,7 +54,7 @@ export enum ActionName {
* considered to be a third party analysis and is treated differently when calculating SLOs. To ensure
* misconfigured workflows are not treated as third party, only the upload-sarif action can return false.
*/
export function isFirstPartyAnalysis(actionName: ActionName): boolean {
function isFirstPartyAnalysis(actionName: ActionName): boolean {
if (actionName !== ActionName.UploadSarif) {
return true;
}

BIN
src/testdata/debug-artifacts-with-fake-token.zip vendored Normal file
View File

Binary file not shown.

31
src/testing-utils.ts
View File

@@ -152,27 +152,38 @@ export interface LoggedMessage {
message: string | Error;
}
export function getRecordingLogger(messages: LoggedMessage[]): Logger {
export function getRecordingLogger(
messages: LoggedMessage[],
{ logToConsole }: { logToConsole?: boolean } = { logToConsole: true },
): Logger {
return {
debug: (message: string) => {
messages.push({ type: "debug", message });
// eslint-disable-next-line no-console
console.debug(message);
if (logToConsole) {
// eslint-disable-next-line no-console
console.debug(message);
}
},
info: (message: string) => {
messages.push({ type: "info", message });
// eslint-disable-next-line no-console
console.info(message);
if (logToConsole) {
// eslint-disable-next-line no-console
console.info(message);
}
},
warning: (message: string | Error) => {
messages.push({ type: "warning", message });
// eslint-disable-next-line no-console
console.warn(message);
if (logToConsole) {
// eslint-disable-next-line no-console
console.warn(message);
}
},
error: (message: string | Error) => {
messages.push({ type: "error", message });
// eslint-disable-next-line no-console
console.error(message);
if (logToConsole) {
// eslint-disable-next-line no-console
console.error(message);
}
},
isDebug: () => true,
startGroup: () => undefined,
@@ -305,7 +316,7 @@ export function createFeatures(enabledFeatures: Feature[]): FeatureEnablement {
throw new Error("not implemented");
},
getValue: async (feature) => {
return enabledFeatures.includes(feature);
return enabledFeatures.includes(feature as Feature);
},
};
}

2
src/tools-download.ts
View File

@@ -17,7 +17,7 @@ import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
/**
* High watermark to use when streaming the download and extraction of the CodeQL tools.
*/
export const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
/**
* The name of the tool cache directory for the CodeQL tools.

1
src/tools-features.ts
View File

@@ -4,6 +4,7 @@ import type { VersionInfo } from "./codeql";
export enum ToolsFeature {
BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries",
BundleSupportsOverlay = "bundleSupportsOverlay",
DatabaseInterpretResultsSupportsSarifRunProperty = "databaseInterpretResultsSupportsSarifRunProperty",
ForceOverwrite = "forceOverwrite",
IndirectTracingSupportsStaticBinaries = "indirectTracingSupportsStaticBinaries",

2
src/tracer-config.ts
View File

@@ -76,7 +76,7 @@ export async function endTracingForCluster(
}
}
export async function getTracerConfigForCluster(
async function getTracerConfigForCluster(
config: Config,
): Promise<TracerConfig> {
const tracingEnvVariables = JSON.parse(

2
src/upload-lib.ts
View File

@@ -412,7 +412,7 @@ export function findSarifFilesInDir(
return sarifFiles;
}
export function getSarifFilePaths(
function getSarifFilePaths(
sarifPath: string,
isSarif: (name: string) => boolean,
) {

10
src/util.test.ts
View File

@@ -433,8 +433,8 @@ function formatGitHubVersion(version: util.GitHubVersion): string {
switch (version.type) {
case util.GitHubVariant.DOTCOM:
return "dotcom";
case util.GitHubVariant.GHE_DOTCOM:
return "GHE dotcom";
case util.GitHubVariant.GHEC_DR:
return "GHEC-DR";
case util.GitHubVariant.GHES:
return `GHES ${version.version}`;
default:
@@ -445,12 +445,12 @@ function formatGitHubVersion(version: util.GitHubVersion): string {
const CHECK_ACTION_VERSION_TESTS: Array<[string, util.GitHubVersion, boolean]> =
[
["2.2.1", { type: util.GitHubVariant.DOTCOM }, true],
["2.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, true],
["2.2.1", { type: util.GitHubVariant.GHEC_DR }, true],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.10" }, false],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.11" }, false],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.12" }, false],
["3.2.1", { type: util.GitHubVariant.DOTCOM }, true],
["3.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, true],
["3.2.1", { type: util.GitHubVariant.GHEC_DR }, true],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.10" }, false],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.11" }, false],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.12" }, false],
@@ -458,7 +458,7 @@ const CHECK_ACTION_VERSION_TESTS: Array<[string, util.GitHubVersion, boolean]> =
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.20" }, true],
["3.2.1", { type: util.GitHubVariant.GHES, version: "3.21" }, true],
["4.2.1", { type: util.GitHubVariant.DOTCOM }, false],
["4.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, false],
["4.2.1", { type: util.GitHubVariant.GHEC_DR }, false],
["4.2.1", { type: util.GitHubVariant.GHES, version: "3.19" }, false],
["4.2.1", { type: util.GitHubVariant.GHES, version: "3.20" }, false],
["4.2.1", { type: util.GitHubVariant.GHES, version: "3.21" }, false],

107
src/util.ts
View File

@@ -4,7 +4,6 @@ import * as os from "os";
import * as path from "path";
import * as core from "@actions/core";
import * as exec from "@actions/exec/lib/exec";
import * as io from "@actions/io";
import getFolderSize from "get-folder-size";
import * as yaml from "js-yaml";
@@ -310,13 +309,13 @@ function getCgroupMemoryLimitBytes(
}
/**
* Get the value of the codeql `--ram` flag as configured by the `ram` input.
* If no value was specified, the total available memory will be used minus a
* Get the maximum amount of memory CodeQL is allowed to use. If no limit has been
* configured by the user, then the total available memory will be used minus a
* threshold reserved for the OS.
*
* @returns {number} the amount of RAM to use, in megabytes
* @returns {number} the amount of RAM CodeQL is allowed to use, in megabytes
*/
export function getMemoryFlagValue(
export function getCodeQLMemoryLimit(
userInput: string | undefined,
logger: Logger,
): number {
@@ -338,7 +337,7 @@ export function getMemoryFlag(
userInput: string | undefined,
logger: Logger,
): string {
const megabytes = getMemoryFlagValue(userInput, logger);
const megabytes = getCodeQLMemoryLimit(userInput, logger);
return `--ram=${megabytes}`;
}
@@ -557,13 +556,17 @@ const CODEQL_ACTION_WARNED_ABOUT_VERSION_ENV_VAR =
let hasBeenWarnedAboutVersion = false;
export enum GitHubVariant {
DOTCOM,
GHES,
GHE_DOTCOM,
/** [GitHub.com](https://github.com) */
DOTCOM = "GitHub.com",
/** [GitHub Enterprise Server](https://docs.github.com/en/enterprise-server@latest/admin/overview/about-github-enterprise-server) */
GHES = "GitHub Enterprise Server",
/** [GitHub Enterprise Cloud with data residency](https://docs.github.com/en/enterprise-cloud@latest/admin/data-residency/about-github-enterprise-cloud-with-data-residency) */
GHEC_DR = "GitHub Enterprise Cloud with data residency",
}
export type GitHubVersion =
| { type: GitHubVariant.DOTCOM }
| { type: GitHubVariant.GHE_DOTCOM }
| { type: GitHubVariant.GHEC_DR }
| { type: GitHubVariant.GHES; version: string };
export function checkGitHubVersionInRange(
@@ -1026,34 +1029,6 @@ export function fixInvalidNotifications(
return newSarif;
}
/**
* Removes duplicates from the sarif file.
*
* When `CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX` is set to true, this will
* simply rename the input file to the output file. Otherwise, it will parse the
* input file as JSON, remove duplicate locations from the SARIF notification
* objects, and write the result to the output file.
*
* For context, see documentation of:
* `CODEQL_ACTION_DISABLE_DUPLICATE_LOCATION_FIX`. */
export function fixInvalidNotificationsInFile(
inputPath: string,
outputPath: string,
logger: Logger,
): void {
if (process.env[EnvVar.DISABLE_DUPLICATE_LOCATION_FIX] === "true") {
logger.info(
"SARIF notification object duplicate location fix disabled by the " +
`${EnvVar.DISABLE_DUPLICATE_LOCATION_FIX} environment variable.`,
);
fs.renameSync(inputPath, outputPath);
} else {
let sarif = JSON.parse(fs.readFileSync(inputPath, "utf8")) as SarifFile;
sarif = fixInvalidNotifications(sarif, logger);
fs.writeFileSync(outputPath, JSON.stringify(sarif));
}
}
export function wrapError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
@@ -1134,7 +1109,7 @@ export function checkActionVersion(
// and should update to CodeQL Action v4.
if (
githubVersion.type === GitHubVariant.DOTCOM ||
githubVersion.type === GitHubVariant.GHE_DOTCOM ||
githubVersion.type === GitHubVariant.GHEC_DR ||
(githubVersion.type === GitHubVariant.GHES &&
semver.satisfies(
semver.coerce(githubVersion.version) ?? "0.0.0",
@@ -1197,49 +1172,6 @@ export function cloneObject<T>(obj: T): T {
return JSON.parse(JSON.stringify(obj)) as T;
}
// The first time this function is called, it runs `csrutil status` to determine
// whether System Integrity Protection is enabled; and saves the result in an
// environment variable. Afterwards, simply return the value of the environment
// variable.
export async function checkSipEnablement(
logger: Logger,
): Promise<boolean | undefined> {
if (
process.env[EnvVar.IS_SIP_ENABLED] !== undefined &&
["true", "false"].includes(process.env[EnvVar.IS_SIP_ENABLED])
) {
return process.env[EnvVar.IS_SIP_ENABLED] === "true";
}
try {
const sipStatusOutput = await exec.getExecOutput("csrutil status");
if (sipStatusOutput.exitCode === 0) {
if (
sipStatusOutput.stdout.includes(
"System Integrity Protection status: enabled.",
)
) {
core.exportVariable(EnvVar.IS_SIP_ENABLED, "true");
return true;
}
if (
sipStatusOutput.stdout.includes(
"System Integrity Protection status: disabled.",
)
) {
core.exportVariable(EnvVar.IS_SIP_ENABLED, "false");
return false;
}
}
return undefined;
} catch (e) {
logger.warning(
`Failed to determine if System Integrity Protection was enabled: ${e}`,
);
return undefined;
}
}
export async function cleanUpPath(file: string, name: string, logger: Logger) {
logger.debug(`Cleaning up ${name}.`);
try {
@@ -1291,17 +1223,6 @@ export function isDefined<T>(value: T | null | undefined): value is T {
return value !== undefined && value !== null;
}
/** Like `Object.keys`, but typed so that the elements of the resulting array have the
* same type as the keys of the input object. Note that this may not be sound if the input
* object has been cast to `T` from a subtype of `T` and contains additional keys that
* are not represented by `keyof T`.
*/
export function unsafeKeysInvariant<T extends Record<string, any>>(
object: T,
): Array<keyof T> {
return Object.keys(object) as Array<keyof T>;
}
/** Like `Object.entries`, but typed so that the key elements of the result have the
* same type as the keys of the input object. Note that this may not be sound if the input
* object has been cast to `T` from a subtype of `T` and contains additional keys that