diff --git a/src/cli/output-cache.test.ts b/src/cli/output-cache.test.ts index 2504f484f..d81c613c3 100644 --- a/src/cli/output-cache.test.ts +++ b/src/cli/output-cache.test.ts @@ -4,13 +4,13 @@ import path from "path"; import test from "ava"; -import { isVersionInfo } from "../codeql"; import { setupTests } from "../testing-utils"; import { cacheCommandOutput, getCachedCommandOutput, resetCachedCommandOutputs, + type VersionInfo, } from "./output-cache"; setupTests(test); @@ -60,7 +60,6 @@ test.serial( getCachedCommandOutput( "version", "/path/to/codeql", - isVersionInfo, ), { version: "2.20.0" }, ); @@ -82,7 +81,6 @@ test.serial( getCachedCommandOutput( "version", "/path/to/codeql", - isVersionInfo, ), undefined, ); @@ -99,7 +97,6 @@ test.serial( getCachedCommandOutput( "version", "/path/to/codeql", - isVersionInfo, ), undefined, ); @@ -115,7 +112,6 @@ test.serial( getCachedCommandOutput( "version", "/path/to/codeql", - isVersionInfo, ), undefined, ); @@ -141,7 +137,6 @@ test.serial( getCachedCommandOutput( "version", "/path/to/codeql", - isVersionInfo, ), undefined, JSON.stringify(output), @@ -162,7 +157,6 @@ test.serial( getCachedCommandOutput( "version", "/path/to/codeql", - isVersionInfo, ), undefined, ); @@ -172,15 +166,11 @@ test.serial( test.serial("cacheCommandOutput persists the output to the memo", async (t) => { await withCacheDir(() => { - cacheCommandOutput("version", "/path/to/codeql", { - hello: "world", - }); + const output: VersionInfo = { version: "2.20.0" }; + cacheCommandOutput("version", "/path/to/codeql", output); // Tier 1: the value is immediately available from the memo. - t.deepEqual( - getCachedCommandOutput("version", "/path/to/codeql"), - { hello: "world" }, - ); + t.deepEqual(getCachedCommandOutput("version", "/path/to/codeql"), output); }); }); @@ -188,21 +178,17 @@ test.serial( "getCachedCommandOutput prefers the in-memory memo over the file", async (t) => { await withCacheDir((cacheFilePath) => { - cacheCommandOutput("version", "/path/to/codeql", { - value: 1, - }); + const output: VersionInfo = { version: "2.20.0", overlayVersion: 1 }; + cacheCommandOutput("version", "/path/to/codeql", output); // Overwrite the file with a different value; the memo (tier 1) should win. writeCacheFile(cacheFilePath, { version: { cmd: "/path/to/codeql", - output: { value: 2 }, + output: { version: "2.21.0" }, }, }); - t.deepEqual( - getCachedCommandOutput("version", "/path/to/codeql"), - { value: 1 }, - ); + t.deepEqual(getCachedCommandOutput("version", "/path/to/codeql"), output); }); }, ); diff --git a/src/cli/output-cache.ts b/src/cli/output-cache.ts index 4acb055da..78dbbe4f2 100644 --- a/src/cli/output-cache.ts +++ b/src/cli/output-cache.ts @@ -14,15 +14,97 @@ const COMMAND_CACHE_FILENAME = "codeql-action-command-cache.json"; /** A key used to identify cached command output. */ export type CommandCacheKey = "version" | "resolve languages"; +export interface VersionInfo { + version: string; + features?: { [name: string]: boolean }; + /** + * The overlay version helps deal with backward incompatible changes for + * overlay analysis. When a precompiled query pack reports the same overlay + * version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay + * analysis with that pack. Otherwise, if the overlay versions are different, + * or if either the pack or the CLI does not report an overlay version, + * we need to revert to non-overlay analysis. + */ + overlayVersion?: number; +} + +/** Returns true if `x` is a {@link VersionInfo}. */ +export function isVersionInfo(x: unknown): x is VersionInfo { + return ( + json.isObject(x) && + json.validateSchema( + { + version: json.string, + features: json.optional(json.object), + overlayVersion: json.optional(json.number), + }, + x, + ) + ); +} + +export interface ResolveLanguagesOutput { + aliases?: { + [alias: string]: string; + }; + extractors: { + [language: string]: Array<{ + extractor_root: string; + extractor_options?: any; + }>; + }; +} + +/** Returns true if `x` is a {@link ResolveLanguagesOutput}. */ +export function isResolveLanguagesOutput( + x: unknown, +): x is ResolveLanguagesOutput { + return ( + json.isObject(x) && + json.isObject(x.extractors) && + Object.values(x.extractors).every( + (extractorList) => + json.isArray(extractorList) && + extractorList.every( + (extractor) => + json.isObject<{ extractor_root: unknown }>(extractor) && + json.isString(extractor.extractor_root), + ), + ) && + (x.aliases === undefined || + (json.isObject(x.aliases) && + Object.values(x.aliases).every((alias) => json.isString(alias)))) + ); +} + +export type CommandCacheKeyOutputMap = { + version: VersionInfo; + "resolve languages": ResolveLanguagesOutput; +}; + +const commandCacheValidators: { + [K in CommandCacheKey]: ( + output: unknown, + ) => output is CommandCacheKeyOutputMap[K]; +} = { + version: isVersionInfo, + "resolve languages": isResolveLanguagesOutput, +}; + +interface StoredCommandCacheEntry { + cmd: string; + output: unknown; +} + /** A single cached command output together with the CLI path it came from. */ -interface CommandCacheEntry { +export interface CommandCacheEntry { /** * The path to the CodeQL CLI that produced `output`. Persisted so that a * different step using a different CodeQL bundle doesn't pick up a stale * value. */ cmd: string; - output: unknown; + output: CommandCacheKeyOutputMap[K]; } /** @@ -30,7 +112,10 @@ interface CommandCacheEntry { * whenever a value is read from the file (tier 2) or computed via the CLI * (tier 3). */ -const inMemoryCache = new Map(); +const inMemoryCache = new Map< + CommandCacheKey, + CommandCacheEntry +>(); const logger = getActionsLogger(); function getCommandCacheFilePath(): string { @@ -41,7 +126,7 @@ function getCommandCacheFilePath(): string { * Reads and parses the temporary cache file. Best-effort: a missing, malformed, * or otherwise unreadable file is treated as an empty cache. */ -function readCommandCacheFile(): Record { +function readCommandCacheFile(): Record { if (!fs.existsSync(getCommandCacheFilePath())) { return {}; } @@ -75,12 +160,12 @@ export function writeCommandCacheFile(): void { /** * Stores the output of a CLI command under `key` in a module-global object. */ -export function cacheCommandOutput( - key: CommandCacheKey, +export function cacheCommandOutput( + key: K, cmd: string, - output: unknown, + output: CommandCacheKeyOutputMap[K], ): void { - const entry: CommandCacheEntry = { cmd, output }; + const entry: CommandCacheEntry = { cmd, output }; inMemoryCache.set(key, entry); } @@ -95,32 +180,33 @@ export function cacheCommandOutput( * A return value of `undefined` signals the caller to fall back to tier 3 (the * CLI). */ -export function getCachedCommandOutput( - key: CommandCacheKey, +export function getCachedCommandOutput( + key: K, cmd?: string, - validate?: (output: unknown) => output is T, -): T | undefined { +): CommandCacheKeyOutputMap[K] | undefined { // Tier 1: the in-memory variable. const memoized = inMemoryCache.get(key); if (memoized !== undefined) { - return memoized.output as T; + return memoized.output as CommandCacheKeyOutputMap[K]; } // Tier 2: the temporary file persisted by an earlier step, if any. const entry = readCommandCacheFile()[key] as unknown; if ( - !json.isObject(entry) || + !json.isObject(entry) || !json.isString(entry.cmd) || (cmd !== undefined && entry.cmd !== cmd) || - (validate !== undefined && !validate(entry.output)) + !commandCacheValidators[key](entry.output) ) { logger.warning("Received invalid data from the command-cache file."); return undefined; } // Memoize so subsequent lookups in this process hit tier 1. - cacheCommandOutput(key, entry.cmd, entry.output); - return entry.output as T; + const cachedEntry = entry as StoredCommandCacheEntry; + const output = cachedEntry.output as CommandCacheKeyOutputMap[K]; + cacheCommandOutput(key, cachedEntry.cmd, output); + return output; } /** diff --git a/src/codeql.ts b/src/codeql.ts index 9bfeba911..b93078286 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -15,7 +15,10 @@ import * as api from "./api-client"; import { cacheCommandOutput, getCachedCommandOutput, + type CommandCacheKeyOutputMap, type CommandCacheKey, + type ResolveLanguagesOutput, + type VersionInfo, } from "./cli/output-cache"; import { CliError, wrapCliConfigurationError } from "./cli-errors"; import { appendExtraQueryExclusions, type Config } from "./config-utils"; @@ -27,7 +30,6 @@ import { FeatureEnablement, } from "./feature-flags"; import { isAnalyzingDefaultBranch } from "./git-utils"; -import * as json from "./json"; import { Language } from "./languages"; import { Logger } from "./logging"; import { writeBaseDatabaseOidsFile, writeOverlayChangesFile } from "./overlay"; @@ -42,11 +44,6 @@ import { BuildMode, CleanupLevel, getErrorMessage } from "./util"; type Options = Array; -type CommandCacheKeyOutputMap = { - version: VersionInfo; - "resolve languages": ResolveLanguagesOutput; -}; - /** * Extra command line options for the codeql commands. */ @@ -225,73 +222,10 @@ export interface CodeQL { ): Promise; } -export interface VersionInfo { - version: string; - features?: { [name: string]: boolean }; - /** - * The overlay version helps deal with backward incompatible changes for - * overlay analysis. When a precompiled query pack reports the same overlay - * version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay - * analysis with that pack. Otherwise, if the overlay versions are different, - * or if either the pack or the CLI does not report an overlay version, - * we need to revert to non-overlay analysis. - */ - overlayVersion?: number; -} - -/** Returns true if `x` is a {@link VersionInfo}. */ -export function isVersionInfo(x: unknown): x is VersionInfo { - return ( - json.isObject(x) && - json.validateSchema( - { - version: json.string, - features: json.optional(json.object), - overlayVersion: json.optional(json.number), - }, - x, - ) - ); -} - export interface ResolveDatabaseOutput { overlayBaseSpecifier?: string; } -export interface ResolveLanguagesOutput { - aliases?: { - [alias: string]: string; - }; - extractors: { - [language: string]: Array<{ - extractor_root: string; - extractor_options?: any; - }>; - }; -} - -/** Returns true if `x` is a {@link ResolveLanguagesOutput}. */ -export function isResolveLanguagesOutput( - x: unknown, -): x is ResolveLanguagesOutput { - return ( - json.isObject(x) && - json.isObject(x.extractors) && - Object.values(x.extractors).every( - (extractorList) => - json.isArray(extractorList) && - extractorList.every( - (extractor) => - json.isObject<{ extractor_root: unknown }>(extractor) && - json.isString(extractor.extractor_root), - ), - ) && - (x.aliases === undefined || - (json.isObject(x.aliases) && - Object.values(x.aliases).every((alias) => json.isString(alias)))) - ); -} - export interface ResolveBuildEnvironmentOutput { configuration?: { [language: string]: { @@ -550,19 +484,13 @@ export async function getCodeQLForTesting( * @param key The cache key identifying the command's output. * @param cmd The path to the CodeQL CLI the output is obtained from. * @param run Invokes the CLI to compute the output when it isn't cached. - * @param validate Optional type guard for values loaded from the temporary file. */ async function getCachedOrRun( key: K, cmd: string, run: () => Promise, - validate?: (output: unknown) => output is CommandCacheKeyOutputMap[K], ): Promise { - let result = getCachedCommandOutput( - key, - cmd, - validate, - ); + let result = getCachedCommandOutput(key, cmd); if (result === undefined) { result = await run(); cacheCommandOutput(key, cmd, result); @@ -587,14 +515,10 @@ async function getCodeQLForCmd( return cmd; }, async getVersion() { - return getCachedOrRun( - "version", - cmd, - () => - runCliJson(cmd, ["version", "--format=json"], { - noStreamStdout: true, - }), - isVersionInfo, + return getCachedOrRun("version", cmd, () => + runCliJson(cmd, ["version", "--format=json"], { + noStreamStdout: true, + }), ); }, async printVersion() { @@ -794,31 +718,26 @@ async function getCodeQLForCmd( await runCli(cmd, args); }, async resolveLanguages() { - return getCachedOrRun( - "resolve languages", - cmd, - async () => { - const isFilterToLanguagesWithQueriesSupported = - await this.supportsFeature( - ToolsFeature.BuiltinExtractorsSpecifyDefaultQueries, - ); - return runCliJson(cmd, [ - "resolve", - "languages", - "--format=betterjson", - "--extractor-options-verbosity=4", - "--extractor-include-aliases", - // TODO: Unconditionally include `--filter-to-languages-with-queries` - // once CODEQL_MINIMUM_VERSION is at least v2.23.0 - // — the first version to support this flag. - ...(isFilterToLanguagesWithQueriesSupported - ? ["--filter-to-languages-with-queries"] - : []), - ...getExtraOptionsFromEnv(["resolve", "languages"]), - ]); - }, - isResolveLanguagesOutput, - ); + return getCachedOrRun("resolve languages", cmd, async () => { + const isFilterToLanguagesWithQueriesSupported = + await this.supportsFeature( + ToolsFeature.BuiltinExtractorsSpecifyDefaultQueries, + ); + return runCliJson(cmd, [ + "resolve", + "languages", + "--format=betterjson", + "--extractor-options-verbosity=4", + "--extractor-include-aliases", + // TODO: Unconditionally include `--filter-to-languages-with-queries` + // once CODEQL_MINIMUM_VERSION is at least v2.23.0 + // — the first version to support this flag. + ...(isFilterToLanguagesWithQueriesSupported + ? ["--filter-to-languages-with-queries"] + : []), + ...getExtraOptionsFromEnv(["resolve", "languages"]), + ]); + }); }, async resolveBuildEnvironment( workingDir: string | undefined, diff --git a/src/status-report.ts b/src/status-report.ts index 43b279f1b..7cf152ce1 100644 --- a/src/status-report.ts +++ b/src/status-report.ts @@ -13,7 +13,6 @@ import { } from "./actions-util"; import { getAnalysisKey, getApiClient } from "./api-client"; import { getCachedCommandOutput } from "./cli/output-cache"; -import { isVersionInfo } from "./codeql"; import { parseRegistriesWithoutCredentials, type Config } from "./config-utils"; import { DependencyCacheRestoreStatusReport } from "./dependency-caching"; import { DocUrl } from "./doc-url"; @@ -284,11 +283,7 @@ export async function createStatusReportBase( core.exportVariable(EnvVar.WORKFLOW_STARTED_AT, workflowStartedAt); } const runnerOs = getRequiredEnvParam("RUNNER_OS"); - const codeQlCliVersion = getCachedCommandOutput( - "version", - undefined, - isVersionInfo, - ); + const codeQlCliVersion = getCachedCommandOutput("version", undefined); const actionRef = process.env["GITHUB_ACTION_REF"] || ""; const testingEnvironment = getTestingEnvironment(); // re-export the testing environment variable so that it is available to subsequent steps, diff --git a/src/testing-utils.ts b/src/testing-utils.ts index d47f3be74..50876b21f 100644 --- a/src/testing-utils.ts +++ b/src/testing-utils.ts @@ -15,6 +15,7 @@ import { AnalysisKind } from "./analyses"; import * as apiClient from "./api-client"; import { GitHubApiDetails } from "./api-client"; import { CachingKind } from "./caching-utils"; +import type { VersionInfo } from "./cli/output-cache"; import { resetCachedCommandOutputs } from "./cli/output-cache"; import * as codeql from "./codeql"; import { Config } from "./config-utils"; @@ -479,7 +480,7 @@ export const makeVersionInfo = ( version: string, features?: { [name: string]: boolean }, overlayVersion?: number, -): codeql.VersionInfo => ({ +): VersionInfo => ({ version, features, overlayVersion, diff --git a/src/tools-features.ts b/src/tools-features.ts index ff87b754d..55be17883 100644 --- a/src/tools-features.ts +++ b/src/tools-features.ts @@ -1,6 +1,6 @@ import * as semver from "semver"; -import type { VersionInfo } from "./codeql"; +import type { VersionInfo } from "./cli/output-cache"; export enum ToolsFeature { BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries",