Merge branch 'main' into henrymercer/avoid-waiting-for-timeout

This commit is contained in:
Henry Mercer
2023-01-20 16:29:49 +00:00
9014 changed files with 730797 additions and 508780 deletions

View File

@@ -5,6 +5,7 @@ import * as path from "path";
import * as core from "@actions/core";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as safeWhich from "@chrisgavin/safe-which";
import { JSONSchemaForNPMPackageJsonFiles } from "@schemastore/package";
import * as api from "./api-client";
import { Config } from "./config-utils";
@@ -23,7 +24,7 @@ import {
import { getWorkflowPath } from "./workflow";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
const pkg = require("../package.json") as JSONSchemaForNPMPackageJsonFiles;
/**
* Wrapper around core.getInput for inputs that always have a value.
@@ -382,6 +383,10 @@ export function getActionsStatus(
}
}
export function getActionVersion(): string {
return pkg.version!;
}
/**
* Compose a StatusReport.
*
@@ -445,7 +450,7 @@ export async function createStatusReportBase(
status,
testing_environment: testingEnvironment,
runner_os: runnerOs,
action_version: pkg.version,
action_version: getActionVersion(),
};
// Add optional parameters

View File

@@ -31,9 +31,6 @@ import { UploadResult } from "./upload-lib";
import * as util from "./util";
import { checkForTimeout } from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
interface AnalysisStatusReport
extends upload_lib.UploadStatusReport,
QueriesStatusReport {}
@@ -180,8 +177,7 @@ async function run() {
let trapCacheUploadTime: number | undefined = undefined;
let dbCreationTimings: DatabaseCreationTimings | undefined = undefined;
let didUploadTrapCaches = false;
util.initializeEnvironment(pkg.version);
await util.checkActionVersion(pkg.version);
util.initializeEnvironment(actionsUtil.getActionVersion());
const logger = getActionsLogger();
try {

View File

@@ -282,6 +282,7 @@ test("validateQueryFilters", (t) => {
t.throws(
() => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
return validateQueryFilters({ exclude: "foo" } as any);
},
{

View File

@@ -454,7 +454,7 @@ export function createQuerySuiteContents(
queryFilters: configUtils.QueryFilter[]
) {
return yaml.dump(
queries.map((q: string) => ({ query: q })).concat(queryFilters as any)
queries.map((q: string) => ({ query: q })).concat(queryFilters as any[])
);
}

View File

@@ -7,9 +7,6 @@ import { getApiClient } from "./api-client";
import { setupTests } from "./testing-utils";
import * as util from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
setupTests(test);
let pluginStub: sinon.SinonStub;
@@ -19,7 +16,7 @@ test.beforeEach(() => {
pluginStub = sinon.stub(githubUtils.GitHub, "plugin");
githubStub = sinon.stub();
pluginStub.returns(githubStub);
util.initializeEnvironment(pkg.version);
util.initializeEnvironment(actionsUtil.getActionVersion());
});
test("getApiClient", async (t) => {
@@ -39,7 +36,7 @@ test("getApiClient", async (t) => {
auth: "token xyz",
baseUrl: "http://api.github.localhost",
log: sinon.match.any,
userAgent: `CodeQL-Action/${pkg.version}`,
userAgent: `CodeQL-Action/${actionsUtil.getActionVersion()}`,
})
);
});

View File

@@ -2,13 +2,10 @@ import * as githubUtils from "@actions/github/lib/utils";
import * as retry from "@octokit/plugin-retry";
import consoleLogLevel from "console-log-level";
import { getRequiredInput } from "./actions-util";
import { getActionVersion, getRequiredInput } from "./actions-util";
import * as util from "./util";
import { getRequiredEnvParam, GitHubVersion } from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
export enum DisallowedAPIVersionReason {
ACTION_TOO_OLD,
ACTION_TOO_NEW,
@@ -39,7 +36,7 @@ function createApiClientWithDetails(
return new retryingOctokit(
githubUtils.getOctokitOptions(auth, {
baseUrl: apiDetails.apiURL,
userAgent: `CodeQL-Action/${pkg.version}`,
userAgent: `CodeQL-Action/${getActionVersion()}`,
log: consoleLogLevel({ level: "debug" }),
})
);

View File

@@ -3,6 +3,7 @@ import * as core from "@actions/core";
import {
createStatusReportBase,
getActionsStatus,
getActionVersion,
getOptionalInput,
getTemporaryDirectory,
sendStatusReport,
@@ -15,14 +16,10 @@ import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import {
DID_AUTOBUILD_GO_ENV_VAR_NAME,
checkActionVersion,
checkGitHubVersionInRange,
initializeEnvironment,
} from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
interface AutobuildStatusReport extends StatusReportBase {
/** Comma-separated set of languages being auto-built. */
autobuild_languages: string;
@@ -36,7 +33,7 @@ async function sendCompletedStatusReport(
failingLanguage?: string,
cause?: Error
) {
initializeEnvironment(pkg.version);
initializeEnvironment(getActionVersion());
const status = getActionsStatus(cause, failingLanguage);
const statusReportBase = await createStatusReportBase(
@@ -57,7 +54,6 @@ async function sendCompletedStatusReport(
async function run() {
const startedAt = new Date();
const logger = getActionsLogger();
await checkActionVersion(pkg.version);
let currentLanguage: Language | undefined = undefined;
let languages: Language[] | undefined = undefined;
try {

View File

@@ -646,13 +646,13 @@ const injectedConfigMacro = test.macro({
getRunnerLogger(true)
);
const args = runnerConstructorStub.firstCall.args[1];
const args = runnerConstructorStub.firstCall.args[1] as string[];
// should have used an config file
const configArg = args.find((arg: string) =>
arg.startsWith("--codescanning-config=")
);
t.truthy(configArg, "Should have injected a codescanning config");
const configFile = configArg.split("=")[1];
const configFile = configArg!.split("=")[1];
const augmentedConfig = yaml.load(fs.readFileSync(configFile, "utf8"));
t.deepEqual(augmentedConfig, expectedConfig);

View File

@@ -650,6 +650,9 @@ export async function getCodeQLForCmd(
async extractScannedLanguage(config: Config, language: Language) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Get extractor location
//
// Request it using `format=json` so we don't need to strip the trailing new line generated by
// the CLI.
let extractorPath = "";
await new toolrunner.ToolRunner(
cmd,
@@ -676,7 +679,7 @@ export async function getCodeQLForCmd(
// Set trace command
const ext = process.platform === "win32" ? ".cmd" : ".sh";
const traceCommand = path.resolve(
JSON.parse(extractorPath),
JSON.parse(extractorPath) as string,
"tools",
`autobuild${ext}`
);
@@ -1149,6 +1152,6 @@ async function generateCodeScanningConfig(
return configLocation;
}
function cloneObject(obj: any) {
function cloneObject<T>(obj: T): T {
return JSON.parse(JSON.stringify(obj));
}

View File

@@ -9,6 +9,7 @@ import * as sinon from "sinon";
import * as api from "./api-client";
import { getCachedCodeQL, PackDownloadOutput, setCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { RegistryConfigWithCredentials } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger, Logger } from "./logging";
@@ -51,6 +52,7 @@ function mockGetContents(
};
const spyGetContents = sinon
.stub(client.repos, "getContent")
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
.resolves(response as any);
sinon.stub(api, "getApiClient").value(() => client);
sinon.stub(api, "getApiClientWithExternalAuth").value(() => client);
@@ -66,6 +68,7 @@ function mockListLanguages(languages: string[]) {
for (const language of languages) {
response.data[language] = 123;
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
sinon.stub(client.repos, "listLanguages").resolves(response as any);
sinon.stub(api, "getApiClient").value(() => client);
}
@@ -2310,7 +2313,7 @@ test("downloadPacks-with-registries", async (t) => {
const expectedConfigFile = path.join(tmpDir, "qlconfig.yml");
const packDownloadStub = sinon.stub();
packDownloadStub.callsFake((packs, configFile) => {
packDownloadStub.callsFake((packs, configFile: string) => {
t.deepEqual(configFile, expectedConfigFile);
// verify the env vars were set correctly
t.deepEqual(process.env.GITHUB_TOKEN, sampleApiDetails.auth);
@@ -2439,7 +2442,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
codeQL,
[Language.javascript, Language.java, Language.python],
{},
registries as any,
registries as RegistryConfigWithCredentials[] | undefined,
sampleApiDetails,
tmpDir,
logger

View File

@@ -1931,7 +1931,9 @@ export async function downloadPacks(
}
if (numPacksDownloaded > 0) {
logger.info(
`Downloaded ${numPacksDownloaded} ${packs === 1 ? "pack" : "packs"}`
`Downloaded ${numPacksDownloaded} ${
numPacksDownloaded === 1 ? "pack" : "packs"
}`
);
} else {
logger.info("No packs to download");

View File

@@ -36,7 +36,9 @@ export async function uploadDebugArtifacts(
const matrix = getRequiredInput("matrix");
if (matrix) {
try {
for (const [, matrixVal] of Object.entries(JSON.parse(matrix)).sort())
for (const [, matrixVal] of Object.entries(
JSON.parse(matrix) as any[][]
).sort())
suffix += `-${matrixVal}`;
} catch (e) {
core.info(
@@ -163,7 +165,7 @@ export async function uploadDatabaseBundleDebugArtifact(
) {
for (const language of config.languages) {
try {
let databaseBundlePath;
let databaseBundlePath: string;
if (!dbIsFinalized(config, language, logger)) {
databaseBundlePath = await createPartialDatabaseBundle(
config,

View File

@@ -367,5 +367,6 @@ class GitHubFeatureFlags implements FeatureEnablement {
);
}
}
return {};
}
}

View File

@@ -201,7 +201,7 @@ test("addFingerprints", async (t) => {
fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting.input.sarif`)
.toString()
);
) as util.SarifFile;
const expected = JSON.parse(
fs
.readFileSync(
@@ -229,7 +229,7 @@ test("missingRegions", async (t) => {
fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting2.input.sarif`)
.toString()
);
) as util.SarifFile;
const expected = JSON.parse(
fs
.readFileSync(

View File

@@ -41,12 +41,12 @@ type hashCallback = (lineNumber: number, hash: string) => void;
*/
export async function hash(callback: hashCallback, filepath: string) {
// A rolling view in to the input
const window = Array(BLOCK_SIZE).fill(0);
const window: number[] = Array(BLOCK_SIZE).fill(0);
// If the character in the window is the start of a new line
// then records the line number, otherwise will be -1.
// Indexes match up with those from the window variable.
const lineNumbers = Array(BLOCK_SIZE).fill(-1);
const lineNumbers: number[] = Array(BLOCK_SIZE).fill(-1);
// The current hash value, updated as we read each character
let hashRaw = Long.ZERO;
@@ -120,7 +120,7 @@ export async function hash(callback: hashCallback, filepath: string) {
const readStream = fs.createReadStream(filepath, "utf8");
for await (const data of readStream) {
for (let i = 0; i < data.length; ++i) {
processCharacter(data.charCodeAt(i));
processCharacter((data as string).charCodeAt(i));
}
}
processCharacter(EOF);
@@ -201,7 +201,7 @@ export function resolveUriToFile(
logger.debug(`Ignoring location as URI "${location.uri}" is invalid`);
return undefined;
}
let uri = decodeURIComponent(location.uri);
let uri = decodeURIComponent(location.uri as string);
// Remove a file scheme, and abort if the scheme is anything else
const fileUriPrefix = "file://";

View File

@@ -5,6 +5,7 @@ import * as core from "@actions/core";
import {
createStatusReportBase,
getActionsStatus,
getActionVersion,
getOptionalInput,
getRequiredInput,
getTemporaryDirectory,
@@ -27,7 +28,6 @@ import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { getTotalCacheSize } from "./trap-caching";
import {
checkActionVersion,
checkForTimeout,
checkGitHubVersionInRange,
codeQlVersionAbove,
@@ -44,9 +44,6 @@ import {
} from "./util";
import { validateWorkflow } from "./workflow";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
interface InitSuccessStatusReport extends StatusReportBase {
/** Comma-separated list of languages where the default queries are disabled. */
disable_default_queries: string;
@@ -138,8 +135,7 @@ async function sendSuccessStatusReport(
async function run() {
const startedAt = new Date();
const logger = getActionsLogger();
initializeEnvironment(pkg.version);
await checkActionVersion(pkg.version);
initializeEnvironment(getActionVersion());
let config: configUtils.Config;
let codeql: CodeQL;

View File

@@ -1,3 +1,5 @@
import { TextDecoder } from "node:util";
import * as github from "@actions/github";
import { TestFn } from "ava";
import * as nock from "nock";
@@ -178,6 +180,7 @@ export function mockLanguagesInRepo(languages: string[]) {
url: "GET /repos/:owner/:repo/languages",
});
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
mockClient.returns({
repos: {
listLanguages,

View File

@@ -68,7 +68,7 @@ export async function downloadTrapCaches(
languages: Language[],
logger: Logger
): Promise<Partial<Record<Language, string>>> {
const result = {};
const result: Partial<Record<Language, string>> = {};
const languagesSupportingCaching = await getLanguagesSupportingCaching(
codeql,
languages,

View File

@@ -226,7 +226,8 @@ export function countResultsInSarif(sarif: string): number {
// Throws an error if the file is invalid.
export function validateSarifFileSchema(sarifFilePath: string, logger: Logger) {
const sarif = JSON.parse(fs.readFileSync(sarifFilePath, "utf8"));
const schema = require("../src/sarif_v2.1.0_schema.json");
const schema =
require("../src/sarif_v2.1.0_schema.json") as jsonschema.Schema;
const result = new jsonschema.Validator().validate(sarif, schema);
if (!result.valid) {

View File

@@ -1,19 +1,16 @@
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
import { getActionVersion } from "./actions-util";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as upload_lib from "./upload-lib";
import {
checkActionVersion,
getRequiredEnvParam,
initializeEnvironment,
isInTestMode,
} from "./util";
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
interface UploadSarifStatusReport
extends actionsUtil.StatusReportBase,
upload_lib.UploadStatusReport {}
@@ -36,8 +33,7 @@ async function sendSuccessStatusReport(
async function run() {
const startedAt = new Date();
initializeEnvironment(pkg.version);
await checkActionVersion(pkg.version);
initializeEnvironment(getActionVersion());
if (
!(await actionsUtil.sendStatusReport(
await actionsUtil.createStatusReportBase(

View File

@@ -2,7 +2,6 @@ import * as fs from "fs";
import * as os from "os";
import path from "path";
import * as core from "@actions/core";
import * as github from "@actions/github";
import test from "ava";
import * as sinon from "sinon";
@@ -26,7 +25,7 @@ test("getToolNames", (t) => {
`${__dirname}/../src/testdata/tool-names.sarif`,
"utf8"
);
const toolNames = util.getToolNames(JSON.parse(input));
const toolNames = util.getToolNames(JSON.parse(input) as util.SarifFile);
t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
});
@@ -204,6 +203,7 @@ function mockGetMetaVersionHeader(
};
const spyGetContents = sinon
.stub(client.meta, "get")
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
.resolves(response as any);
sinon.stub(api, "getApiClient").value(() => client);
return spyGetContents;
@@ -311,65 +311,6 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
});
}
function formatGitHubVersion(version: util.GitHubVersion): string {
switch (version.type) {
case util.GitHubVariant.DOTCOM:
return "dotcom";
case util.GitHubVariant.GHAE:
return "GHAE";
case util.GitHubVariant.GHES:
return `GHES ${version.version}`;
default:
util.assertNever(version);
}
}
const CHECK_ACTION_VERSION_TESTS: Array<[string, util.GitHubVersion, boolean]> =
[
["1.2.1", { type: util.GitHubVariant.DOTCOM }, true],
["1.2.1", { type: util.GitHubVariant.GHAE }, true],
["1.2.1", { type: util.GitHubVariant.GHES, version: "3.3" }, true],
["1.2.1", { type: util.GitHubVariant.GHES, version: "3.4" }, true],
["1.2.1", { type: util.GitHubVariant.GHES, version: "3.5" }, true],
["2.2.1", { type: util.GitHubVariant.DOTCOM }, false],
["2.2.1", { type: util.GitHubVariant.GHAE }, false],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.3" }, false],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.4" }, false],
["2.2.1", { type: util.GitHubVariant.GHES, version: "3.5" }, false],
];
for (const [
version,
githubVersion,
shouldReportError,
] of CHECK_ACTION_VERSION_TESTS) {
const reportErrorDescription = shouldReportError
? "reports error"
: "doesn't report error";
const versionsDescription = `CodeQL Action version ${version} and GitHub version ${formatGitHubVersion(
githubVersion
)}`;
test(`checkActionVersion ${reportErrorDescription} for ${versionsDescription}`, async (t) => {
const errorSpy = sinon.spy(core, "error");
const versionStub = sinon
.stub(api, "getGitHubVersion")
.resolves(githubVersion);
await util.checkActionVersion(version);
if (shouldReportError) {
t.true(
errorSpy.calledOnceWithExactly(
sinon.match(
"This version of the CodeQL Action was deprecated on January 18th, 2023"
)
)
);
} else {
t.false(errorSpy.called);
}
versionStub.restore();
});
}
test("doesDirectoryExist", async (t) => {
// Returns false if no file/dir of this name exists
t.false(util.doesDirectoryExist("non-existent-file.txt"));

View File

@@ -630,24 +630,6 @@ export function getMlPoweredJsQueriesStatus(config: Config): string {
}
}
/**
* Prompt the customer to upgrade to CodeQL Action v2, if appropriate.
*
* Check whether a customer is running v1. If they are, and we can determine that the GitHub
* instance supports v2, then log an error that v1 is deprecated and prompt the customer to
* upgrade to v2.
*/
export async function checkActionVersion(version: string) {
if (!semver.satisfies(version, ">=2")) {
core.error(
"This version of the CodeQL Action was deprecated on January 18th, 2023, and is no longer " +
"updated or supported. For better performance, improved security, and new features, " +
"upgrade to v2. For more information, see " +
"https://github.blog/changelog/2023-01-18-code-scanning-codeql-action-v1-is-now-deprecated/"
);
}
}
/*
* Returns whether we are in test mode.
*

View File

@@ -144,7 +144,7 @@ test("getWorkflowErrors() for a range of malformed workflows", (t) => {
push: 1,
pull_request: 1,
},
} as any),
} as Workflow),
[]
)
);
@@ -153,13 +153,14 @@ test("getWorkflowErrors() for a range of malformed workflows", (t) => {
...errorCodes(
getWorkflowErrors({
on: 1,
} as any),
} as Workflow),
[]
)
);
t.deepEqual(
...errorCodes(
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
getWorkflowErrors({
on: 1,
jobs: 1,
@@ -170,6 +171,7 @@ test("getWorkflowErrors() for a range of malformed workflows", (t) => {
t.deepEqual(
...errorCodes(
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
getWorkflowErrors({
on: 1,
jobs: [1],
@@ -183,7 +185,7 @@ test("getWorkflowErrors() for a range of malformed workflows", (t) => {
getWorkflowErrors({
on: 1,
jobs: { 1: 1 },
} as any),
} as Workflow),
[]
)
);
@@ -193,7 +195,7 @@ test("getWorkflowErrors() for a range of malformed workflows", (t) => {
getWorkflowErrors({
on: 1,
jobs: { test: 1 },
} as any),
} as Workflow),
[]
)
);
@@ -203,13 +205,14 @@ test("getWorkflowErrors() for a range of malformed workflows", (t) => {
getWorkflowErrors({
on: 1,
jobs: { test: [1] },
} as any),
} as Workflow),
[]
)
);
t.deepEqual(
...errorCodes(
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
getWorkflowErrors({
on: 1,
jobs: { test: { steps: 1 } },
@@ -220,6 +223,7 @@ test("getWorkflowErrors() for a range of malformed workflows", (t) => {
t.deepEqual(
...errorCodes(
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
getWorkflowErrors({
on: 1,
jobs: { test: { steps: [{ notrun: "git checkout HEAD^2" }] } },
@@ -233,15 +237,16 @@ test("getWorkflowErrors() for a range of malformed workflows", (t) => {
getWorkflowErrors({
on: 1,
jobs: { test: [undefined] },
} as any),
} as Workflow),
[]
)
);
t.deepEqual(...errorCodes(getWorkflowErrors(1 as any), []));
t.deepEqual(...errorCodes(getWorkflowErrors(1 as Workflow), []));
t.deepEqual(
...errorCodes(
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
getWorkflowErrors({
on: {
push: {