Compare commits

..

4 Commits

Author SHA1 Message Date
Michael B. Gale
af8f50bf94 Use applyAnalysisKindConfig in loadUserConfig 2026-07-28 13:52:29 +01:00
Michael B. Gale
f638e8b5fb Add applyAnalysisKindConfig 2026-07-28 13:49:56 +01:00
Michael B. Gale
391a05d414 Reshuffle loadUserConfig slightly 2026-07-28 12:46:49 +01:00
Michael B. Gale
fdf46cd70c Add ScopedConfigurations feature 2026-07-28 12:46:47 +01:00
75 changed files with 2964 additions and 4871 deletions

View File

@@ -25,6 +25,17 @@ runs:
shell: bash
run: npm ci
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: '3.12'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install PyGithub==2.3.0 requests
shell: bash
- name: Update git config
run: |
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"

View File

@@ -63,7 +63,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install Java
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
with:
java-version: ${{ inputs.java-version || '17' }}
distribution: temurin

View File

@@ -63,7 +63,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install Java
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
with:
java-version: ${{ inputs.java-version || '17' }}
distribution: temurin

View File

@@ -71,8 +71,8 @@ jobs:
run: |
cd "$RUNNER_TEMP/results"
actual=$(jq -r '.runs[0].properties.jobRunUuid' javascript.sarif)
if [[ "$actual" != "$CODEQL_ACTION_JOB_RUN_UUID" ]]; then
echo "Expected SARIF output to contain job run UUID '$CODEQL_ACTION_JOB_RUN_UUID', but found '$actual'."
if [[ "$actual" != "$JOB_RUN_UUID" ]]; then
echo "Expected SARIF output to contain job run UUID '$JOB_RUN_UUID', but found '$actual'."
exit 1
else
echo "Found job run UUID '$actual'."

View File

@@ -54,7 +54,7 @@ jobs:
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- name: Set up Ruby
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
uses: ruby/setup-ruby@003a5c4d8d6321bd302e38f6f0ec593f77f06600 # v1.319.0
with:
ruby-version: 2.6
- name: Install Code Scanning integration

View File

@@ -113,6 +113,7 @@ jobs:
matrix:
include:
- language: actions
- language: python
permissions:
contents: read

View File

@@ -51,9 +51,9 @@ jobs:
with:
node-version: 24
cache: 'npm'
- name: Install JavaScript dependencies
run: npm ci
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.12'
- name: Update git config
run: |
@@ -127,7 +127,7 @@ jobs:
env:
PARTIAL_CHANGELOG: "${{ runner.temp }}/partial_changelog.md"
run: |
npx tsx pr-checks/prepare-changelog.ts --output="$PARTIAL_CHANGELOG"
python .github/workflows/script/prepare_changelog.py CHANGELOG.md > $PARTIAL_CHANGELOG
echo "::group::Partial CHANGELOG"
cat $PARTIAL_CHANGELOG

View File

@@ -93,7 +93,7 @@ jobs:
LATEST_TAG: ${{ needs.prepare.outputs.latest_tag }}
VERSION: "${{ needs.prepare.outputs.version }}"
run: |
npx tsx pr-checks/rollback-changelog.ts \
python .github/workflows/script/rollback_changelog.py \
--target-version "${ROLLBACK_TAG:1}" \
--rollback-version "${LATEST_TAG:1}" \
--new-version "$VERSION" > $NEW_CHANGELOG
@@ -128,9 +128,7 @@ jobs:
NEW_CHANGELOG: "${{ runner.temp }}/new_changelog.md"
PARTIAL_CHANGELOG: "${{ runner.temp }}/partial_changelog.md"
run: |
npx tsx pr-checks/prepare-changelog.ts \
--changelog="$NEW_CHANGELOG" \
--output="$PARTIAL_CHANGELOG"
python .github/workflows/script/prepare_changelog.py $NEW_CHANGELOG > $PARTIAL_CHANGELOG
echo "::group::Partial CHANGELOG"
cat $PARTIAL_CHANGELOG

23
.github/workflows/script/bundle_changelog.py vendored Executable file
View File

@@ -0,0 +1,23 @@
#!/usr/bin/env python3
import os
import re
cli_version = os.environ['CLI_VERSION']
# The GitHub Release for the new bundle version.
bundle_release_url = f"https://github.com/github/codeql-action/releases/tag/codeql-bundle-v{cli_version}"
# Get the PR number from the PR URL.
pr_number = os.environ['PR_URL'].split('/')[-1]
changelog_note = f"- Update default CodeQL bundle version to [{cli_version}]({bundle_release_url}). [#{pr_number}]({os.environ['PR_URL']})"
# If the "[UNRELEASED]" section starts with "no user facing changes", remove that line.
with open('CHANGELOG.md', 'r') as f:
changelog = f.read()
changelog = changelog.replace('## [UNRELEASED]\n\nNo user facing changes.', '## [UNRELEASED]\n')
# Add the changelog note to the bottom of the "[UNRELEASED]" section.
changelog = re.sub(r'\n## (\d+\.\d+\.\d+)', f'{changelog_note}\n\n## \\1', changelog, count=1)
with open('CHANGELOG.md', 'w') as f:
f.write(changelog)

35
.github/workflows/script/prepare_changelog.py vendored Executable file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/env python3
import os
import sys
EMPTY_CHANGELOG = 'No changes.\n\n'
# Prepare the changelog for the new release
# This function will extract the part of the changelog that
# we want to include in the new release.
def extract_changelog_snippet(changelog_file):
output = ''
if (not os.path.exists(changelog_file)):
output = EMPTY_CHANGELOG
else:
with open(changelog_file, 'r') as f:
lines = f.readlines()
# Include only the contents of the first section
found_first_section = False
for line in lines:
if line.startswith('## '):
if found_first_section:
break
found_first_section = True
elif found_first_section:
output += line
return output.strip()
if len(sys.argv) < 2:
raise Exception('Expecting argument: changelog_file')
changelog_file = sys.argv[1]
print(extract_changelog_snippet(changelog_file))

View File

@@ -0,0 +1,62 @@
import datetime
import os
import argparse
EMPTY_CHANGELOG = """# CodeQL Action Changelog
"""
def get_today_string():
today = datetime.datetime.today()
return '{:%d %b %Y}'.format(today)
# Include everything up to and after the first heading,
# but not the first heading and body.
def drop_unreleased_section(lines: list[str]):
before_first_section = ''
after_first_section = ''
found_first_section = False
skipped_first_section = False
for i, line in enumerate(lines):
if line.startswith('## ') and not found_first_section:
found_first_section = True
elif line.startswith('## ') and found_first_section:
skipped_first_section = True
if not found_first_section:
before_first_section += line
if skipped_first_section:
after_first_section += line
return (before_first_section, after_first_section)
def update_changelog(target_version, rollback_version, new_version):
before_first_section = EMPTY_CHANGELOG
after_first_section = ''
if (os.path.exists('CHANGELOG.md')):
with open('CHANGELOG.md', 'r') as f:
(before_first_section, after_first_section) = drop_unreleased_section(f.readlines())
newHeader = f'## {new_version} - {get_today_string()}\n'
print(before_first_section, end="")
print(newHeader)
print(f"This release rolls back {rollback_version} due to issues with that release. It is identical to {target_version}.\n")
print(after_first_section)
# We expect three version strings as input:
#
# - target_version: the version that we are re-releasing as `new_version`
# - rollback_version: the version that we are rolling back, typically the one that followed `target_version`
# - new_version: the new version that we are releasing `target_version` as, typically the one that follows `rollback_version`
#
# Example: python3 .github/workflows/script/rollback_changelog.py --target-version "1.2.3" --rollback-version "1.2.4" --new-version "1.2.5"
parser = argparse.ArgumentParser(description="Update CHANGELOG.md for a rollback release.")
parser.add_argument("--target-version", "-t", required=True, help="Version to re-release as new_version.")
parser.add_argument("--rollback-version", "-r", required=True, help="Version being rolled back.")
parser.add_argument("--new-version", "-n", required=True, help="New version to publish for target_version.")
args = parser.parse_args()
update_changelog(args.target_version, args.rollback_version, args.new_version)

View File

@@ -40,6 +40,11 @@ jobs:
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.12'
- name: Set up Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
@@ -115,7 +120,7 @@ jobs:
- name: Create changelog note
run: |
npx tsx pr-checks/bundle-changelog.ts
python .github/workflows/script/bundle_changelog.py
- name: Push changelog note
run: |

View File

@@ -22,6 +22,11 @@ jobs:
pull-requests: write # needed to create pull request
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.13"
- name: Checkout CodeQL Action
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -38,7 +43,7 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: github/enterprise-releases
token: ${{ secrets.CODEQL_CI_ENTERPRISE_RELEASE_PAT }}
token: ${{ secrets.ENTERPRISE_RELEASE_TOKEN }}
path: ${{ github.workspace }}/enterprise-releases/
sparse-checkout: releases.json

View File

@@ -2,30 +2,9 @@
See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs.
## 4.37.9 - 26 Aug 2026
- Update default CodeQL bundle version to [2.26.4](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.26.4). [#4106](https://github.com/github/codeql-action/pull/4106)
## 4.37.8 - 21 Aug 2026
No user facing changes.
## 4.37.7 - 13 Aug 2026
- Update default CodeQL bundle version to [2.26.3](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.26.3). [#4085](https://github.com/github/codeql-action/pull/4085)
## 4.37.6 - 04 Aug 2026
- Changed the default filepath for the new remote file address format that was introduced in CodeQL Action 4.37.0 / 3.37.0 to `.github/codeql-config.yml` to align it with the suggested path that is used elsewhere. [#4070](https://github.com/github/codeql-action/pull/4070)
## 4.37.5 - 03 Aug 2026
- Fixed a bug where a network error while streaming the download of the CodeQL bundle could terminate the `init` Action instead of falling back to downloading the bundle before extracting it. [#4061](https://github.com/github/codeql-action/pull/4061)
## 4.37.4 - 29 Jul 2026
## [UNRELEASED]
- This version of the CodeQL Action adds support for the `tools` input for the `codeql-action/init` step to be specified using a `github-codeql-tools` [repository property](https://docs.github.com/en/organizations/managing-organization-settings/managing-custom-properties-for-repositories-in-your-organization). This feature will gradually be rolled out following the release of this version. Once rolled out, this allows for the CodeQL CLI version that is used in GitHub-managed workflows, such as Default Setup, to be set to a custom value. For example, customers who run into issues with rate limits when a new CodeQL CLI version is released can set the value to `toolcache` to always use the CodeQL CLI version that is available in the runner toolcache. For Advanced Setup workflows, the value provided for `tools` in the workflow definition always takes precedence unless the value of the repository property starts with `!`. [#4037](https://github.com/github/codeql-action/pull/4037)
- Update default CodeQL bundle version to [2.26.2](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.26.2). [#4051](https://github.com/github/codeql-action/pull/4051)
## 4.37.3 - 22 Jul 2026

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.26.4",
"cliVersion": "2.26.4",
"priorBundleVersion": "codeql-bundle-v2.26.3",
"priorCliVersion": "2.26.3"
"bundleVersion": "codeql-bundle-v2.26.1",
"cliVersion": "2.26.1",
"priorBundleVersion": "codeql-bundle-v2.26.0",
"priorCliVersion": "2.26.0"
}

4349
lib/entry-points.js generated

File diff suppressed because it is too large Load Diff

736
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "4.37.9",
"version": "4.37.4",
"private": true,
"description": "CodeQL action",
"scripts": {
@@ -30,22 +30,22 @@
"@actions/http-client": "^3.0.0",
"@actions/io": "^2.0.0",
"@actions/tool-cache": "^3.0.1",
"@octokit/core": "^7.0.7",
"@octokit/core": "^7.0.6",
"@octokit/plugin-paginate-rest": "^14.0.0",
"@octokit/plugin-rest-endpoint-methods": "^17.0.0",
"@octokit/plugin-retry": "^8.1.1",
"@octokit/plugin-retry": "^8.1.0",
"archiver": "^8.0.0",
"fast-deep-equal": "^3.1.3",
"follow-redirects": "^1.16.0",
"get-folder-size": "^5.0.0",
"https-proxy-agent": "^7.0.6",
"js-yaml": "^5.2.3",
"js-yaml": "^5.2.1",
"jsonschema": "1.5.0",
"long": "^5.3.2",
"node-forge": "^1.4.0",
"semver": "^7.8.5",
"uuid": "^14.0.1",
"undici": "^6.28.0"
"undici": "^6.24.0"
},
"devDependencies": {
"@ava/typescript": "6.0.0",
@@ -58,22 +58,22 @@
"@types/node": "^20.19.43",
"@types/node-forge": "^1.3.14",
"@types/sarif": "^2.1.7",
"@types/semver": "^7.8.0",
"@types/semver": "^7.7.1",
"@types/sinon": "^22.0.0",
"ava": "^6.4.1",
"esbuild": "^0.28.2",
"esbuild": "^0.28.1",
"eslint": "^9.39.5",
"eslint-import-resolver-typescript": "^4.4.5",
"eslint-plugin-github": "^6.1.2",
"eslint-plugin-github": "^6.1.1",
"eslint-plugin-import-x": "^4.17.1",
"eslint-plugin-jsdoc": "^62.9.0",
"eslint-plugin-no-async-foreach": "^0.1.1",
"glob": "^13.0.6",
"globals": "^17.11.0",
"nock": "^14.0.17",
"sinon": "^22.1.0",
"globals": "^17.7.0",
"nock": "^14.0.16",
"sinon": "^22.0.0",
"typescript": "^6.0.3",
"typescript-eslint": "^8.67.0"
"typescript-eslint": "^8.64.0"
},
"overrides": {
"@actions/tool-cache": {
@@ -95,6 +95,6 @@
"semver": ">=6.3.1"
},
"glob": "^13.0.6",
"undici": "^6.28.0"
"undici": "^6.24.0"
}
}

View File

@@ -1,142 +0,0 @@
/**
* Tests for `bundle-changelog.ts`.
*/
import * as assert from "node:assert/strict";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { afterEach, beforeEach, describe, it } from "node:test";
import {
CLI_VERSION_ENV_VAR,
getCLIVersion,
getPRNumber,
getPRUrl,
PR_URL_ENV_VAR,
updateChangelog,
} from "./bundle-changelog";
import {
EMPTY_CHANGELOG,
NO_CHANGES_STR,
UNRELEASED_PLACEHOLDER,
} from "./changelog";
let testDir: string;
beforeEach(() => {
// Set up a temporary directory for testing
testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bundle-changelog-test-"));
});
afterEach(() => {
/** Clean up temporary directories. */
fs.rmSync(testDir, { recursive: true, force: true });
});
describe("getCLIVersion", async () => {
await it("throws if the environment variable is not set", async () => {
delete process.env[CLI_VERSION_ENV_VAR];
assert.throws(() => getCLIVersion());
});
await it("throws if the environment variable is empty", async () => {
process.env[CLI_VERSION_ENV_VAR] = " ";
assert.throws(() => getCLIVersion());
});
await it("returns value of the environment variable if set", async () => {
const testValue = "1.2.3";
process.env[CLI_VERSION_ENV_VAR] = testValue;
assert.deepEqual(getCLIVersion(), testValue);
});
});
const testPrUrl = "https://github.com/github/codeql-action/pulls/42";
describe("getPRUrl", async () => {
await it("throws if the environment variable is not set", async () => {
delete process.env[PR_URL_ENV_VAR];
assert.throws(() => getPRUrl());
});
await it("throws if the environment variable is empty", async () => {
process.env[PR_URL_ENV_VAR] = " ";
assert.throws(() => getPRUrl());
});
await it("returns value of the environment variable if set", async () => {
process.env[PR_URL_ENV_VAR] = testPrUrl;
assert.deepEqual(getPRUrl(), testPrUrl);
});
});
describe("getPRNumber", async () => {
await it("throws if the last part of the input is not a number", async () => {
assert.throws(() => getPRNumber(`${testPrUrl}/foo`));
});
await it("throws if the last part of the input is not a positive number", async () => {
assert.throws(() => getPRNumber(`${testPrUrl}/-100`));
});
await it("returns the PR number from an URL", async () => {
assert.equal(getPRNumber(testPrUrl), 42);
});
});
const testChangelog = `${EMPTY_CHANGELOG.trimEnd()}
## 4.23.7
- Other change
## 4.23.6
${NO_CHANGES_STR}`;
const expectedChangelog = `# CodeQL Action Changelog
## ${UNRELEASED_PLACEHOLDER}
- Update default CodeQL bundle version to
## 4.23.7
- Other change
## 4.23.6
${NO_CHANGES_STR}`;
describe("updateChangelog", async () => {
await it("removes `NO_CHANGES_STR` if present in [UNRELEASED] section", async () => {
const result = updateChangelog(EMPTY_CHANGELOG, "");
assert.ok(!result.includes(NO_CHANGES_STR.trim()));
});
await it("doesn't remove `NO_CHANGES_STR` if present in versioned section", async () => {
const result = updateChangelog(
EMPTY_CHANGELOG.replace(UNRELEASED_PLACEHOLDER, "1.2.3"),
"",
);
assert.ok(result.includes(NO_CHANGES_STR.trim()));
});
await it("throws if there are no sections", async () => {
assert.throws(() => {
updateChangelog(
"# CodeQL Action Changelog",
"- Update default CodeQL bundle version to",
);
});
});
await it("adds note at the end of the first section", async () => {
const result = updateChangelog(
testChangelog,
"- Update default CodeQL bundle version to",
);
assert.deepEqual(result, expectedChangelog);
});
});

View File

@@ -1,127 +0,0 @@
#!/usr/bin/env npx tsx
/**
* Updates the changelog with a change note for an updated CodeQL CLI bundle.
*/
import * as fs from "node:fs";
import {
parseChangelog,
renderChangelog,
UNRELEASED_PLACEHOLDER,
} from "./changelog";
import { CHANGELOG_FILE, CLI_BUNDLE_RELEASE_URL_PREFIX } from "./config";
import { getErrorMessage } from "./util";
export const CLI_VERSION_ENV_VAR = "CLI_VERSION";
export const PR_URL_ENV_VAR = "PR_URL";
/** Gets the CLI version from the environment. */
export function getCLIVersion() {
const cliVersion = process.env[CLI_VERSION_ENV_VAR];
if (cliVersion === undefined || cliVersion.trim() === "") {
throw new Error(`No CLI version was set in '${CLI_VERSION_ENV_VAR}'.`);
}
return cliVersion;
}
/** Gets the PR URL from the environment. */
export function getPRUrl() {
const prUrl = process.env[PR_URL_ENV_VAR];
if (prUrl === undefined || prUrl.trim() === "") {
throw new Error(`No PR URL was set in '${PR_URL_ENV_VAR}'.`);
}
return prUrl;
}
/**
* Gets the PR number from something like a PR URL.
*/
export function getPRNumber(prUrl: string) {
const prUrlParts = prUrl.split("/");
const prNumberStr = prUrlParts[prUrlParts.length - 1];
const prNumber = Number.parseInt(prNumberStr, 10);
if (!Number.isInteger(prNumber) || prNumber <= 0) {
throw new Error(
`Invalid PR URL '${prUrl}': last part is not a positive number`,
);
}
return prNumber;
}
/**
* Updates `changelog` by adding `changelogNote` to the first section.
*
* @param contents The existing changelog contents.
* @param changelogNote The note to add to the first section.
*/
export function updateChangelog(contents: string, changelogNote: string) {
// If the "[UNRELEASED]" section starts with "no user facing changes", remove that line.
contents = contents.replace(
`## ${UNRELEASED_PLACEHOLDER}\n\nNo user facing changes.`,
`## ${UNRELEASED_PLACEHOLDER}\n`,
);
const changelog = parseChangelog(contents);
if (changelog.sections.length === 0) {
throw new Error("The changelog contains no existing sections.");
}
// Add the changelog note to the bottom of the first section.
const firstSection = changelog.sections[0];
const lastLine = firstSection.bodyLines.pop();
if (lastLine !== undefined && lastLine.trim() !== "") {
// We expect the last line to be empty. If it isn't for some reason,
// add it back.
firstSection.bodyLines.push(lastLine);
}
firstSection.bodyLines.push(changelogNote);
// If the last line is empty as expected, then add it back in after the new note.
if (lastLine?.trim() === "") {
firstSection.bodyLines.push(lastLine);
}
return renderChangelog(changelog);
}
function main() {
try {
const cliVersion = getCLIVersion();
const prUrl = getPRUrl();
// The GitHub Release for the new bundle version.
const bundleReleaseUrl = `${CLI_BUNDLE_RELEASE_URL_PREFIX}${cliVersion}`;
// Get the PR number from the PR URL.
const prNumber = getPRNumber(prUrl);
const changelogNote = `- Update default CodeQL bundle version to [${cliVersion}](${bundleReleaseUrl}). [#${prNumber}](${prUrl})`;
let changelog = fs.readFileSync(CHANGELOG_FILE, "utf-8");
changelog = updateChangelog(changelog, changelogNote);
fs.writeFileSync(CHANGELOG_FILE, changelog);
return 0;
} catch (err) {
console.error(`Failed to bundle changelog: ${getErrorMessage(err)}`);
return -1;
}
}
// Only call `main` if this script was run directly.
if (require.main === module) {
process.exit(main());
}

View File

@@ -5,18 +5,14 @@
*/
import * as assert from "node:assert/strict";
import * as fs from "node:fs";
import { describe, it } from "node:test";
import {
EMPTY_CHANGELOG,
getReleaseDateString,
parseChangelog,
processChangelogForBackports,
renderChangelog,
setVersionAndDate,
} from "./changelog";
import { CHANGELOG_FILE } from "./config";
const testDate = new Date(2026, 7, 14);
@@ -41,14 +37,6 @@ describe("setVersionAndDate", async () => {
});
});
describe("parseChangelog + renderChangelog", async () => {
await it("renderChangelog(parseChangelog(c)) == c", async () => {
const actualChangelog = fs.readFileSync(CHANGELOG_FILE, "utf-8");
const roundtrip = renderChangelog(parseChangelog(actualChangelog));
assert.deepEqual(roundtrip.split("\n"), actualChangelog.split("\n"));
});
});
const testChangelog = `# CodeQL Action Changelog
## 4.12.3 - 14 Aug 2026

View File

@@ -2,34 +2,14 @@ import * as fs from "node:fs";
import { CHANGELOG_FILE, DryRunOption } from "./config";
/** The placeholder in the header for unreleased changes. */
export const UNRELEASED_PLACEHOLDER = "[UNRELEASED]";
/** The default contents for a section in the changelog. */
export const NO_CHANGES_STR = "No user facing changes.\n\n";
/** Placeholder changelog content for a new release. */
export const EMPTY_CHANGELOG = `# CodeQL Action Changelog
## ${UNRELEASED_PLACEHOLDER}
## [UNRELEASED]
${NO_CHANGES_STR}`;
No user facing changes.
/**
* Represents sections in a changelog.
*/
export interface ChangelogSection {
headerLine: string;
bodyLines: string[];
}
/**
* Represents a changelog.
*/
export interface Changelog {
preamble: string[];
sections: ChangelogSection[];
}
`;
/** Returns `date` formatted as `DD Mon YYYY`. */
export function getReleaseDateString(today: Date = new Date()): string {
@@ -73,76 +53,7 @@ export function setVersionAndDate(
date: Date = new Date(),
): string {
const versionAndDate = `${version} - ${getReleaseDateString(date)}`;
return content.replace(UNRELEASED_PLACEHOLDER, versionAndDate);
}
/**
* Parses `content` into a structured representation of a changelog.
*
* @param content The contents of the changelog file.
*/
export function parseChangelog(content: string): Changelog {
const lines = content.split("\n");
let i = 0;
const preamble: string[] = [];
const sections: ChangelogSection[] = [];
let currentSection: ChangelogSection | undefined = undefined;
// Process all lines of the input file.
while (i < lines.length) {
const line = lines[i];
// Sections of the changelog start with `## `.
if (line.startsWith("## ")) {
// We have discovered a new section. If `currentSection` is already defined,
// then this marks the end of that section. Push it to the array of sections
// in the changelog.
if (currentSection !== undefined) {
sections.push(currentSection);
}
// Initialise the new section.
currentSection = { headerLine: line, bodyLines: [] };
} else if (currentSection !== undefined) {
// Add lines between the section header and the next to the current section.
currentSection.bodyLines.push(line);
} else {
// This is neither a section header nor are we in a section already,
// so this line is part of the preamble.
preamble.push(line);
}
i++;
}
// Push the current section to the array of completed sections, if there is
// still one unfinished.
if (currentSection !== undefined) {
sections.push(currentSection);
}
return { preamble, sections };
}
/**
* Combines an array of lines into a single string by adding line breaks.
*/
export function unlines(lines: string[]): string {
return `${lines.join("\n")}`;
}
/**
* Renders a given changelog to a string.
*/
export function renderChangelog(changelog: Changelog): string {
let result = unlines(changelog.preamble);
for (const section of changelog.sections) {
result += `\n${section.headerLine}\n${unlines(section.bodyLines)}`;
}
return result;
return content.replace("[UNRELEASED]", versionAndDate);
}
/**
@@ -155,58 +66,70 @@ export function processChangelogForBackports(
targetBranchMajorVersion: string,
content: string,
): string {
const lines = content.split("\n");
// Changelog entries can use the following format to indicate
// that they only apply to newer versions
const someVersionsOnlyRegex = /\[v(\d+)\+ only\]/;
// Parse the changelog.
const changelog = parseChangelog(content);
let output = "";
let i = 0;
if (changelog.sections.length === 0) {
// Copy lines until we find the first section heading.
let foundFirstSection = false;
while (!foundFirstSection && i < lines.length) {
let line = lines[i];
if (line.startsWith("## ")) {
line = line.replace(
`## ${sourceBranchMajorVersion}`,
`## ${targetBranchMajorVersion}`,
);
foundFirstSection = true;
}
output += `${line}\n`;
i++;
}
if (!foundFirstSection) {
throw new Error("Could not find any change sections in CHANGELOG.md");
}
// Filter out changelog entries that only apply to newer versions and
// update the section headings with the backport major version for
// sections we keep.
for (const section of changelog.sections) {
// Update the section headings with the backport major version.
section.headerLine = section.headerLine.replace(
`## ${sourceBranchMajorVersion}`,
`## ${targetBranchMajorVersion}`,
);
// Process remaining lines.
// `foundContent` tracks whether we hit two headings in a row
let foundContent = false;
output += "\n";
const filteredEntries: string[] = [];
let foundContent = false;
while (i < lines.length) {
let line = lines[i];
i++;
for (const line of section.bodyLines) {
// Skip the entry if `someVersionsOnlyRegex` matches and the major version
// of the target branch is smaller than the required version.
const match = someVersionsOnlyRegex.exec(line);
// Filter out changelog entries that only apply to newer versions.
const match = someVersionsOnlyRegex.exec(line);
if (match) {
if (
match &&
Number.parseInt(targetBranchMajorVersion) < Number.parseInt(match[1])
) {
continue;
}
// Keep the line.
filteredEntries.push(line);
// Set `foundContent` to `true` if the line is not empty.
if (line.trim() !== "") {
foundContent = true;
}
}
// Update the section with the retained entries.
section.bodyLines = filteredEntries;
// Add an entry if we didn't keep any.
if (!foundContent) {
section.bodyLines.push(NO_CHANGES_STR.trim());
if (line.startsWith("## ")) {
line = line.replace(
`## ${sourceBranchMajorVersion}`,
`## ${targetBranchMajorVersion}`,
);
if (!foundContent) {
output += "No user facing changes.\n";
}
foundContent = false;
output += `\n${line}\n\n`;
} else {
if (line.trim() !== "") {
foundContent = true;
output += `${line}\n`;
}
}
}
return renderChangelog(changelog);
return output;
}

View File

@@ -21,8 +21,8 @@ steps:
run: |
cd "$RUNNER_TEMP/results"
actual=$(jq -r '.runs[0].properties.jobRunUuid' javascript.sarif)
if [[ "$actual" != "$CODEQL_ACTION_JOB_RUN_UUID" ]]; then
echo "Expected SARIF output to contain job run UUID '$CODEQL_ACTION_JOB_RUN_UUID', but found '$actual'."
if [[ "$actual" != "$JOB_RUN_UUID" ]]; then
echo "Expected SARIF output to contain job run UUID '$JOB_RUN_UUID', but found '$actual'."
exit 1
else
echo "Found job run UUID '$actual'."

View File

@@ -5,7 +5,7 @@ versions:
- default
steps:
- name: Set up Ruby
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
uses: ruby/setup-ruby@003a5c4d8d6321bd302e38f6f0ec593f77f06600 # v1.319.0
with:
ruby-version: 2.6
- name: Install Code Scanning integration

View File

@@ -37,10 +37,6 @@ export const API_COMPATIBILITY_FILE = path.join(
"api-compatibility.json",
);
/** The prefix of CodeQL CLI bundle release URLs. */
export const CLI_BUNDLE_RELEASE_URL_PREFIX =
"https://github.com/github/codeql-action/releases/tag/codeql-bundle-v";
/** A common interface for operations that support dry runs. */
export interface DryRunOption {
/** A value indicating whether to perform operations with side effects. */

View File

@@ -4,7 +4,7 @@
"dependencies": {
"@actions/core": "^2.0.3",
"@actions/github": "^8.0.1",
"@octokit/core": "^7.0.7",
"@octokit/core": "^7.0.6",
"@octokit/plugin-paginate-rest": ">=9.2.2",
"@octokit/plugin-rest-endpoint-methods": "^17.0.0",
"semver": "^7.8.5",
@@ -12,6 +12,6 @@
},
"devDependencies": {
"@types/node": "^20.19.43",
"tsx": "^4.23.12"
"tsx": "^4.23.1"
}
}

View File

@@ -1,54 +0,0 @@
/**
* Tests for `prepare-changelog.ts`.
*/
import * as assert from "node:assert/strict";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { afterEach, beforeEach, describe, it } from "node:test";
import { EMPTY_CHANGELOG, NO_CHANGES_STR } from "./changelog";
import { extractChangelogSnippet } from "./prepare-changelog";
let testDir: string;
beforeEach(() => {
// Set up a temporary directory for testing
testDir = fs.mkdtempSync(path.join(os.tmpdir(), "prepare-changelog-test-"));
});
afterEach(() => {
/** Clean up temporary directories. */
fs.rmSync(testDir, { recursive: true, force: true });
});
const testBody = `- Test change`;
const testChangelog = `${EMPTY_CHANGELOG.replace(NO_CHANGES_STR, testBody)}
## Another section
- Other change`;
describe("extractChangelogSnippet", async () => {
await it("returns the default body if the input doesn't exist", async () => {
const result = extractChangelogSnippet(path.join(testDir, "not-here.md"));
assert.deepEqual(result, NO_CHANGES_STR);
});
await it("returns the first section if the input exists", async () => {
const changelogPath = path.join(testDir, "test-readme.md");
fs.writeFileSync(changelogPath, testChangelog);
const result = extractChangelogSnippet(changelogPath);
assert.deepEqual(result, testBody);
});
await it("returns an empty string if there is no first section", async () => {
const changelogPath = path.join(testDir, "test-readme.md");
fs.writeFileSync(changelogPath, "# CodeQL Action Changelog\n");
const result = extractChangelogSnippet(changelogPath);
assert.deepEqual(result, "");
});
});

View File

@@ -1,82 +0,0 @@
#!/usr/bin/env npx tsx
/**
* Extracts the body of the first changelog section and outputs it to either
* stdout or a file.
*/
import * as fs from "node:fs";
import { parseArgs } from "node:util";
import { NO_CHANGES_STR, parseChangelog } from "./changelog";
import { CHANGELOG_FILE } from "./config";
import { getErrorMessage } from "./util";
/**
* Prepare the changelog for the new release
* This function will extract the part of the changelog that
* we want to include in the new release.
*
* @param changelogPath The path to the changelog file.
*/
export function extractChangelogSnippet(changelogPath: string) {
try {
const content = fs.readFileSync(changelogPath, "utf-8");
const changelog = parseChangelog(content);
// Return an empty string if we couldn't find the first section.
if (changelog.sections.length === 0) {
return "";
}
return changelog.sections[0].bodyLines.join("\n").trim();
} catch (err) {
if (err instanceof Error && "code" in err && err.code === "ENOENT") {
console.error(`Changelog file at '${changelogPath}' does not exist.`);
return NO_CHANGES_STR;
} else {
throw Error(
`Failed to open changelog file at '${changelogPath}': ${getErrorMessage(err)}`,
);
}
}
}
function main() {
try {
const { values } = parseArgs({
options: {
changelog: {
type: "string",
short: "f",
default: CHANGELOG_FILE,
},
output: {
type: "string",
short: "o",
},
},
strict: true,
});
const body = extractChangelogSnippet(values.changelog);
// If no `output` argument was provided, output to stdout. Otherwise,
// write a file to the specified path.
if (values.output === undefined) {
console.info(body);
} else {
fs.writeFileSync(values.output, body);
}
return 0;
} catch (err) {
console.error(`Failed to prepare changelog: ${getErrorMessage(err)}`);
return -1;
}
}
// Only call `main` if this script was run directly.
if (require.main === module) {
process.exit(main());
}

View File

@@ -1,45 +0,0 @@
/**
* Tests for `rollback-changelog.ts`.
*/
import * as assert from "node:assert/strict";
import * as fs from "node:fs";
import { describe, it } from "node:test";
import { getReleaseDateString, parseChangelog } from "./changelog";
import { CHANGELOG_FILE } from "./config";
import { updateChangelog } from "./rollback-changelog";
describe("updateChangelog", async () => {
await it("replaces the first section with one for the rollback release", async () => {
const actualChangelog = parseChangelog(
fs.readFileSync(CHANGELOG_FILE, "utf-8"),
);
const existingFirstSection = actualChangelog.sections[0];
const today = new Date();
updateChangelog(actualChangelog, {
"new-version": "Test.1.3",
"rollback-version": "Test.1.2",
"target-version": "Test.1.1",
today,
});
// Check that the old, first section is gone.
for (const section of actualChangelog.sections) {
assert.notDeepEqual(section, existingFirstSection);
}
// Check that the new, first section matches our expectations.
const newFirstSection = actualChangelog.sections[0];
assert.deepEqual(
newFirstSection.headerLine,
`## Test.1.3 - ${getReleaseDateString(today)}`,
);
assert.equal(newFirstSection.bodyLines.length, 3);
assert.deepEqual(
newFirstSection.bodyLines[1],
`This release rolls back Test.1.2 due to issues with that release. It is identical to Test.1.1.`,
);
});
});

View File

@@ -1,84 +0,0 @@
#!/usr/bin/env npx tsx
/**
* Replaces the current, first section of the changelog with a new one for the rollback release.
*/
import * as fs from "node:fs";
import { parseArgs } from "node:util";
import {
Changelog,
ChangelogSection,
getReleaseDateString,
parseChangelog,
renderChangelog,
} from "./changelog";
import { CHANGELOG_FILE } from "./config";
import { getErrorMessage } from "./util";
export interface RollbackChangelogInputs {
"target-version": string;
"rollback-version": string;
"new-version": string;
today?: Date;
}
/**
* Replaces the current, first section of the changelog with a new one for the rollback release.
*/
export function updateChangelog(
changelog: Changelog,
versions: RollbackChangelogInputs,
) {
// Drop the existing first section.
changelog.sections.shift();
// Construct the section for the rollback version.
const newSection: ChangelogSection = {
headerLine: `## ${versions["new-version"]} - ${getReleaseDateString(versions.today)}`,
bodyLines: [
"",
`This release rolls back ${versions["rollback-version"]} due to issues with that release. It is identical to ${versions["target-version"]}.`,
"",
],
};
// Add the new section at the top of the changelog.
changelog.sections.unshift(newSection);
}
function main() {
try {
const options = {
"target-version": { type: "string", short: "t" },
"rollback-version": { type: "string", short: "r" },
"new-version": { type: "string", short: "n" },
} as const;
const { values } = parseArgs({ options, strict: true });
for (const key of Object.keys(options)) {
const val = values[key as keyof typeof values];
if (val === undefined || val.trim() === "") {
throw new Error(`Argument '--${key}' is required.`);
}
}
const changelog = parseChangelog(fs.readFileSync(CHANGELOG_FILE, "utf-8"));
updateChangelog(changelog, values as RollbackChangelogInputs);
console.info(renderChangelog(changelog));
return 0;
} catch (err) {
console.error(
`Failed to prepare rollback changelog: ${getErrorMessage(err)}`,
);
return -1;
}
}
// Only call `main` if this script was run directly.
if (require.main === module) {
process.exit(main());
}

View File

@@ -253,8 +253,8 @@ const languageSetups: LanguageSetups = {
name: "Install Java",
uses: pinnedUses(
"actions/setup-java",
"b6effb05e454b25005698d916606bdc6ffcbf961",
"v5.7.0",
"03ad4de0992f5dab5e18fcb136590ce7c4a0ac95",
"v5.6.0",
),
with: {
"java-version": `\${{ inputs.java-version || '${defaultLanguageVersions.java}' }}`,

View File

@@ -1,9 +0,0 @@
/**
* Returns an appropriate message for the error.
*
* If the error is an `Error` instance, this returns the error message without
* an `Error: ` prefix.
*/
export function getErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}

View File

@@ -1,123 +0,0 @@
import * as core from "@actions/core";
import test from "ava";
import sinon from "sinon";
import * as common from "./action-common";
import * as actionsUtil from "./actions-util";
import * as environment from "./environment";
import * as logging from "./logging";
import { ActionName } from "./status-report";
import * as statusReport from "./status-report";
import {
getTestActionsEnv,
getTestEnv,
makeMacro,
RecordingLogger,
setupTests,
} from "./testing-utils";
import { getErrorMessage } from "./util";
setupTests(test);
interface RunInActionsTestOpts {
runFn?: () => Promise<any>;
expectedErrorMessage?: string;
expectedTelemetryError?: string;
}
const runInActionsMacro = makeMacro({
exec: async (t, opts: RunInActionsTestOpts) => {
const expectFailure = opts?.expectedErrorMessage !== undefined;
const logger = new RecordingLogger();
const getActionsLogger = sinon
.stub(logging, "getActionsLogger")
.returns(logger);
const env = getTestEnv();
const getEnv = sinon.stub(environment, "getEnv").returns(env);
const actionsEnv = getTestActionsEnv(env);
const getActionsEnv = sinon
.stub(actionsUtil, "getActionsEnv")
.returns(actionsEnv);
const getJobUUID = sinon
.stub(statusReport, "getJobUUID")
.returns("test-job-uuid");
const setFailed = sinon.stub(core, "setFailed");
const sendUnhandledErrorStatusReport = sinon.stub(
statusReport,
"sendUnhandledErrorStatusReport",
);
const name = ActionName.Init;
const run = sinon.stub();
if (opts?.runFn) {
run.callsFake(opts.runFn);
}
const transformTelemetryError = sinon
.stub()
.callsFake((err) => opts?.expectedTelemetryError ?? getErrorMessage(err));
const testAction: common.Action = {
name,
run,
transformTelemetryError,
};
await common.runInActions(testAction);
// These always should have been called once.
t.true(getActionsLogger.calledOnce);
t.true(getEnv.calledOnce);
t.true(getActionsEnv.calledOnce);
const expectedActionState = {
actions: actionsEnv,
env,
logger,
name: ActionName.Init,
};
t.true(getJobUUID.calledOnceWithExactly(sinon.match(expectedActionState)));
t.true(run.calledOnceWithExactly(sinon.match(expectedActionState)));
t.is(setFailed.calledOnce, expectFailure ?? false);
t.is(sendUnhandledErrorStatusReport.calledOnce, expectFailure ?? false);
if (expectFailure) {
t.true(
setFailed.calledOnceWithExactly(
`${statusReport.getDisplayActionName(name)} action failed: ${opts?.expectedErrorMessage}`,
),
);
t.true(
sendUnhandledErrorStatusReport.calledOnceWithExactly(
name,
sinon.match.any,
opts?.expectedTelemetryError ?? opts?.expectedErrorMessage,
logger,
),
);
}
},
title: (providedTitle) => `runInActions - ${providedTitle}`,
});
runInActionsMacro.serial("calls run", {});
runInActionsMacro.serial("handles run exceptions", {
runFn: () => {
throw new Error("Test failure");
},
expectedErrorMessage: "Test failure",
});
runInActionsMacro.serial("transforms run exceptions", {
runFn: () => {
throw new Error("Test failure");
},
expectedErrorMessage: "Test failure",
expectedTelemetryError: "Transformed failure message",
});

View File

@@ -8,10 +8,9 @@ import { getActionsLogger, Logger } from "./logging";
import {
ActionName,
getDisplayActionName,
getJobUUID,
sendUnhandledErrorStatusReport,
} from "./status-report";
import { getEnv, getErrorMessage, wrapError } from "./util";
import { getEnv, getErrorMessage } from "./util";
/** Base state that is available to an Action on startup. */
export interface BaseState {
@@ -79,12 +78,6 @@ export interface Action {
name: ActionName;
/** The entry point for the Action. */
run: ActionMain;
/**
* An optional function that transforms a caught error into a message suitable for
* inclusion in a status report. This is primarily intended for the `start-proxy`
* action to replace the thrown `Error`'s message with a safe one.
*/
transformTelemetryError?: (error: Error) => string;
}
/** A generic entry point that sets up the basic environment for the `action` and runs it. */
@@ -95,32 +88,17 @@ export async function runInActions(action: Action) {
const actionsEnv = getActionsEnv();
try {
const actionState = {
await action.run({
name: action.name,
startedAt,
logger,
env,
actions: actionsEnv,
};
// Create a unique identifier for this run.
getJobUUID(actionState);
await action.run(actionState);
});
} catch (error) {
core.setFailed(
`${getDisplayActionName(action.name)} action failed: ${getErrorMessage(error)}`,
);
const statusReportError =
action.transformTelemetryError !== undefined
? action.transformTelemetryError(wrapError(error))
: error;
await sendUnhandledErrorStatusReport(
action.name,
startedAt,
statusReportError,
logger,
);
await sendUnhandledErrorStatusReport(action.name, startedAt, error, logger);
}
}

View File

@@ -27,20 +27,14 @@ declare const __CODEQL_ACTION_VERSION__: string;
* global functions in tests.
*/
export interface ActionsEnv {
getRequiredInput: (name: string) => string;
getOptionalInput: (name: string) => string | undefined;
exportVariable: (name: string, value: string) => void;
}
/**
* Gets the real `ActionsEnv` used by production code.
*/
export function getActionsEnv(): ActionsEnv {
return {
getRequiredInput,
getOptionalInput,
exportVariable: core.exportVariable,
};
return { getOptionalInput };
}
/**

View File

@@ -38,7 +38,7 @@ export async function runWrapper() {
logger,
);
if (config !== undefined) {
const codeql = await getCodeQL(logger, config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd);
const version = await codeql.getVersion();
await debugArtifacts.uploadCombinedSarifArtifacts(
logger,

View File

@@ -255,7 +255,7 @@ async function run({ startedAt, logger }: ActionState<["Base", "Logger"]>) {
);
}
const codeql = await getCodeQL(logger, config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd);
if (hasBadExpectErrorInput()) {
throw new util.ConfigurationError(

View File

@@ -1 +1 @@
{"maximumVersion":"3.23","minimumVersion":"3.17"}
{"maximumVersion": "3.22", "minimumVersion": "3.17"}

View File

@@ -99,7 +99,7 @@ async function run({ startedAt, logger }: ActionState<["Base", "Logger"]>) {
);
}
const codeql = await getCodeQL(logger, config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd);
languages = await determineAutobuildLanguages(codeql, config, logger);
if (languages !== undefined) {

View File

@@ -155,7 +155,7 @@ export async function runAutobuild(
logger: Logger,
) {
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = await getCodeQL(logger, config.codeQLCmd);
const codeQL = await getCodeQL(config.codeQLCmd);
if (language === BuiltInLanguage.cpp) {
await setupCppAutobuild(codeQL, logger);
}

View File

@@ -1,128 +0,0 @@
import * as fs from "fs";
import path from "path";
import test from "ava";
import { EnvVar } from "../environment";
import { getRunnerLogger } from "../logging";
import { getTestEnv, setupTests } from "../testing-utils";
import * as util from "../util";
import * as outputCache from "./output-cache";
setupTests(test);
const logger = getRunnerLogger(true);
test.serial(
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "codeql-action-command-cache.json");
fs.writeFileSync(
cacheFile,
JSON.stringify({
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0" } },
}),
"utf8",
);
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.deepEqual(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
{
version: "2.20.0",
},
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
fs.writeFileSync(
cacheFile,
JSON.stringify({
cmd: "/path/to/other-codeql",
version: { version: "2.20.0" },
}),
"utf8",
);
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a malformed persisted value",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
fs.writeFileSync(cacheFile, "not valid json", "utf8");
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const cacheFile = path.join(tmpDir, "version.json");
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
const testValues = [
{ cmd: "/path/to/codeql" },
{ entries: { version: { version: "2.20.0" } } },
{ cmd: "/path/to/codeql", entries: {} },
{ cmd: "/path/to/codeql", entries: null },
{ cmd: "/path/to/codeql", entries: { version: {} } },
{ cmd: "/path/to/codeql", entries: { version: null } },
{ cmd: "/path/to/codeql", entries: { version: "2.20.0" } },
{ cmd: "/path/to/codeql", entries: { version: { version: null } } },
{ cmd: "/path/to/codeql", entries: { version: { version: 2.2 } } },
{ cmd: "/path/to/codeql", entries: { version: { version: 2 } } },
{
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0", overlayVersion: "1" } },
},
{
cmd: "/path/to/codeql",
entries: { version: { version: "2.20.0", features: "nope" } },
},
].map((v) => JSON.stringify(v));
for (const value of testValues) {
fs.writeFileSync(cacheFile, value, "utf8");
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
value,
);
}
});
},
);
test.serial("getCachedCodeQlVersion ignores non-existent file", async (t) => {
await util.withTmpDir(async (tmpDir: string) => {
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
t.notThrows(() => {
t.is(
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
undefined,
);
});
});
});

View File

@@ -1,156 +0,0 @@
import * as fs from "fs";
import path from "path";
import { getTemporaryDirectory } from "../actions-util";
import { Env } from "../environment";
import { Logger } from "../logging";
import type { VersionInfo } from "./types";
/**
* The keys of the command cache. Each key corresponds to a command whose output we cache.
*/
export type CommandCacheKey = string;
/**
* The type of the command cache that is persisted to disk.
*/
export interface OutputCache {
cmd: string;
entries: Record<CommandCacheKey, unknown>;
}
/**
* The name of the temporary file that backs the on-disk cache of
* CLI responses between workflow steps.
*/
const COMMAND_CACHE_FILENAME = "codeql-action-command-cache.json";
/**
* The module-global variable that caches the CodeQL CLI version in-memory.
*/
let cachedCodeQlVersion: undefined | VersionInfo = undefined;
/**
* Resets the in-process cache of the CodeQL CLI version. Only for use in tests,
* which exercise multiple "steps" within a single process.
*/
export function resetCachedCodeQlVersion(): void {
cachedCodeQlVersion = undefined;
}
/**
* Returns the path to the temporary file that backs the
* on-disk cache of CLI responses between workflow steps.
*/
function getCommandCacheFilePath(env: Env): string {
return path.join(getTemporaryDirectory(env), COMMAND_CACHE_FILENAME);
}
/**
* Caches the CodeQL CLI version both in-memory and on disk.
* @param env The environment variables to use.
* @param cmd The path to the CodeQL CLI.
* @param version The version information to cache.
*/
export function cacheCodeQlVersion(
env: Env,
cmd: string,
version: VersionInfo,
): void {
if (cachedCodeQlVersion !== undefined) {
throw new Error("cacheCodeQlVersion() should be called only once");
}
cachedCodeQlVersion = version;
const outputCache = {
cmd,
entries: { version },
} satisfies OutputCache;
// Persist the version so that subsequent Actions steps, which run in separate
// processes, can reuse it rather than invoking `codeql version` again. We
// record the CLI path so that a different step using a different CodeQL bundle
// doesn't pick up a stale version.
fs.writeFileSync(
getCommandCacheFilePath(env),
JSON.stringify(outputCache),
"utf8",
);
}
/**
* Returns the cached CodeQL CLI version, if any.
* @param logger The logger to use for logging messages.
* @param env The environment variables to use.
* @param cmd The path to the CodeQL CLI.
*/
export function getCachedCodeQlVersion(
logger: Logger,
env: Env,
cmd?: string,
): undefined | VersionInfo {
if (cachedCodeQlVersion !== undefined) {
return cachedCodeQlVersion;
}
// Fall back to the value persisted by an earlier Actions step, if any. This is
// best-effort: any malformed or mismatched value is ignored so that the caller
// invokes `codeql version` instead.
let serialized: string;
try {
serialized = fs.readFileSync(getCommandCacheFilePath(env), "utf8");
} catch (e) {
logger.debug(
`Cannot read CLI-cache file ${getCommandCacheFilePath(env)}: ${e}`,
);
return undefined;
}
let persisted: unknown;
try {
persisted = JSON.parse(serialized);
} catch (e) {
logger.debug(`Cannot parse CLI-cache data as JSON: ${e}`);
return undefined;
}
if (
!isOutputCache(persisted) ||
(cmd !== undefined && persisted.cmd !== cmd)
) {
return undefined;
}
// Memoize the parsed value so that subsequent calls in this process don't
// re-parse the environment variable.
cachedCodeQlVersion = persisted.entries.version as VersionInfo;
return cachedCodeQlVersion;
}
/**
* Determines whether a value is a `VersionInfo` object.
* @param x The value to test
*/
function isVersionInfo(x: unknown): x is VersionInfo {
const candidate = x as Partial<VersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.version === "string" &&
(candidate.features === undefined ||
(typeof candidate.features === "object" &&
candidate.features !== null)) &&
(candidate.overlayVersion === undefined ||
typeof candidate.overlayVersion === "number")
);
}
/**
* Determines whether a value is a `OutputCache` object.
* @param x The value to test
*/
function isOutputCache(x: unknown): x is OutputCache {
const candidate = x as Partial<OutputCache> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.cmd === "string" &&
candidate.entries !== undefined &&
isVersionInfo(candidate.entries.version)
);
}

View File

@@ -1,13 +0,0 @@
export interface VersionInfo {
version: string;
features?: { [name: string]: boolean };
/**
* The overlay version helps deal with backward incompatible changes for
* overlay analysis. When a precompiled query pack reports the same overlay
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
* analysis with that pack. Otherwise, if the overlay versions are different,
* or if either the pack or the CLI does not report an overlay version,
* we need to revert to non-overlay analysis.
*/
overlayVersion?: number;
}

View File

@@ -51,31 +51,6 @@ test.beforeEach(() => {
});
});
test("isDiskConfigurationError - true for expected errors", async (t) => {
t.true(
codeql.isDiskConfigurationError(new Error("ENOSPC: Out of disk space")),
);
t.true(
codeql.isDiskConfigurationError(
new Error(
"EACCES: permission denied, mkdir /opt/hostedtoolcache/CodeQL/",
),
),
);
});
test("isDiskConfigurationError - false for other errors", async (t) => {
t.false(codeql.isDiskConfigurationError("Not an Error instance"));
const otherMessages = [
"Does not contain an error code we test for",
"ENOSP: Not quite the full error code",
];
for (const otherMessage of otherMessages) {
t.false(codeql.isDiskConfigurationError(new Error(otherMessage)));
}
});
async function installIntoToolcache({
apiDetails = SAMPLE_DOTCOM_API_DETAILS,
cliVersion,
@@ -605,6 +580,7 @@ const injectedConfigMacro = makeMacro({
"",
undefined,
undefined,
getRunnerLogger(true),
);
const args = runnerConstructorStub.firstCall.args[1] as string[];
@@ -880,6 +856,7 @@ test.serial(
"",
undefined,
"/path/to/qlconfig.yml",
getRunnerLogger(true),
);
const args = runnerConstructorStub.firstCall.args[1] as string[];
@@ -910,6 +887,7 @@ test.serial(
"",
undefined,
undefined, // undefined qlconfigFile
getRunnerLogger(true),
);
const args = runnerConstructorStub.firstCall.args[1] as any[];
@@ -1088,6 +1066,7 @@ test.serial(
"sourceRoot",
undefined,
undefined,
getRunnerLogger(false),
);
t.true(runnerConstructorStub.calledOnce);

View File

@@ -12,12 +12,10 @@ import {
runTool,
} from "./actions-util";
import * as api from "./api-client";
import * as outputCache from "./cli/output-cache";
import type { VersionInfo } from "./cli/types";
import { CliError, wrapCliConfigurationError } from "./cli-errors";
import { appendExtraQueryExclusions, type Config } from "./config-utils";
import { DocUrl } from "./doc-url";
import { EnvVar, getEnv } from "./environment";
import { EnvVar } from "./environment";
import {
CodeQLDefaultVersionInfo,
Feature,
@@ -25,7 +23,7 @@ import {
} from "./feature-flags";
import { isAnalyzingDefaultBranch } from "./git-utils";
import { Language } from "./languages";
import { getRunnerLogger, Logger } from "./logging";
import { Logger } from "./logging";
import { writeBaseDatabaseOidsFile, writeOverlayChangesFile } from "./overlay";
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
import * as setupCodeql from "./setup-codeql";
@@ -93,6 +91,7 @@ export interface CodeQL {
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
logger: Logger,
): Promise<void>;
/**
* Runs the autobuilder for the given language.
@@ -216,6 +215,20 @@ export interface CodeQL {
): Promise<void>;
}
export interface VersionInfo {
version: string;
features?: { [name: string]: boolean };
/**
* The overlay version helps deal with backward incompatible changes for
* overlay analysis. When a precompiled query pack reports the same overlay
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
* analysis with that pack. Otherwise, if the overlay versions are different,
* or if either the pack or the CLI does not report an overlay version,
* we need to revert to non-overlay analysis.
*/
overlayVersion?: number;
}
export interface ResolveDatabaseOutput {
overlayBaseSpecifier?: string;
}
@@ -273,26 +286,6 @@ const GHES_MOST_RECENT_DEPRECATION_DATE = "2026-07-01";
/** The CLI verbosity level to use for extraction in debug mode. */
const EXTRACTION_DEBUG_MODE_VERBOSITY = "progress++";
/**
* Decides whether `e` is a disk-related error outside of our control
* that should be classified as a `ConfigurationError`.
*
* @param e The error to check.
* @returns True if the error should be treated as a `ConfigurationError` or false if not.
*/
export function isDiskConfigurationError(e: unknown): boolean {
if (!(e instanceof Error)) {
return false;
}
return (
// out of disk space
e.message.includes("ENOSPC") ||
// access denied
e.message.includes("EACCES")
);
}
/**
* Set up CodeQL CLI access.
*
@@ -353,7 +346,7 @@ export async function setupCodeQL(
);
}
cachedCodeQL = await getCodeQLForCmd(logger, codeqlCmd, checkVersion);
cachedCodeQL = await getCodeQLForCmd(codeqlCmd, checkVersion);
return {
codeql: cachedCodeQL,
toolsDownloadStatusReport,
@@ -363,7 +356,8 @@ export async function setupCodeQL(
} catch (rawError) {
const e = api.wrapApiConfigurationError(rawError);
const ErrorClass =
e instanceof util.ConfigurationError || isDiskConfigurationError(e)
e instanceof util.ConfigurationError ||
(e instanceof Error && e.message.includes("ENOSPC")) // out of disk space
? util.ConfigurationError
: Error;
@@ -378,9 +372,9 @@ export async function setupCodeQL(
/**
* Use the CodeQL executable located at the given path.
*/
export async function getCodeQL(logger: Logger, cmd: string): Promise<CodeQL> {
export async function getCodeQL(cmd: string): Promise<CodeQL> {
if (cachedCodeQL === undefined) {
cachedCodeQL = await getCodeQLForCmd(logger, cmd, true);
cachedCodeQL = await getCodeQLForCmd(cmd, true);
}
return cachedCodeQL;
}
@@ -487,9 +481,8 @@ export function createStubCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
*/
export async function getCodeQLForTesting(
cmd = "codeql-for-testing",
logger: Logger = getRunnerLogger(true),
): Promise<CodeQL> {
return getCodeQLForCmd(logger, cmd, false);
return getCodeQLForCmd(cmd, false);
}
/**
@@ -501,7 +494,6 @@ export async function getCodeQLForTesting(
* @returns A new CodeQL object
*/
async function getCodeQLForCmd(
logger: Logger,
cmd: string,
checkVersion: boolean,
): Promise<CodeQL> {
@@ -510,7 +502,7 @@ async function getCodeQLForCmd(
return cmd;
},
async getVersion() {
let result = outputCache.getCachedCodeQlVersion(logger, getEnv(), cmd);
let result = util.getCachedCodeQlVersion(cmd);
if (result === undefined) {
result = await runCliJson<VersionInfo>(
cmd,
@@ -519,7 +511,7 @@ async function getCodeQLForCmd(
noStreamStdout: true,
},
);
outputCache.cacheCodeQlVersion(getEnv(), cmd, result);
util.cacheCodeQlVersion(cmd, result);
}
return result;
},
@@ -547,6 +539,7 @@ async function getCodeQLForCmd(
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
logger: Logger,
) {
const extraArgs = config.languages.map(
(language) => `--language=${language}`,

View File

@@ -12,7 +12,7 @@ import { AnalysisKind, supportedAnalysisKinds } from "./analyses";
import * as api from "./api-client";
import { CachingKind } from "./caching-utils";
import { createStubCodeQL } from "./codeql";
import { UserConfig } from "./config/db-config";
import { UserConfig, UserConfigWithActionExtensions } from "./config/db-config";
import * as file from "./config/file";
import * as configUtils from "./config-utils";
import * as errorMessages from "./error-messages";
@@ -1295,12 +1295,13 @@ checkOverlayEnablementMacro.serial(
);
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if runner disk space is below minimum",
"No overlay-base database on default branch if runner disk space is below v2 limit and v2 resource checks enabled",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisResourceChecksV2,
],
isDefaultBranch: true,
diskUsage: {
@@ -1314,12 +1315,13 @@ checkOverlayEnablementMacro.serial(
);
checkOverlayEnablementMacro.serial(
"Overlay-base database on default branch if runner disk space is above minimum",
"Overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks enabled",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisResourceChecksV2,
],
isDefaultBranch: true,
diskUsage: {
@@ -1333,93 +1335,17 @@ checkOverlayEnablementMacro.serial(
},
);
// Check that each feature flag lowers the limit to the threshold that its name
// declares. Both sides of the boundary are needed to pin the threshold down: a
// mapping to a lower value would still pass the case at the limit, and one to a
// higher value would still fail the case below it.
for (const [feature, thresholdGb] of [
[Feature.OverlayAnalysisMinDisk8Gb, 8],
[Feature.OverlayAnalysisMinDisk9Gb, 9],
[Feature.OverlayAnalysisMinDisk10Gb, 10],
[Feature.OverlayAnalysisMinDisk11Gb, 11],
[Feature.OverlayAnalysisMinDisk12Gb, 12],
[Feature.OverlayAnalysisMinDisk13Gb, 13],
] as Array<[Feature, number]>) {
const features = [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
feature,
];
checkOverlayEnablementMacro.serial(
`Overlay-base database on default branch if ${feature} is enabled and runner disk space is at its limit`,
{
languages: [BuiltInLanguage.javascript],
features,
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: thresholdGb * 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
checkOverlayEnablementMacro.serial(
`No overlay-base database on default branch if ${feature} is enabled and runner disk space is below its limit`,
{
languages: [BuiltInLanguage.javascript],
features,
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: thresholdGb * 1_000_000_000 - 1_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
},
);
}
checkOverlayEnablementMacro.serial(
"Overlay-base database on default branch if runner disk space is exactly at the lowest limit enabled by a feature flag",
"No overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks not enabled",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisMinDisk9Gb,
Feature.OverlayAnalysisMinDisk12Gb,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 9_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if runner disk space is below the lowest limit enabled by a feature flag",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisMinDisk9Gb,
Feature.OverlayAnalysisMinDisk12Gb,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 8_500_000_000,
numAvailableBytes: 15_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
@@ -2618,6 +2544,7 @@ test("loadUserConfig - loads local configuration files", async (t) => {
) =>
configUtils.loadUserConfig(
actionState,
[AnalysisKind.CodeScanning],
filePath,
workspaceDir,
SAMPLE_DOTCOM_API_DETAILS,
@@ -2661,7 +2588,13 @@ test.serial("loadUserConfig - loads remote configuration files", async (t) => {
const remoteAddress = "owner/repo/file@ref";
await callee(configUtils.loadUserConfig)
.withArgs(remoteAddress, tmpDir, SAMPLE_DOTCOM_API_DETAILS, tmpDir)
.withArgs(
[AnalysisKind.CodeScanning],
remoteAddress,
tmpDir,
SAMPLE_DOTCOM_API_DETAILS,
tmpDir,
)
.passes(t.deepEqual, {});
t.true(
@@ -2739,6 +2672,7 @@ test.serial(
// Prepare the test call to `loadUserConfig`.
const targetWithArgs = target.withArgs(
[AnalysisKind.CodeScanning],
address,
tmpDir,
SAMPLE_DOTCOM_API_DETAILS,
@@ -2779,3 +2713,57 @@ test.serial(
});
},
);
test("loadUserConfig - applies analysis-specific settings", async (t) => {
await withTmpDir(async (workspaceDir) => {
await withTmpDir(async (tmpDir) => {
// Construct the test target.
const loadUserConfig = (
actionState: ActionState<["Logger", "Env", "FeatureFlags"]>,
filePath: string,
analysisKind: AnalysisKind,
) =>
configUtils.loadUserConfig(
actionState,
[analysisKind],
filePath,
workspaceDir,
SAMPLE_DOTCOM_API_DETAILS,
tmpDir,
);
const target = callee(loadUserConfig);
for (const analysisKind of Object.values(AnalysisKind)) {
const analysisSpecificConfig: UserConfig = {
paths: ["custom"],
queries: [{ uses: "foo" }],
};
const baseConfig: UserConfig = { "threat-models": ["remote"] };
// Write a local configuration file with an `analysisKind`-specific section.
const configPath = path.join(workspaceDir, "codeql.yml");
fs.writeFileSync(
configPath,
JSON.stringify({
...baseConfig,
[analysisKind]: analysisSpecificConfig,
} satisfies UserConfigWithActionExtensions),
"utf8",
);
// The resulting configuration should be the `baseConfig` with the `analysisSpecificConfig`
// merged into it if the FF is enabled.
await target
.withArgs(configPath, analysisKind)
.withFeatures([Feature.ScopedConfigurations])
.passes(t.deepEqual, { ...baseConfig, ...analysisSpecificConfig });
// Without the FF, it should be the `baseConfig`. This also validates that the
// `AnalysisKind`-specific sections are removed even if the FF is off.
await target
.withArgs(configPath, analysisKind)
.passes(t.deepEqual, baseConfig);
}
});
});
});

View File

@@ -24,11 +24,13 @@ import { getCachingKind } from "./caching-utils";
import { type CodeQL } from "./codeql";
import { type Config } from "./config/action-config";
import {
applyAnalysisKindConfig,
calculateAugmentation,
ExcludeQueryFilter,
generateCodeScanningConfig,
mergeDefaultSetupAndUserConfigs,
parseUserConfig,
removeAnalysisKindConfigs,
UserConfig,
} from "./config/db-config";
import {
@@ -48,7 +50,7 @@ import {
import { prepareDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import * as errorMessages from "./error-messages";
import { Feature, FeatureEnablement, FeatureWithoutCLI } from "./feature-flags";
import { Feature, FeatureEnablement } from "./feature-flags";
import {
RepositoryProperties,
RepositoryPropertyName,
@@ -101,23 +103,19 @@ export { type Config } from "./config/action-config";
* whether to perform overlay analysis, then the action will not perform overlay
* analysis unless overlay analysis has been explicitly enabled via environment
* variable.
*
* This threshold can be lowered by the feature flags in
* `OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE`.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
/**
* Minimum available disk space (in MB) enabled by each overlay feature flag.
* The v2 minimum available disk space (in MB) required to perform overlay
* analysis. This is a lower threshold than the v1 limit, allowing overlay
* analysis to run on runners with less available disk space.
*/
const OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE = {
[Feature.OverlayAnalysisMinDisk8Gb]: 8000,
[Feature.OverlayAnalysisMinDisk9Gb]: 9000,
[Feature.OverlayAnalysisMinDisk10Gb]: 10000,
[Feature.OverlayAnalysisMinDisk11Gb]: 11000,
[Feature.OverlayAnalysisMinDisk12Gb]: 12000,
[Feature.OverlayAnalysisMinDisk13Gb]: 13000,
} satisfies Partial<Record<FeatureWithoutCLI, number>>;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB = 14000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB * 1_000_000;
/**
* The minimum memory (in MB) that must be available for CodeQL to perform overlay analysis. If
@@ -488,11 +486,14 @@ async function downloadCacheWithTime(
*/
export async function loadUserConfig(
actionState: ActionState<["Logger", "Env", "FeatureFlags"]>,
analysisKinds: AnalysisKind[],
configFile: string,
workspacePath: string,
apiDetails: api.GitHubApiCombinedDetails,
tempDir: string,
): Promise<UserConfig> {
let config: UserConfig;
if (isLocal(configFile)) {
if (configFile !== userConfigFromActionPath(tempDir)) {
// If the config file is not generated by the Action, it should be relative to the workspace.
@@ -507,7 +508,7 @@ export async function loadUserConfig(
const validateConfig = await actionState.features.getValue(
Feature.ValidateDbConfig,
);
return getLocalConfig(actionState.logger, configFile, validateConfig);
config = getLocalConfig(actionState.logger, configFile, validateConfig);
} else {
// Drop the explicit prefix if it is present. Since `REMOTE_PATH_PREFIX` is chosen
// to not conflict with permissible characters in "owner" or "repo" components,
@@ -515,8 +516,29 @@ export async function loadUserConfig(
if (isExplicitRemotePath(configFile)) {
configFile = configFile.substring(REMOTE_PATH_PREFIX.length);
}
return await getRemoteConfig(actionState, configFile, apiDetails);
config = await getRemoteConfig(actionState, configFile, apiDetails);
}
// If the `ScopedConfigurations` feature is enabled, we allow options specific to the
// current `AnalysisKind` to be applied on top of a base configuration.
const allowScopedConfig = await actionState.features.getValue(
Feature.ScopedConfigurations,
);
if (allowScopedConfig && analysisKinds.length === 1) {
// Apply the `analysisKind`-specific configuration.
return applyAnalysisKindConfig(analysisKinds[0], config);
} else if (allowScopedConfig) {
actionState.logger.info(
`Ignoring '${Feature.ScopedConfigurations}' feature, because multiple analysis kinds are enabled.`,
);
}
// Remove `AnalysisKind`-specific sections from `config` even if the FF is not enabled,
// because the CLI will warn about unknown keys in the configuration. This is to guard
// against the case where we ship the feature, enable the FF, start using the new keys,
// and then need to roll the FF back.
return removeAnalysisKindConfigs(config);
}
/**
@@ -592,44 +614,24 @@ async function checkOverlayAnalysisFeatureEnabled(
return new Success(undefined);
}
/**
* Returns the minimum available disk space (in MB) required to perform overlay
* analysis, which is the lowest threshold enabled by a feature flag, or the
* default threshold if no such feature flag is enabled.
*/
async function getMinimumDiskSpaceMb(
features: FeatureEnablement,
): Promise<number> {
let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB;
for (const [feature, thresholdMb] of Object.entries(
OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE,
)) {
if (await features.getValue(feature as FeatureWithoutCLI)) {
minimumMb = Math.min(minimumMb, thresholdMb);
}
}
return minimumMb;
}
/** Checks if the runner has enough disk space for overlay analysis. */
function runnerHasSufficientDiskSpace(
diskUsage: DiskUsage,
logger: Logger,
minimumDiskSpaceMb: number,
useV2ResourceChecks: boolean,
): boolean {
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1_000_000) {
const minimumDiskSpaceBytes = useV2ResourceChecks
? OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES
: OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES;
if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) {
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000);
logger.info(
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
`due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`,
);
return false;
}
logger.debug(
`Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the ` +
`minimum of ${minimumDiskSpaceMb} MB.`,
);
return true;
}
@@ -661,7 +663,7 @@ async function runnerHasSufficientMemory(
}
logger.debug(
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
);
return true;
}
@@ -672,13 +674,12 @@ async function runnerHasSufficientMemory(
*/
async function checkRunnerResources(
codeql: CodeQL,
features: FeatureEnablement,
diskUsage: DiskUsage,
ramInput: string | undefined,
logger: Logger,
useV2ResourceChecks: boolean,
): Promise<Result<void, OverlayDisabledReason>> {
const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features);
if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) {
if (!runnerHasSufficientDiskSpace(diskUsage, logger, useV2ResourceChecks)) {
return new Failure(OverlayDisabledReason.InsufficientDiskSpace);
}
if (!(await runnerHasSufficientMemory(codeql, ramInput, logger))) {
@@ -777,6 +778,9 @@ export async function checkOverlayEnablement(
Feature.OverlayAnalysisSkipResourceChecks,
codeql,
));
const useV2ResourceChecks = await features.getValue(
Feature.OverlayAnalysisResourceChecksV2,
);
const checkOverlayStatus = await features.getValue(
Feature.OverlayAnalysisStatusCheck,
);
@@ -792,10 +796,10 @@ export async function checkOverlayEnablement(
performResourceChecks && diskUsage !== undefined
? await checkRunnerResources(
codeql,
features,
diskUsage,
ramInput,
logger,
useV2ResourceChecks,
)
: new Success<void>(undefined);
if (resourceResult.isFailure()) {
@@ -1093,6 +1097,7 @@ export async function determineUserConfig(
);
const fromConfigFile = await loadUserConfig(
action,
inputs.analysisKinds,
inputs.configFile,
inputs.workspacePath,
inputs.apiDetails,
@@ -1140,6 +1145,7 @@ export async function determineUserConfig(
action.logger.debug(`Using configuration file: ${inputs.configFile}`);
return await loadUserConfig(
action,
inputs.analysisKinds,
inputs.configFile,
inputs.workspacePath,
inputs.apiDetails,

View File

@@ -1,5 +1,6 @@
import test, { ExecutionContext } from "ava";
import { AnalysisKind } from "../analyses";
import { RepositoryProperties } from "../feature-flags/properties";
import { BuiltInLanguage, Language } from "../languages";
import { getRunnerLogger } from "../logging";
@@ -14,6 +15,117 @@ import { ConfigurationError, prettyPrintPack } from "../util";
import * as dbConfig from "./db-config";
function makeUserConfigWithActionExtensions(options?: {
analysisConfig?: dbConfig.UserConfig;
analysisKinds?: AnalysisKind[];
baseConfig?: dbConfig.UserConfig;
}) {
const testConfig = { ...options?.baseConfig };
const allAnalysisKinds = Object.values(AnalysisKind);
const analysisKinds = options?.analysisKinds ?? allAnalysisKinds;
for (const analysisKind of analysisKinds) {
testConfig[analysisKind] = { ...options?.analysisConfig };
}
return testConfig as dbConfig.UserConfigWithActionExtensions;
}
test("hasAnalysisKindKey - finds `AnalysisKind` sections", async (t) => {
const testConfig = makeUserConfigWithActionExtensions();
for (const analysisKind of Object.values(AnalysisKind)) {
// Check that it finds the `analysisKind`-specific section among multiple.
t.true(dbConfig.hasAnalysisKindKey(analysisKind, testConfig));
// And for just the `analysisKind`-specific section.
t.true(dbConfig.hasAnalysisKindKey(analysisKind, { [analysisKind]: {} }));
}
});
test("hasAnalysisKindKey - returns false if there is no relevant section", async (t) => {
const testConfig = makeUserConfigWithActionExtensions();
for (const analysisKind of Object.values(AnalysisKind)) {
// Clone the `testConfig` and remove the `analysisKind`-specific key.
const testConfigClone = { ...testConfig };
delete testConfigClone[analysisKind];
// Check that it doesn't find an `analysisKind`-specific section.
t.false(dbConfig.hasAnalysisKindKey(analysisKind, testConfigClone));
}
});
test("applyAnalysisKindConfig - applies analysis-specific settings", async (t) => {
for (const analysisKind of Object.values(AnalysisKind)) {
const testConfig = makeUserConfigWithActionExtensions({
analysisKinds: [analysisKind],
analysisConfig: {
"paths-ignore": [],
},
baseConfig: {
"paths-ignore": ["a", "b", "c"],
"threat-models": ["local"],
},
});
const clonedTestConfig = { ...testConfig };
const result = dbConfig.applyAnalysisKindConfig(analysisKind, testConfig);
// The `analysisKind`-specific section should have been removed.
// `threat-models` should be unchanged from the `baseConfig`.
// `paths-ignore` should have been overwritten with the `analysisKind`-specific config.
t.deepEqual(result, {
"threat-models": ["local"],
"paths-ignore": [],
});
// The input `testConfig` should not have changed.
t.deepEqual(testConfig, clonedTestConfig);
}
});
test("applyAnalysisKindConfig - removes all `AnalysisKind`-specific keys", async (t) => {
for (const analysisKind of Object.values(AnalysisKind)) {
const testConfig = makeUserConfigWithActionExtensions({
analysisConfig: {
"paths-ignore": ["a", "b", "c"],
},
baseConfig: {},
});
const result = dbConfig.applyAnalysisKindConfig(analysisKind, testConfig);
// All `AnalysisKind`-specific sections should have been removed.
// `paths-ignore` should have been overwritten with the analysis-specific config.
t.deepEqual(result, {
"paths-ignore": ["a", "b", "c"],
});
}
});
test("applyAnalysisKindConfig - returns the `baseConfig` if there is no `analysisKind`-specific key", async (t) => {
const baseConfig = {
"disable-default-queries": true,
"paths-ignore": ["a", "b", "c"],
queries: [{ name: "foo", uses: "something" }],
packs: { test: ["bar"] },
"default-setup": { org: { "model-packs": ["pack"] } },
} satisfies dbConfig.UserConfig;
const testConfig = makeUserConfigWithActionExtensions({
analysisKinds: [],
baseConfig,
});
const result = dbConfig.applyAnalysisKindConfig(
AnalysisKind.CodeScanning,
testConfig,
);
// The result should be the same as the `baseConfig`.
t.deepEqual(result, baseConfig);
});
/**
* Test macro for ensuring the packs block is valid
*/

View File

@@ -4,6 +4,7 @@ import * as yaml from "js-yaml";
import * as jsonschema from "jsonschema";
import * as semver from "semver";
import { AnalysisKind } from "../analyses";
import {
addNoLanguageDiagnostic,
makeTelemetryDiagnostic,
@@ -77,6 +78,68 @@ export interface UserConfig {
"default-setup"?: DefaultSetupConfig;
}
/** Represents a `UserConfig` with CodeQL Action extensions. */
export type UserConfigWithActionExtensions = {
/**
* We add a key for each analysis kind that may contain options
* specific to that kind of analysis.
*/
[T in keyof typeof AnalysisKind as (typeof AnalysisKind)[T]]?: UserConfig;
} & UserConfig;
/**
* Determines whether `config` contains an `analysisKind`-specific section.
*
* @param analysisKind The analysis section to check for.
* @param config The base `UserConfig` value.
*/
export function hasAnalysisKindKey(
analysisKind: AnalysisKind,
config: UserConfig,
): config is UserConfigWithActionExtensions {
return analysisKind in config && typeof config[analysisKind] === "object";
}
/**
* Clones `config` and removes any `AnalysisKind`-specific sections.
*/
export function removeAnalysisKindConfigs(config: UserConfig): UserConfig {
// Clone the configuration and remove any `AnalysisKind`-specific sections.
const configWithoutScopes = { ...config };
for (const kind of Object.values(AnalysisKind)) {
delete configWithoutScopes[kind];
}
return configWithoutScopes;
}
/**
* Applies an `analysisKind`-specific configuration, if any, to the resulting configuration
* and removes all `AnalysisKind`-specific configuration sections.
*
* @param analysisKind The `AnalysisKind` to apply specific configuration options for.
* @param config The loaded configuration, possibly including `analysisKind`-specific settings.
*/
export function applyAnalysisKindConfig(
analysisKind: AnalysisKind,
config: UserConfig,
): UserConfig {
// Clone the configuration and remove any `AnalysisKind`-specific sections.
const configWithoutScopes = removeAnalysisKindConfigs(config);
// If there is no `analysisKind`-specific section, return the input configuration
// without any `AnalysisKind`-specific sections.
if (!hasAnalysisKindKey(analysisKind, config)) {
return configWithoutScopes;
}
// Otherwise, extract the `analysisKind`-specific configuration from the input `config`.
const analysisKindSpecificConfig = config[analysisKind];
// Return the base configuration (without any `AnalysisKind`-specific sections) and
// override the top-level properties with the `analysisKind`-specific settings.
return { ...configWithoutScopes, ...analysisKindSpecificConfig };
}
/** A subset of the `UserConfig` schema that is used by Default Setup. */
const DEFAULT_SETUP_CONFIG_SCHEMA = {
"threat-models": json.optional(json.array(json.string)),

View File

@@ -1,7 +1,7 @@
import test from "ava";
import sinon from "sinon";
import { ActionsEnv } from "../actions-util";
import { getActionsEnv } from "../actions-util";
import { Feature } from "../feature-flags";
import { RepositoryPropertyName } from "../feature-flags/properties";
import { callee } from "../testing-utils";
@@ -22,26 +22,32 @@ const expectedRepositoryPropertyResult: ComputedInput = {
value: "repo-property-input-value",
};
function stubGetToolsInput(actions: ActionsEnv) {
function stubGetToolsInput() {
const actions = getActionsEnv();
sinon
.stub(actions, "getOptionalInput")
.withArgs(InputName.Tools)
.returns(expectedWorkflowResult.value);
return actions;
}
const workflowLogMessage = `Using ${InputName.Tools} input from workflow:`;
test("getToolsInput - returns workflow input if available", async (t) => {
const actions = stubGetToolsInput();
await callee(getToolsInput)
.withActions(stubGetToolsInput)
.withActions(actions)
.withArgs({})
.logs(t, workflowLogMessage)
.passes(t.deepEqual, expectedWorkflowResult);
});
test("getToolsInput - returns repository property value if enforced", async (t) => {
const actions = stubGetToolsInput();
const target = callee(getToolsInput)
.withActions(stubGetToolsInput)
.withActions(actions)
.withArgs({
[RepositoryPropertyName.TOOLS]: `!${expectedRepositoryPropertyResult.value}`,
});
@@ -59,8 +65,10 @@ test("getToolsInput - returns repository property value if enforced", async (t)
});
test("getToolsInput - prefers workflow input", async (t) => {
const actions = stubGetToolsInput();
const target = callee(getToolsInput)
.withActions(stubGetToolsInput)
.withActions(actions)
.withArgs({
[RepositoryPropertyName.TOOLS]: expectedRepositoryPropertyResult.value,
});

View File

@@ -16,7 +16,7 @@ export interface RemoteFileAddress {
}
/** The default file path to use in configuration file shorthands. */
export const DEFAULT_CONFIG_FILE_NAME = ".github/codeql-config.yml";
export const DEFAULT_CONFIG_FILE_NAME = ".github/codeql-action.yaml";
/** The default ref to use in configuration file shorthands. */
export const DEFAULT_CONFIG_FILE_REF = "main";

View File

@@ -1,6 +1,6 @@
{
"bundleVersion": "codeql-bundle-v2.26.4",
"cliVersion": "2.26.4",
"priorBundleVersion": "codeql-bundle-v2.26.3",
"priorCliVersion": "2.26.3"
"bundleVersion": "codeql-bundle-v2.26.1",
"cliVersion": "2.26.1",
"priorBundleVersion": "codeql-bundle-v2.26.0",
"priorCliVersion": "2.26.0"
}

View File

@@ -39,6 +39,12 @@ export enum EnvVar {
*/
CODE_SCANNING_REF = "CODE_SCANNING_REF",
/**
* `PersistedVersionInfo` for the CodeQL CLI, so later Actions steps can reuse it instead of
* invoking `codeql version` again.
*/
CODEQL_VERSION_INFO = "CODEQL_ACTION_CLI_VERSION_INFO",
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
@@ -82,7 +88,7 @@ export enum EnvVar {
LOG_VERSION_DEPRECATION = "CODEQL_ACTION_DID_LOG_VERSION_DEPRECATION",
/** UUID representing the current job run. */
JOB_RUN_UUID = "CODEQL_ACTION_JOB_RUN_UUID",
JOB_RUN_UUID = "JOB_RUN_UUID",
/** Status for the entire job, submitted to the status report in `init-post` */
JOB_STATUS = "CODEQL_ACTION_JOB_STATUS",
@@ -264,11 +270,6 @@ export class ReadOnlyEnv<T extends string | undefined = string | undefined> {
return Object.create(this, { vars: { value: { ...this.vars } } }) as this;
}
/** Gets a copy of the underlying environment. */
public get(): Record<string, T> {
return { ...this.vars };
}
/** Tries to get the value for `name` and throws if there isn't one. */
public getRequired(name: string): string {
return getRequiredEnvVar(this.vars, name);

View File

@@ -121,37 +121,12 @@ export enum Feature {
* `OverlayAnalysisMatchCodeqlVersion` overrides this flag.
*/
OverlayAnalysisMatchCodeqlVersionDryRun = "overlay_analysis_match_codeql_version_dry_run",
/**
* Lowers the overlay minimum available disk space to 8 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk8Gb = "overlay_analysis_min_disk_8_gb",
/**
* Lowers the overlay minimum available disk space to 9 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk9Gb = "overlay_analysis_min_disk_9_gb",
/**
* Lowers the overlay minimum available disk space to 10 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk10Gb = "overlay_analysis_min_disk_10_gb",
/**
* Lowers the overlay minimum available disk space to 11 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk11Gb = "overlay_analysis_min_disk_11_gb",
/**
* Lowers the overlay minimum available disk space to 12 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk12Gb = "overlay_analysis_min_disk_12_gb",
/**
* Lowers the overlay minimum available disk space to 13 GB. The lowest enabled limit wins; if
* none are enabled, the default applies.
*/
OverlayAnalysisMinDisk13Gb = "overlay_analysis_min_disk_13_gb",
OverlayAnalysisPython = "overlay_analysis_python",
/**
* Controls whether lower disk space requirements are used for overlay hardware checks.
* Has no effect if `OverlayAnalysisSkipResourceChecks` is enabled.
*/
OverlayAnalysisResourceChecksV2 = "overlay_analysis_resource_checks_v2",
OverlayAnalysisRuby = "overlay_analysis_ruby",
/** Controls whether hardware checks are skipped for overlay analysis. */
OverlayAnalysisSkipResourceChecks = "overlay_analysis_skip_resource_checks",
@@ -162,6 +137,8 @@ export enum Feature {
QaTelemetryEnabled = "qa_telemetry_enabled",
/** Routes (some) API requests through the registry proxy. */
ProxyApiRequests = "proxy_api_requests",
/** Allows sections specific to analysis kinds in configuration files. */
ScopedConfigurations = "scoped_configurations",
/** Note that this currently only disables baseline file coverage information. */
SkipFileCoverageOnPrs = "skip_file_coverage_on_prs",
StartProxyUseFeaturesRelease = "start_proxy_use_features_release",
@@ -379,34 +356,9 @@ export const featureConfig = {
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk8Gb]: {
[Feature.OverlayAnalysisResourceChecksV2]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk9Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk10Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk11Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk12Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk13Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB",
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisStatusCheck]: {
@@ -435,6 +387,11 @@ export const featureConfig = {
envVar: "CODEQL_ACTION_PROXY_API_REQUESTS",
minimumVersion: undefined,
},
[Feature.ScopedConfigurations]: {
defaultValue: false,
envVar: "CODEQL_ACTION_SCOPED_CONFIGURATIONS",
minimumVersion: undefined,
},
[Feature.SkipFileCoverageOnPrs]: {
defaultValue: false,
envVar: "CODEQL_ACTION_SKIP_FILE_COVERAGE_ON_PRS",

View File

@@ -123,7 +123,6 @@ async function prepareFailedSarif(
const category = `/language:${language}`;
const checkoutPath = ".";
const result = await generateFailedSarif(
logger,
features,
config,
category,
@@ -147,7 +146,6 @@ async function prepareFailedSarif(
const checkoutPath = getCheckoutPathInputOrThrow(workflow, jobName, matrix);
const result = await generateFailedSarif(
logger,
features,
config,
category,
@@ -158,7 +156,6 @@ async function prepareFailedSarif(
}
async function generateFailedSarif(
logger: Logger,
features: FeatureEnablement,
config: Config,
category: string | undefined,
@@ -166,7 +163,7 @@ async function generateFailedSarif(
sarifFile?: string,
) {
const databasePath = config.dbLocation;
const codeql = await getCodeQL(logger, config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd);
// Set the filename for the SARIF file if not already set.
if (sarifFile === undefined) {

View File

@@ -75,7 +75,7 @@ async function run(startedAt: Date) {
"Debugging artifacts are unavailable since the 'init' Action failed before it could produce any.",
);
} else {
const codeql = await getCodeQL(logger, config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd);
uploadFailedSarifResult = await initActionPostHelper.uploadFailureInfo(
debugArtifacts.tryUploadAllAvailableDebugArtifacts,

View File

@@ -4,6 +4,7 @@ import * as path from "path";
import * as core from "@actions/core";
import * as io from "@actions/io";
import * as semver from "semver";
import { v4 as uuidV4 } from "uuid";
import { Action, ActionState, runInActions } from "./action-common";
import {
@@ -254,6 +255,11 @@ async function run(
);
const repositoryProperties = repositoryPropertiesResult.orElse({});
// Create a unique identifier for this run.
const jobRunUuid = uuidV4();
logger.info(`Job run UUID is ${jobRunUuid}.`);
core.exportVariable(EnvVar.JOB_RUN_UUID, jobRunUuid);
core.exportVariable(EnvVar.INIT_ACTION_HAS_RUN, "true");
// path.resolve() respects the intended semantics of source-root. If
@@ -689,6 +695,7 @@ async function run(
sourceRoot,
"Runner.Worker.exe",
qlconfigFile,
logger,
);
// To check custom query packs for compatibility with overlay analysis, we
@@ -717,6 +724,7 @@ async function run(
sourceRoot,
"Runner.Worker.exe",
qlconfigFile,
logger,
);
}

View File

@@ -89,6 +89,7 @@ export async function runDatabaseInitCluster(
sourceRoot: string,
processName: string | undefined,
qlconfigFile: string | undefined,
logger: Logger,
): Promise<void> {
fs.mkdirSync(config.dbLocation, { recursive: true });
await configUtils.wrapEnvironment(
@@ -99,6 +100,7 @@ export async function runDatabaseInitCluster(
sourceRoot,
processName,
qlconfigFile,
logger,
),
);
}

View File

@@ -35,11 +35,6 @@ export function isNumber(value: unknown): value is number {
return typeof value === "number";
}
/** Asserts that `value` is a boolean. */
export function isBoolean(value: unknown): value is boolean {
return typeof value === "boolean";
}
/** Asserts that `value` is either a string or undefined. */
export function isStringOrUndefined(
value: unknown,
@@ -67,11 +62,14 @@ function defaultCheck(
return (arg) => ({ unknownKeys: [], invalidKeys: [], valid: validate(arg) });
}
function makeValidator<T>(validate: (arg: unknown) => arg is T) {
function makeValidator<T>(
validate: (arg: unknown) => arg is T,
required: boolean = true,
) {
return {
validate,
check: defaultCheck(validate),
required: true,
required,
} as const satisfies Validator<T>;
}
@@ -84,9 +82,6 @@ export const string = makeValidator(isString);
/** A validator for number fields in schemas. */
export const number = makeValidator(isNumber);
/** A validator for boolean fields in schemas. */
export const boolean = makeValidator(isBoolean);
/** A validator for arrays. */
export function array<T>(validator: Validator<T>) {
const validate = (val: unknown) => {
@@ -226,23 +221,6 @@ export function validateSchema<
return result.valid;
}
/**
* Validates that `arr` is an array whose elements satisfy at least `elementSchema`.
* Additional keys are accepted in each element.
*
* @param elementSchema The schema to validate the elements against.
* @param arr The array to validate.
* @returns Asserts that `arr` has elements of `schema`'s type if validation is successful.
*/
export function validateArray<
S extends Schema,
T extends UnvalidatedArray = Array<FromSchema<S>>,
>(elementSchema: S, arr: UnvalidatedArray): arr is T {
const elementValidator = object(elementSchema);
return array(elementValidator).validate(arr);
}
export interface CheckSchemaOptions {
/** Whether to stop validation after the first error. */
failFast?: boolean;

View File

@@ -9,7 +9,7 @@ export async function runResolveBuildEnvironment(
) {
logger.startGroup(`Attempting to resolve build environment for ${language}`);
const codeql = await getCodeQL(logger, cmd);
const codeql = await getCodeQL(cmd);
if (workingDir !== undefined) {
logger.info(`Using ${workingDir} as the working directory.`);

View File

@@ -1,4 +1,5 @@
import * as core from "@actions/core";
import { v4 as uuidV4 } from "uuid";
import { Action, ActionState, runInActions } from "./action-common";
import {
@@ -94,7 +95,7 @@ async function sendCompletedStatusReport(
/** The main behaviour of this action. */
async function run(
actionState: ActionState<["Base", "Logger", "Env", "Actions"]>,
actionState: ActionState<["Base", "Logger", "Actions"]>,
): Promise<void> {
// To capture errors appropriately, keep as much code within the try-catch as
// possible, and only use safe functions outside.
@@ -139,6 +140,10 @@ async function run(
const actionStateWithFeatures = { ...actionState, features };
const jobRunUuid = uuidV4();
logger.info(`Job run UUID is ${jobRunUuid}.`);
core.exportVariable(EnvVar.JOB_RUN_UUID, jobRunUuid);
const statusReportBase = await createStatusReportBase(
ActionName.SetupCodeQL,
"starting",

View File

@@ -3,12 +3,11 @@ import * as path from "path";
import * as core from "@actions/core";
import { Action, ActionState, runInActions } from "./action-common";
import * as actionsUtil from "./actions-util";
import { getGitHubVersion } from "./api-client";
import { FeatureEnablement, initFeatures } from "./feature-flags";
import { BuiltInLanguage, parseBuiltInLanguage } from "./languages";
import { Logger } from "./logging";
import { getActionsLogger, Logger } from "./logging";
import { getRepositoryNwo } from "./repository";
import {
credentialToStr,
@@ -24,14 +23,14 @@ import {
import { generateCertificateAuthority } from "./start-proxy/ca";
import { checkProxyEnvironment } from "./start-proxy/environment";
import { checkConnections } from "./start-proxy/reachability";
import { ActionName } from "./status-report";
import { ActionName, sendUnhandledErrorStatusReport } from "./status-report";
import * as util from "./util";
async function run(action: ActionState<["Base", "Logger", "Env", "Actions"]>) {
async function run(startedAt: Date) {
// To capture errors appropriately, keep as much code within the try-catch as
// possible, and only use safe functions outside.
const startedAt = action.startedAt;
const logger = action.logger;
const logger = getActionsLogger();
let features: FeatureEnablement | undefined;
let language: BuiltInLanguage | undefined;
@@ -123,15 +122,21 @@ async function run(action: ActionState<["Base", "Logger", "Env", "Actions"]>) {
}
}
/** Defines the `start-proxy` Action. */
const startProxyAction: Action = {
name: ActionName.StartProxy,
run,
transformTelemetryError: getSafeErrorMessage,
};
export async function runWrapper() {
await runInActions(startProxyAction);
const startedAt = new Date();
const logger = getActionsLogger();
try {
await run(startedAt);
} catch (error) {
core.setFailed(`start-proxy action failed: ${util.getErrorMessage(error)}`);
await sendUnhandledErrorStatusReport(
ActionName.StartProxy,
startedAt,
getSafeErrorMessage(util.wrapError(error)),
logger,
);
}
}
async function startProxy(

View File

@@ -83,6 +83,12 @@ export class StartProxyError extends Error {
}
}
interface StartProxyStatus extends StatusReportBase {
// A comma-separated list of registry types which are configured for CodeQL.
// This only includes registry types we support, not all that are configured.
registry_types: string;
}
/**
* Sends a status report for the `start-proxy` action indicating a successful outcome.
*
@@ -106,7 +112,7 @@ export async function sendSuccessStatusReport(
logger,
);
if (statusReportBase !== undefined) {
const statusReport: StatusReportBase = {
const statusReport: StartProxyStatus = {
...statusReportBase,
registry_types: registry_types.join(","),
};

View File

@@ -254,19 +254,13 @@ export function credentialToStr(credential: Credential): string {
return result;
}
/** The schema for `RegistryBase` objects. */
export const registryBaseSchema = {
/** The type of the package registry. */
type: json.string,
/** Whether the registry replaces the base registry for the ecosystem. */
"replaces-base": json.optional(json.boolean),
} as const satisfies json.Schema;
/** Information about a registry, other than its address. */
export type RegistryBase = json.FromSchema<typeof registryBaseSchema>;
/** A package registry is identified by its type and address. */
export type Registry = RegistryBase & Address;
export type Registry = {
/** The type of the package registry. */
type: string;
/** Whether the registry replaces the base registry for the ecosystem. */
"replaces-base"?: boolean;
} & Address;
// If a registry has an `url`, then that takes precedence over the `host` which may or may
// not be defined.

View File

@@ -1,21 +1,17 @@
import test from "ava";
import * as sinon from "sinon";
import * as uuid from "uuid";
import * as actionsUtil from "./actions-util";
import { Config } from "./config-utils";
import { EnvVar, RegistryProxyVars } from "./environment";
import { EnvVar } from "./environment";
import { BuiltInLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import { ToolsSource } from "./setup-codeql";
import type { Registry } from "./start-proxy";
import {
ActionName,
createInitWithConfigStatusReport,
createStatusReportBase,
getActionsStatus,
getRegistryTypesFromEnv,
getJobUUID,
InitStatusReport,
InitWithConfigStatusReport,
} from "./status-report";
@@ -24,106 +20,11 @@ import {
setupActionsVars,
createTestConfig,
makeMacro,
getTestEnv,
RecordingLogger,
callee,
} from "./testing-utils";
import { BuildMode, ConfigurationError, withTmpDir, wrapError } from "./util";
setupTests(test);
test("getRegistryTypesFromEnv - gets unique registry types from environment", async (t) => {
const logger = new RecordingLogger(true);
const env = getTestEnv({
[RegistryProxyVars.PROXY_URLS]: JSON.stringify([
{ type: "git_source", url: "https://example.com" },
{ type: "git_source", url: "https://github.com" },
{ type: "docker_registry", url: "https://registry.example.com" },
] satisfies Array<Partial<Registry>>),
});
const result = getRegistryTypesFromEnv(logger, env);
t.deepEqual(result, ["git_source", "docker_registry"].sort().join(","));
});
test("getRegistryTypesFromEnv - returns undefined if the env var is not set", async (t) => {
const logger = new RecordingLogger(true);
const env = getTestEnv({});
const result = getRegistryTypesFromEnv(logger, env);
t.is(result, undefined);
});
test("getRegistryTypesFromEnv - returns undefined if the env var is not valid JSON", async (t) => {
const logger = new RecordingLogger(true);
const env = getTestEnv({ [RegistryProxyVars.PROXY_URLS]: "[" });
const result = getRegistryTypesFromEnv(logger, env);
t.is(result, undefined);
});
test("getRegistryTypesFromEnv - returns undefined if the env var is unexpected JSON", async (t) => {
const logger = new RecordingLogger(true);
t.is(
getRegistryTypesFromEnv(
logger,
getTestEnv({
// Top-level object rather than an array of objects.
[RegistryProxyVars.PROXY_URLS]: JSON.stringify({ type: "git_source" }),
}),
),
undefined,
);
t.is(
getRegistryTypesFromEnv(
logger,
getTestEnv({
// Object has no "type" key.
[RegistryProxyVars.PROXY_URLS]: JSON.stringify([{}]),
}),
),
undefined,
);
});
test("getJobUUID - generates valid UUIDs", async (t) => {
await callee(getJobUUID)
.withArgs()
.logs(t, "Job run UUID is ")
.hasEnv(t, (val) => {
return {
[EnvVar.JOB_RUN_UUID]: val,
};
})
.passes((val) => {
t.true(uuid.validate(val));
});
});
test("getJobUUID - retrieves existing job UUIDs", async (t) => {
const existingJobUuid = uuid.v4();
await callee(getJobUUID)
.withArgs()
.withEnv((env) => {
env.set(EnvVar.JOB_RUN_UUID, existingJobUuid);
})
.logs(t, `Existing job run UUID is ${existingJobUuid}.`)
.passes(t.deepEqual, existingJobUuid);
});
test("getJobUUID - doesn't retrieve invalid UUIDs", async (t) => {
const existingJobUuid = "not-a-uuid";
await callee(getJobUUID)
.withArgs()
.withEnv((env) => {
env.set(EnvVar.JOB_RUN_UUID, existingJobUuid);
})
.logs(t, `Job run UUID is `)
.notLogs(t, `Existing job run UUID is ${existingJobUuid}.`)
.passes(t.notDeepEqual, existingJobUuid);
});
function setupEnvironmentAndStub(tmpDir: string) {
setupActionsVars(tmpDir, tmpDir, {
GITHUB_EVENT_NAME: "dynamic",
@@ -133,9 +34,6 @@ function setupEnvironmentAndStub(tmpDir: string) {
process.env[EnvVar.ANALYSIS_KEY] = "analysis-key";
process.env["ImageVersion"] = "2023.05.19.1";
process.env[RegistryProxyVars.PROXY_URLS] = JSON.stringify([
{ type: "maven_repository" },
] satisfies Array<Partial<Registry>>);
const getRequiredInput = sinon.stub(actionsUtil, "getRequiredInput");
getRequiredInput.withArgs("matrix").resolves("input/matrix");
@@ -179,7 +77,6 @@ test.serial("createStatusReportBase", async (t) => {
t.is(typeof statusReport.job_run_uuid, "string");
t.is(statusReport.languages, "java,swift");
t.is(statusReport.ref, process.env["GITHUB_REF"]!);
t.is(statusReport.registry_types, "maven_repository");
t.is(statusReport.runner_available_disk_space_bytes, 100);
t.is(statusReport.runner_image_version, process.env["ImageVersion"]);
t.is(statusReport.runner_os, process.env["RUNNER_OS"]!);

View File

@@ -1,9 +1,7 @@
import * as os from "os";
import * as core from "@actions/core";
import * as uuid from "uuid";
import type { ActionState } from "./action-common";
import {
getWorkflowEventName,
getOptionalInput,
@@ -14,23 +12,21 @@ import {
isSelfHostedRunner,
} from "./actions-util";
import { getAnalysisKey, getApiClient } from "./api-client";
import { getCachedCodeQlVersion } from "./cli/output-cache";
import type { Config } from "./config/action-config";
import type { ComputedInput, InputName } from "./config/inputs";
import { parseRegistriesWithoutCredentials } from "./config/pack-registries";
import type { DependencyCacheRestoreStatusReport } from "./dependency-caching";
import { DocUrl } from "./doc-url";
import { EnvVar, getEnv, ReadOnlyEnv, RegistryProxyVars } from "./environment";
import { EnvVar } from "./environment";
import { getRef } from "./git-utils";
import * as json from "./json";
import type { Logger } from "./logging";
import type { OverlayBaseDatabaseDownloadStats } from "./overlay/caching";
import { getRepositoryNwo } from "./repository";
import type { ToolsSource } from "./setup-codeql";
import { registryBaseSchema } from "./start-proxy/types";
import {
ConfigurationError,
getRequiredEnvParam,
getCachedCodeQlVersion,
isInTestMode,
GITHUB_DOTCOM_URL,
DiskUsage,
@@ -63,30 +59,6 @@ export function getDisplayActionName(actionName: ActionName): string {
return actionName;
}
/**
* Either creates a UUIDv4 for the analysis or retrieves an existing one from the
* environment and returns it.
* If a new UUID is generated, it is also exported as an environment variable.
*/
export function getJobUUID(
action: ActionState<["Logger", "ReadOnlyEnv", "Actions"]>,
) {
// Check if we already have a UUID for the analysis and return it if so.
const existingJobRunUuid = action.env.getOptional(EnvVar.JOB_RUN_UUID);
if (existingJobRunUuid !== undefined && uuid.validate(existingJobRunUuid)) {
action.logger.info(`Existing job run UUID is ${existingJobRunUuid}.`);
return existingJobRunUuid;
}
// Otherwise generate a new UUID.
const jobRunUuid = uuid.v4();
action.logger.info(`Job run UUID is ${jobRunUuid}.`);
action.actions.exportVariable(EnvVar.JOB_RUN_UUID, jobRunUuid);
return jobRunUuid;
}
/**
* @returns a boolean indicating whether the analysis is considered to be first party.
*
@@ -187,12 +159,6 @@ export interface StatusReportBase {
ml_powered_javascript_queries?: string;
/** Ref that the workflow was triggered on. */
ref: string;
/**
* A comma-separated list of private registry types which are configured for CodeQL.
* This only includes registry types we support (as determined by the `start-proxy` action),
* not all that are configured.
*/
registry_types?: string;
/** Action runner hardware architecture (context runner.arch). */
runner_arch?: string;
/** Available disk space on the runner, in bytes. */
@@ -296,50 +262,6 @@ export interface EventReport {
started_at: string;
}
/**
* Attempts to retrieve a list of private registry types from the `CODEQL_PROXY_URLS` environment
* variable and returns it as a comma-separated string if successful. Returns `undefined` otherwise.
*/
export function getRegistryTypesFromEnv(
logger: Logger,
env: ReadOnlyEnv = getEnv(),
): string | undefined {
// Try to get the value of the environment variable.
const value = env.getOptional(RegistryProxyVars.PROXY_URLS);
if (value === undefined) {
return undefined;
}
// Try to parse the JSON we expect to find in it and return the comma-separated list of
// (unique) registry types.
try {
const data = JSON.parse(value) as unknown;
// Check that the parsed JSON meets our expectations.
if (!json.isArray(data)) {
logger.debug(
`Expected '${RegistryProxyVars.PROXY_URLS}' to contain a JSON array, but got '${typeof data}'.`,
);
return undefined;
}
if (!json.validateArray(registryBaseSchema, data)) {
logger.debug(
`Expected '${RegistryProxyVars.PROXY_URLS}' to contain a JSON array of registry objects, but got something else.`,
);
return undefined;
}
const types = new Set(data.map((r) => r.type));
return Array.from(types).sort().join(",");
} catch (err) {
logger.debug(
`Failed to parse '${RegistryProxyVars.PROXY_URLS}': ${getErrorMessage(err)}.`,
);
return undefined;
}
}
/**
* Compose a StatusReport.
*
@@ -376,7 +298,7 @@ export async function createStatusReportBase(
core.exportVariable(EnvVar.WORKFLOW_STARTED_AT, workflowStartedAt);
}
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const codeQlCliVersion = getCachedCodeQlVersion(logger, getEnv());
const codeQlCliVersion = getCachedCodeQlVersion();
const actionRef = process.env["GITHUB_ACTION_REF"] || "";
const testingEnvironment = getTestingEnvironment();
// re-export the testing environment variable so that it is available to subsequent steps,
@@ -402,7 +324,6 @@ export async function createStatusReportBase(
job_name: jobName,
job_run_uuid: jobRunUUID,
ref,
registry_types: getRegistryTypesFromEnv(logger),
runner_os: runnerOs,
started_at: workflowStartedAt,
status,

View File

@@ -1,33 +0,0 @@
import * as path from "path";
import * as stream from "stream";
import test from "ava";
import { getRunnerLogger } from "./logging";
import { extractTarZst } from "./tar";
import { setupTests } from "./testing-utils";
import { withTmpDir } from "./util";
setupTests(test);
test("extractTarZst rejects if the input stream errors", async (t) => {
await withTmpDir(async (tmpDir) => {
const archive = new stream.PassThrough();
const promise = extractTarZst(
archive,
path.join(tmpDir, "dest"),
{ type: "gnu", version: "1.34" },
getRunnerLogger(true),
);
archive.destroy(
Object.assign(new Error("socket hang up"), {
code: "ECONNRESET",
}),
);
await t.throwsAsync(promise, {
message: /Error while downloading and extracting tar/,
});
});
});

View File

@@ -194,15 +194,10 @@ export async function extractTarZst(
});
if (tar instanceof stream.Readable) {
// Use `pipeline` rather than `pipe` so that an error on either stream is reported here
// rather than being emitted as an unhandled `error` event, and so that `tar`'s standard
// input is closed if the download fails partway through.
stream.pipeline(tar, tarProcess.stdin, (err) => {
if (err) {
reject(
new Error(`Error while downloading and extracting tar: ${err}`),
);
}
tar.pipe(tarProcess.stdin).on("error", (err) => {
reject(
new Error(`Error while downloading and extracting tar: ${err}`),
);
});
}

View File

@@ -18,8 +18,6 @@ import { AnalysisKind } from "./analyses";
import * as apiClient from "./api-client";
import { GitHubApiDetails } from "./api-client";
import { CachingKind } from "./caching-utils";
import { resetCachedCodeQlVersion } from "./cli/output-cache";
import type { VersionInfo } from "./cli/types";
import * as codeql from "./codeql";
import { Config } from "./config-utils";
import * as defaults from "./defaults.json";
@@ -36,13 +34,11 @@ import { ActionName } from "./status-report";
import {
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
Failure,
getEnv,
GitHubVariant,
GitHubVersion,
HTTPError,
Result,
Success,
resetCachedCodeQlVersion,
} from "./util";
export const SAMPLE_DOTCOM_API_DETAILS = {
@@ -186,32 +182,13 @@ export function getTestEnv(testEnv: NodeJS.ProcessEnv = {}): Env {
return getEnv(testEnv);
}
/** An implementation of `ActionsEnv` for use in tests. */
class TestActionsEnv implements ActionsEnv {
constructor(private readonly env: Env) {}
public clone(env: Env): this {
return Object.create(this, { env: { value: env } }) as this;
}
public getRequiredInput(name: string): string {
throw new Error(`Input required and not supplied: ${name}`);
}
public getOptionalInput(_name: string): string | undefined {
return undefined;
}
public exportVariable(name: string, value: string): void {
this.env.set(name, value);
}
}
/**
* Gets an `ActionsEnv` instance for use in tests.
*/
export function getTestActionsEnv(env: Env): TestActionsEnv {
return new TestActionsEnv(env);
export function getTestActionsEnv(): ActionsEnv {
return {
getOptionalInput: () => undefined,
};
}
/** For testing purposes, we make all available state features accessible in `TestEnv`. */
@@ -229,13 +206,12 @@ type AllState = [
export function initAllState(
overrides?: Partial<ActionState<AllState>>,
): ActionState<AllState> {
const env = getTestEnv();
return {
name: ActionName.Init,
startedAt: new Date(),
logger: new RecordingLogger(),
env,
actions: getTestActionsEnv(env),
env: getTestEnv(),
actions: getTestActionsEnv(),
apiClient: github.getOctokit("123"),
features: createFeatures([]),
...overrides,
@@ -246,13 +222,9 @@ type DelayedCheck<
Args extends readonly any[],
R,
Fs extends ReadonlyArray<AllState[number]>,
> = (
env: Readonly<BaseEnvBuilder<Args, R, Fs>>,
result: Result<Awaited<R>, ThrownError<ErrorConstructor | Error>>,
) => Promise<any>;
> = (env: Readonly<BaseEnvBuilder<Args, R, Fs>>) => Promise<any>;
export type Mutation<T> = (val: T) => void;
export type ValueOrMutation<T> = T | Mutation<T>;
export type ValueOrMutation<T> = T | ((val: T) => void);
/**
* Wraps a function that accepts an `ActionState` for testing in different environments.
@@ -264,7 +236,6 @@ abstract class BaseEnvBuilder<
> {
protected readonly fn: (state: ActionState<Fs>, ...args: Args) => R;
private logger: RecordingLogger;
private actions: TestActionsEnv;
protected state: ActionState<AllState>;
protected checks: Array<DelayedCheck<Args, R, Fs>>;
@@ -274,26 +245,15 @@ abstract class BaseEnvBuilder<
) {
this.fn = fn;
this.logger = new RecordingLogger();
if (cloneFrom !== undefined) {
const env = cloneFrom.state.env.clone();
this.actions = cloneFrom.actions.clone(env);
this.state = {
...cloneFrom.state,
env,
actions: this.actions,
logger: this.logger,
} satisfies ActionState<AllState>;
} else {
const env = getTestEnv();
this.actions = getTestActionsEnv(env);
this.state = initAllState({
logger: this.logger,
env,
actions: this.actions,
});
}
this.state =
cloneFrom !== undefined
? ({
...cloneFrom.state,
env: cloneFrom.state.env.clone(),
actions: Object.create(cloneFrom.state.actions),
logger: this.logger,
} satisfies ActionState<AllState>)
: initAllState({ logger: this.logger });
this.checks = [...(cloneFrom?.checks ?? [])];
}
@@ -360,10 +320,13 @@ abstract class BaseEnvBuilder<
return result;
}
/** Applies `fn` to the `ActionsEnv`. */
public withActions(fn: Mutation<ActionsEnv>): this {
public withActions(arg: ValueOrMutation<ActionsEnv>): this {
const result = this.clone();
fn(result.state.actions);
if (typeof arg === "function") {
arg(result.state.actions);
} else {
result.state.actions = arg;
}
return result;
}
@@ -379,28 +342,6 @@ abstract class BaseEnvBuilder<
return result;
}
/**
* Adds a delayed check that the environment variables returned by `fn`
* are present in the environment after the main assertion passes.
*/
public hasEnv(
t: ExecutionContext<unknown>,
fn: (
value: Awaited<R> | undefined,
error: ThrownError<ErrorConstructor | Error> | undefined,
) => Record<string, string | undefined>,
): this {
const result = this.clone();
result.checks.push(async (env, r) => {
const value = r.orElse(undefined);
const error = r.isFailure() ? r.value : undefined;
const expected = fn(value, error);
t.like(env.getState().env.get(), expected);
});
return result;
}
/**
* Adds a delayed check that `messages` are not logged. The check will be
* performed after the main assertion passes.
@@ -498,7 +439,7 @@ class CallableEnvBuilder<
// Run other delayed checks.
for (const delayedCheck of this.checks) {
await delayedCheck(this, new Success(result));
await delayedCheck(this);
}
// Return the results of the function call and the main assertion.
@@ -524,7 +465,7 @@ class CallableEnvBuilder<
// Run other delayed checks.
for (const delayedCheck of this.checks) {
await delayedCheck(this, new Failure(error));
await delayedCheck(this);
}
// Return the error.
@@ -873,7 +814,7 @@ export const makeVersionInfo = (
version: string,
features?: { [name: string]: boolean },
overlayVersion?: number,
): VersionInfo => ({
): codeql.VersionInfo => ({
version,
features,
overlayVersion,

View File

@@ -38,43 +38,6 @@ test.serial(
},
);
test.serial(
"downloadAndExtract falls back to downloading before extracting if streaming fails",
async (t) => {
await withTmpDir(async (tmpDir) => {
sinon.stub(process, "platform").value("linux");
const archivePath = path.join(tmpDir, "codeql-bundle.tar.zst");
const destination = path.join(tmpDir, "codeql");
const downloadTool = sinon
.stub(toolcache, "downloadTool")
.resolves(archivePath);
const extract = sinon.stub(tar, "extract").resolves(destination);
const extractTarZst = sinon.stub(tar, "extractTarZst").resolves();
const request = nock("https://example.com")
.get("/codeql-bundle.tar.zst")
.replyWithError(
Object.assign(new Error("socket hang up"), { code: "ECONNRESET" }),
);
const statusReport = await downloadAndExtract(
"https://example.com/codeql-bundle.tar.zst",
"zstd",
destination,
undefined,
{},
{ type: "gnu", version: "1.34" },
getRunnerLogger(true),
);
t.assert(Number.isInteger(statusReport.downloadDurationMs));
t.true(request.isDone());
t.false(extractTarZst.called);
t.true(downloadTool.calledOnce);
t.true(extract.calledOnce);
});
},
);
test.serial(
"downloadAndExtract omits the download duration when streaming extraction",
async (t) => {

View File

@@ -19,12 +19,6 @@ import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
*/
const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
/**
* How long the streaming download of the CodeQL tools may stall for before we abort it. This
* applies both to establishing the connection and to gaps between chunks of the response body.
*/
const STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
/**
* The name of the tool cache directory for the CodeQL tools.
*/
@@ -143,8 +137,8 @@ async function downloadAndExtractZstdWithStreaming(
authorization ? { authorization } : {},
headers,
);
const response = await new Promise<IncomingMessage>((resolve, reject) => {
const request = https.get(
const response = await new Promise<IncomingMessage>((resolve) =>
https.get(
codeqlURL,
{
headers,
@@ -154,24 +148,10 @@ async function downloadAndExtractZstdWithStreaming(
agent,
} as unknown as RequestOptions,
(r) => resolve(r),
);
// Without this listener, connection failures such as `ECONNRESET` are emitted as unhandled
// `error` events, which terminate the process instead of letting us fall back to downloading
// the bundle before extracting it. This listener stays attached after the response arrives, so
// it also handles errors that occur while the response is being streamed.
request.on("error", reject);
request.setTimeout(STREAMING_STALL_TIMEOUT_MS, () => {
request.destroy(
new Error(
`No data received for ${formatDuration(STREAMING_STALL_TIMEOUT_MS)}.`,
),
);
});
});
),
);
if (response.statusCode !== 200) {
// Discard the response body so that the connection can be released.
response.resume();
throw new Error(
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
);

View File

@@ -1,6 +1,6 @@
import * as semver from "semver";
import type { VersionInfo } from "./cli/types";
import type { VersionInfo } from "./codeql";
export enum ToolsFeature {
BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries",

View File

@@ -140,7 +140,7 @@ async function combineSarifFilesUsingCLI(
const config = await getConfig(tempDir, logger);
if (config !== undefined) {
codeQL = await getCodeQL(logger, config.codeQLCmd);
codeQL = await getCodeQL(config.codeQLCmd);
tempDir = config.tempDir;
} else {
logger.info(

View File

@@ -532,3 +532,58 @@ test("Failure.orElse returns the default value for a failure result", (t) => {
const result = new util.Failure(new Error("test error"));
t.is(result.orElse("default value"), "default value");
});
test.serial(
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0" },
});
t.deepEqual(util.getCachedCodeQlVersion("/path/to/codeql"), {
version: "2.20.0",
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = JSON.stringify({
cmd: "/path/to/other-codeql",
version: { version: "2.20.0" },
});
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined);
},
);
test.serial(
"getCachedCodeQlVersion ignores a malformed persisted value",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = "not valid json";
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined);
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
(t) => {
for (const value of [
JSON.stringify({ cmd: "/path/to/codeql" }),
JSON.stringify({ cmd: "/path/to/codeql", version: {} }),
JSON.stringify({ cmd: "/path/to/codeql", version: { version: 2 } }),
JSON.stringify({ version: { version: "2.20.0" } }),
JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0", overlayVersion: "1" },
}),
JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0", features: "nope" },
}),
]) {
process.env[EnvVar.CODEQL_VERSION_INFO] = value;
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined, value);
}
},
);

View File

@@ -10,7 +10,7 @@ import * as yaml from "js-yaml";
import * as semver from "semver";
import * as apiCompatibility from "./api-compatibility.json";
import type { CodeQL } from "./codeql";
import type { CodeQL, VersionInfo } from "./codeql";
import type { Pack } from "./config/db-config";
import type { Config } from "./config-utils";
import { EnvVar, getRequiredEnvParam } from "./environment";
@@ -598,6 +598,90 @@ export function asHTTPError(arg: any): HTTPError | undefined {
return undefined;
}
let cachedCodeQlVersion: undefined | VersionInfo = undefined;
/**
* Resets the in-process cache of the CodeQL CLI version. Only for use in tests,
* which exercise multiple "steps" within a single process.
*/
export function resetCachedCodeQlVersion(): void {
cachedCodeQlVersion = undefined;
}
/** The persisted version together with the CLI path it was obtained from. */
interface PersistedVersionInfo {
cmd: string;
version: VersionInfo;
}
function isVersionInfo(x: unknown): x is VersionInfo {
const candidate = x as Partial<VersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.version === "string" &&
(candidate.features === undefined ||
(typeof candidate.features === "object" &&
candidate.features !== null)) &&
(candidate.overlayVersion === undefined ||
typeof candidate.overlayVersion === "number")
);
}
function isPersistedVersionInfo(x: unknown): x is PersistedVersionInfo {
const candidate = x as Partial<PersistedVersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.cmd === "string" &&
isVersionInfo(candidate.version)
);
}
export function cacheCodeQlVersion(cmd: string, version: VersionInfo): void {
if (cachedCodeQlVersion !== undefined) {
throw new Error("cacheCodeQlVersion() should be called only once");
}
cachedCodeQlVersion = version;
// Persist the version so that subsequent Actions steps, which run in separate
// processes, can reuse it rather than invoking `codeql version` again. We
// record the CLI path so that a different step using a different CodeQL bundle
// doesn't pick up a stale version.
core.exportVariable(
EnvVar.CODEQL_VERSION_INFO,
JSON.stringify({ cmd, version }),
);
}
export function getCachedCodeQlVersion(cmd?: string): undefined | VersionInfo {
if (cachedCodeQlVersion !== undefined) {
return cachedCodeQlVersion;
}
// Fall back to the value persisted by an earlier Actions step, if any. This is
// best-effort: any malformed or mismatched value is ignored so that the caller
// invokes `codeql version` instead.
const serialized = process.env[EnvVar.CODEQL_VERSION_INFO];
if (!serialized) {
return undefined;
}
let persisted: unknown;
try {
persisted = JSON.parse(serialized);
} catch {
return undefined;
}
if (
!isPersistedVersionInfo(persisted) ||
(cmd !== undefined && persisted.cmd !== cmd)
) {
return undefined;
}
// Memoize the parsed value so that subsequent calls in this process don't
// re-parse the environment variable.
cachedCodeQlVersion = persisted.version;
return cachedCodeQlVersion;
}
export async function codeQlVersionAtLeast(
codeql: CodeQL,
requiredVersion: string,