mirror of
https://github.com/github/codeql-action.git
synced 2026-11-04 12:09:46 +00:00
Compare commits
95 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b96794f015 | ||
|
|
02d5093871 | ||
|
|
7e08580a93 | ||
|
|
bfcc52b4f5 | ||
|
|
8c251e757c | ||
|
|
0b7ca400df | ||
|
|
40484b3395 | ||
|
|
977e6ceaea | ||
|
|
40a6b38247 | ||
|
|
deece8f852 | ||
|
|
034db721dd | ||
|
|
313a0b9922 | ||
|
|
31da345c07 | ||
|
|
46dfb14111 | ||
|
|
e06b60f75f | ||
|
|
a48f2d3077 | ||
|
|
657964c39f | ||
|
|
d2fe508a6d | ||
|
|
16da74769e | ||
|
|
e13c3dc834 | ||
|
|
1331773b9a | ||
|
|
2681b03bd6 | ||
|
|
a7a90f3ffb | ||
|
|
4a0b22ec25 | ||
|
|
762a5ed7f7 | ||
|
|
ceb85f25b5 | ||
|
|
9fddc16f0d | ||
|
|
36cbf13b5e | ||
|
|
38dd4a088a | ||
|
|
2f3c1c964a | ||
|
|
5914b031b1 | ||
|
|
102499482f | ||
|
|
9401a335a3 | ||
|
|
3ee8a9398b | ||
|
|
25da1b1495 | ||
|
|
9509fd0822 | ||
|
|
20ec60c2d9 | ||
|
|
4cccb3aa86 | ||
|
|
b724a86493 | ||
|
|
2a03009e3f | ||
|
|
da21771e2e | ||
|
|
bb36f8049c | ||
|
|
276e2ce25d | ||
|
|
ccd5275aa4 | ||
|
|
a0c73122a6 | ||
|
|
50e244824e | ||
|
|
025009006c | ||
|
|
6f530319d8 | ||
|
|
1bcdd0c019 | ||
|
|
f049ecb9ce | ||
|
|
25694c08ea | ||
|
|
0963041ab4 | ||
|
|
1a8ddd7325 | ||
|
|
abc579a511 | ||
|
|
4d1d53ec73 | ||
|
|
6d4d5dfcd6 | ||
|
|
7acc225a21 | ||
|
|
76ba03be44 | ||
|
|
6441b26a49 | ||
|
|
cacb7b37ed | ||
|
|
e2cd31e32e | ||
|
|
be555cf0a3 | ||
|
|
0447ab23b5 | ||
|
|
50ec5a5c18 | ||
|
|
5aea817801 | ||
|
|
580da893f6 | ||
|
|
426c10420b | ||
|
|
dc5bc53050 | ||
|
|
d75af13705 | ||
|
|
0006c77502 | ||
|
|
ec1bd1999f | ||
|
|
4d40c93072 | ||
|
|
3e93618b99 | ||
|
|
42c2ea9ef7 | ||
|
|
f37565646f | ||
|
|
0f2e2bde5c | ||
|
|
b5383aad77 | ||
|
|
9d89e2d1d6 | ||
|
|
bee82de8ba | ||
|
|
c05e445696 | ||
|
|
a952c0706c | ||
|
|
97b7459e09 | ||
|
|
fcd8d74cda | ||
|
|
b5d34388b7 | ||
|
|
1f46830a1f | ||
|
|
b60777a215 | ||
|
|
f3deecb42d | ||
|
|
c8ba2d2a05 | ||
|
|
d7d901a925 | ||
|
|
cb9d39fcc1 | ||
|
|
93d793c5fe | ||
|
|
9875827ada | ||
|
|
b4c390c362 | ||
|
|
43ae5e70b0 | ||
|
|
4c96123771 |
16
.github/instructions/release-merge.instructions.md
vendored
Normal file
16
.github/instructions/release-merge.instructions.md
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
---
|
||||
applyTo: "CHANGELOG.md,src/defaults.json,lib/defaults.json,src/api-compatibility.json"
|
||||
---
|
||||
|
||||
# Merging release, mergeback, and backport PRs
|
||||
|
||||
The release process creates a cascade of PRs (`main` → `releases/vN`, then
|
||||
`releases/vN` → `main` mergeback, then `releases/vN` → `releases/v(N-1)`
|
||||
backport). These PRs reliably touch `CHANGELOG.md`, `src/defaults.json` /
|
||||
`lib/defaults.json` (bundle/CLI version bump), and `src/api-compatibility.json`.
|
||||
|
||||
Such PRs **must be merged with a merge commit**. Never squash or rebase, as
|
||||
that breaks the branch linkage the release automation relies on.
|
||||
|
||||
When arming auto-merge on these PRs, use `--merge` (e.g. `gh pr merge --merge`),
|
||||
not `--squash` or `--rebase`.
|
||||
2
.github/workflows/__all-platform-bundle.yml
generated
vendored
2
.github/workflows/__all-platform-bundle.yml
generated
vendored
@@ -56,7 +56,7 @@ jobs:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
version: nightly-latest
|
||||
- os: macos-latest
|
||||
- os: macos-latest-xlarge
|
||||
version: nightly-latest
|
||||
- os: windows-latest
|
||||
version: nightly-latest
|
||||
|
||||
2
.github/workflows/__autobuild-direct-tracing-with-working-dir.yml
generated
vendored
2
.github/workflows/__autobuild-direct-tracing-with-working-dir.yml
generated
vendored
@@ -63,7 +63,7 @@ jobs:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Install Java
|
||||
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
|
||||
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
|
||||
with:
|
||||
java-version: ${{ inputs.java-version || '17' }}
|
||||
distribution: temurin
|
||||
|
||||
2
.github/workflows/__build-mode-autobuild.yml
generated
vendored
2
.github/workflows/__build-mode-autobuild.yml
generated
vendored
@@ -63,7 +63,7 @@ jobs:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Install Java
|
||||
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
|
||||
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
|
||||
with:
|
||||
java-version: ${{ inputs.java-version || '17' }}
|
||||
distribution: temurin
|
||||
|
||||
106
.github/workflows/__linux-arm64.yml
generated
vendored
Normal file
106
.github/workflows/__linux-arm64.yml
generated
vendored
Normal file
@@ -0,0 +1,106 @@
|
||||
# Warning: This file is generated automatically, and should not be modified.
|
||||
# Instead, please modify the template in the pr-checks directory and run:
|
||||
# pr-checks/sync.sh
|
||||
# to regenerate this file.
|
||||
|
||||
name: PR Check - Linux Arm64
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GO111MODULE: auto
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- releases/v*
|
||||
pull_request: {}
|
||||
merge_group:
|
||||
types:
|
||||
- checks_requested
|
||||
schedule:
|
||||
- cron: '0 5 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dotnet-version:
|
||||
type: string
|
||||
description: The version of .NET to install
|
||||
required: false
|
||||
default: 9.x
|
||||
go-version:
|
||||
type: string
|
||||
description: The version of Go to install
|
||||
required: false
|
||||
default: '>=1.21.0'
|
||||
workflow_call:
|
||||
inputs:
|
||||
dotnet-version:
|
||||
type: string
|
||||
description: The version of .NET to install
|
||||
required: false
|
||||
default: 9.x
|
||||
go-version:
|
||||
type: string
|
||||
description: The version of Go to install
|
||||
required: false
|
||||
default: '>=1.21.0'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
concurrency:
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || false }}
|
||||
group: linux-arm64-${{github.ref}}-${{inputs.dotnet-version}}-${{inputs.go-version}}
|
||||
jobs:
|
||||
linux-arm64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-24.04-arm
|
||||
version: nightly-latest
|
||||
name: Linux Arm64
|
||||
if: github.triggering_actor != 'dependabot[bot]'
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: read
|
||||
timeout-minutes: 45
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Install .NET
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
||||
with:
|
||||
dotnet-version: ${{ inputs.dotnet-version || '9.x' }}
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version: ${{ inputs.go-version || '>=1.21.0' }}
|
||||
cache: false
|
||||
- name: Prepare test
|
||||
id: prepare-test
|
||||
uses: ./.github/actions/prepare-test
|
||||
with:
|
||||
version: ${{ matrix.version }}
|
||||
use-all-platform-bundle: 'false'
|
||||
setup-kotlin: 'true'
|
||||
- uses: ./../action/init
|
||||
with:
|
||||
languages: ${{ env.LANGUAGES }}
|
||||
tools: ${{ steps.prepare-test.outputs.tools-url }}
|
||||
- name: Build code
|
||||
run: ./build.sh
|
||||
- uses: ./../action/analyze
|
||||
with:
|
||||
upload-database: false
|
||||
- name: Assert databases exist
|
||||
run: |
|
||||
cd "$RUNNER_TEMP/codeql_databases"
|
||||
for lang in ${LANGUAGES//,/ }; do
|
||||
if [[ ! -d "$lang" ]]; then
|
||||
echo "Did not find a database for $lang"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found database for $lang"
|
||||
done
|
||||
env:
|
||||
LANGUAGES: cpp,csharp,go,java,javascript,python,ruby
|
||||
CODEQL_ACTION_TEST_MODE: true
|
||||
1
.github/workflows/__multi-language-autodetect.yml
generated
vendored
1
.github/workflows/__multi-language-autodetect.yml
generated
vendored
@@ -191,5 +191,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
env:
|
||||
CODEQL_ACTION_CLEANUP_TOOLCACHE_BUNDLES: true
|
||||
CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI: true
|
||||
CODEQL_ACTION_TEST_MODE: true
|
||||
|
||||
6
.github/workflows/__swift-custom-build.yml
generated
vendored
6
.github/workflows/__swift-custom-build.yml
generated
vendored
@@ -54,11 +54,11 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-latest
|
||||
- os: macos-latest-xlarge
|
||||
version: linked
|
||||
- os: macos-latest
|
||||
- os: macos-latest-xlarge
|
||||
version: default
|
||||
- os: macos-latest
|
||||
- os: macos-latest-xlarge
|
||||
version: nightly-latest
|
||||
name: Swift analysis using a custom build command
|
||||
if: github.triggering_actor != 'dependabot[bot]'
|
||||
|
||||
34
.github/workflows/pr-checks.yml
vendored
34
.github/workflows/pr-checks.yml
vendored
@@ -67,11 +67,43 @@ jobs:
|
||||
|
||||
- name: Upload sarif
|
||||
uses: ./upload-sarif
|
||||
if: matrix.os == 'ubuntu-latest' && matrix.node-version == 24
|
||||
# The merge queue deletes its `gh-readonly-queue` ref as soon as the queue entry resolves,
|
||||
# so uploading against it races with that deletion. Both the `merge_group` run and the
|
||||
# paired `push` run that the queue branch creates use that ref, so gate on the ref itself
|
||||
# rather than the event. The same results are uploaded by the `pull_request` run and again
|
||||
# by the `push` run on `main`.
|
||||
if: matrix.os == 'ubuntu-latest' && matrix.node-version == 24 && !startsWith(github.ref, 'refs/heads/gh-readonly-queue/')
|
||||
with:
|
||||
sarif_file: eslint.sarif
|
||||
category: eslint
|
||||
|
||||
changetool-tests:
|
||||
name: changetool unit tests
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-slim
|
||||
timeout-minutes: 10
|
||||
|
||||
concurrency:
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || false }}
|
||||
group: pr-checks-changetool-tests-${{ github.ref }}-${{ github.event_name }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 24
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run changetool unit tests
|
||||
run: npm --workspace changetool test
|
||||
|
||||
# These checks do not need to be run as part of the same matrix that we use for the `unit-tests`
|
||||
# job.
|
||||
other-checks:
|
||||
|
||||
@@ -2,6 +2,12 @@
|
||||
|
||||
See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs.
|
||||
|
||||
## 4.38.0 - 09 Sept 2026
|
||||
|
||||
- On GitHub-hosted runners, the CodeQL Action now deletes unused CodeQL bundles from the toolcache before downloading a different bundle, which frees up disk space for the analysis. We expect to roll this change out to everyone in September. [#4124](https://github.com/github/codeql-action/pull/4124)
|
||||
- The CodeQL Action now supports CodeQL releases that are compatible with Linux Arm64 and downloads the native `linux-arm64` CodeQL bundle when available. [#4072](https://github.com/github/codeql-action/pull/4072)
|
||||
- Update default CodeQL bundle version to [2.27.0](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.27.0). [#4129](https://github.com/github/codeql-action/pull/4129)
|
||||
|
||||
## 4.37.9 - 26 Aug 2026
|
||||
|
||||
- Update default CodeQL bundle version to [2.26.4](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.26.4). [#4106](https://github.com/github/codeql-action/pull/4106)
|
||||
|
||||
@@ -60,10 +60,13 @@ Here are a few things you can do that will increase the likelihood of your pull
|
||||
This workflow goes through the pull requests that have been merged to `main` since the last release, creates a changelog, then opens a pull request to merge the changes since the last release into the `releases/v3` release branch.
|
||||
|
||||
You can start a release by triggering this workflow via [workflow dispatch](https://github.com/github/codeql-action/actions/workflows/update-release-branch.yml).
|
||||
1. The workflow run will open a pull request titled "Merge main into releases/v3". Follow the steps on the checklist in the pull request. Once you've checked off all but the last two of these, approve the PR and automerge it.
|
||||
1. The workflow run will open a pull request titled "Merge main into releases/v3". Follow the steps on the checklist in the pull request. Once you've checked off all but the last two of these, approve the PR and automerge it **with a merge commit** (`gh pr merge --merge`).
|
||||
1. When the "Merge main into releases/v3" pull request is merged into the `releases/v3` branch, a mergeback pull request to `main` will be automatically created. This mergeback pull request incorporates the changelog updates into `main`, tags the release using the merge commit of the "Merge main into releases/v3" pull request, and bumps the patch version of the CodeQL Action.
|
||||
1. If a backport to an older major version is required, a pull request targeting that version's branch will also be automatically created.
|
||||
1. Approve the mergeback and backport pull request (if applicable) and automerge them.
|
||||
1. Approve the mergeback and backport pull request (if applicable) and automerge them **with a merge commit** (`gh pr merge --merge`).
|
||||
|
||||
> [!NOTE]
|
||||
> The release, mergeback, and backport pull requests must always be merged with a merge commit — **never squash or rebase**. The mergeback tags the release using the merge commit of the "Merge main into releases/v3" pull request, so squashing or rebasing breaks tagging and the branch linkage the release automation relies on.
|
||||
|
||||
Once the mergeback and backport pull request have been merged, the release is complete.
|
||||
|
||||
|
||||
@@ -209,4 +209,18 @@ export default [
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
files: ["scripts/changetool/**/*.ts"],
|
||||
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
project: "./scripts/changetool/tsconfig.json",
|
||||
},
|
||||
},
|
||||
|
||||
rules: {
|
||||
"no-console": "off",
|
||||
"import/extensions": "off",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
@@ -164,6 +164,13 @@ inputs:
|
||||
[Internal] The ID of the check run, as provided by the Actions runtime environment. Do not set this value manually.
|
||||
default: ${{ job.check_run_id }}
|
||||
required: false
|
||||
job-status:
|
||||
description: >-
|
||||
[Internal] The status of the job, as provided by the Actions runtime environment. This is how the
|
||||
post step learns whether the job as a whole succeeded, failed, or was cancelled. Do not set this
|
||||
value manually.
|
||||
default: ${{ job.status }}
|
||||
required: false
|
||||
outputs:
|
||||
codeql-path:
|
||||
description: The path of the CodeQL binary used for analysis
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"bundleVersion": "codeql-bundle-v2.26.4",
|
||||
"cliVersion": "2.26.4",
|
||||
"priorBundleVersion": "codeql-bundle-v2.26.3",
|
||||
"priorCliVersion": "2.26.3"
|
||||
"bundleVersion": "codeql-bundle-v2.27.0",
|
||||
"cliVersion": "2.27.0",
|
||||
"priorBundleVersion": "codeql-bundle-v2.26.4",
|
||||
"priorCliVersion": "2.26.4"
|
||||
}
|
||||
|
||||
2190
lib/entry-points.js
generated
2190
lib/entry-points.js
generated
File diff suppressed because it is too large
Load Diff
1517
package-lock.json
generated
1517
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
19
package.json
19
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "codeql",
|
||||
"version": "4.37.9",
|
||||
"version": "4.38.0",
|
||||
"private": true,
|
||||
"description": "CodeQL action",
|
||||
"scripts": {
|
||||
@@ -17,7 +17,8 @@
|
||||
},
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
"pr-checks"
|
||||
"pr-checks",
|
||||
"scripts/changetool"
|
||||
],
|
||||
"dependencies": {
|
||||
"@actions/artifact": "^5.0.3",
|
||||
@@ -31,27 +32,27 @@
|
||||
"@actions/io": "^2.0.0",
|
||||
"@actions/tool-cache": "^3.0.1",
|
||||
"@octokit/core": "^7.0.7",
|
||||
"@octokit/plugin-paginate-rest": "^14.0.0",
|
||||
"@octokit/plugin-rest-endpoint-methods": "^17.0.0",
|
||||
"@octokit/plugin-paginate-rest": "^15.0.0",
|
||||
"@octokit/plugin-rest-endpoint-methods": "^18.0.0",
|
||||
"@octokit/plugin-retry": "^8.1.1",
|
||||
"archiver": "^8.0.0",
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"follow-redirects": "^1.16.0",
|
||||
"get-folder-size": "^5.0.0",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"js-yaml": "^5.2.3",
|
||||
"js-yaml": "^5.4.0",
|
||||
"jsonschema": "1.5.0",
|
||||
"long": "^5.3.2",
|
||||
"node-forge": "^1.4.0",
|
||||
"semver": "^7.8.5",
|
||||
"uuid": "^14.0.1",
|
||||
"uuid": "^14.0.2",
|
||||
"undici": "^6.28.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ava/typescript": "6.0.0",
|
||||
"@eslint/compat": "^2.1.0",
|
||||
"@microsoft/eslint-formatter-sarif": "^3.1.0",
|
||||
"@octokit/types": "^16.0.0",
|
||||
"@octokit/types": "^17.0.0",
|
||||
"@types/archiver": "^8.0.0",
|
||||
"@types/follow-redirects": "^1.14.4",
|
||||
"@types/js-yaml": "^4.0.9",
|
||||
@@ -66,14 +67,14 @@
|
||||
"eslint-import-resolver-typescript": "^4.4.5",
|
||||
"eslint-plugin-github": "^6.1.2",
|
||||
"eslint-plugin-import-x": "^4.17.1",
|
||||
"eslint-plugin-jsdoc": "^62.9.0",
|
||||
"eslint-plugin-jsdoc": "^64.2.1",
|
||||
"eslint-plugin-no-async-foreach": "^0.1.1",
|
||||
"glob": "^13.0.6",
|
||||
"globals": "^17.11.0",
|
||||
"nock": "^14.0.17",
|
||||
"sinon": "^22.1.0",
|
||||
"typescript": "^6.0.3",
|
||||
"typescript-eslint": "^8.67.0"
|
||||
"typescript-eslint": "^8.68.0"
|
||||
},
|
||||
"overrides": {
|
||||
"@actions/tool-cache": {
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
import * as githubUtils from "@actions/github/lib/utils";
|
||||
import { type Octokit } from "@octokit/core";
|
||||
import { type PaginateInterface } from "@octokit/plugin-paginate-rest";
|
||||
import { type Api } from "@octokit/plugin-rest-endpoint-methods";
|
||||
|
||||
/** The type of the Octokit client. */
|
||||
export type ApiClient = Octokit & Api & { paginate: PaginateInterface };
|
||||
export type ApiClient = InstanceType<typeof githubUtils.GitHub>;
|
||||
|
||||
/** Constructs an `ApiClient` using `token` for authentication. */
|
||||
export function getApiClient(token: string): ApiClient {
|
||||
|
||||
@@ -2,7 +2,8 @@ name: "All-platform bundle"
|
||||
description: "Tests using an all-platform CodeQL Bundle"
|
||||
operatingSystems:
|
||||
- ubuntu
|
||||
- macos
|
||||
- os: macos
|
||||
runner-image: macos-latest-xlarge
|
||||
- windows
|
||||
versions:
|
||||
- nightly-latest
|
||||
|
||||
35
pr-checks/checks/linux-arm64.yml
Normal file
35
pr-checks/checks/linux-arm64.yml
Normal file
@@ -0,0 +1,35 @@
|
||||
name: "Linux Arm64"
|
||||
description: "An end-to-end integration test running on a Linux Arm64 runner, checking that the native linux-arm64 CodeQL bundle is downloaded and can analyze interpreted and compiled code"
|
||||
operatingSystems:
|
||||
- os: ubuntu
|
||||
runner-image: ubuntu-24.04-arm
|
||||
# The native linux-arm64 CodeQL bundle is only available in recent CLI releases, so we restrict this
|
||||
# check to `nightly-latest`, which is guaranteed to ship it. Older stable versions do not have an
|
||||
# arm64 asset, and `prepare-test` would resolve an x64 bundle URL for them on this runner.
|
||||
versions:
|
||||
- nightly-latest
|
||||
installGo: true
|
||||
installDotNet: true
|
||||
# The set of languages CodeQL supports on this platform, excluding Swift (macOS only).
|
||||
env:
|
||||
LANGUAGES: cpp,csharp,go,java,javascript,python,ruby
|
||||
steps:
|
||||
- uses: ./../action/init
|
||||
with:
|
||||
languages: ${{ env.LANGUAGES }}
|
||||
tools: ${{ steps.prepare-test.outputs.tools-url }}
|
||||
- name: Build code
|
||||
run: ./build.sh
|
||||
- uses: ./../action/analyze
|
||||
with:
|
||||
upload-database: false
|
||||
- name: Assert databases exist
|
||||
run: |
|
||||
cd "$RUNNER_TEMP/codeql_databases"
|
||||
for lang in ${LANGUAGES//,/ }; do
|
||||
if [[ ! -d "$lang" ]]; then
|
||||
echo "Did not find a database for $lang"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found database for $lang"
|
||||
done
|
||||
@@ -15,6 +15,7 @@ operatingSystems:
|
||||
- stable-v2.21.4
|
||||
- stable-v2.22.4
|
||||
env:
|
||||
CODEQL_ACTION_CLEANUP_TOOLCACHE_BUNDLES: true
|
||||
CODEQL_ACTION_RESOLVE_SUPPORTED_LANGUAGES_USING_CLI: true
|
||||
installGo: true
|
||||
installDotNet: true
|
||||
|
||||
@@ -5,7 +5,8 @@ versions:
|
||||
- default
|
||||
- nightly-latest
|
||||
operatingSystems:
|
||||
- macos
|
||||
- os: macos
|
||||
runner-image: macos-latest-xlarge
|
||||
installGo: true
|
||||
installDotNet: true
|
||||
env:
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
"@actions/core": "^2.0.3",
|
||||
"@actions/github": "^8.0.1",
|
||||
"@octokit/core": "^7.0.7",
|
||||
"@octokit/plugin-paginate-rest": ">=9.2.2",
|
||||
"@octokit/plugin-rest-endpoint-methods": "^17.0.0",
|
||||
"@octokit/plugin-paginate-rest": ">=15.0.0",
|
||||
"@octokit/plugin-rest-endpoint-methods": "^18.0.0",
|
||||
"semver": "^7.8.5",
|
||||
"yaml": "^2.9.0"
|
||||
},
|
||||
|
||||
@@ -253,8 +253,8 @@ const languageSetups: LanguageSetups = {
|
||||
name: "Install Java",
|
||||
uses: pinnedUses(
|
||||
"actions/setup-java",
|
||||
"b6effb05e454b25005698d916606bdc6ffcbf961",
|
||||
"v5.7.0",
|
||||
"dd06d9cba3e5552c54d9f8ea23572deb30010f7c",
|
||||
"v6.0.0",
|
||||
),
|
||||
with: {
|
||||
"java-version": `\${{ inputs.java-version || '${defaultLanguageVersions.java}' }}`,
|
||||
|
||||
200
scripts/changetool/cli/validate.test.ts
Normal file
200
scripts/changetool/cli/validate.test.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, it } from "node:test";
|
||||
|
||||
import {
|
||||
isValidChangenoteContent,
|
||||
isValidChangenoteFile,
|
||||
isValidChangenoteFilename,
|
||||
hasValidChangenoteCategory,
|
||||
VALID_CHANGE_NOTE_CATEGORIES,
|
||||
} from "./validate.ts";
|
||||
|
||||
async function withTmpFile<T>(
|
||||
baseFileName: string,
|
||||
contents: string,
|
||||
body: (filePath: string) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const tmpDir = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "changetool-validate-test-"),
|
||||
);
|
||||
try {
|
||||
const filePath = path.join(tmpDir, baseFileName);
|
||||
fs.writeFileSync(filePath, contents);
|
||||
return await body(filePath);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
await describe("isValidChangenoteContent", async () => {
|
||||
await it("recognizes an unordered Markdown list", async () => {
|
||||
const inputs = [
|
||||
"- One changenote entry",
|
||||
"- First item\n- Second item",
|
||||
"\n\n\n\n- Fixed a bug\n- Added a feature",
|
||||
];
|
||||
|
||||
for (const input of inputs) {
|
||||
assert.equal(isValidChangenoteContent(input), true);
|
||||
}
|
||||
});
|
||||
|
||||
await it("does not recognize non-Markdown text", async () => {
|
||||
const inputs = [
|
||||
"This is not a list.",
|
||||
'["this", "is", "JSON"]',
|
||||
"---",
|
||||
"***",
|
||||
"___",
|
||||
"paragraph",
|
||||
];
|
||||
|
||||
for (const input of inputs) {
|
||||
assert.equal(isValidChangenoteContent(input), false);
|
||||
}
|
||||
});
|
||||
|
||||
await it("does not recognize ordered Markdown lists", async () => {
|
||||
const inputs = [
|
||||
"1. First item\n2. Second item",
|
||||
"\n\n\n1. First item\n1. Second item",
|
||||
];
|
||||
|
||||
for (const input of inputs) {
|
||||
assert.equal(isValidChangenoteContent(input), false);
|
||||
}
|
||||
});
|
||||
|
||||
await it("requires all list items to use a hyphen bullet", async () => {
|
||||
const inputs = [
|
||||
"* Fixed a bug\n* Added feature",
|
||||
"+ Fixed a bug\n+ Added feature",
|
||||
"- Fixed a bug\n* Added feature",
|
||||
"- Fixed a bug\n+ Added feature",
|
||||
"- Fixed a bug\n * Added feature\n + Updated docs",
|
||||
"\n\n\n* Fixed a bug",
|
||||
"\n\n\n+ Fixed a bug",
|
||||
"---\n* Fixed a bug\n* Added feature",
|
||||
] as const;
|
||||
|
||||
for (const input of inputs) {
|
||||
assert.equal(isValidChangenoteContent(input), false);
|
||||
}
|
||||
});
|
||||
|
||||
await it("does not contain other Markdown elements", async () => {
|
||||
const inputs = [
|
||||
"- Fixed a bug\n\nParagraph of text",
|
||||
"- Fixed a bug\n\n* Added a feature",
|
||||
"# Header\n- Fixed a bug",
|
||||
"- Fixed a bug\n## Subheader",
|
||||
];
|
||||
|
||||
for (const input of inputs) {
|
||||
assert.equal(isValidChangenoteContent(input), false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
await describe("isValidChangenoteFilename", async () => {
|
||||
await it("accepts valid filenames", async () => {
|
||||
const inputs = [
|
||||
"2023-01-01-fix-bug.md",
|
||||
"2023-12-31-add-feature.md",
|
||||
"2023-06-15-update-docs.md",
|
||||
];
|
||||
|
||||
for (const input of inputs) {
|
||||
assert.equal(isValidChangenoteFilename(input), true);
|
||||
}
|
||||
});
|
||||
|
||||
await it("rejects invalid filenames", async () => {
|
||||
const inputs = [
|
||||
"missing-date-from-filename.md",
|
||||
"2021-01-01.md",
|
||||
"2026-12-19-wrong-file-name-extension.txt",
|
||||
];
|
||||
|
||||
for (const input of inputs) {
|
||||
assert.equal(isValidChangenoteFilename(input), false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
await describe("hasValidChangenoteCategory", async () => {
|
||||
await it("accepts valid categories", async () => {
|
||||
for (const category of Object.keys(VALID_CHANGE_NOTE_CATEGORIES)) {
|
||||
const frontmatter = { category };
|
||||
assert.equal(hasValidChangenoteCategory(frontmatter), true);
|
||||
}
|
||||
});
|
||||
|
||||
await it("rejects invalid categories", async () => {
|
||||
const inputs = [
|
||||
"",
|
||||
"invalid-category",
|
||||
"bug-fix",
|
||||
"new-feature",
|
||||
"security-patch",
|
||||
"miscellaneous",
|
||||
"documentation",
|
||||
];
|
||||
|
||||
for (const category of inputs) {
|
||||
const frontmatter = { category };
|
||||
assert.equal(hasValidChangenoteCategory(frontmatter), false);
|
||||
}
|
||||
});
|
||||
|
||||
await it("reject missing category", async () => {
|
||||
assert.equal(hasValidChangenoteCategory({}), false);
|
||||
assert.equal(hasValidChangenoteCategory({ category: null }), false);
|
||||
assert.equal(hasValidChangenoteCategory({ category: undefined }), false);
|
||||
});
|
||||
});
|
||||
|
||||
await describe("isValidChangenoteFile", async () => {
|
||||
await it("accepts a valid change-note file", async () => {
|
||||
await withTmpFile(
|
||||
"2026-01-01-fix-bug.md",
|
||||
"---\ncategory: fix\n---\n- Fixed a bug\n",
|
||||
async (filePath) => {
|
||||
assert.equal(isValidChangenoteFile(filePath), true);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
await it("rejects invalid filename", async () => {
|
||||
await withTmpFile(
|
||||
"fix-bug.md",
|
||||
"---\ncategory: fix\n---\n- Fixed a bug\n",
|
||||
async (filePath) => {
|
||||
assert.equal(isValidChangenoteFile(filePath), false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
await it("rejects missing frontmatter", async () => {
|
||||
await withTmpFile(
|
||||
"2026-01-01-fix-bug.md",
|
||||
"- Fixed a bug\n",
|
||||
async (filePath) => {
|
||||
assert.equal(isValidChangenoteFile(filePath), false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
await it("rejects invalid Markdown", async () => {
|
||||
await withTmpFile(
|
||||
"2026-01-01-fix-bug.md",
|
||||
"---\ncategory: fix\n---\n* Fixed a bug\n",
|
||||
async (filePath) => {
|
||||
assert.equal(isValidChangenoteFile(filePath), false);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
121
scripts/changetool/cli/validate.ts
Normal file
121
scripts/changetool/cli/validate.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
|
||||
import { matter } from "lite-matter";
|
||||
import type { List, ListItem } from "mdast";
|
||||
import { fromMarkdown } from "mdast-util-from-markdown";
|
||||
|
||||
// Regex for filename: YYYY-MM-DD-id.md
|
||||
const VALID_CHANGE_NOTE_FILENAME_PATTERN =
|
||||
/^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])-([a-z0-9]+(?:-[a-z0-9]+)*)\.md$/;
|
||||
|
||||
export const VALID_CHANGE_NOTE_CATEGORIES = {
|
||||
breaking: "Breaking Changes",
|
||||
feature: "New Features",
|
||||
improvement: "Improvements",
|
||||
securityFix: "Security Fixes",
|
||||
fix: "Bug Fixes",
|
||||
unship: "Removed Features",
|
||||
deprecation: "Deprecations",
|
||||
knownIssue: "Known Issues",
|
||||
misc: "Miscellaneous",
|
||||
};
|
||||
|
||||
/**
|
||||
* Validates that the given Markdown string meets the criteria for a change-note, which is:
|
||||
* - A single unordered list
|
||||
* - Each list item must start with a hyphen (-)
|
||||
* - No other Markdown elements are allowed
|
||||
* @param content The Markdown string to validate
|
||||
* @returns True if the string is a valid change-note, false otherwise
|
||||
*/
|
||||
export function isValidChangenoteContent(content: string): boolean {
|
||||
const ast = fromMarkdown(content);
|
||||
const lines = content.split("\n");
|
||||
|
||||
function listHasHyphenBullets(node: List | ListItem): boolean {
|
||||
if (node.type === "list") {
|
||||
return node.children.every(listHasHyphenBullets);
|
||||
}
|
||||
|
||||
const line = lines[node.position!.start.line - 1].trim();
|
||||
return (
|
||||
line.startsWith("-") &&
|
||||
node.children.every(
|
||||
(child) => child.type !== "list" || listHasHyphenBullets(child),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
ast.children.length === 1 &&
|
||||
ast.children[0].type === "list" &&
|
||||
ast.children[0].ordered === false &&
|
||||
listHasHyphenBullets(ast.children[0])
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that the given filename meets the criteria for a change-note filename.
|
||||
* @param filename The name of the change-note file to validate.
|
||||
* @returns True if the filename is valid, false otherwise.
|
||||
*/
|
||||
export function isValidChangenoteFilename(filename: string): boolean {
|
||||
return filename.match(VALID_CHANGE_NOTE_FILENAME_PATTERN) !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that the given frontmatter has a valid change-note category.
|
||||
* @param frontmatter The frontmatter object to validate.
|
||||
* @returns True if the frontmatter has a valid category, false otherwise.
|
||||
*/
|
||||
export function hasValidChangenoteCategory(
|
||||
frontmatter: Record<string, unknown>,
|
||||
): boolean {
|
||||
const category = frontmatter["category"];
|
||||
return (
|
||||
typeof category === "string" &&
|
||||
Object.hasOwn(VALID_CHANGE_NOTE_CATEGORIES, category)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that the given change-note file meets all of the criteria for a change-note.
|
||||
* @param filename The name of the change-note file to validate.
|
||||
* @returns True if the file is a valid change-note, false otherwise.
|
||||
*/
|
||||
export function isValidChangenoteFile(filename: string): boolean {
|
||||
let isValid: boolean = true;
|
||||
|
||||
let fileData: string | undefined;
|
||||
try {
|
||||
fileData = fs.readFileSync(filename, "utf8");
|
||||
} catch (error) {
|
||||
console.error(`${filename}: failed to read file`, error);
|
||||
return false;
|
||||
}
|
||||
|
||||
const { data: frontmatter, content } = matter(fileData);
|
||||
|
||||
if (!isValidChangenoteFilename(path.basename(filename))) {
|
||||
isValid = false;
|
||||
console.error(
|
||||
`${filename}: invalid filename; must match pattern YYYY-MM-DD-id.md`,
|
||||
);
|
||||
}
|
||||
if (!hasValidChangenoteCategory(frontmatter)) {
|
||||
isValid = false;
|
||||
const categories = Object.keys(VALID_CHANGE_NOTE_CATEGORIES).join(", ");
|
||||
console.error(
|
||||
`${filename}: invalid category; must be one of: ${categories}`,
|
||||
);
|
||||
}
|
||||
if (!isValidChangenoteContent(content)) {
|
||||
isValid = false;
|
||||
console.error(
|
||||
`${filename}: invalid Markdown; content must be a single unordered list with hyphen bullets and no other Markdown elements`,
|
||||
);
|
||||
}
|
||||
|
||||
return isValid;
|
||||
}
|
||||
51
scripts/changetool/index.ts
Normal file
51
scripts/changetool/index.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
import { isValidChangenoteFile } from "./cli/validate.ts";
|
||||
|
||||
const entryPoint = process.argv[1];
|
||||
if (entryPoint && import.meta.url === pathToFileURL(entryPoint).href) {
|
||||
try {
|
||||
process.exit(main());
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
function main(): number {
|
||||
const { positionals } = parseArgs({
|
||||
allowPositionals: true,
|
||||
strict: true,
|
||||
});
|
||||
const [command, ...paths] = positionals;
|
||||
switch (command) {
|
||||
case undefined:
|
||||
case "help":
|
||||
return usage();
|
||||
case "validate":
|
||||
return validate(paths);
|
||||
default:
|
||||
console.error(`Unknown command: ${command}`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
function usage(): number {
|
||||
console.log("Usage: changetool validate <path> [<path> ...]");
|
||||
return 0;
|
||||
}
|
||||
|
||||
function validate(paths: string[]): number {
|
||||
let valid = true;
|
||||
if (paths.length === 0) {
|
||||
console.error("error: no paths provided (see 'help' command for usage)");
|
||||
return 1;
|
||||
}
|
||||
for (const path of paths) {
|
||||
if (!isValidChangenoteFile(path)) {
|
||||
valid = false;
|
||||
}
|
||||
}
|
||||
return valid ? 0 : 1;
|
||||
}
|
||||
21
scripts/changetool/package.json
Normal file
21
scripts/changetool/package.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "changetool",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "Validates change-notes and merges them into CHANGELOG.md",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "tsx index.ts",
|
||||
"test": "node --test --experimental-strip-types cli/*.test.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^26.2.0",
|
||||
"tsx": "^4.23.12",
|
||||
"typescript": "^7.0.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"lite-matter": "^0.1.2",
|
||||
"mdast-util-from-markdown": "^2.0.3"
|
||||
}
|
||||
}
|
||||
11
scripts/changetool/tsconfig.json
Normal file
11
scripts/changetool/tsconfig.json
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"module": "preserve",
|
||||
"allowImportingTsExtensions": true,
|
||||
"rootDir": ".",
|
||||
"sourceMap": false
|
||||
},
|
||||
"include": ["./**/*.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
@@ -283,6 +283,19 @@ export function isSelfHostedRunner(env: Env = getEnv()) {
|
||||
return env.getOptional(ActionsEnvVars.RUNNER_ENVIRONMENT) === "self-hosted";
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the job is running on a runner that GitHub hosts, and whose toolcache is therefore thrown
|
||||
* away once the job has finished.
|
||||
*
|
||||
* Unlike `looksLikeHostedRunner`, this is based on what the service reports for the job rather than
|
||||
* on how the runner's filesystem happens to be laid out, so it does not match self-hosted runners
|
||||
* that are configured to resemble hosted ones, such as those that mount a persistent volume at
|
||||
* `/opt/hostedtoolcache`.
|
||||
*/
|
||||
export function isGitHubHostedRunner(env: Env = getEnv()) {
|
||||
return env.getOptional(ActionsEnvVars.RUNNER_ENVIRONMENT) === "github-hosted";
|
||||
}
|
||||
|
||||
/** Determines whether the workflow trigger is `dynamic`. */
|
||||
export function isDynamicWorkflow(env: Env = getEnv()): boolean {
|
||||
return getWorkflowEventName(env) === "dynamic";
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import * as core from "@actions/core";
|
||||
import * as githubUtils from "@actions/github/lib/utils";
|
||||
import { type Octokit } from "@octokit/core";
|
||||
import { type PaginateInterface } from "@octokit/plugin-paginate-rest";
|
||||
import { type Api } from "@octokit/plugin-rest-endpoint-methods";
|
||||
import * as retry from "@octokit/plugin-retry";
|
||||
import { RequestRequestOptions } from "@octokit/types";
|
||||
import {
|
||||
@@ -128,7 +125,7 @@ export function makeProxyRequestOptions(
|
||||
}
|
||||
|
||||
/** The type of GitHub API client we use. */
|
||||
export type ApiClient = Octokit & Api & { paginate: PaginateInterface };
|
||||
export type ApiClient = InstanceType<typeof githubUtils.GitHub>;
|
||||
|
||||
/** Options for `createApiClientWithDetails`. */
|
||||
interface CreateApiClientOptions {
|
||||
|
||||
@@ -5,7 +5,7 @@ import * as core from "@actions/core";
|
||||
import { getOptionalInput, isDefaultSetup } from "./actions-util";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Logger } from "./logging";
|
||||
import { isHostedRunner, tryGetFolderBytes } from "./util";
|
||||
import { looksLikeHostedRunner, tryGetFolderBytes } from "./util";
|
||||
|
||||
/**
|
||||
* Returns the total size of all the specified paths.
|
||||
@@ -109,7 +109,7 @@ export function getDependencyCachingEnabled(): CachingKind {
|
||||
if (dependencyCaching !== undefined) return getCachingKind(dependencyCaching);
|
||||
|
||||
// On self-hosted runners which may have dependencies installed centrally, disable caching by default
|
||||
if (!isHostedRunner()) return CachingKind.None;
|
||||
if (!looksLikeHostedRunner()) return CachingKind.None;
|
||||
|
||||
// Disable in advanced workflows by default.
|
||||
if (!isDefaultSetup()) return CachingKind.None;
|
||||
|
||||
@@ -128,7 +128,6 @@ test("CliError constructor with empty stderr", (t) => {
|
||||
|
||||
for (const [platform, arch] of [
|
||||
["weird_plat", "x64"],
|
||||
["linux", "arm64"],
|
||||
["win32", "arm64"],
|
||||
]) {
|
||||
test.serial(
|
||||
@@ -157,20 +156,34 @@ for (const [platform, arch] of [
|
||||
);
|
||||
}
|
||||
|
||||
test("wrapCliConfigurationError - supported platform", (t) => {
|
||||
const commandError = new CommandInvocationError(
|
||||
"codeql",
|
||||
["version"],
|
||||
1,
|
||||
"Some error",
|
||||
for (const [platform, arch] of [
|
||||
["linux", "x64"],
|
||||
["linux", "arm64"],
|
||||
["win32", "x64"],
|
||||
["darwin", "x64"],
|
||||
["darwin", "arm64"],
|
||||
]) {
|
||||
test.serial(
|
||||
`wrapCliConfigurationError - ${platform}/${arch} supported`,
|
||||
(t) => {
|
||||
sinon.stub(process, "platform").value(platform);
|
||||
sinon.stub(process, "arch").value(arch);
|
||||
const commandError = new CommandInvocationError(
|
||||
"codeql",
|
||||
["version"],
|
||||
1,
|
||||
"Some error",
|
||||
);
|
||||
const cliError = new CliError(commandError);
|
||||
|
||||
const wrappedError = wrapCliConfigurationError(cliError);
|
||||
|
||||
// Should return the original error since the platform is supported, rather
|
||||
// than replacing it with the unsupported-platform ConfigurationError.
|
||||
t.is(wrappedError, cliError);
|
||||
},
|
||||
);
|
||||
const cliError = new CliError(commandError);
|
||||
|
||||
const wrappedError = wrapCliConfigurationError(cliError);
|
||||
|
||||
// Should return the original error since platform is supported
|
||||
t.is(wrappedError, cliError);
|
||||
});
|
||||
}
|
||||
|
||||
test("wrapCliConfigurationError - autobuild error", (t) => {
|
||||
const commandError = new CommandInvocationError(
|
||||
|
||||
@@ -8,6 +8,7 @@ import { ConfigurationError } from "./util";
|
||||
|
||||
const SUPPORTED_PLATFORMS = [
|
||||
["linux", "x64"],
|
||||
["linux", "arm64"],
|
||||
["win32", "x64"],
|
||||
["darwin", "x64"],
|
||||
["darwin", "arm64"],
|
||||
|
||||
@@ -3,12 +3,11 @@ import path from "path";
|
||||
|
||||
import test from "ava";
|
||||
|
||||
import { EnvVar } from "../environment";
|
||||
import { getRunnerLogger } from "../logging";
|
||||
import { getTestEnv, setupTests } from "../testing-utils";
|
||||
import { setupTests } from "../testing-utils";
|
||||
import * as util from "../util";
|
||||
|
||||
import * as outputCache from "./output-cache";
|
||||
import { getCachedCodeQlVersion } from "./output-cache";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
@@ -18,18 +17,18 @@ test.serial(
|
||||
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
|
||||
async (t) => {
|
||||
await util.withTmpDir(async (tmpDir: string) => {
|
||||
const cacheFile = path.join(tmpDir, "codeql-action-command-cache.json");
|
||||
const cacheFilePath = path.join(tmpDir, "cache.json");
|
||||
|
||||
fs.writeFileSync(
|
||||
cacheFile,
|
||||
cacheFilePath,
|
||||
JSON.stringify({
|
||||
cmd: "/path/to/codeql",
|
||||
entries: { version: { version: "2.20.0" } },
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
|
||||
t.deepEqual(
|
||||
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
|
||||
getCachedCodeQlVersion(logger, cacheFilePath, "/path/to/codeql"),
|
||||
{
|
||||
version: "2.20.0",
|
||||
},
|
||||
@@ -42,18 +41,17 @@ test.serial(
|
||||
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
|
||||
async (t) => {
|
||||
await util.withTmpDir(async (tmpDir: string) => {
|
||||
const cacheFile = path.join(tmpDir, "version.json");
|
||||
const cacheFilePath = path.join(tmpDir, "cache.json");
|
||||
fs.writeFileSync(
|
||||
cacheFile,
|
||||
cacheFilePath,
|
||||
JSON.stringify({
|
||||
cmd: "/path/to/other-codeql",
|
||||
version: { version: "2.20.0" },
|
||||
entries: { version: { version: "2.20.0" } },
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
|
||||
t.is(
|
||||
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
|
||||
getCachedCodeQlVersion(logger, cacheFilePath, "/path/to/codeql"),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
@@ -64,11 +62,10 @@ test.serial(
|
||||
"getCachedCodeQlVersion ignores a malformed persisted value",
|
||||
async (t) => {
|
||||
await util.withTmpDir(async (tmpDir: string) => {
|
||||
const cacheFile = path.join(tmpDir, "version.json");
|
||||
fs.writeFileSync(cacheFile, "not valid json", "utf8");
|
||||
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
|
||||
const cacheFilePath = path.join(tmpDir, "cache.json");
|
||||
fs.writeFileSync(cacheFilePath, "not valid json", "utf8");
|
||||
t.is(
|
||||
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
|
||||
getCachedCodeQlVersion(logger, cacheFilePath, "/path/to/codeql"),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
@@ -79,9 +76,7 @@ test.serial(
|
||||
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
|
||||
async (t) => {
|
||||
await util.withTmpDir(async (tmpDir: string) => {
|
||||
const cacheFile = path.join(tmpDir, "version.json");
|
||||
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
|
||||
|
||||
const cacheFilePath = path.join(tmpDir, "cache.json");
|
||||
const testValues = [
|
||||
{ cmd: "/path/to/codeql" },
|
||||
{ entries: { version: { version: "2.20.0" } } },
|
||||
@@ -104,9 +99,9 @@ test.serial(
|
||||
].map((v) => JSON.stringify(v));
|
||||
|
||||
for (const value of testValues) {
|
||||
fs.writeFileSync(cacheFile, value, "utf8");
|
||||
fs.writeFileSync(cacheFilePath, value, "utf8");
|
||||
t.is(
|
||||
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
|
||||
getCachedCodeQlVersion(logger, cacheFilePath, "/path/to/codeql"),
|
||||
undefined,
|
||||
value,
|
||||
);
|
||||
@@ -117,10 +112,10 @@ test.serial(
|
||||
|
||||
test.serial("getCachedCodeQlVersion ignores non-existent file", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir: string) => {
|
||||
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
|
||||
const cacheFilePath = path.join(tmpDir, "cache.json");
|
||||
t.notThrows(() => {
|
||||
t.is(
|
||||
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
|
||||
getCachedCodeQlVersion(logger, cacheFilePath, "/path/to/codeql"),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -3,9 +3,10 @@ import path from "path";
|
||||
|
||||
import { getTemporaryDirectory } from "../actions-util";
|
||||
import { Env } from "../environment";
|
||||
import * as json from "../json";
|
||||
import { Logger } from "../logging";
|
||||
|
||||
import type { VersionInfo } from "./types";
|
||||
import { VersionInfo, versionInfoBaseSchema } from "./types";
|
||||
|
||||
/**
|
||||
* The keys of the command cache. Each key corresponds to a command whose output we cache.
|
||||
@@ -13,12 +14,19 @@ import type { VersionInfo } from "./types";
|
||||
export type CommandCacheKey = string;
|
||||
|
||||
/**
|
||||
* The type of the command cache that is persisted to disk.
|
||||
* The JSON schema of the command cache that is persisted to disk.
|
||||
*/
|
||||
export interface OutputCache {
|
||||
cmd: string;
|
||||
entries: Record<CommandCacheKey, unknown>;
|
||||
}
|
||||
const outputCacheSchema = {
|
||||
cmd: json.string,
|
||||
entries: json.object({}),
|
||||
} as const satisfies json.Schema;
|
||||
|
||||
/**
|
||||
* The type that describes the command cache that is persisted to disk.
|
||||
*/
|
||||
export type OutputCache = json.FromSchema<typeof outputCacheSchema> & {
|
||||
entries: { version: VersionInfo };
|
||||
};
|
||||
|
||||
/**
|
||||
* The name of the temporary file that backs the on-disk cache of
|
||||
@@ -43,18 +51,18 @@ export function resetCachedCodeQlVersion(): void {
|
||||
* Returns the path to the temporary file that backs the
|
||||
* on-disk cache of CLI responses between workflow steps.
|
||||
*/
|
||||
function getCommandCacheFilePath(env: Env): string {
|
||||
export function getCommandCacheFilePath(env: Env): string {
|
||||
return path.join(getTemporaryDirectory(env), COMMAND_CACHE_FILENAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches the CodeQL CLI version both in-memory and on disk.
|
||||
* @param env The environment variables to use.
|
||||
* @param cacheFilePath The path to the cache file.
|
||||
* @param cmd The path to the CodeQL CLI.
|
||||
* @param version The version information to cache.
|
||||
*/
|
||||
export function cacheCodeQlVersion(
|
||||
env: Env,
|
||||
cacheFilePath: string,
|
||||
cmd: string,
|
||||
version: VersionInfo,
|
||||
): void {
|
||||
@@ -70,22 +78,18 @@ export function cacheCodeQlVersion(
|
||||
// processes, can reuse it rather than invoking `codeql version` again. We
|
||||
// record the CLI path so that a different step using a different CodeQL bundle
|
||||
// doesn't pick up a stale version.
|
||||
fs.writeFileSync(
|
||||
getCommandCacheFilePath(env),
|
||||
JSON.stringify(outputCache),
|
||||
"utf8",
|
||||
);
|
||||
fs.writeFileSync(cacheFilePath, JSON.stringify(outputCache), "utf8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the cached CodeQL CLI version, if any.
|
||||
* @param logger The logger to use for logging messages.
|
||||
* @param env The environment variables to use.
|
||||
* @param cacheFilePath The path to the cache file.
|
||||
* @param cmd The path to the CodeQL CLI.
|
||||
*/
|
||||
export function getCachedCodeQlVersion(
|
||||
logger: Logger,
|
||||
env: Env,
|
||||
cacheFilePath: string,
|
||||
cmd?: string,
|
||||
): undefined | VersionInfo {
|
||||
if (cachedCodeQlVersion !== undefined) {
|
||||
@@ -96,11 +100,9 @@ export function getCachedCodeQlVersion(
|
||||
// invokes `codeql version` instead.
|
||||
let serialized: string;
|
||||
try {
|
||||
serialized = fs.readFileSync(getCommandCacheFilePath(env), "utf8");
|
||||
serialized = fs.readFileSync(cacheFilePath, "utf8");
|
||||
} catch (e) {
|
||||
logger.debug(
|
||||
`Cannot read CLI-cache file ${getCommandCacheFilePath(env)}: ${e}`,
|
||||
);
|
||||
logger.debug(`Cannot read CLI-cache file ${cacheFilePath}: ${e}`);
|
||||
return undefined;
|
||||
}
|
||||
let persisted: unknown;
|
||||
@@ -127,17 +129,7 @@ export function getCachedCodeQlVersion(
|
||||
* @param x The value to test
|
||||
*/
|
||||
function isVersionInfo(x: unknown): x is VersionInfo {
|
||||
const candidate = x as Partial<VersionInfo> | null;
|
||||
return (
|
||||
typeof candidate === "object" &&
|
||||
candidate !== null &&
|
||||
typeof candidate.version === "string" &&
|
||||
(candidate.features === undefined ||
|
||||
(typeof candidate.features === "object" &&
|
||||
candidate.features !== null)) &&
|
||||
(candidate.overlayVersion === undefined ||
|
||||
typeof candidate.overlayVersion === "number")
|
||||
);
|
||||
return json.isObject(x) && json.validateSchema(versionInfoBaseSchema, x);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -145,12 +137,10 @@ function isVersionInfo(x: unknown): x is VersionInfo {
|
||||
* @param x The value to test
|
||||
*/
|
||||
function isOutputCache(x: unknown): x is OutputCache {
|
||||
const candidate = x as Partial<OutputCache> | null;
|
||||
return (
|
||||
typeof candidate === "object" &&
|
||||
candidate !== null &&
|
||||
typeof candidate.cmd === "string" &&
|
||||
candidate.entries !== undefined &&
|
||||
isVersionInfo(candidate.entries.version)
|
||||
json.isObject(x) &&
|
||||
json.validateSchema(outputCacheSchema, x) &&
|
||||
json.isObject<{ version: unknown }>(x.entries) &&
|
||||
isVersionInfo(x.entries.version)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
export interface VersionInfo {
|
||||
version: string;
|
||||
features?: { [name: string]: boolean };
|
||||
import * as json from "../json";
|
||||
|
||||
/**
|
||||
* The JSON schema of the expected output of the `codeql version` command.
|
||||
*/
|
||||
export const versionInfoBaseSchema = {
|
||||
version: json.string,
|
||||
features: json.optional(json.object({})),
|
||||
/**
|
||||
* The overlay version helps deal with backward incompatible changes for
|
||||
* overlay analysis. When a precompiled query pack reports the same overlay
|
||||
@@ -9,5 +14,17 @@ export interface VersionInfo {
|
||||
* or if either the pack or the CLI does not report an overlay version,
|
||||
* we need to revert to non-overlay analysis.
|
||||
*/
|
||||
overlayVersion?: number;
|
||||
}
|
||||
overlayVersion: json.optional(json.number),
|
||||
} as const satisfies json.Schema;
|
||||
|
||||
/**
|
||||
* The base type that describes the expected output of the `codeql version` command.
|
||||
*/
|
||||
export type VersionInfoBase = json.FromSchema<typeof versionInfoBaseSchema>;
|
||||
|
||||
/**
|
||||
* The full type that describes the expected output of the `codeql version` command.
|
||||
*/
|
||||
export type VersionInfo = Omit<VersionInfoBase, "features"> & {
|
||||
features?: { [name: string]: boolean };
|
||||
};
|
||||
|
||||
@@ -510,7 +510,12 @@ async function getCodeQLForCmd(
|
||||
return cmd;
|
||||
},
|
||||
async getVersion() {
|
||||
let result = outputCache.getCachedCodeQlVersion(logger, getEnv(), cmd);
|
||||
const cacheFilePath = outputCache.getCommandCacheFilePath(getEnv());
|
||||
let result = outputCache.getCachedCodeQlVersion(
|
||||
logger,
|
||||
cacheFilePath,
|
||||
cmd,
|
||||
);
|
||||
if (result === undefined) {
|
||||
result = await runCliJson<VersionInfo>(
|
||||
cmd,
|
||||
@@ -519,7 +524,7 @@ async function getCodeQLForCmd(
|
||||
noStreamStdout: true,
|
||||
},
|
||||
);
|
||||
outputCache.cacheCodeQlVersion(getEnv(), cmd, result);
|
||||
outputCache.cacheCodeQlVersion(cacheFilePath, cmd, result);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
|
||||
@@ -90,9 +90,8 @@ import {
|
||||
Result,
|
||||
Success,
|
||||
Failure,
|
||||
isHostedRunner,
|
||||
looksLikeHostedRunner,
|
||||
} from "./util";
|
||||
|
||||
export { type Config } from "./config/action-config";
|
||||
|
||||
/**
|
||||
@@ -938,7 +937,7 @@ export async function isTrapCachingEnabled(
|
||||
if (trapCaching !== undefined) return trapCaching === "true";
|
||||
|
||||
// On self-hosted runners which may have slow network access, disable TRAP caching by default.
|
||||
if (!isHostedRunner()) return false;
|
||||
if (!looksLikeHostedRunner()) return false;
|
||||
|
||||
// If overlay analysis is enabled, then disable TRAP caching since overlay analysis supersedes it.
|
||||
// This change is gated behind a feature flag.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"bundleVersion": "codeql-bundle-v2.26.4",
|
||||
"cliVersion": "2.26.4",
|
||||
"priorBundleVersion": "codeql-bundle-v2.26.3",
|
||||
"priorCliVersion": "2.26.3"
|
||||
"bundleVersion": "codeql-bundle-v2.27.0",
|
||||
"cliVersion": "2.27.0",
|
||||
"priorBundleVersion": "codeql-bundle-v2.26.4",
|
||||
"priorCliVersion": "2.26.4"
|
||||
}
|
||||
|
||||
@@ -63,6 +63,12 @@ export enum EnvVar {
|
||||
/** Whether the CodeQL Action has already warned the user about low disk space. */
|
||||
HAS_WARNED_ABOUT_DISK_SPACE = "CODEQL_ACTION_HAS_WARNED_ABOUT_DISK_SPACE",
|
||||
|
||||
/**
|
||||
* Whether a step in this job has already set up CodeQL. Steps that run afterwards may be holding
|
||||
* a path into the toolcache, so we must not delete anything from it.
|
||||
*/
|
||||
HAS_SET_UP_CODEQL = "CODEQL_ACTION_HAS_SET_UP_CODEQL",
|
||||
|
||||
/** Whether the `setup-codeql` action has been run. */
|
||||
SETUP_CODEQL_ACTION_HAS_RUN = "CODEQL_ACTION_SETUP_CODEQL_HAS_RUN",
|
||||
|
||||
|
||||
@@ -74,6 +74,11 @@ export enum Feature {
|
||||
AllowMergeConfigFiles = "allow_merge_config_files",
|
||||
/** Controls whether we allow multiple values for the `analysis-kinds` input. */
|
||||
AllowMultipleAnalysisKinds = "allow_multiple_analysis_kinds",
|
||||
/**
|
||||
* Controls whether we delete CodeQL bundles that we are not going to use from the toolcache
|
||||
* before downloading a different bundle, in order to reclaim disk space.
|
||||
*/
|
||||
CleanupToolcacheBundles = "cleanup_toolcache_bundles",
|
||||
CleanupTrapCaches = "cleanup_trap_caches",
|
||||
/** Whether to allow the `config-file` input to be specified via a repository property. */
|
||||
ConfigFileRepositoryProperty = "config_file_repository_property",
|
||||
@@ -211,6 +216,11 @@ export const featureConfig = {
|
||||
envVar: "CODEQL_ACTION_ALLOW_MULTIPLE_ANALYSIS_KINDS",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.CleanupToolcacheBundles]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_CLEANUP_TOOLCACHE_BUNDLES",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.CleanupTrapCaches]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_CLEANUP_TRAP_CACHES",
|
||||
|
||||
@@ -15,10 +15,12 @@ import { getRunnerLogger } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import * as overlayStatus from "./overlay/status";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import { JobStatus } from "./status-report";
|
||||
import {
|
||||
createFeatures,
|
||||
createTestConfig,
|
||||
DEFAULT_ACTIONS_VARS,
|
||||
getTestEnv,
|
||||
makeMacro,
|
||||
makeVersionInfo,
|
||||
RecordingLogger,
|
||||
@@ -58,6 +60,8 @@ test.serial("init-post action with debug mode off", async (t) => {
|
||||
createTestConfig({ debugMode: false }),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([]),
|
||||
"success",
|
||||
getTestEnv(),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
@@ -80,6 +84,8 @@ test.serial("init-post action with debug mode on", async (t) => {
|
||||
createTestConfig({ debugMode: true }),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([]),
|
||||
"success",
|
||||
getTestEnv(),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
@@ -375,6 +381,8 @@ test.serial(
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([Feature.OverlayAnalysisStatusSave]),
|
||||
"success",
|
||||
getTestEnv(),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
@@ -443,6 +451,8 @@ test.serial(
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([]),
|
||||
"success",
|
||||
getTestEnv(),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
@@ -457,8 +467,13 @@ test.serial(
|
||||
test.serial("does not save overlay status when build successful", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
// Mark analyze as having completed successfully.
|
||||
// Mark analyze as having completed successfully. `tryUploadSarifIfRunFailed` reads this from
|
||||
// the process environment, while `recordOverlayStatus` reads it from the environment it is
|
||||
// given.
|
||||
process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY] = "true";
|
||||
const env = getTestEnv({
|
||||
[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY]: "true",
|
||||
});
|
||||
|
||||
sinon.stub(util, "checkDiskUsage").resolves({
|
||||
numAvailableBytes: 100 * NUM_BYTES_PER_GIB,
|
||||
@@ -480,6 +495,8 @@ test.serial("does not save overlay status when build successful", async (t) => {
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([Feature.OverlayAnalysisStatusSave]),
|
||||
"success",
|
||||
env,
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
@@ -517,6 +534,8 @@ test.serial(
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([]),
|
||||
"success",
|
||||
getTestEnv(),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
@@ -528,6 +547,137 @@ test.serial(
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* Runs `uploadFailureInfo` for an overlay-base job that did not complete successfully, with the
|
||||
* given job status from the Actions runtime environment.
|
||||
*/
|
||||
async function runOverlayPostStep({
|
||||
jobStatus,
|
||||
codeQlReportedError = false,
|
||||
}: {
|
||||
jobStatus: string | undefined;
|
||||
codeQlReportedError?: boolean;
|
||||
}) {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
delete process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY];
|
||||
const env = getTestEnv(
|
||||
codeQlReportedError
|
||||
? { [EnvVar.JOB_STATUS]: JobStatus.FailureStatus }
|
||||
: {},
|
||||
);
|
||||
|
||||
sinon.stub(util, "checkDiskUsage").resolves({
|
||||
numAvailableBytes: 100 * NUM_BYTES_PER_GIB,
|
||||
numTotalBytes: 200 * NUM_BYTES_PER_GIB,
|
||||
});
|
||||
|
||||
const saveOverlayStatusStub = sinon
|
||||
.stub(overlayStatus, "saveOverlayStatus")
|
||||
.resolves(true);
|
||||
|
||||
await initActionPostHelper.uploadFailureInfo(
|
||||
sinon.spy(),
|
||||
sinon.spy(),
|
||||
codeql.createStubCodeQL({}),
|
||||
createTestConfig({
|
||||
debugMode: false,
|
||||
languages: ["javascript"],
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([Feature.OverlayAnalysisStatusSave]),
|
||||
jobStatus,
|
||||
env,
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
return { saveOverlayStatusStub };
|
||||
});
|
||||
}
|
||||
|
||||
test.serial(
|
||||
"does not save overlay status when the job was cancelled",
|
||||
async (t) => {
|
||||
const { saveOverlayStatusStub } = await runOverlayPostStep({
|
||||
jobStatus: "cancelled",
|
||||
});
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.notCalled,
|
||||
"a cancellation tells us nothing about whether the analysis would have succeeded",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"does not save overlay status when the job status is not recognised",
|
||||
async (t) => {
|
||||
const { saveOverlayStatusStub } = await runOverlayPostStep({
|
||||
jobStatus: "some-new-status",
|
||||
});
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.notCalled,
|
||||
"a status we do not recognise tells us nothing about whether the analysis would have succeeded",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"does not save overlay status when the job status is unavailable",
|
||||
async (t) => {
|
||||
const { saveOverlayStatusStub } = await runOverlayPostStep({
|
||||
jobStatus: undefined,
|
||||
});
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.notCalled,
|
||||
"without a job status we cannot tell whether the analysis would have succeeded",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"saves overlay status when the job failed rather than being cancelled",
|
||||
async (t) => {
|
||||
const { saveOverlayStatusStub } = await runOverlayPostStep({
|
||||
jobStatus: "failure",
|
||||
});
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.calledOnce,
|
||||
"a failed job indicates that the analysis itself failed",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial("saves overlay status when the job succeeded", async (t) => {
|
||||
const { saveOverlayStatusStub } = await runOverlayPostStep({
|
||||
jobStatus: "success",
|
||||
});
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.calledOnce,
|
||||
"the analysis did not complete successfully even though the job as a whole succeeded",
|
||||
);
|
||||
});
|
||||
|
||||
test.serial(
|
||||
"saves overlay status when a CodeQL Action reported an error before the run was cancelled",
|
||||
async (t) => {
|
||||
const { saveOverlayStatusStub } = await runOverlayPostStep({
|
||||
jobStatus: "cancelled",
|
||||
codeQlReportedError: true,
|
||||
});
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.calledOnce,
|
||||
"the analysis genuinely failed, even though the run was later cancelled",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
function createTestWorkflow(
|
||||
steps: workflow.WorkflowJobStep[],
|
||||
): workflow.Workflow {
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
sanitizeArtifactName,
|
||||
} from "./debug-artifacts";
|
||||
import * as dependencyCaching from "./dependency-caching";
|
||||
import { EnvVar } from "./environment";
|
||||
import { EnvVar, ReadOnlyEnv } from "./environment";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import { Logger } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
@@ -316,6 +316,8 @@ export async function tryUploadSarifIfRunFailed(
|
||||
* @param config The CodeQL Action configuration.
|
||||
* @param repositoryNwo The name and owner of the repository.
|
||||
* @param features Information about enabled features.
|
||||
* @param jobStatus The status of the job, as reported by the Actions runtime environment.
|
||||
* @param env The environment to read variables from.
|
||||
* @param logger The logger to use.
|
||||
* @returns The results of uploading the SARIF file for the failure.
|
||||
*/
|
||||
@@ -331,9 +333,11 @@ export async function uploadFailureInfo(
|
||||
config: Config,
|
||||
repositoryNwo: RepositoryNwo,
|
||||
features: FeatureEnablement,
|
||||
jobStatus: string | undefined,
|
||||
env: ReadOnlyEnv,
|
||||
logger: Logger,
|
||||
): Promise<UploadFailedSarifResult> {
|
||||
await recordOverlayStatus(codeql, config, features, logger);
|
||||
await recordOverlayStatus(codeql, config, features, jobStatus, env, logger);
|
||||
|
||||
const uploadFailedSarifResult = await tryUploadSarifIfRunFailed(
|
||||
config,
|
||||
@@ -412,6 +416,37 @@ export async function uploadFailureInfo(
|
||||
return uploadFailedSarifResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether one of the CodeQL Actions reported an error for this job, which means the analysis
|
||||
* genuinely failed.
|
||||
*
|
||||
* Note that the converse does not hold: an Action that is terminated abruptly, or that fails before
|
||||
* it can gather telemetry, does not get to report anything.
|
||||
*/
|
||||
function didCodeQlReportError(env: ReadOnlyEnv): boolean {
|
||||
const jobStatus = env.getOptional(EnvVar.JOB_STATUS);
|
||||
return (
|
||||
jobStatus === JobStatus.FailureStatus ||
|
||||
jobStatus === JobStatus.ConfigErrorStatus
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the job status tells us anything about whether the analysis itself would have succeeded.
|
||||
*
|
||||
* We check for the statuses we know to be meaningful rather than excluding the ones that are not,
|
||||
* so that a status we do not recognise is treated as inconclusive.
|
||||
*/
|
||||
function isConclusiveJobStatus(jobStatus: string | undefined): boolean {
|
||||
switch (jobStatus?.trim().toLowerCase()) {
|
||||
case "failure":
|
||||
case "success":
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* If overlay base database creation was attempted but the analysis did not complete
|
||||
* successfully, save the failure status to the Actions cache so that subsequent runs
|
||||
@@ -421,16 +456,30 @@ async function recordOverlayStatus(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
features: FeatureEnablement,
|
||||
jobStatus: string | undefined,
|
||||
env: ReadOnlyEnv,
|
||||
logger: Logger,
|
||||
) {
|
||||
if (
|
||||
config.overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase ||
|
||||
process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY] === "true" ||
|
||||
env.getOptional(EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY) === "true" ||
|
||||
!(await features.getValue(Feature.OverlayAnalysisStatusSave))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Only record a failure when the job outcome tells us something about the analysis. A cancelled
|
||||
// job, or a status we do not recognise, says nothing about whether the analysis would have
|
||||
// succeeded, so recording a failure would disable overlay analysis needlessly. We still record
|
||||
// one if a CodeQL Action reported an error before the job ended.
|
||||
if (!isConclusiveJobStatus(jobStatus) && !didCodeQlReportError(env)) {
|
||||
logger.info(
|
||||
"Not recording an improved incremental analysis failure for this job because the job " +
|
||||
`status (${jobStatus ?? "unset"}) does not tell us whether the analysis itself failed.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const checkRunIdInput = actionsUtil.getOptionalInput("check-run-id");
|
||||
const checkRunId =
|
||||
checkRunIdInput !== undefined ? parseInt(checkRunIdInput, 10) : undefined;
|
||||
|
||||
@@ -8,6 +8,7 @@ import * as core from "@actions/core";
|
||||
|
||||
import {
|
||||
restoreInputs,
|
||||
getOptionalInput,
|
||||
getTemporaryDirectory,
|
||||
printDebugLogs,
|
||||
} from "./actions-util";
|
||||
@@ -20,7 +21,7 @@ import {
|
||||
DependencyCachingUsageReport,
|
||||
getDependencyCacheUsage,
|
||||
} from "./dependency-caching";
|
||||
import { EnvVar } from "./environment";
|
||||
import { EnvVar, getEnv } from "./environment";
|
||||
import { initFeatures } from "./feature-flags";
|
||||
import * as gitUtils from "./git-utils";
|
||||
import * as initActionPostHelper from "./init-action-post-helper";
|
||||
@@ -55,6 +56,11 @@ async function run(startedAt: Date) {
|
||||
| undefined;
|
||||
let dependencyCachingUsage: DependencyCachingUsageReport | undefined;
|
||||
try {
|
||||
// Read the job status before restoring inputs, since it is provided by the Actions runtime
|
||||
// environment for this step and would otherwise be overwritten by the value that the `init`
|
||||
// Action saw, which is always a success.
|
||||
const jobStatus = getOptionalInput("job-status");
|
||||
|
||||
// Restore inputs from `init` Action.
|
||||
restoreInputs();
|
||||
|
||||
@@ -84,6 +90,8 @@ async function run(startedAt: Date) {
|
||||
config,
|
||||
repositoryNwo,
|
||||
features,
|
||||
jobStatus,
|
||||
getEnv(),
|
||||
logger,
|
||||
);
|
||||
|
||||
|
||||
@@ -174,6 +174,14 @@ async function sendCompletedStatusReport(
|
||||
initToolsDownloadFields.tools_download_duration_ms =
|
||||
toolsDownloadStatusReport.downloadDurationMs;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.extractionDurationMs !== undefined) {
|
||||
initToolsDownloadFields.tools_extraction_duration_ms =
|
||||
toolsDownloadStatusReport.extractionDurationMs;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.totalDurationMs !== undefined) {
|
||||
initToolsDownloadFields.tools_total_duration_ms =
|
||||
toolsDownloadStatusReport.totalDurationMs;
|
||||
}
|
||||
if (toolsFeatureFlagsValid !== undefined) {
|
||||
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
|
||||
}
|
||||
|
||||
@@ -85,6 +85,14 @@ async function sendCompletedStatusReport(
|
||||
initToolsDownloadFields.tools_download_duration_ms =
|
||||
toolsDownloadStatusReport.downloadDurationMs;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.extractionDurationMs !== undefined) {
|
||||
initToolsDownloadFields.tools_extraction_duration_ms =
|
||||
toolsDownloadStatusReport.extractionDurationMs;
|
||||
}
|
||||
if (toolsDownloadStatusReport?.totalDurationMs !== undefined) {
|
||||
initToolsDownloadFields.tools_total_duration_ms =
|
||||
toolsDownloadStatusReport.totalDurationMs;
|
||||
}
|
||||
if (toolsFeatureFlagsValid !== undefined) {
|
||||
initToolsDownloadFields.tools_feature_flags_valid = toolsFeatureFlagsValid;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import * as fs from "fs";
|
||||
import * as os from "os";
|
||||
import * as path from "path";
|
||||
|
||||
import * as github from "@actions/github";
|
||||
@@ -7,7 +9,8 @@ import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import * as api from "./api-client";
|
||||
import { EnvVar } from "./environment";
|
||||
import * as diagnostics from "./diagnostics";
|
||||
import { ActionsEnvVars, EnvVar, ReadOnlyEnv } from "./environment";
|
||||
import { Feature } from "./feature-flags";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { getCacheRestoreKeyPrefix } from "./overlay/caching";
|
||||
@@ -27,6 +30,7 @@ import {
|
||||
setupActionsVars,
|
||||
setupTests,
|
||||
} from "./testing-utils";
|
||||
import * as toolsDownload from "./tools-download";
|
||||
import {
|
||||
getErrorMessage,
|
||||
GitHubVariant,
|
||||
@@ -120,24 +124,42 @@ test.serial(
|
||||
const LINKED_BUNDLE_TEST_CASES = [
|
||||
{
|
||||
platform: "linux",
|
||||
arch: "x64",
|
||||
tarSupportsZstd: true,
|
||||
expectedBundleName: "codeql-bundle-linux64.tar.zst",
|
||||
expectedCompressionMethod: "zstd",
|
||||
},
|
||||
{
|
||||
platform: "linux",
|
||||
arch: "arm64",
|
||||
tarSupportsZstd: true,
|
||||
expectedBundleName: "codeql-bundle-linux-arm64.tar.zst",
|
||||
expectedCompressionMethod: "zstd",
|
||||
},
|
||||
{
|
||||
platform: "darwin",
|
||||
arch: "arm64",
|
||||
tarSupportsZstd: true,
|
||||
expectedBundleName: "codeql-bundle-osx64.tar.zst",
|
||||
expectedCompressionMethod: "zstd",
|
||||
},
|
||||
{
|
||||
platform: "darwin",
|
||||
arch: "x64",
|
||||
tarSupportsZstd: true,
|
||||
expectedBundleName: "codeql-bundle-osx64.tar.zst",
|
||||
expectedCompressionMethod: "zstd",
|
||||
},
|
||||
{
|
||||
platform: "win32",
|
||||
arch: "x64",
|
||||
tarSupportsZstd: true,
|
||||
expectedBundleName: "codeql-bundle-win64.tar.gz",
|
||||
expectedCompressionMethod: "gzip",
|
||||
},
|
||||
{
|
||||
platform: "linux",
|
||||
arch: "x64",
|
||||
tarSupportsZstd: false,
|
||||
expectedBundleName: "codeql-bundle-linux64.tar.gz",
|
||||
expectedCompressionMethod: "gzip",
|
||||
@@ -146,15 +168,17 @@ const LINKED_BUNDLE_TEST_CASES = [
|
||||
|
||||
for (const {
|
||||
platform,
|
||||
arch,
|
||||
tarSupportsZstd,
|
||||
expectedBundleName,
|
||||
expectedCompressionMethod,
|
||||
} of LINKED_BUNDLE_TEST_CASES) {
|
||||
test.serial(
|
||||
`getCodeQLSource selects ${expectedBundleName} for linked tools`,
|
||||
`getCodeQLSource selects ${expectedBundleName} for linked tools on ${platform}/${arch}`,
|
||||
async (t) => {
|
||||
const features = createFeatures([]);
|
||||
sinon.stub(process, "platform").value(platform);
|
||||
sinon.stub(process, "arch").value(arch);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
@@ -234,6 +258,7 @@ test.serial(
|
||||
codeqlFolder: "codeql",
|
||||
statusReport: {
|
||||
downloadDurationMs: 200,
|
||||
totalDurationMs: 300,
|
||||
},
|
||||
toolsVersion: LINKED_CLI_VERSION.cliVersion,
|
||||
});
|
||||
@@ -286,6 +311,7 @@ test.serial(
|
||||
codeqlFolder: "codeql",
|
||||
statusReport: {
|
||||
downloadDurationMs: 200,
|
||||
totalDurationMs: 300,
|
||||
},
|
||||
toolsVersion: expectedVersion,
|
||||
});
|
||||
@@ -917,3 +943,616 @@ test.serial(
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
/** The CLI version that the toolcache cleanup tests download. */
|
||||
const CLEANUP_CLI_VERSION = "2.21.0";
|
||||
/** The bundle version that the toolcache cleanup tests download. */
|
||||
const CLEANUP_BUNDLE_VERSION = "20240101";
|
||||
/** A version of the CodeQL tools that is already in the toolcache but that we are not going to use. */
|
||||
const CLEANUP_STALE_VERSION = "2.20.0";
|
||||
|
||||
/** Creates a directory in the toolcache that looks like a tool that `tool-cache` has cached. */
|
||||
function createToolcacheEntry(
|
||||
toolcacheRoot: string,
|
||||
tool: string,
|
||||
version: string,
|
||||
): string {
|
||||
const versionDirectory = path.join(toolcacheRoot, tool, version);
|
||||
const archDirectory = path.join(versionDirectory, os.arch());
|
||||
fs.mkdirSync(archDirectory, { recursive: true });
|
||||
fs.writeFileSync(path.join(archDirectory, "contents"), "x".repeat(1024));
|
||||
fs.writeFileSync(`${archDirectory}.complete`, "");
|
||||
return versionDirectory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stubs out the download and the diagnostic sink, then downloads the CodeQL tools.
|
||||
*
|
||||
* @returns the extraction directory and the toolcache cleanup diagnostic, if emitted.
|
||||
*/
|
||||
async function runDownloadCodeQL(
|
||||
toolcacheRoot: string,
|
||||
features: Feature[],
|
||||
bundleVersion: string | undefined,
|
||||
): Promise<{
|
||||
codeqlFolder: string;
|
||||
cleanupDiagnostic: toolsDownload.ToolcacheCleanupResult | undefined;
|
||||
}> {
|
||||
sinon
|
||||
.stub(toolsDownload, "downloadAndExtract")
|
||||
.callsFake(async (_url, _compressionMethod, dest) => {
|
||||
// The real implementation creates the destination directory, which matters here because the
|
||||
// cleanup deletes it first and `writeToolcacheMarkerFile` writes into its parent afterwards.
|
||||
fs.mkdirSync(dest, { recursive: true });
|
||||
return { totalDurationMs: 1 };
|
||||
});
|
||||
const addDiagnostic = sinon.stub(diagnostics, "addNoLanguageDiagnostic");
|
||||
|
||||
const { codeqlFolder } = await setupCodeql.downloadCodeQL(
|
||||
"https://example.com/codeql-bundle.tar.gz",
|
||||
"gzip",
|
||||
bundleVersion,
|
||||
CLEANUP_CLI_VERSION,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
undefined, // tarVersion
|
||||
toolcacheRoot, // tempDir
|
||||
createFeatures(features),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
const diagnostic = addDiagnostic
|
||||
.getCalls()
|
||||
.map((call) => call.args[1])
|
||||
.find((d) => d.source?.id === "codeql-action/toolcache-bundle-cleanup");
|
||||
|
||||
return {
|
||||
codeqlFolder,
|
||||
cleanupDiagnostic: diagnostic?.attributes as
|
||||
| toolsDownload.ToolcacheCleanupResult
|
||||
| undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets up a toolcache containing the version of the CodeQL tools that we are about to download, a
|
||||
* different version of the CodeQL tools, and an unrelated tool, then downloads the CodeQL tools.
|
||||
*/
|
||||
async function testToolcacheCleanup(
|
||||
t: ExecutionContext<unknown>,
|
||||
{
|
||||
features,
|
||||
runnerEnvironment,
|
||||
setUp,
|
||||
}: {
|
||||
features: Feature[];
|
||||
runnerEnvironment: string | undefined;
|
||||
setUp?: () => void;
|
||||
},
|
||||
check: (context: {
|
||||
cleanupDiagnostic: toolsDownload.ToolcacheCleanupResult | undefined;
|
||||
destinationDirectory: string;
|
||||
staleDirectory: string;
|
||||
}) => void,
|
||||
) {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
if (runnerEnvironment === undefined) {
|
||||
delete process.env[ActionsEnvVars.RUNNER_ENVIRONMENT];
|
||||
} else {
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = runnerEnvironment;
|
||||
}
|
||||
setUp?.();
|
||||
|
||||
// The extraction of the bundle would normally create this directory.
|
||||
const destinationDirectory = createToolcacheEntry(
|
||||
tmpDir,
|
||||
"CodeQL",
|
||||
CLEANUP_CLI_VERSION,
|
||||
);
|
||||
const staleDirectory = createToolcacheEntry(
|
||||
tmpDir,
|
||||
"CodeQL",
|
||||
CLEANUP_STALE_VERSION,
|
||||
);
|
||||
const otherToolDirectory = createToolcacheEntry(tmpDir, "Node", "20.0.0");
|
||||
|
||||
const { cleanupDiagnostic } = await runDownloadCodeQL(
|
||||
tmpDir,
|
||||
features,
|
||||
CLEANUP_BUNDLE_VERSION,
|
||||
);
|
||||
|
||||
t.true(
|
||||
fs.existsSync(otherToolDirectory),
|
||||
"Should never delete other tools from the toolcache.",
|
||||
);
|
||||
|
||||
check({ cleanupDiagnostic, destinationDirectory, staleDirectory });
|
||||
});
|
||||
}
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL does not clean up the toolcache when the feature flag is disabled",
|
||||
async (t) => {
|
||||
await testToolcacheCleanup(
|
||||
t,
|
||||
{ features: [], runnerEnvironment: "github-hosted" },
|
||||
({ cleanupDiagnostic, destinationDirectory, staleDirectory }) => {
|
||||
t.true(fs.existsSync(staleDirectory));
|
||||
t.true(fs.existsSync(destinationDirectory));
|
||||
t.is(cleanupDiagnostic, undefined);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL does not clean up the toolcache when the runner is not GitHub-hosted",
|
||||
async (t) => {
|
||||
await testToolcacheCleanup(
|
||||
t,
|
||||
{
|
||||
features: [Feature.CleanupToolcacheBundles],
|
||||
runnerEnvironment: "self-hosted",
|
||||
},
|
||||
({ cleanupDiagnostic, destinationDirectory, staleDirectory }) => {
|
||||
t.true(fs.existsSync(staleDirectory));
|
||||
t.true(fs.existsSync(destinationDirectory));
|
||||
t.is(cleanupDiagnostic, undefined);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL does not clean up the toolcache when the runner environment is unknown",
|
||||
async (t) => {
|
||||
// A runner that doesn't report its environment must be treated as not GitHub-hosted, since its
|
||||
// toolcache may well outlive the job.
|
||||
await testToolcacheCleanup(
|
||||
t,
|
||||
{
|
||||
features: [Feature.CleanupToolcacheBundles],
|
||||
runnerEnvironment: undefined,
|
||||
},
|
||||
({ cleanupDiagnostic, destinationDirectory, staleDirectory }) => {
|
||||
t.true(fs.existsSync(staleDirectory));
|
||||
t.true(fs.existsSync(destinationDirectory));
|
||||
t.is(cleanupDiagnostic, undefined);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL deletes other CodeQL bundles from the toolcache when enabled on a GitHub-hosted runner",
|
||||
async (t) => {
|
||||
await testToolcacheCleanup(
|
||||
t,
|
||||
{
|
||||
features: [Feature.CleanupToolcacheBundles],
|
||||
runnerEnvironment: "github-hosted",
|
||||
},
|
||||
({ cleanupDiagnostic, destinationDirectory, staleDirectory }) => {
|
||||
t.false(
|
||||
fs.existsSync(staleDirectory),
|
||||
"Should delete the version directory, including the `tool-cache` marker file it contains.",
|
||||
);
|
||||
t.false(
|
||||
fs.existsSync(path.join(destinationDirectory, os.arch(), "contents")),
|
||||
"Should also delete a partial entry for the version we are about to download, rather " +
|
||||
"than extracting over it.",
|
||||
);
|
||||
t.deepEqual(cleanupDiagnostic, {
|
||||
deletedVersions: [CLEANUP_STALE_VERSION, CLEANUP_CLI_VERSION].sort(),
|
||||
failed: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL reports no deleted versions when the toolcache has no CodeQL bundles",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
// A toolcache with other tools in it, but no CodeQL.
|
||||
const otherToolDirectory = createToolcacheEntry(tmpDir, "Node", "20.0.0");
|
||||
|
||||
const { cleanupDiagnostic } = await runDownloadCodeQL(
|
||||
tmpDir,
|
||||
[Feature.CleanupToolcacheBundles],
|
||||
CLEANUP_BUNDLE_VERSION,
|
||||
);
|
||||
|
||||
t.true(fs.existsSync(otherToolDirectory));
|
||||
t.deepEqual(cleanupDiagnostic, { deletedVersions: [], failed: false });
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL continues when deleting a CodeQL bundle from the toolcache fails",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
createToolcacheEntry(tmpDir, "CodeQL", CLEANUP_CLI_VERSION);
|
||||
const staleDirectory = createToolcacheEntry(
|
||||
tmpDir,
|
||||
"CodeQL",
|
||||
CLEANUP_STALE_VERSION,
|
||||
);
|
||||
|
||||
const rmStub = sinon
|
||||
.stub(fs.promises, "rm")
|
||||
.rejects(new Error("EACCES: permission denied"));
|
||||
|
||||
const { cleanupDiagnostic } = await runDownloadCodeQL(
|
||||
tmpDir,
|
||||
[Feature.CleanupToolcacheBundles],
|
||||
CLEANUP_BUNDLE_VERSION,
|
||||
);
|
||||
|
||||
// Restore before `withTmpDir` cleans up after itself.
|
||||
rmStub.restore();
|
||||
|
||||
t.true(fs.existsSync(staleDirectory));
|
||||
t.deepEqual(
|
||||
cleanupDiagnostic,
|
||||
{ deletedVersions: [], failed: true },
|
||||
"Should not report versions that we failed to delete.",
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"deleteToolcacheBundles reports a failure when the toolcache location is unknown",
|
||||
async (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
|
||||
const result = await toolsDownload.deleteToolcacheBundles({
|
||||
env: new ReadOnlyEnv({}),
|
||||
logger: getRecordingLogger(messages),
|
||||
});
|
||||
|
||||
t.deepEqual(
|
||||
result,
|
||||
{ deletedVersions: [], failed: true },
|
||||
"Should report a failure rather than throwing, so the download can continue.",
|
||||
);
|
||||
checkExpectedLogMessages(t, messages, [
|
||||
"Unable to determine toolcache directory: RUNNER_TOOL_CACHE environment variable must be set",
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"deleteToolcacheBundles uses the supplied environment rather than process.env",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const ambientRoot = path.join(tmpDir, "ambient");
|
||||
const injectedRoot = path.join(tmpDir, "injected");
|
||||
setupActionsVars(tmpDir, ambientRoot);
|
||||
|
||||
const ambientVersion = createToolcacheEntry(
|
||||
ambientRoot,
|
||||
"CodeQL",
|
||||
CLEANUP_STALE_VERSION,
|
||||
);
|
||||
const injectedVersion = createToolcacheEntry(
|
||||
injectedRoot,
|
||||
"CodeQL",
|
||||
CLEANUP_STALE_VERSION,
|
||||
);
|
||||
const fileEntry = path.join(injectedRoot, "CodeQL", "not-a-directory");
|
||||
fs.writeFileSync(fileEntry, "keep");
|
||||
|
||||
const result = await toolsDownload.deleteToolcacheBundles({
|
||||
env: new ReadOnlyEnv({
|
||||
[ActionsEnvVars.RUNNER_TOOL_CACHE]: injectedRoot,
|
||||
}),
|
||||
logger: getRunnerLogger(true),
|
||||
});
|
||||
|
||||
t.deepEqual(result, {
|
||||
deletedVersions: [CLEANUP_STALE_VERSION],
|
||||
failed: false,
|
||||
});
|
||||
t.false(fs.existsSync(injectedVersion));
|
||||
t.true(fs.existsSync(ambientVersion));
|
||||
t.is(fs.readFileSync(fileEntry, "utf8"), "keep");
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"deleteToolcacheBundles reports a failure when the toolcache directory cannot be read",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const versionDirectory = createToolcacheEntry(
|
||||
tmpDir,
|
||||
"CodeQL",
|
||||
CLEANUP_STALE_VERSION,
|
||||
);
|
||||
const messages: LoggedMessage[] = [];
|
||||
const readdir = sinon
|
||||
.stub(fs.promises, "readdir")
|
||||
.rejects(new Error("permission denied"));
|
||||
|
||||
try {
|
||||
const result = await toolsDownload.deleteToolcacheBundles({
|
||||
env: new ReadOnlyEnv({
|
||||
[ActionsEnvVars.RUNNER_TOOL_CACHE]: tmpDir,
|
||||
}),
|
||||
logger: getRecordingLogger(messages),
|
||||
});
|
||||
|
||||
t.deepEqual(result, { deletedVersions: [], failed: true });
|
||||
t.true(fs.existsSync(versionDirectory));
|
||||
checkExpectedLogMessages(t, messages, [
|
||||
`Failed to clean up the CodeQL toolcache at '${path.join(tmpDir, "CodeQL")}': permission denied`,
|
||||
]);
|
||||
} finally {
|
||||
readdir.restore();
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL does not follow a symlinked CodeQL toolcache directory",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const toolcacheRoot = path.join(tmpDir, "toolcache");
|
||||
setupActionsVars(tmpDir, toolcacheRoot);
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
// Somewhere the toolcache cleanup must never reach.
|
||||
const outsideDirectory = path.join(tmpDir, "outside");
|
||||
createToolcacheEntry(outsideDirectory, "CodeQL", CLEANUP_STALE_VERSION);
|
||||
createToolcacheEntry(outsideDirectory, "CodeQL", CLEANUP_CLI_VERSION);
|
||||
|
||||
fs.mkdirSync(toolcacheRoot, { recursive: true });
|
||||
fs.symlinkSync(
|
||||
path.join(outsideDirectory, "CodeQL"),
|
||||
path.join(toolcacheRoot, "CodeQL"),
|
||||
);
|
||||
|
||||
const { cleanupDiagnostic } = await runDownloadCodeQL(
|
||||
toolcacheRoot,
|
||||
[Feature.CleanupToolcacheBundles],
|
||||
CLEANUP_BUNDLE_VERSION,
|
||||
);
|
||||
|
||||
t.true(
|
||||
fs.existsSync(
|
||||
path.join(outsideDirectory, "CodeQL", CLEANUP_STALE_VERSION),
|
||||
),
|
||||
"Should not delete anything through a symlinked CodeQL directory.",
|
||||
);
|
||||
t.deepEqual(cleanupDiagnostic, { deletedVersions: [], failed: true });
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL does not clean up the toolcache once a step has already set up CodeQL",
|
||||
async (t) => {
|
||||
// `.github/workflows/codeql.yml` sets up CodeQL twice and then runs both returned paths. If the
|
||||
// second setup downloads, it must not delete the bundle the first one handed out.
|
||||
await testToolcacheCleanup(
|
||||
t,
|
||||
{
|
||||
features: [Feature.CleanupToolcacheBundles],
|
||||
runnerEnvironment: "github-hosted",
|
||||
setUp: () => {
|
||||
process.env[EnvVar.HAS_SET_UP_CODEQL] = "true";
|
||||
},
|
||||
},
|
||||
({ cleanupDiagnostic, destinationDirectory, staleDirectory }) => {
|
||||
t.true(fs.existsSync(staleDirectory));
|
||||
t.true(fs.existsSync(destinationDirectory));
|
||||
t.is(cleanupDiagnostic, undefined);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"setupCodeQLBundle records that this job has set up CodeQL",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
delete process.env[EnvVar.HAS_SET_UP_CODEQL];
|
||||
|
||||
sinon.stub(setupCodeql, "downloadCodeQL").resolves({
|
||||
codeqlFolder: "codeql",
|
||||
statusReport: { totalDurationMs: 1 },
|
||||
toolsVersion: LINKED_CLI_VERSION.cliVersion,
|
||||
});
|
||||
|
||||
await setupCodeql.setupCodeQLBundle(
|
||||
"linked",
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
tmpDir,
|
||||
GitHubVariant.DOTCOM,
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
undefined, // rawLanguages
|
||||
false, // useOverlayAwareDefaultCliVersion
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.is(
|
||||
process.env[EnvVar.HAS_SET_UP_CODEQL],
|
||||
"true",
|
||||
"A later step must be able to tell that the toolcache is in use.",
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL cleans up the toolcache even when the download will not be cached",
|
||||
async (t) => {
|
||||
// A `tools` URL we can't derive a bundle version from is extracted to a temporary directory
|
||||
// rather than the toolcache, but the toolcache is on the same filesystem, so emptying it still
|
||||
// frees up space for the analysis.
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
const staleDirectory = createToolcacheEntry(
|
||||
tmpDir,
|
||||
"CodeQL",
|
||||
CLEANUP_STALE_VERSION,
|
||||
);
|
||||
|
||||
const { codeqlFolder, cleanupDiagnostic } = await runDownloadCodeQL(
|
||||
tmpDir,
|
||||
[Feature.CleanupToolcacheBundles],
|
||||
undefined, // bundleVersion
|
||||
);
|
||||
|
||||
t.is(path.dirname(codeqlFolder), tmpDir);
|
||||
t.not(codeqlFolder, path.join(tmpDir, "CodeQL"));
|
||||
t.true(fs.existsSync(codeqlFolder));
|
||||
t.false(fs.existsSync(`${codeqlFolder}.complete`));
|
||||
t.false(fs.existsSync(staleDirectory));
|
||||
t.deepEqual(cleanupDiagnostic, {
|
||||
deletedVersions: [CLEANUP_STALE_VERSION],
|
||||
failed: false,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL reports a failure when the toolcache cannot be inspected",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
createToolcacheEntry(tmpDir, "CodeQL", CLEANUP_STALE_VERSION);
|
||||
|
||||
const lstatStub = sinon.stub(fs.promises, "lstat").rejects(
|
||||
Object.assign(new Error("permission denied"), {
|
||||
code: "EACCES",
|
||||
}),
|
||||
);
|
||||
|
||||
const { cleanupDiagnostic } = await runDownloadCodeQL(
|
||||
tmpDir,
|
||||
[Feature.CleanupToolcacheBundles],
|
||||
CLEANUP_BUNDLE_VERSION,
|
||||
);
|
||||
|
||||
lstatStub.restore();
|
||||
|
||||
t.deepEqual(
|
||||
cleanupDiagnostic,
|
||||
{ deletedVersions: [], failed: true },
|
||||
"An error other than the toolcache being absent must not be reported as success.",
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL does not clean up a toolcache on a different filesystem to the workspace",
|
||||
async (t) => {
|
||||
// Some runner images keep the toolcache on a different volume to the workspace, in which case
|
||||
// deleting the tools frees up disk space that the analysis cannot use.
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
const staleDirectory = createToolcacheEntry(
|
||||
tmpDir,
|
||||
"CodeQL",
|
||||
CLEANUP_STALE_VERSION,
|
||||
);
|
||||
|
||||
sinon
|
||||
.stub(toolsDownload, "isToolcacheOnWorkspaceFilesystem")
|
||||
.returns(false);
|
||||
|
||||
const { cleanupDiagnostic } = await runDownloadCodeQL(
|
||||
tmpDir,
|
||||
[Feature.CleanupToolcacheBundles],
|
||||
CLEANUP_BUNDLE_VERSION,
|
||||
);
|
||||
|
||||
t.true(fs.existsSync(staleDirectory));
|
||||
t.is(cleanupDiagnostic, undefined);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"isToolcacheOnWorkspaceFilesystem assumes a different filesystem when it cannot tell",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const logger = getRunnerLogger(true);
|
||||
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
t.true(toolsDownload.isToolcacheOnWorkspaceFilesystem(logger));
|
||||
|
||||
// If we can't tell, we assume the toolcache is not somewhere we can reclaim space from.
|
||||
process.env[ActionsEnvVars.RUNNER_TOOL_CACHE] = path.join(
|
||||
tmpDir,
|
||||
"does-not-exist",
|
||||
);
|
||||
t.false(toolsDownload.isToolcacheOnWorkspaceFilesystem(logger));
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadCodeQL does not delete through a symlinked version directory",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const toolcacheRoot = path.join(tmpDir, "toolcache");
|
||||
setupActionsVars(tmpDir, toolcacheRoot);
|
||||
process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted";
|
||||
|
||||
createToolcacheEntry(toolcacheRoot, "CodeQL", CLEANUP_STALE_VERSION);
|
||||
|
||||
// Somewhere outside the toolcache that a version directory points at.
|
||||
const outsideDirectory = path.join(tmpDir, "outside");
|
||||
fs.mkdirSync(outsideDirectory, { recursive: true });
|
||||
fs.writeFileSync(path.join(outsideDirectory, "contents"), "x");
|
||||
fs.symlinkSync(
|
||||
outsideDirectory,
|
||||
path.join(toolcacheRoot, "CodeQL", "9.9.9"),
|
||||
);
|
||||
|
||||
const { cleanupDiagnostic } = await runDownloadCodeQL(
|
||||
toolcacheRoot,
|
||||
[Feature.CleanupToolcacheBundles],
|
||||
CLEANUP_BUNDLE_VERSION,
|
||||
);
|
||||
|
||||
t.true(
|
||||
fs.existsSync(path.join(outsideDirectory, "contents")),
|
||||
"Should not delete anything through a symlinked version directory.",
|
||||
);
|
||||
t.true(
|
||||
fs
|
||||
.lstatSync(path.join(toolcacheRoot, "CodeQL", "9.9.9"))
|
||||
.isSymbolicLink(),
|
||||
);
|
||||
t.deepEqual(cleanupDiagnostic, {
|
||||
deletedVersions: [CLEANUP_STALE_VERSION],
|
||||
failed: false,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -2,14 +2,17 @@ import * as fs from "fs";
|
||||
import { OutgoingHttpHeaders } from "http";
|
||||
import * as path from "path";
|
||||
|
||||
import * as core from "@actions/core";
|
||||
import * as toolcache from "@actions/tool-cache";
|
||||
import { default as deepEqual } from "fast-deep-equal";
|
||||
import * as semver from "semver";
|
||||
import { v4 as uuidV4 } from "uuid";
|
||||
|
||||
import { ActionState } from "./action-common";
|
||||
import {
|
||||
isAnalyzingPullRequest,
|
||||
isDynamicWorkflow,
|
||||
isGitHubHostedRunner,
|
||||
isRunningLocalAction,
|
||||
} from "./actions-util";
|
||||
import * as api from "./api-client";
|
||||
@@ -19,6 +22,7 @@ import {
|
||||
makeDiagnostic,
|
||||
makeTelemetryDiagnostic,
|
||||
} from "./diagnostics";
|
||||
import { EnvVar, getEnv } from "./environment";
|
||||
import {
|
||||
CODEQL_VERSION_ZSTD_BUNDLE,
|
||||
CodeQLDefaultVersionInfo,
|
||||
@@ -30,8 +34,10 @@ import { Logger } from "./logging";
|
||||
import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching";
|
||||
import * as tar from "./tar";
|
||||
import {
|
||||
deleteToolcacheBundles,
|
||||
downloadAndExtract,
|
||||
getToolcacheDirectory,
|
||||
isToolcacheOnWorkspaceFilesystem,
|
||||
ToolsDownloadStatusReport,
|
||||
writeToolcacheMarkerFile,
|
||||
} from "./tools-download";
|
||||
@@ -75,7 +81,7 @@ export function getCodeQLBundleName(
|
||||
if (process.platform === "win32") {
|
||||
platform = "win64";
|
||||
} else if (process.platform === "linux") {
|
||||
platform = "linux64";
|
||||
platform = process.arch === "arm64" ? "linux-arm64" : "linux64";
|
||||
} else if (process.platform === "darwin") {
|
||||
platform = "osx64";
|
||||
} else {
|
||||
@@ -784,6 +790,7 @@ export const downloadCodeQL = async function (
|
||||
apiDetails: api.GitHubApiDetails,
|
||||
tarVersion: tar.TarVersion | undefined,
|
||||
tempDir: string,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
): Promise<{
|
||||
codeqlFolder: string;
|
||||
@@ -817,6 +824,8 @@ export const downloadCodeQL = async function (
|
||||
const extractedBundlePath =
|
||||
toolcacheInfo?.path ?? getTempExtractionDir(tempDir);
|
||||
|
||||
await tryDeleteToolcacheBundles({ env: getEnv(), features, logger });
|
||||
|
||||
const statusReport = await downloadAndExtract(
|
||||
codeqlURL,
|
||||
compressionMethod,
|
||||
@@ -869,6 +878,48 @@ function getToolcacheDestinationInfo(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reclaims disk space by deleting the CodeQL tools from the toolcache, if enabled.
|
||||
*
|
||||
* On GitHub-hosted runners the toolcache shares a filesystem with the workspace, so tools left in
|
||||
* the toolcache take up space that the analysis could use instead. This holds wherever we extract
|
||||
* the tools we are obtaining, since the toolcache is on that filesystem either way.
|
||||
*/
|
||||
async function tryDeleteToolcacheBundles({
|
||||
env,
|
||||
features,
|
||||
logger,
|
||||
}: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>): Promise<void> {
|
||||
// A step that has already set up CodeQL may hand out a path into the toolcache that a later step
|
||||
// runs, so only the first step to set it up can know that nothing else relies on the toolcache.
|
||||
if (env.getOptional(EnvVar.HAS_SET_UP_CODEQL) !== undefined) {
|
||||
logger.debug(
|
||||
"Not deleting the CodeQL tools from the toolcache since a previous step in this job has " +
|
||||
"already set up CodeQL.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
!isGitHubHostedRunner() ||
|
||||
!isToolcacheOnWorkspaceFilesystem(logger) ||
|
||||
!(await features.getValue(Feature.CleanupToolcacheBundles))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await deleteToolcacheBundles({ env, logger });
|
||||
|
||||
addNoLanguageDiagnostic(
|
||||
undefined,
|
||||
makeTelemetryDiagnostic(
|
||||
"codeql-action/toolcache-bundle-cleanup",
|
||||
"Toolcache CodeQL bundle cleanup",
|
||||
{ ...result },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function getCodeQLURLVersion(url: string): string {
|
||||
const match = url.match(/\/codeql-bundle-(.*)\//);
|
||||
if (match === null || match.length < 2) {
|
||||
@@ -978,6 +1029,7 @@ export async function setupCodeQLBundle(
|
||||
apiDetails,
|
||||
zstdAvailability.version,
|
||||
tempDir,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
toolsVersion = result.toolsVersion;
|
||||
@@ -989,6 +1041,11 @@ export async function setupCodeQLBundle(
|
||||
default:
|
||||
util.assertNever(source);
|
||||
}
|
||||
|
||||
// Record that this job now has a copy of the CodeQL tools, so that a later step doesn't delete
|
||||
// the toolcache out from under the path we are about to return.
|
||||
core.exportVariable(EnvVar.HAS_SET_UP_CODEQL, "true");
|
||||
|
||||
return {
|
||||
codeqlFolder,
|
||||
toolsDownloadStatusReport,
|
||||
|
||||
@@ -14,7 +14,10 @@ import {
|
||||
isSelfHostedRunner,
|
||||
} from "./actions-util";
|
||||
import { getAnalysisKey, getApiClient } from "./api-client";
|
||||
import { getCachedCodeQlVersion } from "./cli/output-cache";
|
||||
import {
|
||||
getCachedCodeQlVersion,
|
||||
getCommandCacheFilePath,
|
||||
} from "./cli/output-cache";
|
||||
import type { Config } from "./config/action-config";
|
||||
import type { ComputedInput, InputName } from "./config/inputs";
|
||||
import { parseRegistriesWithoutCredentials } from "./config/pack-registries";
|
||||
@@ -376,7 +379,10 @@ export async function createStatusReportBase(
|
||||
core.exportVariable(EnvVar.WORKFLOW_STARTED_AT, workflowStartedAt);
|
||||
}
|
||||
const runnerOs = getRequiredEnvParam("RUNNER_OS");
|
||||
const codeQlCliVersion = getCachedCodeQlVersion(logger, getEnv());
|
||||
const codeQlCliVersion = getCachedCodeQlVersion(
|
||||
logger,
|
||||
getCommandCacheFilePath(getEnv()),
|
||||
);
|
||||
const actionRef = process.env["GITHUB_ACTION_REF"] || "";
|
||||
const testingEnvironment = getTestingEnvironment();
|
||||
// re-export the testing environment variable so that it is available to subsequent steps,
|
||||
@@ -620,8 +626,21 @@ export interface InitWithConfigStatusReport extends InitStatusReport {
|
||||
|
||||
/** Fields of the init status report populated when the tools source is `download`. */
|
||||
export interface InitToolsDownloadFields {
|
||||
/** Time taken to download the bundle, in milliseconds. */
|
||||
/**
|
||||
* Time taken to download the bundle, in milliseconds. Not populated when the bundle is downloaded
|
||||
* and extracted concurrently.
|
||||
*/
|
||||
tools_download_duration_ms?: number;
|
||||
/**
|
||||
* Time taken to extract the bundle, in milliseconds. Not populated when the bundle is downloaded
|
||||
* and extracted concurrently.
|
||||
*/
|
||||
tools_extraction_duration_ms?: number;
|
||||
/**
|
||||
* Total time taken to make the bundle available on disk, in milliseconds. This includes any time
|
||||
* spent on a streaming attempt that failed and fell back to downloading before extracting.
|
||||
*/
|
||||
tools_total_duration_ms?: number;
|
||||
/**
|
||||
* Whether the relevant tools dotcom feature flags have been misconfigured.
|
||||
* Only populated if we attempt to determine the default version based on the dotcom feature flags. */
|
||||
|
||||
@@ -937,7 +937,9 @@ export function mockBundleDownloadApi({
|
||||
process.platform === "win32"
|
||||
? "win64"
|
||||
: process.platform === "linux"
|
||||
? "linux64"
|
||||
? process.arch === "arm64"
|
||||
? "linux-arm64"
|
||||
: "linux64"
|
||||
: "osx64";
|
||||
|
||||
const baseUrl = apiDetails?.url ?? "https://example.com";
|
||||
|
||||
@@ -15,7 +15,7 @@ import { withTmpDir } from "./util";
|
||||
setupTests(test);
|
||||
|
||||
test.serial(
|
||||
"downloadAndExtract reports the duration when downloading before extracting",
|
||||
"downloadAndExtract reports the durations when downloading before extracting",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const archivePath = path.join(tmpDir, "codeql-bundle.tar.gz");
|
||||
@@ -34,6 +34,8 @@ test.serial(
|
||||
);
|
||||
|
||||
t.assert(Number.isInteger(statusReport.downloadDurationMs));
|
||||
t.assert(Number.isInteger(statusReport.extractionDurationMs));
|
||||
t.assert(Number.isInteger(statusReport.totalDurationMs));
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -67,6 +69,7 @@ test.serial(
|
||||
);
|
||||
|
||||
t.assert(Number.isInteger(statusReport.downloadDurationMs));
|
||||
t.assert(Number.isInteger(statusReport.totalDurationMs));
|
||||
t.true(request.isDone());
|
||||
t.false(extractTarZst.called);
|
||||
t.true(downloadTool.calledOnce);
|
||||
@@ -76,7 +79,7 @@ test.serial(
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadAndExtract omits the download duration when streaming extraction",
|
||||
"downloadAndExtract reports only the total duration when streaming extraction",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
@@ -106,7 +109,9 @@ test.serial(
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.deepEqual(statusReport, {});
|
||||
t.assert(Number.isInteger(statusReport.totalDurationMs));
|
||||
t.is(statusReport.downloadDurationMs, undefined);
|
||||
t.is(statusReport.extractionDurationMs, undefined);
|
||||
t.false(downloadTool.called);
|
||||
t.true(extractTarZst.calledOnce);
|
||||
t.true(request.isDone());
|
||||
|
||||
@@ -10,6 +10,8 @@ import * as toolcache from "@actions/tool-cache";
|
||||
import { https } from "follow-redirects";
|
||||
import * as semver from "semver";
|
||||
|
||||
import { ActionState } from "./action-common";
|
||||
import { ActionsEnvVars, getEnv, ReadOnlyEnv } from "./environment";
|
||||
import { formatDuration, Logger } from "./logging";
|
||||
import * as tar from "./tar";
|
||||
import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
|
||||
@@ -31,7 +33,21 @@ const STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const TOOLCACHE_TOOL_NAME = "CodeQL";
|
||||
|
||||
export type ToolsDownloadStatusReport = {
|
||||
/**
|
||||
* Time spent downloading the bundle, in milliseconds. Not populated when the bundle is downloaded
|
||||
* and extracted concurrently, since the two cannot be told apart.
|
||||
*/
|
||||
downloadDurationMs?: number;
|
||||
/**
|
||||
* Time spent extracting the bundle, in milliseconds. Not populated when the bundle is downloaded
|
||||
* and extracted concurrently, since the two cannot be told apart.
|
||||
*/
|
||||
extractionDurationMs?: number;
|
||||
/**
|
||||
* Total time taken to make the bundle available on disk, in milliseconds. This includes any time
|
||||
* spent on a streaming attempt that failed and fell back to downloading before extracting.
|
||||
*/
|
||||
totalDurationMs: number;
|
||||
};
|
||||
|
||||
export async function downloadAndExtract(
|
||||
@@ -47,11 +63,12 @@ export async function downloadAndExtract(
|
||||
`Downloading CodeQL tools from ${codeqlURL} . This may take a while.`,
|
||||
);
|
||||
|
||||
const startTime = performance.now();
|
||||
|
||||
try {
|
||||
if (compressionMethod === "zstd" && process.platform === "linux") {
|
||||
logger.info(`Streaming the extraction of the CodeQL bundle.`);
|
||||
|
||||
const toolsInstallStart = performance.now();
|
||||
await downloadAndExtractZstdWithStreaming(
|
||||
codeqlURL,
|
||||
dest,
|
||||
@@ -61,16 +78,14 @@ export async function downloadAndExtract(
|
||||
logger,
|
||||
);
|
||||
|
||||
const combinedDurationMs = Math.round(
|
||||
performance.now() - toolsInstallStart,
|
||||
);
|
||||
const totalDurationMs = Math.round(performance.now() - startTime);
|
||||
logger.info(
|
||||
`Finished downloading and extracting CodeQL bundle to ${dest} (${formatDuration(
|
||||
combinedDurationMs,
|
||||
totalDurationMs,
|
||||
)}).`,
|
||||
);
|
||||
|
||||
return {};
|
||||
return { totalDurationMs };
|
||||
}
|
||||
} catch (e) {
|
||||
core.warning(
|
||||
@@ -98,7 +113,7 @@ export async function downloadAndExtract(
|
||||
)}).`,
|
||||
);
|
||||
|
||||
let extractionDurationMs: number;
|
||||
let extractionDurationMs: number | undefined;
|
||||
|
||||
try {
|
||||
logger.info("Extracting CodeQL bundle.");
|
||||
@@ -120,7 +135,11 @@ export async function downloadAndExtract(
|
||||
await cleanUpPath(archivedBundlePath, "CodeQL bundle archive", logger);
|
||||
}
|
||||
|
||||
return { downloadDurationMs };
|
||||
return {
|
||||
downloadDurationMs,
|
||||
extractionDurationMs,
|
||||
totalDurationMs: Math.round(performance.now() - startTime),
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadAndExtractZstdWithStreaming(
|
||||
@@ -180,16 +199,154 @@ async function downloadAndExtractZstdWithStreaming(
|
||||
await tar.extractTarZst(response, dest, tarVersion, logger);
|
||||
}
|
||||
|
||||
/** Gets the path to the toolcache directory that holds all versions of the CodeQL tools. */
|
||||
function getToolcacheToolDirectory(env: ReadOnlyEnv): string {
|
||||
return path.join(
|
||||
env.getRequired(ActionsEnvVars.RUNNER_TOOL_CACHE),
|
||||
TOOLCACHE_TOOL_NAME,
|
||||
);
|
||||
}
|
||||
|
||||
/** Gets the name of the toolcache directory that holds the given version of the CodeQL tools. */
|
||||
function getToolcacheVersionDirectoryName(version: string): string {
|
||||
return semver.clean(version) || version;
|
||||
}
|
||||
|
||||
/** Gets the path to the toolcache directory for the specified version of the CodeQL tools. */
|
||||
export function getToolcacheDirectory(version: string): string {
|
||||
return path.join(
|
||||
getRequiredEnvParam("RUNNER_TOOL_CACHE"),
|
||||
TOOLCACHE_TOOL_NAME,
|
||||
semver.clean(version) || version,
|
||||
getToolcacheToolDirectory(getEnv()),
|
||||
getToolcacheVersionDirectoryName(version),
|
||||
os.arch() || "",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the toolcache is on the same filesystem as the workspace, and so whether deleting the
|
||||
* tools frees up disk space that the analysis can use.
|
||||
*
|
||||
* These are separate volumes on some runner images. Windows runners, for example, keep the
|
||||
* toolcache on `C:` while the workspace is on `D:`.
|
||||
*/
|
||||
export function isToolcacheOnWorkspaceFilesystem(logger: Logger): boolean {
|
||||
try {
|
||||
return (
|
||||
fs.statSync(getRequiredEnvParam("RUNNER_TOOL_CACHE")).dev ===
|
||||
fs.statSync(getRequiredEnvParam("GITHUB_WORKSPACE")).dev
|
||||
);
|
||||
} catch (e) {
|
||||
logger.debug(
|
||||
`Could not determine whether the toolcache is on the same filesystem as the workspace: ${getErrorMessage(e)}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** The outcome of trying to reclaim disk space by deleting the CodeQL tools from the toolcache. */
|
||||
export interface ToolcacheCleanupResult {
|
||||
/** The versions of the CodeQL tools that were deleted. */
|
||||
deletedVersions: string[];
|
||||
/**
|
||||
* Whether we hit an error while trying to delete the tools. Distinguishes a toolcache that had
|
||||
* nothing to reclaim from one we failed to clean up.
|
||||
*/
|
||||
failed: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes every version of the CodeQL tools from the toolcache.
|
||||
*
|
||||
* Only safe to call when we are about to download the tools, since that means we did not resolve
|
||||
* them from the toolcache and so nothing in there is in use by this job.
|
||||
*
|
||||
* This only ever touches the CodeQL directory of the toolcache. Cleanup errors are logged and
|
||||
* returned as `failed: true` rather than thrown.
|
||||
*
|
||||
* @returns the versions that were deleted, and whether we hit an error while trying.
|
||||
*/
|
||||
export async function deleteToolcacheBundles({
|
||||
env,
|
||||
logger,
|
||||
}: ActionState<["Logger", "ReadOnlyEnv"]>): Promise<ToolcacheCleanupResult> {
|
||||
let toolDirectory: string;
|
||||
|
||||
try {
|
||||
toolDirectory = getToolcacheToolDirectory(env);
|
||||
} catch (e) {
|
||||
logger.info(
|
||||
`Unable to determine toolcache directory: ${getErrorMessage(e)}`,
|
||||
);
|
||||
return { deletedVersions: [], failed: true };
|
||||
}
|
||||
|
||||
try {
|
||||
// Refuse to follow a symlinked CodeQL directory, so that we can only ever delete paths that are
|
||||
// really inside the toolcache.
|
||||
if ((await fs.promises.lstat(toolDirectory)).isSymbolicLink()) {
|
||||
logger.info(
|
||||
`Not deleting the CodeQL tools from the toolcache since '${toolDirectory}' is a symlink.`,
|
||||
);
|
||||
return { deletedVersions: [], failed: true };
|
||||
}
|
||||
} catch (e: any) {
|
||||
if (e?.code === "ENOENT") {
|
||||
logger.debug(
|
||||
`There are no CodeQL tools at '${toolDirectory}' to delete from the toolcache.`,
|
||||
);
|
||||
return { deletedVersions: [], failed: false };
|
||||
}
|
||||
logger.info(
|
||||
`Failed to inspect the CodeQL tools at '${toolDirectory}': ${getErrorMessage(e)}`,
|
||||
);
|
||||
return { deletedVersions: [], failed: true };
|
||||
}
|
||||
|
||||
try {
|
||||
const entries = await fs.promises.readdir(toolDirectory, {
|
||||
withFileTypes: true,
|
||||
});
|
||||
|
||||
const deletedVersions: string[] = [];
|
||||
let failed = false;
|
||||
|
||||
for (const entry of entries) {
|
||||
// `isDirectory` is false for a symlink, so we never delete a version directory that is
|
||||
// really somewhere else.
|
||||
if (!entry.isDirectory()) {
|
||||
logger.debug(
|
||||
`Not deleting '${entry.name}' from the CodeQL toolcache since it is not a directory.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const versionDirectory = path.join(toolDirectory, entry.name);
|
||||
|
||||
try {
|
||||
await fs.promises.rm(versionDirectory, {
|
||||
force: true,
|
||||
recursive: true,
|
||||
});
|
||||
deletedVersions.push(entry.name);
|
||||
logger.info(
|
||||
`Deleted the CodeQL tools at '${versionDirectory}' from the toolcache to free up disk space.`,
|
||||
);
|
||||
} catch (e) {
|
||||
failed = true;
|
||||
logger.info(
|
||||
`Failed to delete the CodeQL tools at '${versionDirectory}' from the toolcache: ${getErrorMessage(e)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { deletedVersions: deletedVersions.sort(), failed };
|
||||
} catch (e) {
|
||||
logger.info(
|
||||
`Failed to clean up the CodeQL toolcache at '${toolDirectory}': ${getErrorMessage(e)}`,
|
||||
);
|
||||
return { deletedVersions: [], failed: true };
|
||||
}
|
||||
}
|
||||
|
||||
export function writeToolcacheMarkerFile(
|
||||
extractedPath: string,
|
||||
logger: Logger,
|
||||
|
||||
@@ -842,9 +842,13 @@ export async function checkForTimeout() {
|
||||
* directory with the name hostedtoolcache which is present on
|
||||
* GitHub-hosted runners.
|
||||
*
|
||||
* @returns true iff the runner is hosted by GitHub
|
||||
* Since this is a heuristic over how the runner happens to be named and laid out, it also matches
|
||||
* self-hosted runners that are configured to resemble hosted ones. Prefer
|
||||
* `isGitHubHostedRunner` when you need the answer the Actions service reports.
|
||||
*
|
||||
* @returns true iff the runner looks like it is hosted by GitHub
|
||||
*/
|
||||
export function isHostedRunner() {
|
||||
export function looksLikeHostedRunner() {
|
||||
return (
|
||||
// Name of the runner on hosted Windows runners
|
||||
process.env["RUNNER_NAME"]?.includes("Hosted Agent") ||
|
||||
|
||||
@@ -37,5 +37,5 @@
|
||||
"@octokit/core/dist-types/types": ["./node_modules/@octokit/core/dist-types/types.d.ts"]
|
||||
},
|
||||
},
|
||||
"exclude": ["node_modules", "pr-checks"]
|
||||
"exclude": ["node_modules", "pr-checks", "scripts/changetool"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user