Merge pull request #1051 from github/henrymercer/run-atm-on-windows

Run ML-powered queries on Windows with CodeQL CLI 2.9.0+

.github/update-release-branch.py

@@ -19,15 +19,19 @@ V1_MODE = 'v1-release'
 # Value of the mode flag for a v2 release
 V2_MODE = 'v2-release'
 
+SOURCE_BRANCH_FOR_MODE = { V1_MODE: 'releases/v2', V2_MODE: 'main' }
+TARGET_BRANCH_FOR_MODE = { V1_MODE: 'releases/v1', V2_MODE: 'releases/v2' }
+
 # Name of the remote
 ORIGIN = 'origin'
 
 # Runs git with the given args and returns the stdout.
-# Raises an error if git does not exit successfully.
-def run_git(*args):
+# Raises an error if git does not exit successfully (unless passed
+# allow_non_zero_exit_code=True).
+def run_git(*args, allow_non_zero_exit_code=False):
   cmd = ['git', *args]
   p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  if (p.returncode != 0):
+  if not allow_non_zero_exit_code and p.returncode != 0:
     raise Exception('Call to ' + ' '.join(cmd) + ' exited with code ' + str(p.returncode) + ' stderr:' + p.stderr.decode('ascii'))
   return p.stdout.decode('ascii')
 
@@ -36,7 +40,9 @@ def branch_exists_on_remote(branch_name):
   return run_git('ls-remote', '--heads', ORIGIN, branch_name).strip() != ''
 
 # Opens a PR from the given branch to the target branch
-def open_pr(repo, all_commits, source_branch_short_sha, new_branch_name, source_branch, target_branch, conductor, is_v2_release, labels):
+def open_pr(
+  repo, all_commits, source_branch_short_sha, new_branch_name, source_branch, target_branch,
+  conductor, is_v2_release, labels, conflicted_files):
   # Sort the commits into the pull requests that introduced them,
   # and any commits that don't have a pull request
   pull_requests = []
@@ -81,6 +87,12 @@ def open_pr(repo, all_commits, source_branch_short_sha, new_branch_name, source_
 
   body.append('')
   body.append('Please review the following:')
+  if len(conflicted_files) > 0:
+    body.append(' - [ ] You have added commits to this branch that resolve the merge conflicts ' +
+      'in the following files:')
+    body.extend([f'    - [ ] `{file}`' for file in conflicted_files])
+    body.append(' - [ ] Another maintainer has reviewed the additional commits you added to this ' +
+      'branch to resolve the merge conflicts.')
   body.append(' - [ ] The CHANGELOG displays the correct version and date.')
   body.append(' - [ ] The CHANGELOG includes all relevant, user-facing changes since the last release.')
   body.append(' - [ ] There are no unexpected commits being merged into the ' + target_branch + ' branch.')
@@ -191,8 +203,10 @@ def main():
     type=str,
     required=True,
     choices=[V2_MODE, V1_MODE],
-    help=f"Which release to perform. '{V2_MODE}' uses main as the source branch and v2 as the target branch. " +
-      f"'{V1_MODE}' uses v2 as the source branch and v1 as the target branch."
+    help=f"Which release to perform. '{V2_MODE}' uses {SOURCE_BRANCH_FOR_MODE[V2_MODE]} as the source " +
+      f"branch and {TARGET_BRANCH_FOR_MODE[V2_MODE]} as the target branch. " +
+      f"'{V1_MODE}' uses {SOURCE_BRANCH_FOR_MODE[V1_MODE]} as the source branch and " +
+      f"{TARGET_BRANCH_FOR_MODE[V1_MODE]} as the target branch."
   )
   parser.add_argument(
     '--conductor',
@@ -203,14 +217,8 @@ def main():
 
   args = parser.parse_args()
 
-  if args.mode == V2_MODE:
-    source_branch = 'main'
-    target_branch = 'v2'
-  elif args.mode == V1_MODE:
-    source_branch = 'v2'
-    target_branch = 'v1'
-  else:
-    raise ValueError(f"Unexpected value for release mode: '{args.mode}'")
+  source_branch = SOURCE_BRANCH_FOR_MODE[args.mode]
+  target_branch = TARGET_BRANCH_FOR_MODE[args.mode]
 
   repo = Github(args.github_token).get_repo(args.repository_nwo)
   version = get_current_version()
@@ -246,10 +254,15 @@ def main():
   # Create the new branch and push it to the remote
   print('Creating branch ' + new_branch_name)
 
+  # The process of creating the v1 release can run into merge conflicts. We commit the unresolved
+  # conflicts so a maintainer can easily resolve them (vs erroring and requiring maintainers to
+  # reconstruct the release manually)
+  conflicted_files = []
+
   if args.mode == V1_MODE:
-    # If we're performing a backport, start from the v1 branch
-    print(f'Creating {new_branch_name} from the {ORIGIN}/v1 branch')
-    run_git('checkout', '-b', new_branch_name, f'{ORIGIN}/v1')
+    # If we're performing a backport, start from the target branch
+    print(f'Creating {new_branch_name} from the {ORIGIN}/{target_branch} branch')
+    run_git('checkout', '-b', new_branch_name, f'{ORIGIN}/{target_branch}')
 
     # Revert the commit that we made as part of the last release that updated the version number and
     # changelog to refer to 1.x.x variants. This avoids merge conflicts in the changelog and
@@ -274,7 +287,12 @@ def main():
       print('  Nothing to revert.')
 
     print(f'Merging {ORIGIN}/{source_branch} into the release prep branch')
-    run_git('merge', f'{ORIGIN}/{source_branch}', '--no-edit')
+    # Commit any conflicts (see the comment for `conflicted_files`)
+    run_git('merge', f'{ORIGIN}/{source_branch}', allow_non_zero_exit_code=True)
+    conflicted_files = run_git('diff', '--name-only', '--diff-filter', 'U').splitlines()
+    if len(conflicted_files) > 0:
+      run_git('add', '.')
+      run_git('commit', '--no-edit')
 
     # Migrate the package version number from a v2 version number to a v1 version number
     print(f'Setting version number to {version}')
@@ -317,6 +335,7 @@ def main():
     conductor=args.conductor,
     is_v2_release=args.mode == V2_MODE,
     labels=['Update dependencies'] if args.mode == V1_MODE else [],
+    conflicted_files=conflicted_files
   )
 
 if __name__ == '__main__':

.github/workflows/__analyze-ref-input.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__debug-artifacts.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__extractor-ram-threads.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__go-custom-queries.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__go-custom-tracing-autobuild.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__go-custom-tracing.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__javascript-source-root.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__ml-powered-queries.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened
@@ -82,10 +82,20 @@ jobs:
         retention-days: 7
 
     - name: Check results
+    # Running ML-powered queries on Windows requires CodeQL CLI 2.9.0+. We don't run these checks
+    # against Windows and `cached` while CodeQL CLI 2.9.0 makes its way into `cached` to avoid the
+    # test starting to fail when the cached CodeQL Bundle gets updated. Once the CodeQL Bundle
+    # containing CodeQL CLI 2.9.0 has been fully released, we can drop this line and start running
+    # these checks on Windows and `cached`.
+      if: matrix.os != 'windows-latest' || matrix.version != 'cached'
       env:
-        IS_WINDOWS: ${{ matrix.os == 'windows-latest' }}
+      # Running on Windows requires CodeQL CLI 2.9.0+, which has so far only made it to 'latest'.
+        SHOULD_RUN_ML_POWERED_QUERIES: ${{ matrix.os != 'windows-latest' || matrix.version
+          == 'latest' || matrix.version == 'nightly-latest' }}
       shell: bash
       run: |
+        echo "Expecting ML-powered queries to be run: ${SHOULD_RUN_ML_POWERED_QUERIES}"
+
         cd "$RUNNER_TEMP/results"
         # We should run at least the ML-powered queries in `expected_rules`.
         expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
@@ -94,10 +104,10 @@ jobs:
           found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
             flatten | .[].id] | any(. == $rule)' javascript.sarif)
           echo "Did find rule '${rule}': ${found_rule}"
-          if [[ "${found_rule}" != "true" && "${IS_WINDOWS}" != "true" ]]; then
+          if [[ "${found_rule}" != "true" && "${SHOULD_RUN_ML_POWERED_QUERIES}" == "true" ]]; then
             echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
             exit 1
-          elif [[ "${found_rule}" == "true" && "${IS_WINDOWS}" == "true" ]]; then
+          elif [[ "${found_rule}" == "true" && "${SHOULD_RUN_ML_POWERED_QUERIES}" != "true" ]]; then
             echo "Found rule '${rule}' in the SARIF output which shouldn't have been part of the analysis."
             exit 1
           fi
@@ -108,10 +118,10 @@ jobs:
           select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
           javascript.sarif)
         echo "Found ${num_alerts} alerts from ML-powered queries.";
-        if [[ "${num_alerts}" -eq 0 && "${IS_WINDOWS}" != "true" ]]; then
+        if [[ "${num_alerts}" -eq 0 && "${SHOULD_RUN_ML_POWERED_QUERIES}" == "true" ]]; then
           echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
           exit 1
-        elif [[ "${num_alerts}" -ne 0 && "${IS_WINDOWS}" == "true" ]]; then
+        elif [[ "${num_alerts}" -ne 0 && "${SHOULD_RUN_ML_POWERED_QUERIES}" != "true" ]]; then
           echo "Expected not to find any alerts from an ML-powered query but found ${num_alerts}."
           exit 1
         fi

.github/workflows/__multi-language-autodetect.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__packaging-config-inputs-js.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened
@@ -26,9 +26,27 @@ jobs:
       matrix:
         include:
         - os: ubuntu-latest
-          version: nightly-20210831
+          version: latest
         - os: macos-latest
-          version: nightly-20210831
+          version: latest
+        - os: windows-2019
+          version: latest
+        - os: windows-2022
+          version: latest
+        - os: ubuntu-latest
+          version: cached
+        - os: macos-latest
+          version: cached
+        - os: windows-2019
+          version: cached
+        - os: ubuntu-latest
+          version: nightly-latest
+        - os: macos-latest
+          version: nightly-latest
+        - os: windows-2019
+          version: nightly-latest
+        - os: windows-2022
+          version: nightly-latest
     name: 'Packaging: Config and input'
     timeout-minutes: 45
     runs-on: ${{ matrix.os }}
@@ -43,7 +61,7 @@ jobs:
     - uses: ./../action/init
       with:
         config-file: .github/codeql/codeql-config-packaging3.yml
-        packs: +dsp-testing/codeql-pack1@0.1.0
+        packs: +dsp-testing/codeql-pack1@1.0.0
         languages: javascript
         tools: ${{ steps.prepare-test.outputs.tools-url }}
     - name: Build code
@@ -58,11 +76,11 @@ jobs:
       shell: bash
       run: |
         cd "$RUNNER_TEMP/results"
-        # We should have 3 hits from these rules
-        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+        # We should have 4 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
 
         # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
-        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
         echo "Found matching rules '$RULES'"
         if [ "$RULES" != "$EXPECTED_RULES" ]; then
           echo "Did not match expected rules '$EXPECTED_RULES'."

.github/workflows/__packaging-config-js.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened
@@ -26,9 +26,27 @@ jobs:
       matrix:
         include:
         - os: ubuntu-latest
-          version: nightly-20210831
+          version: latest
         - os: macos-latest
-          version: nightly-20210831
+          version: latest
+        - os: windows-2019
+          version: latest
+        - os: windows-2022
+          version: latest
+        - os: ubuntu-latest
+          version: cached
+        - os: macos-latest
+          version: cached
+        - os: windows-2019
+          version: cached
+        - os: ubuntu-latest
+          version: nightly-latest
+        - os: macos-latest
+          version: nightly-latest
+        - os: windows-2019
+          version: nightly-latest
+        - os: windows-2022
+          version: nightly-latest
     name: 'Packaging: Config file'
     timeout-minutes: 45
     runs-on: ${{ matrix.os }}
@@ -57,11 +75,11 @@ jobs:
       shell: bash
       run: |
         cd "$RUNNER_TEMP/results"
-        # We should have 3 hits from these rules
-        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+        # We should have 4 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
 
         # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
-        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
         echo "Found matching rules '$RULES'"
         if [ "$RULES" != "$EXPECTED_RULES" ]; then
           echo "Did not match expected rules '$EXPECTED_RULES'."

.github/workflows/__packaging-inputs-js.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened
@@ -26,9 +26,27 @@ jobs:
       matrix:
         include:
         - os: ubuntu-latest
-          version: nightly-20210831
+          version: latest
         - os: macos-latest
-          version: nightly-20210831
+          version: latest
+        - os: windows-2019
+          version: latest
+        - os: windows-2022
+          version: latest
+        - os: ubuntu-latest
+          version: cached
+        - os: macos-latest
+          version: cached
+        - os: windows-2019
+          version: cached
+        - os: ubuntu-latest
+          version: nightly-latest
+        - os: macos-latest
+          version: nightly-latest
+        - os: windows-2019
+          version: nightly-latest
+        - os: windows-2022
+          version: nightly-latest
     name: 'Packaging: Action input'
     timeout-minutes: 45
     runs-on: ${{ matrix.os }}
@@ -44,7 +62,7 @@ jobs:
       with:
         config-file: .github/codeql/codeql-config-packaging2.yml
         languages: javascript
-        packs: dsp-testing/codeql-pack1@0.1.0, dsp-testing/codeql-pack2
+        packs: dsp-testing/codeql-pack1@1.0.0, dsp-testing/codeql-pack2, dsp-testing/codeql-pack3:other-query.ql
         tools: ${{ steps.prepare-test.outputs.tools-url }}
     - name: Build code
       shell: bash
@@ -58,11 +76,11 @@ jobs:
       shell: bash
       run: |
         cd "$RUNNER_TEMP/results"
-        # We should have 3 hits from these rules
-        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+        # We should have 4 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
 
         # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
-        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
         echo "Found matching rules '$RULES'"
         if [ "$RULES" != "$EXPECTED_RULES" ]; then
           echo "Did not match expected rules '$EXPECTED_RULES'."

.github/workflows/__remote-config.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__rubocop-multi-language.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__split-workflow.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened
@@ -26,9 +26,17 @@ jobs:
       matrix:
         include:
         - os: ubuntu-latest
-          version: nightly-20210831
+          version: latest
         - os: macos-latest
-          version: nightly-20210831
+          version: latest
+        - os: ubuntu-latest
+          version: cached
+        - os: macos-latest
+          version: cached
+        - os: ubuntu-latest
+          version: nightly-latest
+        - os: macos-latest
+          version: nightly-latest
     name: Split workflow
     timeout-minutes: 45
     runs-on: ${{ matrix.os }}
@@ -43,7 +51,7 @@ jobs:
     - uses: ./../action/init
       with:
         config-file: .github/codeql/codeql-config-packaging3.yml
-        packs: +dsp-testing/codeql-pack1@0.1.0
+        packs: +dsp-testing/codeql-pack1@1.0.0
         languages: javascript
         tools: ${{ steps.prepare-test.outputs.tools-url }}
     - name: Build code
@@ -72,11 +80,11 @@ jobs:
       shell: bash
       run: |
         cd "$RUNNER_TEMP/results"
-        # We should have 3 hits from these rules
-        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+        # We should have 4 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
 
         # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
-        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
         echo "Found matching rules '$RULES'"
         if [ "$RULES" != "$EXPECTED_RULES" ]; then
           echo "Did not match expected rules '$EXPECTED_RULES'."

.github/workflows/__test-autobuild-working-dir.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__test-local-codeql.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__test-proxy.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__test-ruby.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__unset-environment.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__upload-ref-sha-input.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/__with-checkout-path.yml

@@ -11,8 +11,8 @@ on:
   push:
     branches:
     - main
-    - v1
-    - v2
+    - releases/v1
+    - releases/v2
   pull_request:
     types:
     - opened

.github/workflows/check-for-conflicts.yml

@@ -0,0 +1,31 @@
+# Checks for any conflict markers created by git. This check is primarily intended to validate that
+# any merge conflicts in the v2 -> v1 backport PR are fixed before the PR is merged.
+name: Check for conflicts
+
+on:
+  pull_request:
+    branches: [main, v1, v2]
+    # Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
+    # by other workflows.
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  check-for-conflicts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Check for conflicts
+        run: |
+          # Use `|| true` since grep returns exit code 1 if there are no matches, and we don't want
+          # this to fail the workflow.
+          FILES_WITH_CONFLICTS=$(grep --extended-regexp --ignore-case --line-number --recursive \
+            '^(<<<<<<<|>>>>>>>)' . || true)
+          if [[ "${FILES_WITH_CONFLICTS}" ]]; then
+            echo "Fail: Found merge conflict markers in the following files:"
+            echo ""
+            echo "${FILES_WITH_CONFLICTS}"
+            exit 1
+          else
+            echo "Success: Found no merge conflict markers."
+          fi

.github/workflows/codeql.yml

@@ -2,9 +2,9 @@ name: "CodeQL action"
 
 on:
   push:
-    branches: [main, v1, v2]
+    branches: [main, releases/v1, releases/v2]
   pull_request:
-    branches: [main, v1, v2]
+    branches: [main, releases/v1, releases/v2]
     # Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
     # by other workflows.
     types: [opened, synchronize, reopened, ready_for_review]

.github/workflows/post-release-mergeback.yml

@@ -1,7 +1,8 @@
-# This workflow runs after a release of the action.
-# It merges any changes from the release back into the
-# main branch. Typically, this is just a single commit
-# that updates the changelog.
+# This workflow runs after a release of the action. For v2 releases, it merges any changes from the
+# release back into the main branch. Typically, this is just a single commit that updates the
+# changelog. For v2 and v1 releases, it then (a) tags the merge commit on the release branch that
+# represents the new release with an `vx.y.z` tag and (b) updates the `vx` tag to refer to this
+# commit.
 name: Tag release and merge back
 
 on:
@@ -14,8 +15,8 @@ on:
 
   push:
     branches:
-      - v1
-      - v2
+      - releases/v1
+      - releases/v2
 
 jobs:
   merge-back:
@@ -32,7 +33,7 @@ jobs:
       - name: Dump GitHub context
         env:
           GITHUB_CONTEXT: '${{ toJson(github) }}'
-        run: echo "$GITHUB_CONTEXT"
+        run: echo "${GITHUB_CONTEXT}"
 
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
@@ -46,25 +47,25 @@ jobs:
         id: getVersion
         run: |
           VERSION="v$(jq '.version' -r 'package.json')"
-          SHORT_SHA="${GITHUB_SHA:0:8}"
-          echo "::set-output name=version::$VERSION"
-          NEW_BRANCH="mergeback/${VERSION}-to-${BASE_BRANCH}-${SHORT_SHA}"
-          echo "::set-output name=newBranch::$NEW_BRANCH"
+          echo "::set-output name=version::${VERSION}"
+          short_sha="${GITHUB_SHA:0:8}"
+          NEW_BRANCH="mergeback/${VERSION}-to-${BASE_BRANCH}-${short_sha}"
+          echo "::set-output name=newBranch::${NEW_BRANCH}"
 
 
       - name: Dump branches
         env:
           NEW_BRANCH: "${{ steps.getVersion.outputs.newBranch }}"
         run: |
-          echo "BASE_BRANCH $BASE_BRANCH"
-          echo "HEAD_BRANCH $HEAD_BRANCH"
-          echo "NEW_BRANCH $NEW_BRANCH"
+          echo "BASE_BRANCH ${BASE_BRANCH}"
+          echo "HEAD_BRANCH ${HEAD_BRANCH}"
+          echo "NEW_BRANCH ${NEW_BRANCH}"
 
       - name: Create mergeback branch
         env:
           NEW_BRANCH: "${{ steps.getVersion.outputs.newBranch }}"
         run: |
-          git checkout -b "$NEW_BRANCH"
+          git checkout -b "${NEW_BRANCH}"
 
       - name: Check for tag
         id: check
@@ -72,13 +73,13 @@ jobs:
           VERSION: "${{ steps.getVersion.outputs.version }}"
         run: |
           set +e # don't fail on an errored command
-          git ls-remote --tags origin | grep "$VERSION"
-          EXISTS="$?"
-          if [ "$EXISTS" -eq 0 ]; then
-            echo "Tag $TAG exists. Not going to re-release."
+          git ls-remote --tags origin | grep "${VERSION}"
+          exists="$?"
+          if [ "${exists}" -eq 0 ]; then
+            echo "Tag ${VERSION} exists. Not going to re-release."
             echo "::set-output name=exists::true"
           else
-            echo "Tag $TAG does not exist yet."
+            echo "Tag ${VERSION} does not exist yet."
           fi
 
       # we didn't tag the release during the update-release-branch workflow because the
@@ -89,20 +90,31 @@ jobs:
         env:
           VERSION: ${{ steps.getVersion.outputs.version }}
         run: |
-          git tag -a "$VERSION" -m "$VERSION"
-          git fetch --unshallow  # unshallow the repo in order to allow pushes
-          git push origin --follow-tags "$VERSION"
+          # Unshallow the repo in order to allow pushes
+          git fetch --unshallow
+          # Create the `vx.y.z` tag
+          git tag --annotate "${VERSION}" --message "${VERSION}"
+          # Update the `vx` tag
+          major_version_tag=$(cut -d '.' -f1 <<< "${VERSION}")
+          # Use `--force` to overwrite the major version tag
+          git tag --annotate "${major_version_tag}" --message "${major_version_tag}" --force
+          # Push the tags, using:
+          # - `--atomic` to make sure we either update both tags or neither (an intermediate state,
+          #   e.g. where we update the v2.x.y tag on the remote but not the v2 tag, could result in
+          #   unwanted Dependabot updates, e.g. from v2 to v2.x.y)
+          # - `--force` since we're overwriting the `vx` tag
+          git push origin --atomic --force refs/tags/"${VERSION}" refs/tags/"${major_version_tag}"
 
       - name: Create mergeback branch
-        if: steps.check.outputs.exists != 'true' && contains(github.ref, 'v2')
+        if: steps.check.outputs.exists != 'true' && contains(github.ref, 'releases/v2')
         env:
           VERSION: "${{ steps.getVersion.outputs.version }}"
           NEW_BRANCH: "${{ steps.getVersion.outputs.newBranch }}"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         run: |
           set -exu
-          PR_TITLE="Mergeback $VERSION $HEAD_BRANCH into $BASE_BRANCH"
-          PR_BODY="Updates version and changelog."
+          pr_title="Mergeback ${VERSION} ${HEAD_BRANCH} into ${BASE_BRANCH}"
+          pr_body="Updates version and changelog."
 
           # Update the version number ready for the next release
           npm version patch --no-git-tag-version
@@ -110,16 +122,16 @@ jobs:
           # Update the changelog
           perl -i -pe 's/^/## \[UNRELEASED\]\n\nNo user facing changes.\n\n/ if($.==3)' CHANGELOG.md
           git add .
-          git commit -m "Update changelog and version after $VERSION"
+          git commit -m "Update changelog and version after ${VERSION}"
 
-          git push origin "$NEW_BRANCH"
+          git push origin "${NEW_BRANCH}"
 
           # PR checks won't be triggered on PRs created by Actions. Therefore mark the PR as draft
           # so that a maintainer can take the PR out of draft, thereby triggering the PR checks.
           gh pr create \
-            --head "$NEW_BRANCH" \
-            --base "$BASE_BRANCH" \
-            --title "$PR_TITLE" \
+            --head "${NEW_BRANCH}" \
+            --base "${BASE_BRANCH}" \
+            --title "${pr_title}" \
             --label "Update dependencies" \
-            --body "$PR_BODY" \
+            --body "${pr_body}" \
             --draft

.github/workflows/pr-checks.yml

@@ -2,7 +2,7 @@ name: PR Checks (Basic Checks and Runner)
 
 on:
   push:
-    branches: [main, v1, v2]
+    branches: [main, releases/v1, releases/v2]
   pull_request:
     # Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
     # by other workflows.

.github/workflows/python-deps.yml

@@ -2,7 +2,7 @@ name: Test Python Package Installation on Linux and Mac
 
 on:
   push:
-    branches: [main, v1, v2]
+    branches: [main, releases/v1, releases/v2]
   pull_request:
     # Run checks on reopened draft PRs to support triggering PR checks on draft PRs that were opened
     # by other workflows.

.github/workflows/script/update-required-checks.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Update the required checks based on the current branch.
+# Typically, this will be main.
+
+if [ -z "$GITHUB_TOKEN" ]; then
+  echo "Failed: No GitHub token found. This script requires admin access to `github/codeql-action`."
+  exit 1
+fi
+
+if [ "$#" -eq 1 ]; then
+  # If we were passed an argument, pass it as a query to fzf
+  GITHUB_SHA="$@"
+elif [ "$#" -gt 1 ]; then
+  echo "Usage: $0 [SHA]"
+  echo "Update the required checks based on the SHA, or main."
+elif [ -z "$GITHUB_SHA" ]; then
+  # If we don't have a SHA, use main
+  GITHUB_SHA="$(git rev-parse main)"
+fi
+
+echo "Getting checks for $GITHUB_SHA"
+
+# Ignore any checks with "https://", CodeQL, LGTM, and Update checks.
+CHECKS="$(gh api repos/github/codeql-action/commits/${GITHUB_SHA}/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs | .[].name | select(contains("https://") or . == "CodeQL" or . == "LGTM.com" or contains("Update") | not)] | sort')"
+
+echo "$CHECKS" | jq
+
+echo "{\"contexts\": ${CHECKS}}" > checks.json
+
+for BRANCH in main releases/v2 releases/v1; do
+  echo "Updating $BRANCH"
+  gh api --silent -X "PATCH" "repos/github/codeql-action/branches/$BRANCH/protection/required_status_checks" --input checks.json
+done
+
+rm checks.json

.github/workflows/update-release-branch.yml

@@ -7,7 +7,7 @@ on:
   # When the v2 release is complete, this workflow will open a PR to update the v1 release branch.
   push:
     branches:
-      - v2
+      - releases/v2
 
 jobs:
   update:

CHANGELOG.md

@@ -2,8 +2,19 @@
 
 ## [UNRELEASED]
 
+No user facing changes.
+
+## 2.1.10 - 10 May 2022
+
+- Update default CodeQL bundle version to 2.9.5. [#1056](https://github.com/github/codeql-action/pull/1056)
+- When `wait-for-processing` is enabled, the workflow will now fail if there were any errors that occurred during processing of the analysis results.
+
+## 2.1.9 - 27 Apr 2022
+
 - Add `working-directory` input to the `autobuild` action. [#1024](https://github.com/github/codeql-action/pull/1024)
 - The `analyze` and `upload-sarif` actions will now wait up to 2 minutes for processing to complete after they have uploaded the results so they can report any processing errors that occurred. This behavior can be disabled by setting the `wait-for-processing` action input to `"false"`. [#1007](https://github.com/github/codeql-action/pull/1007)
+- Update default CodeQL bundle version to 2.9.0.
+- Fix a bug where [status reporting fails on Windows](https://github.com/github/codeql-action/issues/1041). [#1042](https://github.com/github/codeql-action/pull/1042)
 
 ## 2.1.8 - 08 Apr 2022
 

CONTRIBUTING.md

@@ -61,41 +61,30 @@ Here are a few things you can do that will increase the likelihood of your pull
 ## Releasing (write access required)
 
 1. The first step of releasing a new version of the `codeql-action` is running the "Update release branch" workflow.
-    This workflow goes through the pull requests that have been merged to `main` since the last release, creates a changelog, then opens a pull request to merge the changes since the last release into the `v2` release branch.
+    This workflow goes through the pull requests that have been merged to `main` since the last release, creates a changelog, then opens a pull request to merge the changes since the last release into the `releases/v2` release branch.
 
     You can start a release by triggering this workflow via [workflow dispatch](https://github.com/github/codeql-action/actions/workflows/update-release-branch.yml).
-1. The workflow run will open a pull request titled "Merge main into v2". Mark the pull request as [ready for review](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request#marking-a-pull-request-as-ready-for-review) to trigger the PR checks.
+1. The workflow run will open a pull request titled "Merge main into releases/v2". Mark the pull request as [ready for review](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request#marking-a-pull-request-as-ready-for-review) to trigger the PR checks.
 1. Review the checklist items in the pull request description.
     Once you've checked off all but the last two of these, approve the PR and automerge it.
-1. When the "Merge main into v2" pull request is merged into the `v2` branch, the "Tag release and merge back" workflow will create a mergeback PR.
-    This mergeback incorporates the changelog updates into `main`, tags the release using the merge commit of the "Merge main into v2" pull request, and bumps the patch version of the CodeQL Action.
+1. When the "Merge main into releases/v2" pull request is merged into the `releases/v2` branch, the "Tag release and merge back" workflow will create a mergeback PR.
+    This mergeback incorporates the changelog updates into `main`, tags the release using the merge commit of the "Merge main into releases/v2" pull request, and bumps the patch version of the CodeQL Action.
 
     Approve the mergeback PR and automerge it.
-1. When the "Merge main into v2" pull request is merged into the `v2` branch, the "Update release branch" workflow will create a "Merge v2 into v1" pull request to merge the changes since the last release into the `v1` release branch.
-    This ensures we keep both the `v1` and `v2` release branches up to date and fully supported.
+1. When the "Merge main into releases/v2" pull request is merged into the `releases/v2` branch, the "Update release branch" workflow will create a "Merge releases/v2 into releases/v1" pull request to merge the changes since the last release into the `releases/v1` release branch.
+    This ensures we keep both the `releases/v1` and `releases/v2` release branches up to date and fully supported.
 
     Review the checklist items in the pull request description.
     Once you've checked off all the items, approve the PR and automerge it.
-1. Once the mergeback has been merged to `main` and the "Merge v2 into v1" PR has been merged to `v1`, the release is complete.
+1. Once the mergeback has been merged to `main` and the "Merge releases/v2 into releases/v1" PR has been merged to `releases/v1`, the release is complete.
 
 ## Keeping the PR checks up to date (admin access required)
 
-Since the `codeql-action` runs most of its testing through individual Actions workflows, there are over two hundred jobs that need to pass in order for a PR to turn green. Managing these PR checks manually is time consuming and complex. Here is a semi-automated approach.
+Since the `codeql-action` runs most of its testing through individual Actions workflows, there are over two hundred jobs that need to pass in order for a PR to turn green. You can regenerate the checks automatically by running the [update-required-checks.sh](.github/workflows/script/update-required-checks.sh) script:
 
-To regenerate the PR jobs for the action:
-
-1. From a terminal, run the following commands (replace `SHA` with the sha of the commit whose checks you want to use, typically this should be the latest from `main`):
-
-    ```sh
-    SHA= ####
-    CHECKS="$(gh api repos/github/codeql-action/commits/${SHA}/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs | .[].name | select(contains("https://") or . == "CodeQL" or . == "LGTM.com" or . == "Update dependencies" or . == "Update Supported Enterprise Server Versions" | not)]')"
-    echo "{\"contexts\": ${CHECKS}}" > checks.json
-    gh api -X "PATCH" repos/github/codeql-action/branches/main/protection/required_status_checks --input checks.json
-    gh api -X "PATCH" repos/github/codeql-action/branches/v2/protection/required_status_checks --input checks.json
-    gh api -X "PATCH" repos/github/codeql-action/branches/v1/protection/required_status_checks --input checks.json
-    ````
-
-2. Go to the [branch protection rules settings page](https://github.com/github/codeql-action/settings/branches) and validate that the rules have been updated.
+1. By default, this script retrieves the checks from the latest SHA on `main`, so make sure that your `main` branch is up to date.
+2. Run the script. If there's a reason to, you can pass in a different SHA as a CLI argument.
+3. After running, go to the [branch protection rules settings page](https://github.com/github/codeql-action/settings/branches) and validate that the rules for `main`, `v1`, and `v2` have been updated.
 
 ## Resources
 

README.md

@@ -67,7 +67,7 @@ jobs:
         uses: github/codeql-action/autobuild@v2
 
       # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 https://git.io/JvXDl
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
       # ✏️ If the Autobuild fails above, remove it and uncomment the following
       #    three lines and modify them (or add more) to build your code if your

lib/actions-util.js

@@ -584,8 +584,7 @@ async function sendStatusReport(statusReport) {
     const statusReportJSON = JSON.stringify(statusReport);
     core.debug(`Sending status report: ${statusReportJSON}`);
     // If in test mode we don't want to upload the results
-    const testMode = process.env["TEST_MODE"] === "true" || false;
-    if (testMode) {
+    if ((0, util_1.isInTestMode)()) {
         core.debug("In test mode. Status reports are not uploaded.");
         return true;
     }

lib/analyze-action-env.test.js

@@ -38,14 +38,17 @@ const util = __importStar(require("./util"));
 // but the first test would fail.
 (0, ava_1.default)("analyze action with RAM & threads from environment variables", async (t) => {
     await util.withTmpDir(async (tmpDir) => {
-        process.env["GITHUB_SERVER_URL"] = "fake-server-url";
-        process.env["GITHUB_REPOSITORY"] = "fake/repository";
+        process.env["GITHUB_SERVER_URL"] = util.GITHUB_DOTCOM_URL;
+        process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
         sinon
             .stub(actionsUtil, "createStatusReportBase")
             .resolves({});
         sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+        const gitHubVersion = {
+            type: util.GitHubVariant.DOTCOM,
+        };
         sinon.stub(configUtils, "getConfig").resolves({
-            gitHubVersion: { type: util.GitHubVariant.DOTCOM },
+            gitHubVersion,
             languages: [],
             packs: [],
         });
@@ -54,6 +57,7 @@ const util = __importStar(require("./util"));
         requiredInputStub.withArgs("upload-database").returns("false");
         const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
         optionalInputStub.withArgs("cleanup-level").returns("none");
+        sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
         (0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
         (0, testing_utils_1.mockFeatureFlagApiEndpoint)(200, {});
         // When there are no action inputs for RAM and threads, the action uses

lib/analyze-action-env.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze-action-env.test.js","sourceRoot":"","sources":["../src/analyze-action-env.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAIyB;AACzB,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,8DAA8D,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/E,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;YAClD,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACuB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,IAAA,0CAA0B,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAEpC,uEAAuE;QACvE,0EAA0E;QAC1E,iBAAiB;QACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"analyze-action-env.test.js","sourceRoot":"","sources":["../src/analyze-action-env.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAIyB;AACzB,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,8DAA8D,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/E,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,sCAAsC,CAAC;QAC1E,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAuB;YACxC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;SAChC,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,aAAa;YACb,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACuB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC7D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,IAAA,0CAA0B,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAEpC,uEAAuE;QACvE,0EAA0E;QAC1E,iBAAiB;QACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

lib/analyze-action-input.test.js

@@ -38,14 +38,17 @@ const util = __importStar(require("./util"));
 // but the first test would fail.
 (0, ava_1.default)("analyze action with RAM & threads from action inputs", async (t) => {
     await util.withTmpDir(async (tmpDir) => {
-        process.env["GITHUB_SERVER_URL"] = "fake-server-url";
-        process.env["GITHUB_REPOSITORY"] = "fake/repository";
+        process.env["GITHUB_SERVER_URL"] = util.GITHUB_DOTCOM_URL;
+        process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
         sinon
             .stub(actionsUtil, "createStatusReportBase")
             .resolves({});
         sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+        const gitHubVersion = {
+            type: util.GitHubVariant.DOTCOM,
+        };
         sinon.stub(configUtils, "getConfig").resolves({
-            gitHubVersion: { type: util.GitHubVariant.DOTCOM },
+            gitHubVersion,
             languages: [],
             packs: [],
         });
@@ -54,6 +57,7 @@ const util = __importStar(require("./util"));
         requiredInputStub.withArgs("upload-database").returns("false");
         const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
         optionalInputStub.withArgs("cleanup-level").returns("none");
+        sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
         (0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
         (0, testing_utils_1.mockFeatureFlagApiEndpoint)(200, {});
         process.env["CODEQL_THREADS"] = "1";

lib/analyze-action-input.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze-action-input.test.js","sourceRoot":"","sources":["../src/analyze-action-input.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAIyB;AACzB,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,sDAAsD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;YAClD,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACuB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,IAAA,0CAA0B,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAEpC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,4DAA4D;QAC5D,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"analyze-action-input.test.js","sourceRoot":"","sources":["../src/analyze-action-input.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAIyB;AACzB,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,sDAAsD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,sCAAsC,CAAC;QAC1E,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAuB;YACxC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM;SAChC,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,aAAa;YACb,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACuB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC7D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,IAAA,0CAA0B,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAEpC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,4DAA4D;QAC5D,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

lib/analyze-action.js

@@ -57,6 +57,7 @@ async function run() {
     let runStats = undefined;
     let config = undefined;
     util.initializeEnvironment(util.Mode.actions, pkg.version);
+    await util.checkActionVersion(pkg.version);
     try {
         if (!(await actionsUtil.sendStatusReport(await actionsUtil.createStatusReportBase("finish", "starting", startedAt)))) {
             return;
@@ -117,7 +118,11 @@ async function run() {
         }
         // Possibly upload the database bundles for remote queries
         await (0, database_upload_1.uploadDatabases)(repositoryNwo, config, apiDetails, logger);
-        if (uploadResult !== undefined &&
+        // We don't upload results in test mode, so don't wait for processing
+        if (util.isInTestMode()) {
+            core.debug("In test mode. Waiting for processing is disabled.");
+        }
+        else if (uploadResult !== undefined &&
             actionsUtil.getRequiredInput("wait-for-processing") === "true") {
             await upload_lib.waitForProcessing((0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY")), uploadResult.sarifID, apiDetails, (0, logging_1.getActionsLogger)());
         }

lib/analyze.js

@@ -159,7 +159,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
                 }
             }
             if (packsWithVersion.length > 0) {
-                querySuitePaths.push(await runQueryGroup(language, "packs", createPackSuiteContents(packsWithVersion), undefined));
+                querySuitePaths.push(...(await runQueryPacks(language, "packs", packsWithVersion, undefined)));
                 ranCustom = true;
             }
             if (ranCustom) {
@@ -217,21 +217,23 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
         logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
         return querySuitePath;
     }
+    async function runQueryPacks(language, type, packs, searchPath) {
+        const databasePath = util.getCodeQLDatabasePath(config, language);
+        // Run the queries individually instead of all at once to avoid command
+        // line length restrictions, particularly on windows.
+        for (const pack of packs) {
+            logger.debug(`Running query pack for ${language}-${type}: ${pack}`);
+            const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
+            await codeql.databaseRunQueries(databasePath, searchPath, pack, memoryFlag, threadsFlag);
+            logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
+        }
+        return packs;
+    }
 }
 exports.runQueries = runQueries;
 function createQuerySuiteContents(queries) {
     return queries.map((q) => `- query: ${q}`).join("\n");
 }
-function createPackSuiteContents(packsWithVersion) {
-    return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
-}
-function packWithVersionToQuerySuiteEntry(pack) {
-    let text = `- qlpack: ${pack.packName}`;
-    if (pack.version) {
-        text += `\n  version: ${pack.version}`;
-    }
-    return text;
-}
 async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
     if (await util.codeQlVersionAbove(codeql, codeql_1.CODEQL_VERSION_NEW_TRACING)) {

lib/analyze.test.js

@@ -26,7 +26,6 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const ava_1 = __importDefault(require("ava"));
 const yaml = __importStar(require("js-yaml"));
-const semver_1 = require("semver");
 const sinon = __importStar(require("sinon"));
 const analyze_1 = require("./analyze");
 const codeql_1 = require("./codeql");
@@ -53,18 +52,8 @@ const util = __importStar(require("./util"));
         const addSnippetsFlag = "";
         const threadsFlag = "";
         const packs = {
-            [languages_1.Language.cpp]: [
-                {
-                    packName: "a/b",
-                    version: (0, semver_1.clean)("1.0.0"),
-                },
-            ],
-            [languages_1.Language.java]: [
-                {
-                    packName: "c/d",
-                    version: (0, semver_1.clean)("2.0.0"),
-                },
-            ],
+            [languages_1.Language.cpp]: ["a/b@1.0.0"],
+            [languages_1.Language.java]: ["c/d@2.0.0"],
         };
         for (const language of Object.values(languages_1.Language)) {
             (0, codeql_1.setCodeQL)({
@@ -209,32 +198,10 @@ const util = __importStar(require("./util"));
                 query: "bar.ql",
             },
         ];
-        const qlsPackContentCpp = [
-            {
-                qlpack: "a/b",
-                version: "1.0.0",
-            },
-        ];
-        const qlsPackContentJava = [
-            {
-                qlpack: "c/d",
-                version: "2.0.0",
-            },
-        ];
         for (const lang of Object.values(languages_1.Language)) {
             t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
             t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
             t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
-            const packSuiteName = `${lang}-queries-packs.qls`;
-            if (lang === languages_1.Language.cpp) {
-                t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
-            }
-            else if (lang === languages_1.Language.java) {
-                t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
-            }
-            else {
-                t.false(fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName)));
-            }
         }
         function readContents(name) {
             const x = fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8");

lib/autobuild-action.js

@@ -39,8 +39,9 @@ async function sendCompletedStatusReport(startedAt, allLanguages, failingLanguag
     await (0, actions_util_1.sendStatusReport)(statusReport);
 }
 async function run() {
-    const logger = (0, logging_1.getActionsLogger)();
     const startedAt = new Date();
+    const logger = (0, logging_1.getActionsLogger)();
+    await (0, util_1.checkActionVersion)(pkg.version);
     let language = undefined;
     try {
         if (!(await (0, actions_util_1.sendStatusReport)(await (0, actions_util_1.createStatusReportBase)("autobuild", "starting", startedAt)))) {

lib/autobuild-action.js.map

@@ -1 +1 @@
-{"version":3,"file":"autobuild-action.js","sourceRoot":"","sources":["../src/autobuild-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,iDAOwB;AACxB,2CAAuE;AACvE,6DAA+C;AAE/C,uCAA6C;AAC7C,iCAAqD;AAErD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AASvC,KAAK,UAAU,yBAAyB,CACtC,SAAe,EACf,YAAsB,EACtB,eAAwB,EACxB,KAAa;IAEb,IAAA,4BAAqB,EAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAG,IAAA,+BAAgB,EAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IACxD,MAAM,gBAAgB,GAAG,MAAM,IAAA,qCAAsB,EACnD,WAAW,EACX,MAAM,EACN,SAAS,EACT,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,EACd,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAA0B;QAC1C,GAAG,gBAAgB;QACnB,mBAAmB,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAC3C,iBAAiB,EAAE,eAAe;KACnC,CAAC;IACF,MAAM,IAAA,+BAAgB,EAAC,YAAY,CAAC,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,MAAM,GAAG,IAAA,0BAAgB,GAAE,CAAC;IAClC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,QAAQ,GAAyB,SAAS,CAAC;IAC/C,IAAI;QACF,IACE,CAAC,CAAC,MAAM,IAAA,+BAAgB,EACtB,MAAM,IAAA,qCAAsB,EAAC,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CACjE,CAAC,EACF;YACA,OAAO;SACR;QAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,CACzC,IAAA,oCAAqB,GAAE,EACvB,MAAM,CACP,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QACD,QAAQ,GAAG,IAAA,sCAA0B,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,MAAM,gBAAgB,GAAG,IAAA,+BAAgB,EAAC,mBAAmB,CAAC,CAAC;YAC/D,IAAI,gBAAgB,EAAE;gBACpB,MAAM,CAAC,IAAI,CACT,6CAA6C,gBAAgB,EAAE,CAChE,CAAC;gBACF,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;aACjC;YACD,MAAM,IAAA,wBAAY,EAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;SAC9C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CACZ,mIACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,CACH,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,yBAAyB,CAC7B,SAAS,EACT,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,EAC1B,QAAQ,EACR,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC1D,CAAC;QACF,OAAO;KACR;IAED,MAAM,yBAAyB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
+{"version":3,"file":"autobuild-action.js","sourceRoot":"","sources":["../src/autobuild-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,iDAOwB;AACxB,2CAAuE;AACvE,6DAA+C;AAE/C,uCAA6C;AAC7C,iCAAyE;AAEzE,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AASvC,KAAK,UAAU,yBAAyB,CACtC,SAAe,EACf,YAAsB,EACtB,eAAwB,EACxB,KAAa;IAEb,IAAA,4BAAqB,EAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAG,IAAA,+BAAgB,EAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IACxD,MAAM,gBAAgB,GAAG,MAAM,IAAA,qCAAsB,EACnD,WAAW,EACX,MAAM,EACN,SAAS,EACT,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,EACd,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAA0B;QAC1C,GAAG,gBAAgB;QACnB,mBAAmB,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAC3C,iBAAiB,EAAE,eAAe;KACnC,CAAC;IACF,MAAM,IAAA,+BAAgB,EAAC,YAAY,CAAC,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAA,0BAAgB,GAAE,CAAC;IAClC,MAAM,IAAA,yBAAkB,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,QAAQ,GAAyB,SAAS,CAAC;IAC/C,IAAI;QACF,IACE,CAAC,CAAC,MAAM,IAAA,+BAAgB,EACtB,MAAM,IAAA,qCAAsB,EAAC,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CACjE,CAAC,EACF;YACA,OAAO;SACR;QAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,CACzC,IAAA,oCAAqB,GAAE,EACvB,MAAM,CACP,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QACD,QAAQ,GAAG,IAAA,sCAA0B,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,MAAM,gBAAgB,GAAG,IAAA,+BAAgB,EAAC,mBAAmB,CAAC,CAAC;YAC/D,IAAI,gBAAgB,EAAE;gBACpB,MAAM,CAAC,IAAI,CACT,6CAA6C,gBAAgB,EAAE,CAChE,CAAC;gBACF,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;aACjC;YACD,MAAM,IAAA,wBAAY,EAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;SAC9C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CACZ,mIACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,CACH,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,yBAAyB,CAC7B,SAAS,EACT,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,EAC1B,QAAQ,EACR,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC1D,CAAC;QACF,OAAO;KACR;IAED,MAAM,yBAAyB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

lib/codeql.js

@@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_ML_POWERED_QUERIES = exports.CODEQL_VERSION_COUNTS_LINES = exports.CommandInvocationError = void 0;
+exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_ML_POWERED_QUERIES = exports.CODEQL_VERSION_COUNTS_LINES = exports.CommandInvocationError = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@@ -86,6 +86,12 @@ exports.CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
  * versions above that.
  */
 exports.CODEQL_VERSION_NEW_TRACING = "2.7.0";
+/**
+ * Versions 2.9.0+ of the CodeQL CLI run machine learning models from a temporary directory, which
+ * resolves an issue on Windows where TensorFlow models are not correctly loaded due to the path of
+ * some of their files being greater than MAX_PATH (260 characters).
+ */
+exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
 function getCodeQLBundleName() {
     let platform;
     if (process.platform === "win32") {
@@ -394,7 +400,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
         async getVersion() {
             let result = util.getCachedCodeQlVersion();
             if (result === undefined) {
-                result = await runTool(cmd, ["version", "--format=terse"]);
+                result = (await runTool(cmd, ["version", "--format=terse"])).trim();
                 util.cacheCodeQlVersion(result);
             }
             return result;
@@ -641,8 +647,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
                 "pack",
                 "download",
                 "--format=json",
+                "--resolve-query-specs",
                 ...getExtraOptionsFromEnv(["pack", "download"]),
-                ...packs.map(packWithVersionToString),
+                ...packs,
             ];
             const output = await runTool(cmd, codeqlArgs);
             try {
@@ -698,9 +705,6 @@ async function getCodeQLForCmd(cmd, checkVersion) {
     }
     return codeql;
 }
-function packWithVersionToString(pack) {
-    return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
-}
 /**
  * Gets the options for `path` of `options` as an array of extra option strings.
  */

lib/config-utils.js

@@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.parsePacksFromConfig = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPacksRequireLanguage = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = void 0;
+exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePacksSpecification = exports.parsePacksFromConfig = exports.getDefaultConfig = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPacksRequireLanguage = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const yaml = __importStar(require("js-yaml"));
@@ -131,11 +131,12 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
     // opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
     // pack, then add the ML-powered query pack so that we run ML-powered queries.
     if (
-    // Disable ML-powered queries on Windows
-    process.platform !== "win32" &&
+    // Only run ML-powered queries on Windows if we have a CLI that supports it.
+    (process.platform !== "win32" ||
+        (await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS))) &&
         languages.includes("javascript") &&
         (found === "security-extended" || found === "security-and-quality") &&
-        !((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some((pack) => pack.packName === util_1.ML_POWERED_JS_QUERIES_PACK_NAME)) &&
+        !((_a = packs.javascript) === null || _a === void 0 ? void 0 : _a.some(isMlPoweredJsQueriesPack)) &&
         (await featureFlags.getValue(feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled)) &&
         (await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES))) {
         if (!packs.javascript) {
@@ -148,6 +149,11 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
     await runResolveQueries(codeQL, resultMap, suites, undefined);
     return injectedMlQueries;
 }
+function isMlPoweredJsQueriesPack(pack) {
+    return (pack === util_1.ML_POWERED_JS_QUERIES_PACK_NAME ||
+        pack.startsWith(`${util_1.ML_POWERED_JS_QUERIES_PACK_NAME}@`) ||
+        pack.startsWith(`${util_1.ML_POWERED_JS_QUERIES_PACK_NAME}:`));
+}
 /**
  * Retrieve the set of queries at localQueryPath and add them to resultMap.
  */
@@ -634,7 +640,7 @@ function parsePacksFromConfig(packsByLanguage, languages, configFile) {
         }
         packs[lang] = [];
         for (const packStr of packsArr) {
-            packs[lang].push(toPackWithVersion(packStr, configFile));
+            packs[lang].push(validatePacksSpecification(packStr, configFile));
         }
     }
     return packs;
@@ -659,32 +665,74 @@ function parsePacksFromInput(packsInput, languages) {
     }
     return {
         [languages[0]]: packsInput.split(",").reduce((packs, pack) => {
-            packs.push(toPackWithVersion(pack, ""));
+            packs.push(validatePacksSpecification(pack, ""));
             return packs;
         }, []),
     };
 }
-function toPackWithVersion(packStr, configFile) {
+/**
+ * Validates that this package specification is syntactically correct.
+ * It may not point to any real package, but after this function returns
+ * without throwing, we are guaranteed that the package specification
+ * is roughly correct.
+ *
+ * The CLI itself will do a more thorough validation of the package
+ * specification.
+ *
+ * A package specification looks like this:
+ *
+ * `scope/name@version:path`
+ *
+ * Version and path are optional.
+ *
+ * @param packStr the package specification to verify.
+ * @param configFile Config file to use for error reporting
+ */
+function validatePacksSpecification(packStr, configFile) {
     if (typeof packStr !== "string") {
         throw new Error(getPacksStrInvalid(packStr, configFile));
     }
-    const nameWithVersion = packStr.trim().split("@");
-    let version;
-    if (nameWithVersion.length > 2 ||
-        !PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])) {
+    packStr = packStr.trim();
+    const atIndex = packStr.indexOf("@");
+    const colonIndex = packStr.indexOf(":", atIndex);
+    const packStart = 0;
+    const versionStart = atIndex + 1 || undefined;
+    const pathStart = colonIndex + 1 || undefined;
+    const packEnd = Math.min(atIndex > 0 ? atIndex : Infinity, colonIndex > 0 ? colonIndex : Infinity, packStr.length);
+    const versionEnd = versionStart
+        ? Math.min(colonIndex > 0 ? colonIndex : Infinity, packStr.length)
+        : undefined;
+    const pathEnd = pathStart ? packStr.length : undefined;
+    const packName = packStr.slice(packStart, packEnd).trim();
+    const version = versionStart
+        ? packStr.slice(versionStart, versionEnd).trim()
+        : undefined;
+    const packPath = pathStart
+        ? packStr.slice(pathStart, pathEnd).trim()
+        : undefined;
+    if (!PACK_IDENTIFIER_PATTERN.test(packName)) {
         throw new Error(getPacksStrInvalid(packStr, configFile));
     }
-    else if (nameWithVersion.length === 2) {
-        version = semver.clean(nameWithVersion[1]) || undefined;
-        if (!version) {
+    if (version) {
+        try {
+            new semver.Range(version);
+        }
+        catch (e) {
+            // The range string is invalid. OK to ignore the caught error
             throw new Error(getPacksStrInvalid(packStr, configFile));
         }
     }
-    return {
-        packName: nameWithVersion[0].trim(),
-        version,
-    };
+    if (packPath &&
+        (path.isAbsolute(packPath) || path.normalize(packPath) !== packPath)) {
+        throw new Error(getPacksStrInvalid(packStr, configFile));
+    }
+    if (!packPath && pathStart) {
+        // 0 length path
+        throw new Error(getPacksStrInvalid(packStr, configFile));
+    }
+    return (packName + (version ? `@${version}` : "") + (packPath ? `:${packPath}` : ""));
 }
+exports.validatePacksSpecification = validatePacksSpecification;
 // exported for testing
 function parsePacks(rawPacksFromConfig, rawPacksInput, languages, configFile) {
     const packsFromInput = parsePacksFromInput(rawPacksInput, languages);

lib/config-utils.test.js

@@ -26,7 +26,6 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const github = __importStar(require("@actions/github"));
 const ava_1 = __importDefault(require("ava"));
-const semver_1 = require("semver");
 const sinon = __importStar(require("sinon"));
 const api = __importStar(require("./api-client"));
 const codeql_1 = require("./codeql");
@@ -601,12 +600,7 @@ function queriesToResolvedQueryForm(queries) {
         const languages = "javascript";
         const { packs } = await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
         t.deepEqual(packs, {
-            [languages_1.Language.javascript]: [
-                {
-                    packName: "a/b",
-                    version: (0, semver_1.clean)("1.2.3"),
-                },
-            ],
+            [languages_1.Language.javascript]: ["a/b@1.2.3"],
         });
     });
 });
@@ -640,18 +634,8 @@ function queriesToResolvedQueryForm(queries) {
         const languages = "javascript,python,cpp";
         const { packs, queries } = await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example" }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
         t.deepEqual(packs, {
-            [languages_1.Language.javascript]: [
-                {
-                    packName: "a/b",
-                    version: (0, semver_1.clean)("1.2.3"),
-                },
-            ],
-            [languages_1.Language.python]: [
-                {
-                    packName: "c/d",
-                    version: (0, semver_1.clean)("1.2.3"),
-                },
-            ],
+            [languages_1.Language.javascript]: ["a/b@1.2.3"],
+            [languages_1.Language.python]: ["c/d@1.2.3"],
         });
         t.deepEqual(queries, {
             cpp: {
@@ -786,28 +770,47 @@ const invalidPackNameMacro = ava_1.default.macro({
 });
 (0, ava_1.default)("no packs", parsePacksMacro, {}, [], {});
 (0, ava_1.default)("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [languages_1.Language.cpp], {
-    [languages_1.Language.cpp]: [
-        { packName: "a/b", version: undefined },
-        { packName: "c/d", version: (0, semver_1.clean)("1.2.3") },
-    ],
+    [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
 });
 (0, ava_1.default)("two packs with spaces", parsePacksMacro, [" a/b ", " c/d@1.2.3 "], [languages_1.Language.cpp], {
-    [languages_1.Language.cpp]: [
-        { packName: "a/b", version: undefined },
-        { packName: "c/d", version: (0, semver_1.clean)("1.2.3") },
-    ],
+    [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
 });
 (0, ava_1.default)("two packs with language", parsePacksMacro, {
     [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
     [languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
 }, [languages_1.Language.cpp, languages_1.Language.java, languages_1.Language.csharp], {
+    [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
+    [languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
+});
+(0, ava_1.default)("packs with other valid names", parsePacksMacro, [
+    // ranges are ok
+    "c/d@1.0",
+    "c/d@~1.0.0",
+    "c/d@~1.0.0:a/b",
+    "c/d@~1.0.0+abc:a/b",
+    "c/d@~1.0.0-abc:a/b",
+    "c/d:a/b",
+    // whitespace is removed
+    " c/d      @     ~1.0.0    :    b.qls   ",
+    // and it is retained within a path
+    " c/d      @     ~1.0.0    :    b/a path with/spaces.qls   ",
+    // this is valid. the path is '@'. It will probably fail when passed to the CLI
+    "c/d@1.2.3:@",
+    // this is valid, too. It will fail if it doesn't match a path
+    // (globbing is not done)
+    "c/d@1.2.3:+*)_(",
+], [languages_1.Language.cpp], {
     [languages_1.Language.cpp]: [
-        { packName: "a/b", version: undefined },
-        { packName: "c/d", version: (0, semver_1.clean)("1.2.3") },
-    ],
-    [languages_1.Language.java]: [
-        { packName: "d/e", version: undefined },
-        { packName: "f/g", version: (0, semver_1.clean)("1.2.3") },
+        "c/d@1.0",
+        "c/d@~1.0.0",
+        "c/d@~1.0.0:a/b",
+        "c/d@~1.0.0+abc:a/b",
+        "c/d@~1.0.0-abc:a/b",
+        "c/d:a/b",
+        "c/d@~1.0.0:b.qls",
+        "c/d@~1.0.0:b/a path with/spaces.qls",
+        "c/d@1.2.3:@",
+        "c/d@1.2.3:+*)_(",
     ],
 });
 (0, ava_1.default)("no language", parsePacksErrorMacro, ["a/b@1.2.3"], [languages_1.Language.java, languages_1.Language.python], /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/);
@@ -817,7 +820,14 @@ const invalidPackNameMacro = ava_1.default.macro({
 (0, ava_1.default)(invalidPackNameMacro, "c-/d");
 (0, ava_1.default)(invalidPackNameMacro, "-c/d");
 (0, ava_1.default)(invalidPackNameMacro, "c/d_d");
-(0, ava_1.default)(invalidPackNameMacro, "c/d@x");
+(0, ava_1.default)(invalidPackNameMacro, "c/d@@");
+(0, ava_1.default)(invalidPackNameMacro, "c/d@1.0.0:");
+(0, ava_1.default)(invalidPackNameMacro, "c/d:");
+(0, ava_1.default)(invalidPackNameMacro, "c/d:/a");
+(0, ava_1.default)(invalidPackNameMacro, "@1.0.0:a");
+(0, ava_1.default)(invalidPackNameMacro, "c/d@../a");
+(0, ava_1.default)(invalidPackNameMacro, "c/d@b/../a");
+(0, ava_1.default)(invalidPackNameMacro, "c/d:z@1");
 /**
  * Test macro for testing the packs block and the packs input
  */
@@ -834,39 +844,22 @@ function parseInputAndConfigErrorMacro(t, packsFromConfig, packsFromInput, langu
 }
 parseInputAndConfigErrorMacro.title = (providedTitle) => `Parse Packs input and config Error: ${providedTitle}`;
 (0, ava_1.default)("input only", parseInputAndConfigMacro, {}, " c/d ", [languages_1.Language.cpp], {
-    [languages_1.Language.cpp]: [{ packName: "c/d", version: undefined }],
+    [languages_1.Language.cpp]: ["c/d"],
 });
 (0, ava_1.default)("input only with multiple", parseInputAndConfigMacro, {}, "a/b , c/d@1.2.3", [languages_1.Language.cpp], {
-    [languages_1.Language.cpp]: [
-        { packName: "a/b", version: undefined },
-        { packName: "c/d", version: "1.2.3" },
-    ],
+    [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
 });
 (0, ava_1.default)("input only with +", parseInputAndConfigMacro, {}, "  +  a/b , c/d@1.2.3 ", [languages_1.Language.cpp], {
-    [languages_1.Language.cpp]: [
-        { packName: "a/b", version: undefined },
-        { packName: "c/d", version: "1.2.3" },
-    ],
+    [languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
 });
 (0, ava_1.default)("config only", parseInputAndConfigMacro, ["a/b", "c/d"], "  ", [languages_1.Language.cpp], {
-    [languages_1.Language.cpp]: [
-        { packName: "a/b", version: undefined },
-        { packName: "c/d", version: undefined },
-    ],
+    [languages_1.Language.cpp]: ["a/b", "c/d"],
 });
 (0, ava_1.default)("input overrides", parseInputAndConfigMacro, ["a/b", "c/d"], " e/f, g/h@1.2.3 ", [languages_1.Language.cpp], {
-    [languages_1.Language.cpp]: [
-        { packName: "e/f", version: undefined },
-        { packName: "g/h", version: "1.2.3" },
-    ],
+    [languages_1.Language.cpp]: ["e/f", "g/h@1.2.3"],
 });
 (0, ava_1.default)("input and config", parseInputAndConfigMacro, ["a/b", "c/d"], " +e/f, g/h@1.2.3 ", [languages_1.Language.cpp], {
-    [languages_1.Language.cpp]: [
-        { packName: "e/f", version: undefined },
-        { packName: "g/h", version: "1.2.3" },
-        { packName: "a/b", version: undefined },
-        { packName: "c/d", version: undefined },
-    ],
+    [languages_1.Language.cpp]: ["e/f", "g/h@1.2.3", "a/b", "c/d"],
 });
 (0, ava_1.default)("input with no language", parseInputAndConfigErrorMacro, {}, "c/d", [], /No languages specified/);
 (0, ava_1.default)("input with two languages", parseInputAndConfigErrorMacro, {}, "c/d", [languages_1.Language.cpp, languages_1.Language.csharp], /multi-language analysis/);
@@ -895,10 +888,7 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
             if (expectedVersionString !== undefined) {
                 t.deepEqual(packs, {
                     [languages_1.Language.javascript]: [
-                        {
-                            packName: "codeql/javascript-experimental-atm-queries",
-                            version: expectedVersionString,
-                        },
+                        `codeql/javascript-experimental-atm-queries@${expectedVersionString}`,
                     ],
                 });
             }
@@ -916,15 +906,23 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
 (0, ava_1.default)(mlPoweredQueriesMacro, "2.7.4", true, undefined, "security-extended", undefined);
 // Test that ML-powered queries aren't run when the feature flag is off.
 (0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", false, undefined, "security-extended", undefined);
+// Test that the ~0.1.0 version of ML-powered queries is run on v2.8.3 of the CLI.
+(0, ava_1.default)(mlPoweredQueriesMacro, "2.8.3", true, undefined, "security-extended", process.platform === "win32" ? undefined : "~0.1.0");
 // Test that ML-powered queries aren't run when the user hasn't specified that we should run the
 // `security-extended` or `security-and-quality` query suite.
 (0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
-// Test that ML-powered queries are run on non-Windows platforms running `security-extended`.
-(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, "security-extended", process.platform === "win32" ? undefined : "~0.1.0");
-// Test that ML-powered queries are run on non-Windows platforms running `security-and-quality`.
-(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, "security-and-quality", process.platform === "win32" ? undefined : "~0.1.0");
+// Test that ML-powered queries are run on non-Windows platforms running `security-extended` on
+// versions of the CodeQL CLI prior to 2.9.0.
+(0, ava_1.default)(mlPoweredQueriesMacro, "2.8.5", true, undefined, "security-extended", process.platform === "win32" ? undefined : "~0.2.0");
+// Test that ML-powered queries are run on non-Windows platforms running `security-and-quality` on
+// versions of the CodeQL CLI prior to 2.9.0.
+(0, ava_1.default)(mlPoweredQueriesMacro, "2.8.5", true, undefined, "security-and-quality", process.platform === "win32" ? undefined : "~0.2.0");
+// Test that ML-powered queries are run on all platforms running `security-extended` on CodeQL CLI
+// 2.9.0+.
+(0, ava_1.default)(mlPoweredQueriesMacro, "2.9.0", true, undefined, "security-extended", "~0.2.0");
+// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
+// CLI 2.9.0+.
+(0, ava_1.default)(mlPoweredQueriesMacro, "2.9.0", true, undefined, "security-and-quality", "~0.2.0");
 // Test that we don't inject an ML-powered query pack if the user has already specified one.
-(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, "codeql/javascript-experimental-atm-queries@0.0.1", "security-and-quality", process.platform === "win32" ? undefined : "0.0.1");
-// Test that the ~0.2.0 version of ML-powered queries is run on v2.8.4 of the CLI.
-(0, ava_1.default)(mlPoweredQueriesMacro, "2.8.4", true, undefined, "security-extended", process.platform === "win32" ? undefined : "~0.2.0");
+(0, ava_1.default)(mlPoweredQueriesMacro, "2.9.0", true, "codeql/javascript-experimental-atm-queries@0.0.1", "security-and-quality", "0.0.1");
 //# sourceMappingURL=config-utils.test.js.map

lib/defaults.json

@@ -1,3 +1,3 @@
 {
-    "bundleVersion": "codeql-bundle-20220401"
+    "bundleVersion": "codeql-bundle-20220428"
 }

lib/init-action.js

@@ -71,6 +71,7 @@ async function run() {
     const startedAt = new Date();
     const logger = (0, logging_1.getActionsLogger)();
     (0, util_1.initializeEnvironment)(util_1.Mode.actions, pkg.version);
+    await (0, util_1.checkActionVersion)(pkg.version);
     let config;
     let codeql;
     let toolsVersion;

lib/upload-lib.js

@@ -93,8 +93,7 @@ function getAutomationID(category, analysis_key, environment) {
 async function uploadPayload(payload, repositoryNwo, apiDetails, logger) {
     logger.info("Uploading results");
     // If in test mode we don't want to upload the results
-    const testMode = process.env["TEST_MODE"] === "true" || false;
-    if (testMode) {
+    if (util.isInTestMode()) {
         const payloadSaveFile = path.join(actionsUtil.getTemporaryDirectory(), "payload.json");
         logger.info(`In test mode. Results are not uploaded. Saving to ${payloadSaveFile}`);
         logger.info(`Payload: ${JSON.stringify(payload, null, 2)}`);
@@ -311,26 +310,28 @@ async function waitForProcessing(repositoryNwo, sarifID, apiDetails, logger) {
             logger.warning("Timed out waiting for analysis to finish processing. Continuing.");
             break;
         }
+        let response = undefined;
         try {
-            const response = await client.request("GET /repos/:owner/:repo/code-scanning/sarifs/:sarif_id", {
+            response = await client.request("GET /repos/:owner/:repo/code-scanning/sarifs/:sarif_id", {
                 owner: repositoryNwo.owner,
                 repo: repositoryNwo.repo,
                 sarif_id: sarifID,
             });
-            const status = response.data.processing_status;
-            logger.info(`Analysis upload status is ${status}.`);
-            if (status === "complete") {
-                break;
-            }
-            else if (status === "pending") {
-                logger.debug("Analysis processing is still pending...");
-            }
-            else if (status === "failed") {
-                throw new Error(`Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`);
-            }
         }
         catch (e) {
             logger.warning(`An error occurred checking the status of the delivery. ${e} It should still be processed in the background, but errors that occur during processing may not be reported.`);
+            break;
+        }
+        const status = response.data.processing_status;
+        logger.info(`Analysis upload status is ${status}.`);
+        if (status === "complete") {
+            break;
+        }
+        else if (status === "pending") {
+            logger.debug("Analysis processing is still pending...");
+        }
+        else if (status === "failed") {
+            throw new Error(`Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`);
         }
         await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
     }

lib/upload-sarif-action.js

@@ -37,8 +37,9 @@ async function sendSuccessStatusReport(startedAt, uploadStats) {
     await actionsUtil.sendStatusReport(statusReport);
 }
 async function run() {
-    (0, util_1.initializeEnvironment)(util_1.Mode.actions, pkg.version);
     const startedAt = new Date();
+    (0, util_1.initializeEnvironment)(util_1.Mode.actions, pkg.version);
+    await (0, util_1.checkActionVersion)(pkg.version);
     if (!(await actionsUtil.sendStatusReport(await actionsUtil.createStatusReportBase("upload-sarif", "starting", startedAt)))) {
         return;
     }
@@ -50,7 +51,11 @@ async function run() {
         const gitHubVersion = await (0, api_client_1.getGitHubVersionActionsOnly)();
         const uploadResult = await upload_lib.uploadFromActions(actionsUtil.getRequiredInput("sarif_file"), gitHubVersion, apiDetails, (0, logging_1.getActionsLogger)());
         core.setOutput("sarif-id", uploadResult.sarifID);
-        if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
+        // We don't upload results in test mode, so don't wait for processing
+        if ((0, util_1.isInTestMode)()) {
+            core.debug("In test mode. Waiting for processing is disabled.");
+        }
+        else if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
             await upload_lib.waitForProcessing((0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY")), uploadResult.sarifID, apiDetails, (0, logging_1.getActionsLogger)());
         }
         await sendSuccessStatusReport(startedAt, uploadResult.statusReport);

lib/upload-sarif-action.js.map

@@ -1 +1 @@
-{"version":3,"file":"upload-sarif-action.js","sourceRoot":"","sources":["../src/upload-sarif-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,4DAA8C;AAC9C,6CAA2D;AAC3D,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAC3C,iCAA0E;AAE1E,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAMvC,KAAK,UAAU,uBAAuB,CACpC,SAAe,EACf,WAA0C;IAE1C,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,cAAc,EACd,SAAS,EACT,SAAS,CACV,CAAC;IACF,MAAM,YAAY,GAA4B;QAC5C,GAAG,gBAAgB;QACnB,GAAG,WAAW;KACf,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,IAAA,4BAAqB,EAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,cAAc,EACd,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;QACA,OAAO;KACR;IAED,IAAI;QACF,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAA,0BAAmB,EAAC,mBAAmB,CAAC;SAC9C,CAAC;QAEF,MAAM,aAAa,GAAG,MAAM,IAAA,wCAA2B,GAAE,CAAC;QAE1D,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACrD,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAC1C,aAAa,EACb,UAAU,EACV,IAAA,0BAAgB,GAAE,CACnB,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,WAAW,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,KAAK,MAAM,EAAE;YAClE,MAAM,UAAU,CAAC,iBAAiB,CAChC,IAAA,+BAAkB,EAAC,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CAAC,EAC5D,YAAY,CAAC,OAAO,EACpB,UAAU,EACV,IAAA,0BAAgB,GAAE,CACnB,CAAC;SACH;QACD,MAAM,uBAAuB,CAAC,SAAS,EAAE,YAAY,CAAC,YAAY,CAAC,CAAC;KACrE;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,WAAW,CAAC,gBAAgB,CAChC,MAAM,WAAW,CAAC,sBAAsB,CACtC,cAAc,EACd,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,EACnC,SAAS,EACT,OAAO,EACP,KAAK,CACN,CACF,CAAC;QACF,OAAO;KACR;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,sCAAsC,KAAK,EAAE,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
+{"version":3,"file":"upload-sarif-action.js","sourceRoot":"","sources":["../src/upload-sarif-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,4DAA8C;AAC9C,6CAA2D;AAC3D,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAC3C,iCAMgB;AAEhB,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAMvC,KAAK,UAAU,uBAAuB,CACpC,SAAe,EACf,WAA0C;IAE1C,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,cAAc,EACd,SAAS,EACT,SAAS,CACV,CAAC;IACF,MAAM,YAAY,GAA4B;QAC5C,GAAG,gBAAgB;QACnB,GAAG,WAAW;KACf,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAA,4BAAqB,EAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,IAAA,yBAAkB,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,cAAc,EACd,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;QACA,OAAO;KACR;IAED,IAAI;QACF,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAA,0BAAmB,EAAC,mBAAmB,CAAC;SAC9C,CAAC;QAEF,MAAM,aAAa,GAAG,MAAM,IAAA,wCAA2B,GAAE,CAAC;QAE1D,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACrD,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAC1C,aAAa,EACb,UAAU,EACV,IAAA,0BAAgB,GAAE,CACnB,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QAEjD,qEAAqE;QACrE,IAAI,IAAA,mBAAY,GAAE,EAAE;YAClB,IAAI,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;SACjE;aAAM,IAAI,WAAW,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,KAAK,MAAM,EAAE;YACzE,MAAM,UAAU,CAAC,iBAAiB,CAChC,IAAA,+BAAkB,EAAC,IAAA,0BAAmB,EAAC,mBAAmB,CAAC,CAAC,EAC5D,YAAY,CAAC,OAAO,EACpB,UAAU,EACV,IAAA,0BAAgB,GAAE,CACnB,CAAC;SACH;QACD,MAAM,uBAAuB,CAAC,SAAS,EAAE,YAAY,CAAC,YAAY,CAAC,CAAC;KACrE;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,WAAW,CAAC,gBAAgB,CAChC,MAAM,WAAW,CAAC,sBAAsB,CACtC,cAAc,EACd,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,EACnC,SAAS,EACT,OAAO,EACP,KAAK,CACN,CACF,CAAC;QACF,OAAO;KACR;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,sCAAsC,KAAK,EAAE,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

lib/util.js

@@ -22,13 +22,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
+exports.isInTestMode = exports.checkActionVersion = exports.getMlPoweredJsQueriesStatus = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isGitHubGhesVersionBelow = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
 const core = __importStar(require("@actions/core"));
 const del_1 = __importDefault(require("del"));
 const semver = __importStar(require("semver"));
+const api = __importStar(require("./api-client"));
 const api_client_1 = require("./api-client");
 const apiCompatibility = __importStar(require("./api-compatibility.json"));
 const codeql_1 = require("./codeql");
@@ -552,9 +553,9 @@ exports.ML_POWERED_JS_QUERIES_PACK_NAME = "codeql/javascript-experimental-atm-qu
  */
 async function getMlPoweredJsQueriesPack(codeQL) {
     if (await codeQlVersionAbove(codeQL, "2.8.4")) {
-        return { packName: exports.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.2.0" };
+        return `${exports.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.2.0`;
     }
-    return { packName: exports.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" };
+    return `${exports.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`;
 }
 exports.getMlPoweredJsQueriesPack = getMlPoweredJsQueriesPack;
 /**
@@ -579,7 +580,10 @@ exports.getMlPoweredJsQueriesPack = getMlPoweredJsQueriesPack;
  * explanation as to why this is.
  */
 function getMlPoweredJsQueriesStatus(config) {
-    const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter((pack) => pack.packName === exports.ML_POWERED_JS_QUERIES_PACK_NAME);
+    const mlPoweredJsQueryPacks = (config.packs.javascript || [])
+        .map((pack) => pack.split("@"))
+        .filter((packNameVersion) => packNameVersion[0] === "codeql/javascript-experimental-atm-queries" &&
+        packNameVersion.length <= 2);
     switch (mlPoweredJsQueryPacks.length) {
         case 1:
             // We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
@@ -587,7 +591,7 @@ function getMlPoweredJsQueriesStatus(config) {
             // with each version of the CodeQL Action. Therefore in practice we should only hit the
             // `latest` case here when customers have explicitly added the ML-powered query pack to their
             // CodeQL config.
-            return mlPoweredJsQueryPacks[0].version || "latest";
+            return mlPoweredJsQueryPacks[0][1] || "latest";
         case 0:
             return "false";
         default:
@@ -595,4 +599,40 @@ function getMlPoweredJsQueriesStatus(config) {
     }
 }
 exports.getMlPoweredJsQueriesStatus = getMlPoweredJsQueriesStatus;
+/**
+ * Prompt the customer to upgrade to CodeQL Action v2, if appropriate.
+ *
+ * Check whether a customer is running v1. If they are, and we can determine that the GitHub
+ * instance supports v2, then log a warning about v1's upcoming deprecation prompting the customer
+ * to upgrade to v2.
+ */
+async function checkActionVersion(version) {
+    var _a;
+    if (!semver.satisfies(version, ">=2")) {
+        const githubVersion = await api.getGitHubVersionActionsOnly();
+        // Only log a warning for versions of GHES that are compatible with CodeQL Action version 2.
+        //
+        // GHES 3.4 shipped without the v2 tag, but it also shipped without this warning message code.
+        // Therefore users who are seeing this warning message code have pulled in a new version of the
+        // Action, and with it the v2 tag.
+        if (githubVersion.type === GitHubVariant.DOTCOM ||
+            githubVersion.type === GitHubVariant.GHAE ||
+            (githubVersion.type === GitHubVariant.GHES &&
+                semver.satisfies((_a = semver.coerce(githubVersion.version)) !== null && _a !== void 0 ? _a : "0.0.0", ">=3.4"))) {
+            core.warning("CodeQL Action v1 will be deprecated on December 7th, 2022. Please upgrade to v2. For " +
+                "more information, see " +
+                "https://github.blog/changelog/2022-04-27-code-scanning-deprecation-of-codeql-action-v1/");
+        }
+    }
+}
+exports.checkActionVersion = checkActionVersion;
+/*
+ * Returns whether we are in test mode.
+ *
+ * In test mode, we don't upload SARIF results or status reports to the GitHub API.
+ */
+function isInTestMode() {
+    return process.env["TEST_MODE"] === "true" || false;
+}
+exports.isInTestMode = isInTestMode;
 //# sourceMappingURL=util.js.map

lib/util.test.js

@@ -25,6 +25,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const stream = __importStar(require("stream"));
+const core = __importStar(require("@actions/core"));
 const github = __importStar(require("@actions/github"));
 const ava_1 = __importDefault(require("ava"));
 const sinon = __importStar(require("sinon"));
@@ -208,40 +209,28 @@ const ML_POWERED_JS_STATUS_TESTS = [
     // If no packs are loaded, status is false.
     [[], "false"],
     // If another pack is loaded but not the ML-powered query pack, status is false.
-    [[{ packName: "someOtherPack" }], "false"],
+    [["someOtherPack"], "false"],
     // If the ML-powered query pack is loaded with a specific version, status is that version.
-    [
-        [{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" }],
-        "~0.1.0",
-    ],
+    [[`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`], "~0.1.0"],
     // If the ML-powered query pack is loaded with a specific version and another pack is loaded, the
     // status is the version of the ML-powered query pack.
     [
-        [
-            { packName: "someOtherPack" },
-            { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" },
-        ],
+        ["someOtherPack", `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`],
         "~0.1.0",
     ],
     // If the ML-powered query pack is loaded without a version, the status is "latest".
-    [[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "latest"],
+    [[util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
     // If the ML-powered query pack is loaded with two different versions, the status is "other".
     [
         [
-            { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" },
-            { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" },
+            `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.1`,
+            `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.2`,
         ],
         "other",
     ],
     // If the ML-powered query pack is loaded with no specific version, and another pack is loaded,
     // the status is "latest".
-    [
-        [
-            { packName: "someOtherPack" },
-            { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME },
-        ],
-        "latest",
-    ],
+    [["someOtherPack", util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
 ];
 for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
     const packDescriptions = `[${packs
@@ -281,4 +270,50 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
     t.falsy(util.isGitHubGhesVersionBelow({ type: util.GitHubVariant.GHES, version: "3.2.0" }, "3.2.0"));
     t.true(util.isGitHubGhesVersionBelow({ type: util.GitHubVariant.GHES, version: "3.1.2" }, "3.2.0"));
 });
+function formatGitHubVersion(version) {
+    switch (version.type) {
+        case util.GitHubVariant.DOTCOM:
+            return "dotcom";
+        case util.GitHubVariant.GHAE:
+            return "GHAE";
+        case util.GitHubVariant.GHES:
+            return `GHES ${version.version}`;
+        default:
+            util.assertNever(version);
+    }
+}
+const CHECK_ACTION_VERSION_TESTS = [
+    ["1.2.1", { type: util.GitHubVariant.DOTCOM }, true],
+    ["1.2.1", { type: util.GitHubVariant.GHAE }, true],
+    ["1.2.1", { type: util.GitHubVariant.GHES, version: "3.3" }, false],
+    ["1.2.1", { type: util.GitHubVariant.GHES, version: "3.4" }, true],
+    ["1.2.1", { type: util.GitHubVariant.GHES, version: "3.5" }, true],
+    ["2.2.1", { type: util.GitHubVariant.DOTCOM }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHAE }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.3" }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.4" }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.5" }, false],
+];
+for (const [version, githubVersion, shouldReportWarning,] of CHECK_ACTION_VERSION_TESTS) {
+    const reportWarningDescription = shouldReportWarning
+        ? "reports warning"
+        : "doesn't report warning";
+    const versionsDescription = `CodeQL Action version ${version} and GitHub version ${formatGitHubVersion(githubVersion)}`;
+    (0, ava_1.default)(`checkActionVersion ${reportWarningDescription} for ${versionsDescription}`, async (t) => {
+        const warningSpy = sinon.spy(core, "warning");
+        const versionStub = sinon
+            .stub(api, "getGitHubVersionActionsOnly")
+            .resolves(githubVersion);
+        const isActionsStub = sinon.stub(util, "isActions").returns(true);
+        await util.checkActionVersion(version);
+        if (shouldReportWarning) {
+            t.true(warningSpy.calledOnceWithExactly(sinon.match("CodeQL Action v1 will be deprecated")));
+        }
+        else {
+            t.false(warningSpy.called);
+        }
+        versionStub.restore();
+        isActionsStub.restore();
+    });
+}
 //# sourceMappingURL=util.test.js.map

node_modules/.package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.1.9",
+  "version": "2.1.11",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "codeql",
-  "version": "2.1.9",
+  "version": "2.1.11",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "codeql",
-      "version": "2.1.9",
+      "version": "2.1.11",
       "license": "MIT",
       "dependencies": {
         "@actions/artifact": "^1.0.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.1.9",
+  "version": "2.1.11",
   "private": true,
   "description": "CodeQL action",
   "scripts": {

pr-checks/checks/ml-powered-queries.yml

@@ -32,10 +32,19 @@ steps:
       retention-days: 7
 
   - name: Check results
+    # Running ML-powered queries on Windows requires CodeQL CLI 2.9.0+. We don't run these checks
+    # against Windows and `cached` while CodeQL CLI 2.9.0 makes its way into `cached` to avoid the
+    # test starting to fail when the cached CodeQL Bundle gets updated. Once the CodeQL Bundle
+    # containing CodeQL CLI 2.9.0 has been fully released, we can drop this line and start running
+    # these checks on Windows and `cached`.
+    if: matrix.os != 'windows-latest' || matrix.version != 'cached'
     env:
-      IS_WINDOWS: ${{ matrix.os == 'windows-latest' }}
+      # Running on Windows requires CodeQL CLI 2.9.0+, which has so far only made it to 'latest'.
+      SHOULD_RUN_ML_POWERED_QUERIES: ${{ matrix.os != 'windows-latest' || matrix.version == 'latest' || matrix.version == 'nightly-latest' }}
     shell: bash
     run: |
+      echo "Expecting ML-powered queries to be run: ${SHOULD_RUN_ML_POWERED_QUERIES}"
+
       cd "$RUNNER_TEMP/results"
       # We should run at least the ML-powered queries in `expected_rules`.
       expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
@@ -44,10 +53,10 @@ steps:
         found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
           flatten | .[].id] | any(. == $rule)' javascript.sarif)
         echo "Did find rule '${rule}': ${found_rule}"
-        if [[ "${found_rule}" != "true" && "${IS_WINDOWS}" != "true" ]]; then
+        if [[ "${found_rule}" != "true" && "${SHOULD_RUN_ML_POWERED_QUERIES}" == "true" ]]; then
           echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
           exit 1
-        elif [[ "${found_rule}" == "true" && "${IS_WINDOWS}" == "true" ]]; then
+        elif [[ "${found_rule}" == "true" && "${SHOULD_RUN_ML_POWERED_QUERIES}" != "true" ]]; then
           echo "Found rule '${rule}' in the SARIF output which shouldn't have been part of the analysis."
           exit 1
         fi
@@ -58,10 +67,10 @@ steps:
         select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
         javascript.sarif)
       echo "Found ${num_alerts} alerts from ML-powered queries.";
-      if [[ "${num_alerts}" -eq 0 && "${IS_WINDOWS}" != "true" ]]; then
+      if [[ "${num_alerts}" -eq 0 && "${SHOULD_RUN_ML_POWERED_QUERIES}" == "true" ]]; then
         echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
         exit 1
-      elif [[ "${num_alerts}" -ne 0 && "${IS_WINDOWS}" == "true" ]]; then
+      elif [[ "${num_alerts}" -ne 0 && "${SHOULD_RUN_ML_POWERED_QUERIES}" != "true" ]]; then
         echo "Expected not to find any alerts from an ML-powered query but found ${num_alerts}."
         exit 1
       fi

pr-checks/checks/packaging-config-inputs-js.yml

@@ -1,12 +1,11 @@
 name: "Packaging: Config and input"
 description: "Checks that specifying packages using a combination of a config file and input to the Action works"
-versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
-os: ["ubuntu-latest", "macos-latest"]
+versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
 steps:
   - uses: ./../action/init
     with:
       config-file: ".github/codeql/codeql-config-packaging3.yml"
-      packs: +dsp-testing/codeql-pack1@0.1.0
+      packs: +dsp-testing/codeql-pack1@1.0.0
       languages: javascript
       tools: ${{ steps.prepare-test.outputs.tools-url }}
   - name: Build code
@@ -21,11 +20,11 @@ steps:
     shell: bash
     run: |
       cd "$RUNNER_TEMP/results"
-      # We should have 3 hits from these rules
-      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+      # We should have 4 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
 
       # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
-      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
       echo "Found matching rules '$RULES'"
       if [ "$RULES" != "$EXPECTED_RULES" ]; then
         echo "Did not match expected rules '$EXPECTED_RULES'."

pr-checks/checks/packaging-config-js.yml

@@ -1,7 +1,6 @@
 name: "Packaging: Config file"
 description: "Checks that specifying packages using only a config file works"
-versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
-os: ["ubuntu-latest", "macos-latest"]
+versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
 steps:
   - uses: ./../action/init
     with:
@@ -20,11 +19,11 @@ steps:
     shell: bash
     run: |
       cd "$RUNNER_TEMP/results"
-      # We should have 3 hits from these rules
-      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+      # We should have 4 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
 
       # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
-      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
       echo "Found matching rules '$RULES'"
       if [ "$RULES" != "$EXPECTED_RULES" ]; then
         echo "Did not match expected rules '$EXPECTED_RULES'."

pr-checks/checks/packaging-inputs-js.yml

@@ -1,13 +1,12 @@
 name: "Packaging: Action input"
 description: "Checks that specifying packages using the input to the Action works"
-versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
-os: ["ubuntu-latest", "macos-latest"]
+versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
 steps:
   - uses: ./../action/init
     with:
       config-file: ".github/codeql/codeql-config-packaging2.yml"
       languages: javascript
-      packs: dsp-testing/codeql-pack1@0.1.0, dsp-testing/codeql-pack2
+      packs: dsp-testing/codeql-pack1@1.0.0, dsp-testing/codeql-pack2, dsp-testing/codeql-pack3:other-query.ql
       tools: ${{ steps.prepare-test.outputs.tools-url }}
   - name: Build code
     shell: bash
@@ -21,11 +20,11 @@ steps:
     shell: bash
     run: |
       cd "$RUNNER_TEMP/results"
-      # We should have 3 hits from these rules
-      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+      # We should have 4 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
 
       # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
-      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
       echo "Found matching rules '$RULES'"
       if [ "$RULES" != "$EXPECTED_RULES" ]; then
         echo "Did not match expected rules '$EXPECTED_RULES'."

pr-checks/checks/split-workflow.yml

@@ -1,12 +1,12 @@
 name: "Split workflow"
 description: "Tests a split-up workflow in which we first build a database and later analyze it"
-versions: ["nightly-20210831"] # This CLI version is known to work with package used in this test
 os: ["ubuntu-latest", "macos-latest"]
+versions: ["latest", "cached", "nightly-latest"] # This feature is not compatible with old CLIs
 steps:
   - uses: ./../action/init
     with:
       config-file: ".github/codeql/codeql-config-packaging3.yml"
-      packs: +dsp-testing/codeql-pack1@0.1.0
+      packs: +dsp-testing/codeql-pack1@1.0.0
       languages: javascript
       tools: ${{ steps.prepare-test.outputs.tools-url }}
   - name: Build code
@@ -35,11 +35,11 @@ steps:
     shell: bash
     run: |
       cd "$RUNNER_TEMP/results"
-      # We should have 3 hits from these rules
-      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+      # We should have 4 hits from these rules
+      EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/other-query-block javascript/example/two-block"
 
       # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
-      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+      RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n\r" " " | xargs)"
       echo "Found matching rules '$RULES'"
       if [ "$RULES" != "$EXPECTED_RULES" ]; then
         echo "Did not match expected rules '$EXPECTED_RULES'."

pr-checks/sync.py

@@ -108,7 +108,7 @@ for file in os.listdir('checks'):
             },
             'on': {
                 'push': {
-                    'branches': ['main', 'v1', 'v2']
+                    'branches': ['main', 'releases/v1', 'releases/v2']
                 },
                 'pull_request': {
                     'types': ["opened", "synchronize", "reopened", "ready_for_review"]

python-setup/install_tools.sh

@@ -29,9 +29,16 @@ python3 -m pip install --user poetry!=1.0.10
 python3 -m pip install --user pipenv
 
 if command -v python2 >/dev/null 2>&1; then
-	# Setup Python 2 dependency installation tools.
-	# The Ubuntu 20.04 GHA environment does not come with a Python 2 pip
-	curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
+	# Setup Python 2 dependency installation tools. The Ubuntu 20.04 GHA environment
+	# does not come with a Python 2 pip, but if it is already installed, don't try to
+	# install it again (since that causes problems).
+	#
+	# This might seem like a hypothetical situation, but it happens all the time in our
+	# internal testing where we run the action twice in a row.
+	if ! python2 -m pip --version; then
+		echo "Will install pip for python2"
+		curl --location --fail https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2
+	fi
 
 	python2 -m pip install --user --upgrade pip setuptools wheel
 

src/actions-util.ts

@@ -15,6 +15,7 @@ import {
   GITHUB_DOTCOM_URL,
   isGitHubGhesVersionBelow,
   isHTTPError,
+  isInTestMode,
   UserError,
 } from "./util";
 
@@ -763,8 +764,7 @@ export async function sendStatusReport<S extends StatusReportBase>(
   const statusReportJSON = JSON.stringify(statusReport);
   core.debug(`Sending status report: ${statusReportJSON}`);
   // If in test mode we don't want to upload the results
-  const testMode = process.env["TEST_MODE"] === "true" || false;
-  if (testMode) {
+  if (isInTestMode()) {
     core.debug("In test mode. Status reports are not uploaded.");
     return true;
   }

src/analyze-action-env.test.ts

@@ -22,14 +22,17 @@ setupTests(test);
 
 test("analyze action with RAM & threads from environment variables", async (t) => {
   await util.withTmpDir(async (tmpDir) => {
-    process.env["GITHUB_SERVER_URL"] = "fake-server-url";
-    process.env["GITHUB_REPOSITORY"] = "fake/repository";
+    process.env["GITHUB_SERVER_URL"] = util.GITHUB_DOTCOM_URL;
+    process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
     sinon
       .stub(actionsUtil, "createStatusReportBase")
       .resolves({} as actionsUtil.StatusReportBase);
     sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+    const gitHubVersion: util.GitHubVersion = {
+      type: util.GitHubVariant.DOTCOM,
+    };
     sinon.stub(configUtils, "getConfig").resolves({
-      gitHubVersion: { type: util.GitHubVariant.DOTCOM },
+      gitHubVersion,
       languages: [],
       packs: [],
     } as unknown as configUtils.Config);
@@ -38,6 +41,7 @@ test("analyze action with RAM & threads from environment variables", async (t) =
     requiredInputStub.withArgs("upload-database").returns("false");
     const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
     optionalInputStub.withArgs("cleanup-level").returns("none");
+    sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
     setupActionsVars(tmpDir, tmpDir);
     mockFeatureFlagApiEndpoint(200, {});
 

src/analyze-action-input.test.ts

@@ -22,14 +22,17 @@ setupTests(test);
 
 test("analyze action with RAM & threads from action inputs", async (t) => {
   await util.withTmpDir(async (tmpDir) => {
-    process.env["GITHUB_SERVER_URL"] = "fake-server-url";
-    process.env["GITHUB_REPOSITORY"] = "fake/repository";
+    process.env["GITHUB_SERVER_URL"] = util.GITHUB_DOTCOM_URL;
+    process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
     sinon
       .stub(actionsUtil, "createStatusReportBase")
       .resolves({} as actionsUtil.StatusReportBase);
     sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+    const gitHubVersion: util.GitHubVersion = {
+      type: util.GitHubVariant.DOTCOM,
+    };
     sinon.stub(configUtils, "getConfig").resolves({
-      gitHubVersion: { type: util.GitHubVariant.DOTCOM },
+      gitHubVersion,
       languages: [],
       packs: [],
     } as unknown as configUtils.Config);
@@ -38,6 +41,7 @@ test("analyze action with RAM & threads from action inputs", async (t) => {
     requiredInputStub.withArgs("upload-database").returns("false");
     const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
     optionalInputStub.withArgs("cleanup-level").returns("none");
+    sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
     setupActionsVars(tmpDir, tmpDir);
     mockFeatureFlagApiEndpoint(200, {});
 

src/analyze-action.ts

@@ -69,6 +69,7 @@ async function run() {
   let runStats: QueriesStatusReport | undefined = undefined;
   let config: Config | undefined = undefined;
   util.initializeEnvironment(util.Mode.actions, pkg.version);
+  await util.checkActionVersion(pkg.version);
 
   try {
     if (
@@ -195,7 +196,10 @@ async function run() {
     // Possibly upload the database bundles for remote queries
     await uploadDatabases(repositoryNwo, config, apiDetails, logger);
 
-    if (
+    // We don't upload results in test mode, so don't wait for processing
+    if (util.isInTestMode()) {
+      core.debug("In test mode. Waiting for processing is disabled.");
+    } else if (
       uploadResult !== undefined &&
       actionsUtil.getRequiredInput("wait-for-processing") === "true"
     ) {

src/analyze.test.ts

@@ -3,7 +3,6 @@ import * as path from "path";
 
 import test from "ava";
 import * as yaml from "js-yaml";
-import { clean } from "semver";
 import * as sinon from "sinon";
 
 import { runQueries } from "./analyze";
@@ -35,18 +34,8 @@ test("status report fields and search path setting", async (t) => {
     const addSnippetsFlag = "";
     const threadsFlag = "";
     const packs = {
-      [Language.cpp]: [
-        {
-          packName: "a/b",
-          version: clean("1.0.0")!,
-        },
-      ],
-      [Language.java]: [
-        {
-          packName: "c/d",
-          version: clean("2.0.0")!,
-        },
-      ],
+      [Language.cpp]: ["a/b@1.0.0"],
+      [Language.java]: ["c/d@2.0.0"],
     };
 
     for (const language of Object.values(Language)) {
@@ -241,32 +230,10 @@ test("status report fields and search path setting", async (t) => {
         query: "bar.ql",
       },
     ];
-    const qlsPackContentCpp = [
-      {
-        qlpack: "a/b",
-        version: "1.0.0",
-      },
-    ];
-    const qlsPackContentJava = [
-      {
-        qlpack: "c/d",
-        version: "2.0.0",
-      },
-    ];
     for (const lang of Object.values(Language)) {
       t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
       t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
       t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
-      const packSuiteName = `${lang}-queries-packs.qls`;
-      if (lang === Language.cpp) {
-        t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
-      } else if (lang === Language.java) {
-        t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
-      } else {
-        t.false(
-          fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName))
-        );
-      }
     }
 
     function readContents(name: string) {

src/analyze.ts

@@ -242,6 +242,7 @@ export async function runQueries(
         logger.startGroup(`Downloading custom packs for ${language}`);
 
         const results = await codeql.packDownload(packsWithVersion);
+
         logger.info(
           `Downloaded packs: ${results.packs
             .map((r) => `${r.name}@${r.version || "latest"}`)
@@ -283,12 +284,12 @@ export async function runQueries(
       }
       if (packsWithVersion.length > 0) {
         querySuitePaths.push(
-          await runQueryGroup(
+          ...(await runQueryPacks(
             language,
             "packs",
-            createPackSuiteContents(packsWithVersion),
+            packsWithVersion,
             undefined
-          )
+          ))
         );
         ranCustom = true;
       }
@@ -386,27 +387,38 @@ export async function runQueries(
     logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
     return querySuitePath;
   }
+  async function runQueryPacks(
+    language: Language,
+    type: string,
+    packs: string[],
+    searchPath: string | undefined
+  ): Promise<string[]> {
+    const databasePath = util.getCodeQLDatabasePath(config, language);
+    // Run the queries individually instead of all at once to avoid command
+    // line length restrictions, particularly on windows.
+
+    for (const pack of packs) {
+      logger.debug(`Running query pack for ${language}-${type}: ${pack}`);
+
+      const codeql = await getCodeQL(config.codeQLCmd);
+      await codeql.databaseRunQueries(
+        databasePath,
+        searchPath,
+        pack,
+        memoryFlag,
+        threadsFlag
+      );
+
+      logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
+    }
+    return packs;
+  }
 }
+
 function createQuerySuiteContents(queries: string[]) {
   return queries.map((q: string) => `- query: ${q}`).join("\n");
 }
 
-function createPackSuiteContents(
-  packsWithVersion: configUtils.PackWithVersion[]
-) {
-  return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
-}
-
-function packWithVersionToQuerySuiteEntry(
-  pack: configUtils.PackWithVersion
-): string {
-  let text = `- qlpack: ${pack.packName}`;
-  if (pack.version) {
-    text += `\n  version: ${pack.version}`;
-  }
-  return text;
-}
-
 export async function runFinalize(
   outputDir: string,
   threadsFlag: string,

src/autobuild-action.ts

@@ -12,7 +12,7 @@ import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
 import * as config_utils from "./config-utils";
 import { Language } from "./languages";
 import { getActionsLogger } from "./logging";
-import { initializeEnvironment, Mode } from "./util";
+import { checkActionVersion, initializeEnvironment, Mode } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
 const pkg = require("../package.json");
@@ -49,8 +49,9 @@ async function sendCompletedStatusReport(
 }
 
 async function run() {
-  const logger = getActionsLogger();
   const startedAt = new Date();
+  const logger = getActionsLogger();
+  await checkActionVersion(pkg.version);
   let language: Language | undefined = undefined;
   try {
     if (

src/codeql.ts

@@ -9,7 +9,7 @@ import * as semver from "semver";
 
 import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
 import * as api from "./api-client";
-import { Config, PackWithVersion } from "./config-utils";
+import { Config } from "./config-utils";
 import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
 import { errorMatchers } from "./error-matcher";
 import { isTracedLanguage, Language } from "./languages";
@@ -117,7 +117,7 @@ export interface CodeQL {
   /**
    * Run 'codeql pack download'.
    */
-  packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput>;
+  packDownload(packs: string[]): Promise<PackDownloadOutput>;
 
   /**
    * Run 'codeql database cleanup'.
@@ -232,6 +232,13 @@ export const CODEQL_VERSION_ML_POWERED_QUERIES = "2.7.5";
  */
 export const CODEQL_VERSION_NEW_TRACING = "2.7.0";
 
+/**
+ * Versions 2.9.0+ of the CodeQL CLI run machine learning models from a temporary directory, which
+ * resolves an issue on Windows where TensorFlow models are not correctly loaded due to the path of
+ * some of their files being greater than MAX_PATH (260 characters).
+ */
+export const CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
+
 function getCodeQLBundleName(): string {
   let platform: string;
   if (process.platform === "win32") {
@@ -641,7 +648,7 @@ async function getCodeQLForCmd(
     async getVersion() {
       let result = util.getCachedCodeQlVersion();
       if (result === undefined) {
-        result = await runTool(cmd, ["version", "--format=terse"]);
+        result = (await runTool(cmd, ["version", "--format=terse"])).trim();
         util.cacheCodeQlVersion(result);
       }
       return result;
@@ -950,13 +957,14 @@ async function getCodeQLForCmd(
      * downloaded. The check to determine what the latest version is is done
      * each time this package is requested.
      */
-    async packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput> {
+    async packDownload(packs: string[]): Promise<PackDownloadOutput> {
       const codeqlArgs = [
         "pack",
         "download",
         "--format=json",
+        "--resolve-query-specs",
         ...getExtraOptionsFromEnv(["pack", "download"]),
-        ...packs.map(packWithVersionToString),
+        ...packs,
       ];
 
       const output = await runTool(cmd, codeqlArgs);
@@ -1028,9 +1036,6 @@ async function getCodeQLForCmd(
   return codeql;
 }
 
-function packWithVersionToString(pack: PackWithVersion): string {
-  return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
-}
 /**
  * Gets the options for `path` of `options` as an array of extra option strings.
  */

src/config-utils.test.ts

@@ -3,7 +3,6 @@ import * as path from "path";
 
 import * as github from "@actions/github";
 import test, { ExecutionContext } from "ava";
-import { clean } from "semver";
 import * as sinon from "sinon";
 
 import * as api from "./api-client";
@@ -1132,12 +1131,7 @@ test("Config specifies packages", async (t) => {
       getRunnerLogger(true)
     );
     t.deepEqual(packs as unknown, {
-      [Language.javascript]: [
-        {
-          packName: "a/b",
-          version: clean("1.2.3"),
-        },
-      ],
+      [Language.javascript]: ["a/b@1.2.3"],
     });
   });
 });
@@ -1194,18 +1188,8 @@ test("Config specifies packages for multiple languages", async (t) => {
       getRunnerLogger(true)
     );
     t.deepEqual(packs as unknown, {
-      [Language.javascript]: [
-        {
-          packName: "a/b",
-          version: clean("1.2.3"),
-        },
-      ],
-      [Language.python]: [
-        {
-          packName: "c/d",
-          version: clean("1.2.3"),
-        },
-      ],
+      [Language.javascript]: ["a/b@1.2.3"],
+      [Language.python]: ["c/d@1.2.3"],
     });
     t.deepEqual(queries, {
       cpp: {
@@ -1437,7 +1421,7 @@ const parsePacksMacro = test.macro({
     t: ExecutionContext<unknown>,
     packsByLanguage: string[] | Record<string, string[]>,
     languages: Language[],
-    expected: Partial<Record<Language, configUtils.PackWithVersion[]>>
+    expected: Partial<Record<Language, string[]>>
   ) =>
     t.deepEqual(
       configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b"),
@@ -1490,10 +1474,7 @@ const invalidPackNameMacro = test.macro({
 
 test("no packs", parsePacksMacro, {}, [], {});
 test("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [Language.cpp], {
-  [Language.cpp]: [
-    { packName: "a/b", version: undefined },
-    { packName: "c/d", version: clean("1.2.3") as string },
-  ],
+  [Language.cpp]: ["a/b", "c/d@1.2.3"],
 });
 test(
   "two packs with spaces",
@@ -1501,10 +1482,7 @@ test(
   [" a/b ", " c/d@1.2.3 "],
   [Language.cpp],
   {
-    [Language.cpp]: [
-      { packName: "a/b", version: undefined },
-      { packName: "c/d", version: clean("1.2.3") as string },
-    ],
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
   }
 );
 test(
@@ -1515,14 +1493,46 @@ test(
     [Language.java]: ["d/e", "f/g@1.2.3"],
   },
   [Language.cpp, Language.java, Language.csharp],
+  {
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
+    [Language.java]: ["d/e", "f/g@1.2.3"],
+  }
+);
+
+test(
+  "packs with other valid names",
+  parsePacksMacro,
+  [
+    // ranges are ok
+    "c/d@1.0",
+    "c/d@~1.0.0",
+    "c/d@~1.0.0:a/b",
+    "c/d@~1.0.0+abc:a/b",
+    "c/d@~1.0.0-abc:a/b",
+    "c/d:a/b",
+    // whitespace is removed
+    " c/d      @     ~1.0.0    :    b.qls   ",
+    // and it is retained within a path
+    " c/d      @     ~1.0.0    :    b/a path with/spaces.qls   ",
+    // this is valid. the path is '@'. It will probably fail when passed to the CLI
+    "c/d@1.2.3:@",
+    // this is valid, too. It will fail if it doesn't match a path
+    // (globbing is not done)
+    "c/d@1.2.3:+*)_(",
+  ],
+  [Language.cpp],
   {
     [Language.cpp]: [
-      { packName: "a/b", version: undefined },
-      { packName: "c/d", version: clean("1.2.3") as string },
-    ],
-    [Language.java]: [
-      { packName: "d/e", version: undefined },
-      { packName: "f/g", version: clean("1.2.3") as string },
+      "c/d@1.0",
+      "c/d@~1.0.0",
+      "c/d@~1.0.0:a/b",
+      "c/d@~1.0.0+abc:a/b",
+      "c/d@~1.0.0-abc:a/b",
+      "c/d:a/b",
+      "c/d@~1.0.0:b.qls",
+      "c/d@~1.0.0:b/a path with/spaces.qls",
+      "c/d@1.2.3:@",
+      "c/d@1.2.3:+*)_(",
     ],
   }
 );
@@ -1553,7 +1563,14 @@ test(invalidPackNameMacro, "c"); // all packs require at least a scope and a nam
 test(invalidPackNameMacro, "c-/d");
 test(invalidPackNameMacro, "-c/d");
 test(invalidPackNameMacro, "c/d_d");
-test(invalidPackNameMacro, "c/d@x");
+test(invalidPackNameMacro, "c/d@@");
+test(invalidPackNameMacro, "c/d@1.0.0:");
+test(invalidPackNameMacro, "c/d:");
+test(invalidPackNameMacro, "c/d:/a");
+test(invalidPackNameMacro, "@1.0.0:a");
+test(invalidPackNameMacro, "c/d@../a");
+test(invalidPackNameMacro, "c/d@b/../a");
+test(invalidPackNameMacro, "c/d:z@1");
 
 /**
  * Test macro for testing the packs block and the packs input
@@ -1598,7 +1615,7 @@ parseInputAndConfigErrorMacro.title = (providedTitle: string) =>
   `Parse Packs input and config Error: ${providedTitle}`;
 
 test("input only", parseInputAndConfigMacro, {}, " c/d ", [Language.cpp], {
-  [Language.cpp]: [{ packName: "c/d", version: undefined }],
+  [Language.cpp]: ["c/d"],
 });
 
 test(
@@ -1608,10 +1625,7 @@ test(
   "a/b , c/d@1.2.3",
   [Language.cpp],
   {
-    [Language.cpp]: [
-      { packName: "a/b", version: undefined },
-      { packName: "c/d", version: "1.2.3" },
-    ],
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
   }
 );
 
@@ -1622,10 +1636,7 @@ test(
   "  +  a/b , c/d@1.2.3 ",
   [Language.cpp],
   {
-    [Language.cpp]: [
-      { packName: "a/b", version: undefined },
-      { packName: "c/d", version: "1.2.3" },
-    ],
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
   }
 );
 
@@ -1636,10 +1647,7 @@ test(
   "  ",
   [Language.cpp],
   {
-    [Language.cpp]: [
-      { packName: "a/b", version: undefined },
-      { packName: "c/d", version: undefined },
-    ],
+    [Language.cpp]: ["a/b", "c/d"],
   }
 );
 
@@ -1650,10 +1658,7 @@ test(
   " e/f, g/h@1.2.3 ",
   [Language.cpp],
   {
-    [Language.cpp]: [
-      { packName: "e/f", version: undefined },
-      { packName: "g/h", version: "1.2.3" },
-    ],
+    [Language.cpp]: ["e/f", "g/h@1.2.3"],
   }
 );
 
@@ -1664,12 +1669,7 @@ test(
   " +e/f, g/h@1.2.3 ",
   [Language.cpp],
   {
-    [Language.cpp]: [
-      { packName: "e/f", version: undefined },
-      { packName: "g/h", version: "1.2.3" },
-      { packName: "a/b", version: undefined },
-      { packName: "c/d", version: undefined },
-    ],
+    [Language.cpp]: ["e/f", "g/h@1.2.3", "a/b", "c/d"],
   }
 );
 
@@ -1760,10 +1760,7 @@ const mlPoweredQueriesMacro = test.macro({
       if (expectedVersionString !== undefined) {
         t.deepEqual(packs as unknown, {
           [Language.javascript]: [
-            {
-              packName: "codeql/javascript-experimental-atm-queries",
-              version: expectedVersionString,
-            },
+            `codeql/javascript-experimental-atm-queries@${expectedVersionString}`,
           ],
         });
       } else {
@@ -1807,42 +1804,64 @@ test(
   "security-extended",
   undefined
 );
-// Test that ML-powered queries aren't run when the user hasn't specified that we should run the
-// `security-extended` or `security-and-quality` query suite.
-test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
-// Test that ML-powered queries are run on non-Windows platforms running `security-extended`.
+// Test that the ~0.1.0 version of ML-powered queries is run on v2.8.3 of the CLI.
 test(
   mlPoweredQueriesMacro,
-  "2.7.5",
+  "2.8.3",
   true,
   undefined,
   "security-extended",
   process.platform === "win32" ? undefined : "~0.1.0"
 );
-// Test that ML-powered queries are run on non-Windows platforms running `security-and-quality`.
+// Test that ML-powered queries aren't run when the user hasn't specified that we should run the
+// `security-extended` or `security-and-quality` query suite.
+test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
+// Test that ML-powered queries are run on non-Windows platforms running `security-extended` on
+// versions of the CodeQL CLI prior to 2.9.0.
 test(
   mlPoweredQueriesMacro,
-  "2.7.5",
-  true,
-  undefined,
-  "security-and-quality",
-  process.platform === "win32" ? undefined : "~0.1.0"
-);
-// Test that we don't inject an ML-powered query pack if the user has already specified one.
-test(
-  mlPoweredQueriesMacro,
-  "2.7.5",
-  true,
-  "codeql/javascript-experimental-atm-queries@0.0.1",
-  "security-and-quality",
-  process.platform === "win32" ? undefined : "0.0.1"
-);
-// Test that the ~0.2.0 version of ML-powered queries is run on v2.8.4 of the CLI.
-test(
-  mlPoweredQueriesMacro,
-  "2.8.4",
+  "2.8.5",
   true,
   undefined,
   "security-extended",
   process.platform === "win32" ? undefined : "~0.2.0"
 );
+// Test that ML-powered queries are run on non-Windows platforms running `security-and-quality` on
+// versions of the CodeQL CLI prior to 2.9.0.
+test(
+  mlPoweredQueriesMacro,
+  "2.8.5",
+  true,
+  undefined,
+  "security-and-quality",
+  process.platform === "win32" ? undefined : "~0.2.0"
+);
+// Test that ML-powered queries are run on all platforms running `security-extended` on CodeQL CLI
+// 2.9.0+.
+test(
+  mlPoweredQueriesMacro,
+  "2.9.0",
+  true,
+  undefined,
+  "security-extended",
+  "~0.2.0"
+);
+// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
+// CLI 2.9.0+.
+test(
+  mlPoweredQueriesMacro,
+  "2.9.0",
+  true,
+  undefined,
+  "security-and-quality",
+  "~0.2.0"
+);
+// Test that we don't inject an ML-powered query pack if the user has already specified one.
+test(
+  mlPoweredQueriesMacro,
+  "2.9.0",
+  true,
+  "codeql/javascript-experimental-atm-queries@0.0.1",
+  "security-and-quality",
+  "0.0.1"
+);

src/config-utils.ts

@@ -8,6 +8,7 @@ import * as api from "./api-client";
 import {
   CodeQL,
   CODEQL_VERSION_ML_POWERED_QUERIES,
+  CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS,
   ResolveQueriesOutput,
 } from "./codeql";
 import * as externalQueries from "./external-queries";
@@ -154,14 +155,7 @@ export interface Config {
   injectedMlQueries: boolean;
 }
 
-export type Packs = Partial<Record<Language, PackWithVersion[]>>;
-
-export interface PackWithVersion {
-  /** qualified name of a package reference */
-  packName: string;
-  /** version of the package, or undefined, which means latest version */
-  version?: string;
-}
+export type Packs = Partial<Record<Language, string[]>>;
 
 /**
  * A list of queries from https://github.com/github/codeql that
@@ -300,13 +294,15 @@ async function addBuiltinSuiteQueries(
   // opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
   // pack, then add the ML-powered query pack so that we run ML-powered queries.
   if (
-    // Disable ML-powered queries on Windows
-    process.platform !== "win32" &&
+    // Only run ML-powered queries on Windows if we have a CLI that supports it.
+    (process.platform !== "win32" ||
+      (await codeQlVersionAbove(
+        codeQL,
+        CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS
+      ))) &&
     languages.includes("javascript") &&
     (found === "security-extended" || found === "security-and-quality") &&
-    !packs.javascript?.some(
-      (pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME
-    ) &&
+    !packs.javascript?.some(isMlPoweredJsQueriesPack) &&
     (await featureFlags.getValue(FeatureFlag.MlPoweredQueriesEnabled)) &&
     (await codeQlVersionAbove(codeQL, CODEQL_VERSION_ML_POWERED_QUERIES))
   ) {
@@ -322,6 +318,14 @@ async function addBuiltinSuiteQueries(
   return injectedMlQueries;
 }
 
+function isMlPoweredJsQueriesPack(pack: string) {
+  return (
+    pack === ML_POWERED_JS_QUERIES_PACK_NAME ||
+    pack.startsWith(`${ML_POWERED_JS_QUERIES_PACK_NAME}@`) ||
+    pack.startsWith(`${ML_POWERED_JS_QUERIES_PACK_NAME}:`)
+  );
+}
+
 /**
  * Retrieve the set of queries at localQueryPath and add them to resultMap.
  */
@@ -1168,7 +1172,7 @@ export function parsePacksFromConfig(
     }
     packs[lang] = [];
     for (const packStr of packsArr) {
-      packs[lang].push(toPackWithVersion(packStr, configFile));
+      packs[lang].push(validatePacksSpecification(packStr, configFile));
     }
   }
   return packs;
@@ -1202,35 +1206,89 @@ function parsePacksFromInput(
 
   return {
     [languages[0]]: packsInput.split(",").reduce((packs, pack) => {
-      packs.push(toPackWithVersion(pack, ""));
+      packs.push(validatePacksSpecification(pack, ""));
       return packs;
-    }, [] as PackWithVersion[]),
+    }, [] as string[]),
   };
 }
 
-function toPackWithVersion(packStr, configFile?: string): PackWithVersion {
+/**
+ * Validates that this package specification is syntactically correct.
+ * It may not point to any real package, but after this function returns
+ * without throwing, we are guaranteed that the package specification
+ * is roughly correct.
+ *
+ * The CLI itself will do a more thorough validation of the package
+ * specification.
+ *
+ * A package specification looks like this:
+ *
+ * `scope/name@version:path`
+ *
+ * Version and path are optional.
+ *
+ * @param packStr the package specification to verify.
+ * @param configFile Config file to use for error reporting
+ */
+export function validatePacksSpecification(
+  packStr: string,
+  configFile?: string
+): string {
   if (typeof packStr !== "string") {
     throw new Error(getPacksStrInvalid(packStr, configFile));
   }
 
-  const nameWithVersion = packStr.trim().split("@");
-  let version: string | undefined;
-  if (
-    nameWithVersion.length > 2 ||
-    !PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])
-  ) {
+  packStr = packStr.trim();
+  const atIndex = packStr.indexOf("@");
+  const colonIndex = packStr.indexOf(":", atIndex);
+  const packStart = 0;
+  const versionStart = atIndex + 1 || undefined;
+  const pathStart = colonIndex + 1 || undefined;
+  const packEnd = Math.min(
+    atIndex > 0 ? atIndex : Infinity,
+    colonIndex > 0 ? colonIndex : Infinity,
+    packStr.length
+  );
+  const versionEnd = versionStart
+    ? Math.min(colonIndex > 0 ? colonIndex : Infinity, packStr.length)
+    : undefined;
+  const pathEnd = pathStart ? packStr.length : undefined;
+
+  const packName = packStr.slice(packStart, packEnd).trim();
+  const version = versionStart
+    ? packStr.slice(versionStart, versionEnd).trim()
+    : undefined;
+  const packPath = pathStart
+    ? packStr.slice(pathStart, pathEnd).trim()
+    : undefined;
+
+  if (!PACK_IDENTIFIER_PATTERN.test(packName)) {
     throw new Error(getPacksStrInvalid(packStr, configFile));
-  } else if (nameWithVersion.length === 2) {
-    version = semver.clean(nameWithVersion[1]) || undefined;
-    if (!version) {
+  }
+  if (version) {
+    try {
+      new semver.Range(version);
+    } catch (e) {
+      // The range string is invalid. OK to ignore the caught error
       throw new Error(getPacksStrInvalid(packStr, configFile));
     }
   }
 
-  return {
-    packName: nameWithVersion[0].trim(),
-    version,
-  };
+  if (
+    packPath &&
+    (path.isAbsolute(packPath) || path.normalize(packPath) !== packPath)
+  ) {
+    throw new Error(getPacksStrInvalid(packStr, configFile));
+  }
+
+  if (!packPath && pathStart) {
+    // 0 length path
+    throw new Error(getPacksStrInvalid(packStr, configFile));
+  }
+
+  return (
+    packName + (version ? `@${version}` : "") + (packPath ? `:${packPath}` : "")
+  );
 }
 
 // exported for testing

src/defaults.json

@@ -1,3 +1,3 @@
 {
-  "bundleVersion": "codeql-bundle-20220401"
+  "bundleVersion": "codeql-bundle-20220428"
 }

src/init-action.ts

@@ -39,6 +39,7 @@ import {
   DEFAULT_DEBUG_ARTIFACT_NAME,
   DEFAULT_DEBUG_DATABASE_NAME,
   getMlPoweredJsQueriesStatus,
+  checkActionVersion,
 } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -124,6 +125,7 @@ async function run() {
   const startedAt = new Date();
   const logger = getActionsLogger();
   initializeEnvironment(Mode.actions, pkg.version);
+  await checkActionVersion(pkg.version);
 
   let config: configUtils.Config;
   let codeql: CodeQL;

src/upload-lib.ts

@@ -3,6 +3,7 @@ import * as path from "path";
 import zlib from "zlib";
 
 import * as core from "@actions/core";
+import { OctokitResponse } from "@octokit/types";
 import fileUrl from "file-url";
 import * as jsonschema from "jsonschema";
 import * as semver from "semver";
@@ -98,8 +99,7 @@ async function uploadPayload(
   logger.info("Uploading results");
 
   // If in test mode we don't want to upload the results
-  const testMode = process.env["TEST_MODE"] === "true" || false;
-  if (testMode) {
+  if (util.isInTestMode()) {
     const payloadSaveFile = path.join(
       actionsUtil.getTemporaryDirectory(),
       "payload.json"
@@ -472,8 +472,9 @@ export async function waitForProcessing(
       );
       break;
     }
+    let response: OctokitResponse<any> | undefined = undefined;
     try {
-      const response = await client.request(
+      response = await client.request(
         "GET /repos/:owner/:repo/code-scanning/sarifs/:sarif_id",
         {
           owner: repositoryNwo.owner,
@@ -481,22 +482,24 @@ export async function waitForProcessing(
           sarif_id: sarifID,
         }
       );
-      const status = response.data.processing_status;
-      logger.info(`Analysis upload status is ${status}.`);
-      if (status === "complete") {
-        break;
-      } else if (status === "pending") {
-        logger.debug("Analysis processing is still pending...");
-      } else if (status === "failed") {
-        throw new Error(
-          `Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`
-        );
-      }
     } catch (e) {
       logger.warning(
         `An error occurred checking the status of the delivery. ${e} It should still be processed in the background, but errors that occur during processing may not be reported.`
       );
+      break;
     }
+    const status = response.data.processing_status;
+    logger.info(`Analysis upload status is ${status}.`);
+    if (status === "complete") {
+      break;
+    } else if (status === "pending") {
+      logger.debug("Analysis processing is still pending...");
+    } else if (status === "failed") {
+      throw new Error(
+        `Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`
+      );
+    }
+
     await util.delay(STATUS_CHECK_FREQUENCY_MILLISECONDS);
   }
   logger.endGroup();

src/upload-sarif-action.ts

@@ -5,7 +5,13 @@ import { getGitHubVersionActionsOnly } from "./api-client";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
-import { getRequiredEnvParam, initializeEnvironment, Mode } from "./util";
+import {
+  checkActionVersion,
+  getRequiredEnvParam,
+  initializeEnvironment,
+  isInTestMode,
+  Mode,
+} from "./util";
 
 // eslint-disable-next-line import/no-commonjs
 const pkg = require("../package.json");
@@ -31,8 +37,9 @@ async function sendSuccessStatusReport(
 }
 
 async function run() {
-  initializeEnvironment(Mode.actions, pkg.version);
   const startedAt = new Date();
+  initializeEnvironment(Mode.actions, pkg.version);
+  await checkActionVersion(pkg.version);
   if (
     !(await actionsUtil.sendStatusReport(
       await actionsUtil.createStatusReportBase(
@@ -60,7 +67,11 @@ async function run() {
       getActionsLogger()
     );
     core.setOutput("sarif-id", uploadResult.sarifID);
-    if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
+
+    // We don't upload results in test mode, so don't wait for processing
+    if (isInTestMode()) {
+      core.debug("In test mode. Waiting for processing is disabled.");
+    } else if (actionsUtil.getRequiredInput("wait-for-processing") === "true") {
       await upload_lib.waitForProcessing(
         parseRepositoryNwo(getRequiredEnvParam("GITHUB_REPOSITORY")),
         uploadResult.sarifID,

src/util.test.ts

@@ -2,12 +2,13 @@ import * as fs from "fs";
 import * as os from "os";
 import * as stream from "stream";
 
+import * as core from "@actions/core";
 import * as github from "@actions/github";
 import test, { ExecutionContext } from "ava";
 import * as sinon from "sinon";
 
 import * as api from "./api-client";
-import { Config, PackWithVersion } from "./config-utils";
+import { Config } from "./config-utils";
 import { getRunnerLogger, Logger } from "./logging";
 import { setupTests } from "./testing-utils";
 import * as util from "./util";
@@ -293,44 +294,32 @@ async function mockStdInForAuthExpectError(
   );
 }
 
-const ML_POWERED_JS_STATUS_TESTS: Array<[PackWithVersion[], string]> = [
+const ML_POWERED_JS_STATUS_TESTS: Array<[string[], string]> = [
   // If no packs are loaded, status is false.
   [[], "false"],
   // If another pack is loaded but not the ML-powered query pack, status is false.
-  [[{ packName: "someOtherPack" }], "false"],
+  [["someOtherPack"], "false"],
   // If the ML-powered query pack is loaded with a specific version, status is that version.
-  [
-    [{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" }],
-    "~0.1.0",
-  ],
+  [[`${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`], "~0.1.0"],
   // If the ML-powered query pack is loaded with a specific version and another pack is loaded, the
   // status is the version of the ML-powered query pack.
   [
-    [
-      { packName: "someOtherPack" },
-      { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" },
-    ],
+    ["someOtherPack", `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`],
     "~0.1.0",
   ],
   // If the ML-powered query pack is loaded without a version, the status is "latest".
-  [[{ packName: util.ML_POWERED_JS_QUERIES_PACK_NAME }], "latest"],
+  [[util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
   // If the ML-powered query pack is loaded with two different versions, the status is "other".
   [
     [
-      { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.1" },
-      { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME, version: "0.0.2" },
+      `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.1`,
+      `${util.ML_POWERED_JS_QUERIES_PACK_NAME}@~0.0.2`,
     ],
     "other",
   ],
   // If the ML-powered query pack is loaded with no specific version, and another pack is loaded,
   // the status is "latest".
-  [
-    [
-      { packName: "someOtherPack" },
-      { packName: util.ML_POWERED_JS_QUERIES_PACK_NAME },
-    ],
-    "latest",
-  ],
+  [["someOtherPack", util.ML_POWERED_JS_QUERIES_PACK_NAME], "latest"],
 ];
 
 for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
@@ -392,3 +381,62 @@ test("isGitHubGhesVersionBelow", async (t) => {
     )
   );
 });
+
+function formatGitHubVersion(version: util.GitHubVersion): string {
+  switch (version.type) {
+    case util.GitHubVariant.DOTCOM:
+      return "dotcom";
+    case util.GitHubVariant.GHAE:
+      return "GHAE";
+    case util.GitHubVariant.GHES:
+      return `GHES ${version.version}`;
+    default:
+      util.assertNever(version);
+  }
+}
+
+const CHECK_ACTION_VERSION_TESTS: Array<[string, util.GitHubVersion, boolean]> =
+  [
+    ["1.2.1", { type: util.GitHubVariant.DOTCOM }, true],
+    ["1.2.1", { type: util.GitHubVariant.GHAE }, true],
+    ["1.2.1", { type: util.GitHubVariant.GHES, version: "3.3" }, false],
+    ["1.2.1", { type: util.GitHubVariant.GHES, version: "3.4" }, true],
+    ["1.2.1", { type: util.GitHubVariant.GHES, version: "3.5" }, true],
+    ["2.2.1", { type: util.GitHubVariant.DOTCOM }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHAE }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.3" }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.4" }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.5" }, false],
+  ];
+
+for (const [
+  version,
+  githubVersion,
+  shouldReportWarning,
+] of CHECK_ACTION_VERSION_TESTS) {
+  const reportWarningDescription = shouldReportWarning
+    ? "reports warning"
+    : "doesn't report warning";
+  const versionsDescription = `CodeQL Action version ${version} and GitHub version ${formatGitHubVersion(
+    githubVersion
+  )}`;
+  test(`checkActionVersion ${reportWarningDescription} for ${versionsDescription}`, async (t) => {
+    const warningSpy = sinon.spy(core, "warning");
+    const versionStub = sinon
+      .stub(api, "getGitHubVersionActionsOnly")
+      .resolves(githubVersion);
+    const isActionsStub = sinon.stub(util, "isActions").returns(true);
+    await util.checkActionVersion(version);
+    if (shouldReportWarning) {
+      t.true(
+        warningSpy.calledOnceWithExactly(
+          sinon.match("CodeQL Action v1 will be deprecated")
+        )
+      );
+    } else {
+      t.false(warningSpy.called);
+    }
+    versionStub.restore();
+    isActionsStub.restore();
+  });
+}

src/util.ts

@@ -7,10 +7,11 @@ import * as core from "@actions/core";
 import del from "del";
 import * as semver from "semver";
 
+import * as api from "./api-client";
 import { getApiClient, GitHubApiDetails } from "./api-client";
 import * as apiCompatibility from "./api-compatibility.json";
 import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
-import { Config, PackWithVersion } from "./config-utils";
+import { Config } from "./config-utils";
 import { Language } from "./languages";
 import { Logger } from "./logging";
 
@@ -662,11 +663,11 @@ export const ML_POWERED_JS_QUERIES_PACK_NAME =
  */
 export async function getMlPoweredJsQueriesPack(
   codeQL: CodeQL
-): Promise<PackWithVersion> {
+): Promise<string> {
   if (await codeQlVersionAbove(codeQL, "2.8.4")) {
-    return { packName: ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.2.0" };
+    return `${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.2.0`;
   }
-  return { packName: ML_POWERED_JS_QUERIES_PACK_NAME, version: "~0.1.0" };
+  return `${ML_POWERED_JS_QUERIES_PACK_NAME}@~0.1.0`;
 }
 
 /**
@@ -691,9 +692,13 @@ export async function getMlPoweredJsQueriesPack(
  * explanation as to why this is.
  */
 export function getMlPoweredJsQueriesStatus(config: Config): string {
-  const mlPoweredJsQueryPacks = (config.packs.javascript || []).filter(
-    (pack) => pack.packName === ML_POWERED_JS_QUERIES_PACK_NAME
-  );
+  const mlPoweredJsQueryPacks = (config.packs.javascript || [])
+    .map((pack) => pack.split("@"))
+    .filter(
+      (packNameVersion) =>
+        packNameVersion[0] === "codeql/javascript-experimental-atm-queries" &&
+        packNameVersion.length <= 2
+    );
   switch (mlPoweredJsQueryPacks.length) {
     case 1:
       // We should always specify an explicit version string in `getMlPoweredJsQueriesPack`,
@@ -701,10 +706,52 @@ export function getMlPoweredJsQueriesStatus(config: Config): string {
       // with each version of the CodeQL Action. Therefore in practice we should only hit the
       // `latest` case here when customers have explicitly added the ML-powered query pack to their
       // CodeQL config.
-      return mlPoweredJsQueryPacks[0].version || "latest";
+      return mlPoweredJsQueryPacks[0][1] || "latest";
     case 0:
       return "false";
     default:
       return "other";
   }
 }
+
+/**
+ * Prompt the customer to upgrade to CodeQL Action v2, if appropriate.
+ *
+ * Check whether a customer is running v1. If they are, and we can determine that the GitHub
+ * instance supports v2, then log a warning about v1's upcoming deprecation prompting the customer
+ * to upgrade to v2.
+ */
+export async function checkActionVersion(version: string) {
+  if (!semver.satisfies(version, ">=2")) {
+    const githubVersion = await api.getGitHubVersionActionsOnly();
+    // Only log a warning for versions of GHES that are compatible with CodeQL Action version 2.
+    //
+    // GHES 3.4 shipped without the v2 tag, but it also shipped without this warning message code.
+    // Therefore users who are seeing this warning message code have pulled in a new version of the
+    // Action, and with it the v2 tag.
+    if (
+      githubVersion.type === GitHubVariant.DOTCOM ||
+      githubVersion.type === GitHubVariant.GHAE ||
+      (githubVersion.type === GitHubVariant.GHES &&
+        semver.satisfies(
+          semver.coerce(githubVersion.version) ?? "0.0.0",
+          ">=3.4"
+        ))
+    ) {
+      core.warning(
+        "CodeQL Action v1 will be deprecated on December 7th, 2022. Please upgrade to v2. For " +
+          "more information, see " +
+          "https://github.blog/changelog/2022-04-27-code-scanning-deprecation-of-codeql-action-v1/"
+      );
+    }
+  }
+}
+
+/*
+ * Returns whether we are in test mode.
+ *
+ * In test mode, we don't upload SARIF results or status reports to the GitHub API.
+ */
+export function isInTestMode(): boolean {
+  return process.env["TEST_MODE"] === "true" || false;
+}

tests/multi-language-repo/.github/codeql/codeql-config-packaging.yml

@@ -3,8 +3,10 @@ name: Pack testing in the CodeQL Action
 disable-default-queries: true
 packs:
   javascript:
-    - dsp-testing/codeql-pack1@0.1.0
-    - dsp-testing/codeql-pack2 # latest
+    - dsp-testing/codeql-pack1@1.0.0
+    - dsp-testing/codeql-pack2
+    - dsp-testing/codeql-pack3:other-query.ql
+
 paths-ignore:
   - tests
   - lib

tests/multi-language-repo/.github/codeql/codeql-config-packaging3.yml

@@ -3,7 +3,8 @@ name: Pack testing in the CodeQL Action
 disable-default-queries: true
 packs:
   javascript:
-    - dsp-testing/codeql-pack2 # latest
+    - dsp-testing/codeql-pack2
+    - dsp-testing/codeql-pack3:other-query.ql
 paths-ignore:
   - tests
   - lib