Merge pull request #616 from github/update-v1.0.5-1c26d408

Merge main into v1

.github/workflows/pr-checks.yml

@@ -7,6 +7,7 @@ on:
   push:
     branches: [main, v1]
   pull_request:
+  workflow_dispatch:
 
 jobs:
   lint-js:
@@ -119,9 +120,6 @@ jobs:
       with:
         config-file: ".github/codeql/codeql-config-packaging.yml"
         languages: javascript
-        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
-        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
-
     - name: Build code
       shell: bash
       run: ./build.sh
@@ -164,8 +162,6 @@ jobs:
         config-file: ".github/codeql/codeql-config-packaging2.yml"
         languages: javascript
         packs: dsp-testing/codeql-pack1@0.0.4, dsp-testing/codeql-pack2
-        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
-        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
 
     - name: Build code
       shell: bash
@@ -209,8 +205,6 @@ jobs:
         config-file: ".github/codeql/codeql-config-packaging3.yml"
         packs: +dsp-testing/codeql-pack1@0.0.4
         languages: javascript
-        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
-        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
 
     - name: Build code
       shell: bash
@@ -234,6 +228,59 @@ jobs:
           exit 1
         fi
 
+  # Tests a split workflow where database construction and query execution happen in different steps
+  test-split-workflow:
+    needs: [check-js, check-node-modules]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - uses: ./../action/init
+      with:
+        config-file: ".github/codeql/codeql-config-packaging3.yml"
+        packs: +dsp-testing/codeql-pack1@0.0.4
+        languages: javascript
+        tools: latest
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        skip-queries: true
+        output: "${{ runner.temp }}/results"
+      env:
+        TEST_MODE: true
+    - name: Assert No Results
+      run: |
+        if [ "$(ls -A $RUNNER_TEMP/results)" ]; then
+          echo "Expected results directory to be empty after skipping query execution!"
+          exit 1
+        fi
+    - uses: ./../action/analyze
+      with:
+        output: "${{ runner.temp }}/results"
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi
 
   # Identify the CodeQL tool versions to integration test against.
   check-codeql-versions:
@@ -241,6 +288,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       versions: ${{ steps.compare.outputs.versions }}
+      nightly-url: ${{ steps.get-url.outputs.nightly-url }}
 
     steps:
     - uses: actions/checkout@v2
@@ -266,22 +314,45 @@ jobs:
       with:
         tools: latest
         languages: javascript
-    - name: Compare default and latest CodeQL bundle versions
+    - name: Remove empty database
+      # allows us to run init a third time
+      run: |
+        rm -rf "$RUNNER_TEMP/codeql_databases"
+    - name: Get nightly release URL
+      id: get-url
+      env:
+        GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+      shell: bash
+      # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+      run: |
+        export LATEST=`gh release list --repo dsp-testing/codeql-cli-nightlies -L 1 | cut -f 3`
+        echo "::set-output name=nightly-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST/codeql-bundle.tar.gz"
+    - name: Init with a CodeQL bundle from a nightly release
+      id: init-nightly
+      uses: ./../action/init
+      with:
+        tools: ${{ steps.get-url.outputs.nightly-url }}
+        languages: javascript
+    - name: Compare CodeQL bundle versions
       id: compare
       env:
         CODEQL_DEFAULT: ${{ steps.init-default.outputs.codeql-path }}
         CODEQL_LATEST: ${{ steps.init-latest.outputs.codeql-path }}
+        CODEQL_NIGHTLY: ${{ steps.init-nightly.outputs.codeql-path }}
+        NIGHTLY_URL: ${{ steps.get-url.outputs.nightly-url }}
       run: |
         CODEQL_VERSION_DEFAULT="$("$CODEQL_DEFAULT" version --format terse)"
         CODEQL_VERSION_LATEST="$("$CODEQL_LATEST" version --format terse)"
+        CODEQL_VERSION_NIGHTLY="$("$CODEQL_NIGHTLY" version --format terse)"
         echo "Default CodeQL bundle version is $CODEQL_VERSION_DEFAULT"
         echo "Latest CodeQL bundle version is $CODEQL_VERSION_LATEST"
+        echo "Nightly CodeQL bundle version is $CODEQL_VERSION_NIGHTLY"
         if [[ "$CODEQL_VERSION_DEFAULT" == "$CODEQL_VERSION_LATEST" ]]; then
-          # Just use `tools: null` to avoid duplication in the integration tests.
-          VERSIONS_JSON='[null]'
+          # Skip `tools: latest` since it would be the same as `tools: null`
+          VERSIONS_JSON="[null, \"$NIGHTLY_URL\"]"
         else
-          # Use both `tools: null` and `tools: latest` in the integration tests.
-          VERSIONS_JSON='[null, "latest"]'
+          # Run integration tests with all three bundles.
+          VERSIONS_JSON="[null, \"$NIGHTLY_URL\", \"latest\"]"
         fi
         # Output a JSON-encoded list with the distinct versions to test against.
         echo "Suggested matrix config for integration tests: $VERSIONS_JSON"
@@ -803,3 +874,23 @@ jobs:
         # Deliberately don't use TEST_MODE here. This is specifically testing
         # the compatibility with the API.
         runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  multi-language-repo_test-local-codeql:
+    needs: [check-js, check-node-modules, check-codeql-versions]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      run: |
+        wget ${{ needs.check-codeql-versions.outputs.nightly-url }}
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - uses: ./../action/init
+      with:
+        tools: ../action/codeql-bundle.tar.gz
+    - name: Build code
+      run: ./build.sh
+    - uses: ./../action/analyze

CHANGELOG.md

@@ -1,8 +1,9 @@
 # CodeQL Action and CodeQL Runner Changelog
 
-## [UNRELEASED]
+## 1.0.5 - 12 Jul 2021
 
-No user facing changes.
+- The `analyze` step of the Action now supports a `skip-queries` option to merely build the CodeQL database without analyzing. This functionality is not present in the runner. Additionally, the step will no longer fail if it encounters a finalized database, and will instead continue with query execution. [#602](https://github.com/github/codeql-action/pull/602)
+- Update the warning message when the baseline lines of code count is unavailable. [#608](https://github.com/github/codeql-action/pull/608)
 
 ## 1.0.4 - 28 Jun 2021
 

analyze/action.yml

@@ -24,6 +24,10 @@ inputs:
     description: Specify whether or not to add code snippets to the output sarif file.
     required: false
     default: "false"
+  skip-queries:
+    description: If this option is set, the CodeQL database will be built but no queries will be run on it. Thus, no results will be produced.
+    required: false
+    default: "false"
   threads:
     description: The number of threads to be used by CodeQL.
     required: false

lib/analyze-action.js

@@ -34,7 +34,8 @@ async function sendStatusReport(startedAt, stats, error) {
 }
 async function run() {
     const startedAt = new Date();
-    let stats = undefined;
+    let uploadStats = undefined;
+    let runStats = undefined;
     let config = undefined;
     util.initializeEnvironment(util.Mode.actions, pkg.version);
     try {
@@ -51,7 +52,11 @@ async function run() {
             url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
         };
         const outputDir = actionsUtil.getRequiredInput("output");
-        const queriesStats = await analyze_1.runAnalyze(outputDir, util.getMemoryFlag(actionsUtil.getOptionalInput("ram")), util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger), actionsUtil.getOptionalInput("category"), config, logger);
+        const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger);
+        await analyze_1.runFinalize(outputDir, threads, config, logger);
+        if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
+            runStats = await analyze_1.runQueries(outputDir, util.getMemoryFlag(actionsUtil.getOptionalInput("ram")), util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
+        }
         if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
             await analyze_1.runCleanup(config, actionsUtil.getOptionalInput("cleanup-level") || "brutal", logger);
         }
@@ -60,13 +65,11 @@ async function run() {
             dbLocations[language] = util.getCodeQLDatabasePath(config, language);
         }
         core.setOutput("db-locations", dbLocations);
-        if (actionsUtil.getRequiredInput("upload") === "true") {
-            const uploadStats = await upload_lib.uploadFromActions(outputDir, config.gitHubVersion, apiDetails, logger);
-            stats = { ...queriesStats, ...uploadStats };
+        if (runStats && actionsUtil.getRequiredInput("upload") === "true") {
+            uploadStats = await upload_lib.uploadFromActions(outputDir, config.gitHubVersion, apiDetails, logger);
         }
         else {
             logger.info("Not uploading results");
-            stats = { ...queriesStats };
         }
         const repositoryNwo = repository_1.parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
         await database_upload_1.uploadDatabases(repositoryNwo, config, apiDetails, logger);
@@ -75,9 +78,12 @@ async function run() {
         core.setFailed(error.message);
         console.log(error);
         if (error instanceof analyze_1.CodeQLAnalysisError) {
-            stats = { ...error.queriesStatusReport };
+            const stats = { ...error.queriesStatusReport };
+            await sendStatusReport(startedAt, stats, error);
+        }
+        else {
+            await sendStatusReport(startedAt, undefined, error);
         }
-        await sendStatusReport(startedAt, stats, error);
         return;
     }
     finally {
@@ -103,7 +109,15 @@ async function run() {
             }
         }
     }
-    await sendStatusReport(startedAt, stats);
+    if (runStats && uploadStats) {
+        await sendStatusReport(startedAt, { ...runStats, ...uploadStats });
+    }
+    else if (runStats) {
+        await sendStatusReport(startedAt, { ...runStats });
+    }
+    else {
+        await sendStatusReport(startedAt, undefined);
+    }
 }
 async function runWrapper() {
     try {

lib/analyze-action.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAKmB;AACnB,iDAAmD;AACnD,uDAAoD;AACpD,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAqC,SAAS,CAAC;IACxD,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,MAAM,oBAAU,CACnC,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,EACpE,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;QAEF,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,MAAM,EAAE;YAC5D,MAAM,oBAAU,CACd,MAAM,EACN,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,QAAQ,EACzD,MAAM,CACP,CAAC;SACH;QAED,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SACtE;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACrD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACpD,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;YACF,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;SAC7C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC;SAC7B;QAED,MAAM,aAAa,GAAG,+BAAkB,CACtC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAC9C,CAAC;QACF,MAAM,iCAAe,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;KAClE;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;SAC1C;QAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
+{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAMmB;AACnB,iDAAmD;AACnD,uDAAoD;AACpD,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAE3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,WAAW,GAAmC,SAAS,CAAC;IAC5D,IAAI,QAAQ,GAAoC,SAAS,CAAC;IAC1D,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CACjC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EACvC,MAAM,CACP,CAAC;QACF,MAAM,qBAAW,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,KAAK,MAAM,EAAE;YAC3D,QAAQ,GAAG,MAAM,oBAAU,CACzB,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,OAAO,EACP,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;SACH;QAED,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,MAAM,EAAE;YAC5D,MAAM,oBAAU,CACd,MAAM,EACN,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,QAAQ,EACzD,MAAM,CACP,CAAC;SACH;QAED,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SACtE;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,QAAQ,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACjE,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CAC9C,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;SACH;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;SACtC;QAED,MAAM,aAAa,GAAG,+BAAkB,CACtC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAC9C,CAAC;QACF,MAAM,iCAAe,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;KAClE;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,MAAM,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;YAC/C,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SACjD;aAAM;YACL,MAAM,gBAAgB,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;SACrD;QAED,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,IAAI,QAAQ,IAAI,WAAW,EAAE;QAC3B,MAAM,gBAAgB,CAAC,SAAS,EAAE,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;KACpE;SAAM,IAAI,QAAQ,EAAE;QACnB,MAAM,gBAAgB,CAAC,SAAS,EAAE,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;KACpD;SAAM;QACL,MAAM,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;KAC9C;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

lib/analyze.js

@@ -10,6 +10,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
+const yaml = __importStar(require("js-yaml"));
 const analysisPaths = __importStar(require("./analysis-paths"));
 const codeql_1 = require("./codeql");
 const count_loc_1 = require("./count-loc");
@@ -55,7 +56,8 @@ async function createdDBForScannedLanguages(config, logger) {
     analysisPaths.includeAndExcludeAnalysisPaths(config);
     const codeql = codeql_1.getCodeQL(config.codeQLCmd);
     for (const language of config.languages) {
-        if (languages_1.isScannedLanguage(language)) {
+        if (languages_1.isScannedLanguage(language) &&
+            !dbIsFinalized(config, language, logger)) {
             logger.startGroup(`Extracting ${language}`);
             if (language === languages_1.Language.python) {
                 await setupPythonExtractor(logger);
@@ -65,13 +67,29 @@ async function createdDBForScannedLanguages(config, logger) {
         }
     }
 }
+function dbIsFinalized(config, language, logger) {
+    const dbPath = util.getCodeQLDatabasePath(config, language);
+    try {
+        const dbInfo = yaml.load(fs.readFileSync(path.resolve(dbPath, "codeql-database.yml"), "utf8"));
+        return !("inProgress" in dbInfo);
+    }
+    catch (e) {
+        logger.warning(`Could not check whether database for ${language} was finalized. Assuming it is not.`);
+        return false;
+    }
+}
 async function finalizeDatabaseCreation(config, threadsFlag, logger) {
     await createdDBForScannedLanguages(config, logger);
     const codeql = codeql_1.getCodeQL(config.codeQLCmd);
     for (const language of config.languages) {
-        logger.startGroup(`Finalizing ${language}`);
-        await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config, language), threadsFlag);
-        logger.endGroup();
+        if (dbIsFinalized(config, language, logger)) {
+            logger.info(`There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`);
+        }
+        else {
+            logger.startGroup(`Finalizing ${language}`);
+            await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config, language), threadsFlag);
+            logger.endGroup();
+        }
     }
 }
 // Runs queries and creates sarif files in the given folder
@@ -183,15 +201,13 @@ function packWithVersionToQuerySuiteEntry(pack) {
     }
     return text;
 }
-async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
+async function runFinalize(outputDir, threadsFlag, config, logger) {
     // Delete the tracer config env var to avoid tracing ourselves
     delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
     fs.mkdirSync(outputDir, { recursive: true });
     await finalizeDatabaseCreation(config, threadsFlag, logger);
-    const queriesStats = await runQueries(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger);
-    return { ...queriesStats };
 }
-exports.runAnalyze = runAnalyze;
+exports.runFinalize = runFinalize;
 async function runCleanup(config, cleanupLevel, logger) {
     logger.startGroup("Cleaning up databases");
     for (const language of config.languages) {

lib/codeql.js

@@ -154,50 +154,58 @@ async function setupCodeQL(codeqlURL, apiDetails, tempDir, toolCacheDir, variant
         if (forceLatest) {
             codeqlURL = undefined;
         }
-        const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
-        const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
-        // If we find the specified version, we always use that.
-        let codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer, toolCacheDir, logger);
-        // If we don't find the requested version, in some cases we may allow a
-        // different version to save download time if the version hasn't been
-        // specified explicitly (in which case we always honor it).
-        if (!codeqlFolder && !codeqlURL && !forceLatest) {
-            const codeqlVersions = toolcache.findAllVersions("CodeQL", toolCacheDir, logger);
-            if (codeqlVersions.length === 1) {
-                const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0], toolCacheDir, logger);
-                if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
-                    logger.debug(`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`);
-                    codeqlFolder = tmpCodeqlFolder;
-                }
-            }
-        }
-        if (codeqlFolder) {
-            logger.debug(`CodeQL found in cache ${codeqlFolder}`);
+        let codeqlFolder;
+        let codeqlURLVersion;
+        if (codeqlURL && !codeqlURL.startsWith("http")) {
+            codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
+            codeqlURLVersion = "local";
         }
         else {
-            if (!codeqlURL) {
-                codeqlURL = await getCodeQLBundleDownloadURL(apiDetails, variant, logger);
+            codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
+            const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
+            // If we find the specified version, we always use that.
+            codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer, toolCacheDir, logger);
+            // If we don't find the requested version, in some cases we may allow a
+            // different version to save download time if the version hasn't been
+            // specified explicitly (in which case we always honor it).
+            if (!codeqlFolder && !codeqlURL && !forceLatest) {
+                const codeqlVersions = toolcache.findAllVersions("CodeQL", toolCacheDir, logger);
+                if (codeqlVersions.length === 1) {
+                    const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0], toolCacheDir, logger);
+                    if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
+                        logger.debug(`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`);
+                        codeqlFolder = tmpCodeqlFolder;
+                    }
+                }
             }
-            const parsedCodeQLURL = new URL(codeqlURL);
-            const parsedQueryString = query_string_1.default.parse(parsedCodeQLURL.search);
-            const headers = { accept: "application/octet-stream" };
-            // We only want to provide an authorization header if we are downloading
-            // from the same GitHub instance the Action is running on.
-            // This avoids leaking Enterprise tokens to dotcom.
-            // We also don't want to send an authorization header if there's already a token provided in the URL.
-            if (codeqlURL.startsWith(`${apiDetails.url}/`) &&
-                parsedQueryString["token"] === undefined) {
-                logger.debug("Downloading CodeQL bundle with token.");
-                headers.authorization = `token ${apiDetails.auth}`;
+            if (codeqlFolder) {
+                logger.debug(`CodeQL found in cache ${codeqlFolder}`);
             }
             else {
-                logger.debug("Downloading CodeQL bundle without token.");
+                if (!codeqlURL) {
+                    codeqlURL = await getCodeQLBundleDownloadURL(apiDetails, variant, logger);
+                }
+                const parsedCodeQLURL = new URL(codeqlURL);
+                const parsedQueryString = query_string_1.default.parse(parsedCodeQLURL.search);
+                const headers = { accept: "application/octet-stream" };
+                // We only want to provide an authorization header if we are downloading
+                // from the same GitHub instance the Action is running on.
+                // This avoids leaking Enterprise tokens to dotcom.
+                // We also don't want to send an authorization header if there's already a token provided in the URL.
+                if (codeqlURL.startsWith(`${apiDetails.url}/`) &&
+                    parsedQueryString["token"] === undefined) {
+                    logger.debug("Downloading CodeQL bundle with token.");
+                    headers.authorization = `token ${apiDetails.auth}`;
+                }
+                else {
+                    logger.debug("Downloading CodeQL bundle without token.");
+                }
+                logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
+                const codeqlPath = await toolcache.downloadTool(codeqlURL, tempDir, headers);
+                logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
+                const codeqlExtracted = await toolcache.extractTar(codeqlPath, tempDir, logger);
+                codeqlFolder = await toolcache.cacheDir(codeqlExtracted, "CodeQL", codeqlURLSemVer, toolCacheDir, logger);
             }
-            logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
-            const codeqlPath = await toolcache.downloadTool(codeqlURL, tempDir, headers);
-            logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-            const codeqlExtracted = await toolcache.extractTar(codeqlPath, tempDir, logger);
-            codeqlFolder = await toolcache.cacheDir(codeqlExtracted, "CodeQL", codeqlURLSemVer, toolCacheDir, logger);
         }
         let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
         if (process.platform === "win32") {

lib/count-loc.js

@@ -77,14 +77,13 @@ async function countLoc(cwd, include, exclude, dbLanguages, logger) {
         }
     }
     else {
-        logger.info("Could not determine the total number of lines of code in this repository. " +
-            "Because of this, it will not be possible to compare the number of lines " +
-            "of code analyzed by code scanning with the total number of lines of " +
-            "code in the repository. This will not affect the results produced by code " +
-            "scanning. If you have any questions, you can raise an issue at " +
-            "https://github.com/github/codeql-action/issues. Please include a link " +
-            "to the repository if public, or otherwise information about the code scanning " +
-            "workflow you are using.");
+        logger.info("Could not determine the baseline lines of code count in this repository. " +
+            "Because of this, it will not be possible to compare the lines " +
+            "of code analyzed by code scanning with the baseline. This will not affect " +
+            "the results produced by code scanning. If you have any questions, you can " +
+            "raise an issue at https://github.com/github/codeql-action/issues. Please " +
+            "include a link to the repository if public, or otherwise information about " +
+            "the code scanning workflow you are using.");
     }
     return lineCounts;
 }

lib/count-loc.js.map

@@ -1 +1 @@
-{"version":3,"file":"count-loc.js","sourceRoot":"","sources":["../src/count-loc.ts"],"names":[],"mappings":";;AAAA,qDAAyC;AAEzC,2CAAuC;AAEvC,iCAAqC;AAKrC,sFAAsF;AACtF,MAAM,iBAAiB,GAA6B;IAClD,CAAC,EAAE,oBAAQ,CAAC,GAAG;IACf,KAAK,EAAE,oBAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,oBAAQ,CAAC,MAAM;IACrB,EAAE,EAAE,oBAAQ,CAAC,EAAE;IACf,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;IAC/B,MAAM,EAAE,oBAAQ,CAAC,MAAM;IACvB,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAC7D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACd,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAChB;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EACD,EAAgC,CACjC,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAkB;IAC5C,QAAQ,QAAQ,EAAE;QAChB,KAAK,oBAAQ,CAAC,GAAG;YACf,OAAO,KAAK,CAAC;QACf,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,EAAE;YACd,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAQ,CAAC,UAAU;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,IAAI,CAAC;QAEd;YACE,kBAAW,CAAC,QAAQ,CAAC,CAAC;KACzB;AACH,CAAC;AApBD,kCAoBC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAiB,EACjB,OAAiB,EACjB,WAAuB,EACvB,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,IAAI,wBAAM,CAAC;QAC9B,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,OAAO;QACP,iBAAiB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;KACvE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,uDAAuD;IACvD,uDAAuD;IACvD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CACxD,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,eAAe,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;YAC5D,GAAG,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3D;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;SACzC;KACF;SAAM;QACL,MAAM,CAAC,IAAI,CACT,4EAA4E;YAC1E,0EAA0E;YAC1E,sEAAsE;YACtE,4EAA4E;YAC5E,iEAAiE;YACjE,wEAAwE;YACxE,gFAAgF;YAChF,yBAAyB,CAC5B,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA/CD,4BA+CC"}
+{"version":3,"file":"count-loc.js","sourceRoot":"","sources":["../src/count-loc.ts"],"names":[],"mappings":";;AAAA,qDAAyC;AAEzC,2CAAuC;AAEvC,iCAAqC;AAKrC,sFAAsF;AACtF,MAAM,iBAAiB,GAA6B;IAClD,CAAC,EAAE,oBAAQ,CAAC,GAAG;IACf,KAAK,EAAE,oBAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,oBAAQ,CAAC,MAAM;IACrB,EAAE,EAAE,oBAAQ,CAAC,EAAE;IACf,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;IAC/B,MAAM,EAAE,oBAAQ,CAAC,MAAM;IACvB,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAC7D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACd,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAChB;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EACD,EAAgC,CACjC,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAkB;IAC5C,QAAQ,QAAQ,EAAE;QAChB,KAAK,oBAAQ,CAAC,GAAG;YACf,OAAO,KAAK,CAAC;QACf,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,EAAE;YACd,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAQ,CAAC,UAAU;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,IAAI,CAAC;QAEd;YACE,kBAAW,CAAC,QAAQ,CAAC,CAAC;KACzB;AACH,CAAC;AApBD,kCAoBC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAiB,EACjB,OAAiB,EACjB,WAAuB,EACvB,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,IAAI,wBAAM,CAAC;QAC9B,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,OAAO;QACP,iBAAiB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;KACvE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,uDAAuD;IACvD,uDAAuD;IACvD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CACxD,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,eAAe,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;YAC5D,GAAG,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3D;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;SACzC;KACF;SAAM;QACL,MAAM,CAAC,IAAI,CACT,2EAA2E;YACzE,gEAAgE;YAChE,4EAA4E;YAC5E,4EAA4E;YAC5E,2EAA2E;YAC3E,6EAA6E;YAC7E,2CAA2C,CAC9C,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA9CD,4BA8CC"}

lib/defaults.json

@@ -1,3 +1,3 @@
 {
-    "bundleVersion": "codeql-bundle-20210622"
+    "bundleVersion": "codeql-bundle-20210702"
 }

lib/runner.js

@@ -243,7 +243,9 @@ program
             url: util_1.parseGitHubUrl(cmd.githubUrl),
         };
         const outputDir = cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
-        await analyze_1.runAnalyze(outputDir, util_1.getMemoryFlag(cmd.ram), util_1.getAddSnippetsFlag(cmd.addSnippets), util_1.getThreadsFlag(cmd.threads, logger), cmd.category, config, logger);
+        const threads = util_1.getThreadsFlag(cmd.threads, logger);
+        await analyze_1.runFinalize(outputDir, threads, config, logger);
+        await analyze_1.runQueries(outputDir, util_1.getMemoryFlag(cmd.ram), util_1.getAddSnippetsFlag(cmd.addSnippets), threads, cmd.category, config, logger);
         if (!cmd.upload) {
             logger.info("Not uploading results");
             return;

src/analyze-action.ts

@@ -5,16 +5,18 @@ import * as core from "@actions/core";
 
 import * as actionsUtil from "./actions-util";
 import {
-  runAnalyze,
   CodeQLAnalysisError,
   QueriesStatusReport,
   runCleanup,
+  runQueries,
+  runFinalize,
 } from "./analyze";
 import { Config, getConfig } from "./config-utils";
 import { uploadDatabases } from "./database-upload";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
+import { UploadStatusReport } from "./upload-lib";
 import * as util from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -53,7 +55,8 @@ async function sendStatusReport(
 
 async function run() {
   const startedAt = new Date();
-  let stats: AnalysisStatusReport | undefined = undefined;
+  let uploadStats: UploadStatusReport | undefined = undefined;
+  let runStats: QueriesStatusReport | undefined = undefined;
   let config: Config | undefined = undefined;
   util.initializeEnvironment(util.Mode.actions, pkg.version);
 
@@ -82,15 +85,22 @@ async function run() {
       url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
     };
     const outputDir = actionsUtil.getRequiredInput("output");
-    const queriesStats = await runAnalyze(
-      outputDir,
-      util.getMemoryFlag(actionsUtil.getOptionalInput("ram")),
-      util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
-      util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger),
-      actionsUtil.getOptionalInput("category"),
-      config,
+    const threads = util.getThreadsFlag(
+      actionsUtil.getOptionalInput("threads"),
       logger
     );
+    await runFinalize(outputDir, threads, config, logger);
+    if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
+      runStats = await runQueries(
+        outputDir,
+        util.getMemoryFlag(actionsUtil.getOptionalInput("ram")),
+        util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
+        threads,
+        actionsUtil.getOptionalInput("category"),
+        config,
+        logger
+      );
+    }
 
     if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
       await runCleanup(
@@ -106,17 +116,15 @@ async function run() {
     }
     core.setOutput("db-locations", dbLocations);
 
-    if (actionsUtil.getRequiredInput("upload") === "true") {
-      const uploadStats = await upload_lib.uploadFromActions(
+    if (runStats && actionsUtil.getRequiredInput("upload") === "true") {
+      uploadStats = await upload_lib.uploadFromActions(
         outputDir,
         config.gitHubVersion,
         apiDetails,
         logger
       );
-      stats = { ...queriesStats, ...uploadStats };
     } else {
       logger.info("Not uploading results");
-      stats = { ...queriesStats };
     }
 
     const repositoryNwo = parseRepositoryNwo(
@@ -128,10 +136,12 @@ async function run() {
     console.log(error);
 
     if (error instanceof CodeQLAnalysisError) {
-      stats = { ...error.queriesStatusReport };
+      const stats = { ...error.queriesStatusReport };
+      await sendStatusReport(startedAt, stats, error);
+    } else {
+      await sendStatusReport(startedAt, undefined, error);
     }
 
-    await sendStatusReport(startedAt, stats, error);
     return;
   } finally {
     if (core.isDebug() && config !== undefined) {
@@ -161,7 +171,13 @@ async function run() {
     }
   }
 
-  await sendStatusReport(startedAt, stats);
+  if (runStats && uploadStats) {
+    await sendStatusReport(startedAt, { ...runStats, ...uploadStats });
+  } else if (runStats) {
+    await sendStatusReport(startedAt, { ...runStats });
+  } else {
+    await sendStatusReport(startedAt, undefined);
+  }
 }
 
 async function runWrapper() {

src/analyze.ts

@@ -2,6 +2,7 @@ import * as fs from "fs";
 import * as path from "path";
 
 import * as toolrunner from "@actions/exec/lib/toolrunner";
+import * as yaml from "js-yaml";
 
 import * as analysisPaths from "./analysis-paths";
 import { getCodeQL } from "./codeql";
@@ -117,7 +118,10 @@ async function createdDBForScannedLanguages(
 
   const codeql = getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
-    if (isScannedLanguage(language)) {
+    if (
+      isScannedLanguage(language) &&
+      !dbIsFinalized(config, language, logger)
+    ) {
       logger.startGroup(`Extracting ${language}`);
 
       if (language === Language.python) {
@@ -133,6 +137,25 @@ async function createdDBForScannedLanguages(
   }
 }
 
+function dbIsFinalized(
+  config: configUtils.Config,
+  language: Language,
+  logger: Logger
+) {
+  const dbPath = util.getCodeQLDatabasePath(config, language);
+  try {
+    const dbInfo = yaml.load(
+      fs.readFileSync(path.resolve(dbPath, "codeql-database.yml"), "utf8")
+    );
+    return !("inProgress" in dbInfo);
+  } catch (e) {
+    logger.warning(
+      `Could not check whether database for ${language} was finalized. Assuming it is not.`
+    );
+    return false;
+  }
+}
+
 async function finalizeDatabaseCreation(
   config: configUtils.Config,
   threadsFlag: string,
@@ -142,12 +165,18 @@ async function finalizeDatabaseCreation(
 
   const codeql = getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
-    logger.startGroup(`Finalizing ${language}`);
-    await codeql.finalizeDatabase(
-      util.getCodeQLDatabasePath(config, language),
-      threadsFlag
-    );
-    logger.endGroup();
+    if (dbIsFinalized(config, language, logger)) {
+      logger.info(
+        `There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`
+      );
+    } else {
+      logger.startGroup(`Finalizing ${language}`);
+      await codeql.finalizeDatabase(
+        util.getCodeQLDatabasePath(config, language),
+        threadsFlag
+      );
+      logger.endGroup();
+    }
   }
 }
 
@@ -349,33 +378,18 @@ function packWithVersionToQuerySuiteEntry(
   return text;
 }
 
-export async function runAnalyze(
+export async function runFinalize(
   outputDir: string,
-  memoryFlag: string,
-  addSnippetsFlag: string,
   threadsFlag: string,
-  automationDetailsId: string | undefined,
   config: configUtils.Config,
   logger: Logger
-): Promise<QueriesStatusReport> {
+) {
   // Delete the tracer config env var to avoid tracing ourselves
   delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
 
   fs.mkdirSync(outputDir, { recursive: true });
 
   await finalizeDatabaseCreation(config, threadsFlag, logger);
-
-  const queriesStats = await runQueries(
-    outputDir,
-    memoryFlag,
-    addSnippetsFlag,
-    threadsFlag,
-    automationDetailsId,
-    config,
-    logger
-  );
-
-  return { ...queriesStats };
 }
 
 export async function runCleanup(

src/codeql.ts

@@ -318,96 +318,101 @@ export async function setupCodeQL(
     if (forceLatest) {
       codeqlURL = undefined;
     }
-
-    const codeqlURLVersion = getCodeQLURLVersion(
-      codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
-    );
-    const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
-
-    // If we find the specified version, we always use that.
-    let codeqlFolder = toolcache.find(
-      "CodeQL",
-      codeqlURLSemVer,
-      toolCacheDir,
-      logger
-    );
-
-    // If we don't find the requested version, in some cases we may allow a
-    // different version to save download time if the version hasn't been
-    // specified explicitly (in which case we always honor it).
-    if (!codeqlFolder && !codeqlURL && !forceLatest) {
-      const codeqlVersions = toolcache.findAllVersions(
-        "CodeQL",
-        toolCacheDir,
-        logger
-      );
-      if (codeqlVersions.length === 1) {
-        const tmpCodeqlFolder = toolcache.find(
-          "CodeQL",
-          codeqlVersions[0],
-          toolCacheDir,
-          logger
-        );
-        if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
-          logger.debug(
-            `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
-          );
-          codeqlFolder = tmpCodeqlFolder;
-        }
-      }
-    }
-
-    if (codeqlFolder) {
-      logger.debug(`CodeQL found in cache ${codeqlFolder}`);
+    let codeqlFolder: string;
+    let codeqlURLVersion: string;
+    if (codeqlURL && !codeqlURL.startsWith("http")) {
+      codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
+      codeqlURLVersion = "local";
     } else {
-      if (!codeqlURL) {
-        codeqlURL = await getCodeQLBundleDownloadURL(
-          apiDetails,
-          variant,
-          logger
-        );
-      }
+      codeqlURLVersion = getCodeQLURLVersion(
+        codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
+      );
+      const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
 
-      const parsedCodeQLURL = new URL(codeqlURL);
-      const parsedQueryString = queryString.parse(parsedCodeQLURL.search);
-      const headers: IHeaders = { accept: "application/octet-stream" };
-      // We only want to provide an authorization header if we are downloading
-      // from the same GitHub instance the Action is running on.
-      // This avoids leaking Enterprise tokens to dotcom.
-      // We also don't want to send an authorization header if there's already a token provided in the URL.
-      if (
-        codeqlURL.startsWith(`${apiDetails.url}/`) &&
-        parsedQueryString["token"] === undefined
-      ) {
-        logger.debug("Downloading CodeQL bundle with token.");
-        headers.authorization = `token ${apiDetails.auth}`;
-      } else {
-        logger.debug("Downloading CodeQL bundle without token.");
-      }
-      logger.info(
-        `Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
-      );
-      const codeqlPath = await toolcache.downloadTool(
-        codeqlURL,
-        tempDir,
-        headers
-      );
-      logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-
-      const codeqlExtracted = await toolcache.extractTar(
-        codeqlPath,
-        tempDir,
-        logger
-      );
-      codeqlFolder = await toolcache.cacheDir(
-        codeqlExtracted,
+      // If we find the specified version, we always use that.
+      codeqlFolder = toolcache.find(
         "CodeQL",
         codeqlURLSemVer,
         toolCacheDir,
         logger
       );
-    }
 
+      // If we don't find the requested version, in some cases we may allow a
+      // different version to save download time if the version hasn't been
+      // specified explicitly (in which case we always honor it).
+      if (!codeqlFolder && !codeqlURL && !forceLatest) {
+        const codeqlVersions = toolcache.findAllVersions(
+          "CodeQL",
+          toolCacheDir,
+          logger
+        );
+        if (codeqlVersions.length === 1) {
+          const tmpCodeqlFolder = toolcache.find(
+            "CodeQL",
+            codeqlVersions[0],
+            toolCacheDir,
+            logger
+          );
+          if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
+            logger.debug(
+              `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
+            );
+            codeqlFolder = tmpCodeqlFolder;
+          }
+        }
+      }
+
+      if (codeqlFolder) {
+        logger.debug(`CodeQL found in cache ${codeqlFolder}`);
+      } else {
+        if (!codeqlURL) {
+          codeqlURL = await getCodeQLBundleDownloadURL(
+            apiDetails,
+            variant,
+            logger
+          );
+        }
+
+        const parsedCodeQLURL = new URL(codeqlURL);
+        const parsedQueryString = queryString.parse(parsedCodeQLURL.search);
+        const headers: IHeaders = { accept: "application/octet-stream" };
+        // We only want to provide an authorization header if we are downloading
+        // from the same GitHub instance the Action is running on.
+        // This avoids leaking Enterprise tokens to dotcom.
+        // We also don't want to send an authorization header if there's already a token provided in the URL.
+        if (
+          codeqlURL.startsWith(`${apiDetails.url}/`) &&
+          parsedQueryString["token"] === undefined
+        ) {
+          logger.debug("Downloading CodeQL bundle with token.");
+          headers.authorization = `token ${apiDetails.auth}`;
+        } else {
+          logger.debug("Downloading CodeQL bundle without token.");
+        }
+        logger.info(
+          `Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
+        );
+        const codeqlPath = await toolcache.downloadTool(
+          codeqlURL,
+          tempDir,
+          headers
+        );
+        logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
+
+        const codeqlExtracted = await toolcache.extractTar(
+          codeqlPath,
+          tempDir,
+          logger
+        );
+        codeqlFolder = await toolcache.cacheDir(
+          codeqlExtracted,
+          "CodeQL",
+          codeqlURLSemVer,
+          toolCacheDir,
+          logger
+        );
+      }
+    }
     let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
     if (process.platform === "win32") {
       codeqlCmd += ".exe";

src/count-loc.ts

@@ -98,14 +98,13 @@ export async function countLoc(
     }
   } else {
     logger.info(
-      "Could not determine the total number of lines of code in this repository. " +
-        "Because of this, it will not be possible to compare the number of lines " +
-        "of code analyzed by code scanning with the total number of lines of " +
-        "code in the repository. This will not affect the results produced by code " +
-        "scanning. If you have any questions, you can raise an issue at " +
-        "https://github.com/github/codeql-action/issues. Please include a link " +
-        "to the repository if public, or otherwise information about the code scanning " +
-        "workflow you are using."
+      "Could not determine the baseline lines of code count in this repository. " +
+        "Because of this, it will not be possible to compare the lines " +
+        "of code analyzed by code scanning with the baseline. This will not affect " +
+        "the results produced by code scanning. If you have any questions, you can " +
+        "raise an issue at https://github.com/github/codeql-action/issues. Please " +
+        "include a link to the repository if public, or otherwise information about " +
+        "the code scanning workflow you are using."
     );
   }
 

src/defaults.json

@@ -1,3 +1,3 @@
 {
-	"bundleVersion": "codeql-bundle-20210622"
+	"bundleVersion": "codeql-bundle-20210702"
 }

src/runner.ts

@@ -4,7 +4,7 @@ import * as path from "path";
 
 import { Command } from "commander";
 
-import { runAnalyze } from "./analyze";
+import { runFinalize, runQueries } from "./analyze";
 import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
 import { CodeQL, getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
@@ -431,11 +431,13 @@ program
 
       const outputDir =
         cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
-      await runAnalyze(
+      const threads = getThreadsFlag(cmd.threads, logger);
+      await runFinalize(outputDir, threads, config, logger);
+      await runQueries(
         outputDir,
         getMemoryFlag(cmd.ram),
         getAddSnippetsFlag(cmd.addSnippets),
-        getThreadsFlag(cmd.threads, logger),
+        threads,
         cmd.category,
         config,
         logger