Merge pull request #3404 from github/henrymercer/include-oids-in-bundle

Include base database OIDs when bundling database
This commit is contained in:
Henry Mercer
2026-01-20 06:13:13 -08:00
committed by GitHub
15 changed files with 264 additions and 96 deletions

View File

@@ -123725,6 +123725,7 @@ var maximumVersion = "3.20";
var minimumVersion = "3.14";
// src/util.ts
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var GITHUB_DOTCOM_URL = "https://github.com";
var MINIMUM_CGROUP_MEMORY_LIMIT_BYTES = 1024 * 1024;
function getExtraOptionsEnvParam() {
@@ -123831,6 +123832,9 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
function isInTestMode() {
return process.env["CODEQL_ACTION_TEST_MODE" /* TEST_MODE */] === "true";
}
@@ -124493,9 +124497,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path2.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -125161,7 +125162,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -125170,6 +125171,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

45
lib/analyze-action.js generated
View File

@@ -18850,7 +18850,7 @@ var require_io = __commonJS({
if (path16.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -18983,13 +18983,13 @@ var require_io = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -31282,7 +31282,7 @@ var require_io2 = __commonJS({
if (path16.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -31421,13 +31421,13 @@ var require_io2 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -89364,6 +89364,7 @@ var safeDump = renamed("safeDump", "dump");
// src/util.ts
var semver = __toESM(require_semver2());
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var BROKEN_VERSIONS = ["0.0.0-20211207"];
var GITHUB_DOTCOM_URL = "https://github.com";
var DEFAULT_RESERVED_RAM_SCALING_FACTOR = 0.05;
@@ -89684,13 +89685,30 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
async function bundleDb(config, language, codeql, dbName) {
const databasePath = getCodeQLDatabasePath(config, language);
const databaseBundlePath = path.resolve(config.dbLocation, `${dbName}.zip`);
if (fs.existsSync(databaseBundlePath)) {
await fs.promises.rm(databaseBundlePath, { force: true });
}
await codeql.databaseBundle(databasePath, databaseBundlePath, dbName);
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
const additionalFiles = [];
if (fs.existsSync(baseDatabaseOidsFilePath)) {
await fsPromises.copyFile(
baseDatabaseOidsFilePath,
path.join(databasePath, BASE_DATABASE_OIDS_FILE_NAME)
);
additionalFiles.push(BASE_DATABASE_OIDS_FILE_NAME);
}
await codeql.databaseBundle(
databasePath,
databaseBundlePath,
dbName,
additionalFiles
);
return databaseBundlePath;
}
async function delay(milliseconds, opts) {
@@ -90905,9 +90923,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path4.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -93244,7 +93259,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -93253,6 +93268,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

View File

@@ -18850,7 +18850,7 @@ var require_io = __commonJS({
if (path7.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -18983,13 +18983,13 @@ var require_io = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -31282,7 +31282,7 @@ var require_io2 = __commonJS({
if (path7.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -31421,13 +31421,13 @@ var require_io2 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -86463,6 +86463,7 @@ var maximumVersion = "3.20";
var minimumVersion = "3.14";
// src/util.ts
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var GITHUB_DOTCOM_URL = "https://github.com";
var MINIMUM_CGROUP_MEMORY_LIMIT_BYTES = 1024 * 1024;
function getExtraOptionsEnvParam() {
@@ -86594,6 +86595,9 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
function isInTestMode() {
return process.env["CODEQL_ACTION_TEST_MODE" /* TEST_MODE */] === "true";
}
@@ -87392,9 +87396,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path2.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -88362,7 +88363,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -88371,6 +88372,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

View File

@@ -18850,7 +18850,7 @@ var require_io = __commonJS({
if (path16.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -18983,13 +18983,13 @@ var require_io = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -31282,7 +31282,7 @@ var require_io2 = __commonJS({
if (path16.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -31421,13 +31421,13 @@ var require_io2 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -90084,8 +90084,8 @@ var require_graceful_fs = __commonJS({
}
var fs$copyFile = fs19.copyFile;
if (fs$copyFile)
fs19.copyFile = copyFile;
function copyFile(src, dest, flags, cb) {
fs19.copyFile = copyFile2;
function copyFile2(src, dest, flags, cb) {
if (typeof flags === "function") {
cb = flags;
flags = 0;
@@ -117479,7 +117479,7 @@ var require_io3 = __commonJS({
if (path16.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -117618,13 +117618,13 @@ var require_io3 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -126623,6 +126623,7 @@ var maximumVersion = "3.20";
var minimumVersion = "3.14";
// src/util.ts
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var BROKEN_VERSIONS = ["0.0.0-20211207"];
var GITHUB_DOTCOM_URL = "https://github.com";
var MINIMUM_CGROUP_MEMORY_LIMIT_BYTES = 1024 * 1024;
@@ -126768,13 +126769,30 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
async function bundleDb(config, language, codeql, dbName) {
const databasePath = getCodeQLDatabasePath(config, language);
const databaseBundlePath = path.resolve(config.dbLocation, `${dbName}.zip`);
if (fs.existsSync(databaseBundlePath)) {
await fs.promises.rm(databaseBundlePath, { force: true });
}
await codeql.databaseBundle(databasePath, databaseBundlePath, dbName);
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
const additionalFiles = [];
if (fs.existsSync(baseDatabaseOidsFilePath)) {
await fsPromises.copyFile(
baseDatabaseOidsFilePath,
path.join(databasePath, BASE_DATABASE_OIDS_FILE_NAME)
);
additionalFiles.push(BASE_DATABASE_OIDS_FILE_NAME);
}
await codeql.databaseBundle(
databasePath,
databaseBundlePath,
dbName,
additionalFiles
);
return databaseBundlePath;
}
async function delay(milliseconds, opts) {
@@ -127868,9 +127886,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path3.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -129779,7 +129794,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -129788,6 +129803,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

29
lib/init-action.js generated
View File

@@ -18850,7 +18850,7 @@ var require_io = __commonJS({
if (path16.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -18983,13 +18983,13 @@ var require_io = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -31433,7 +31433,7 @@ var require_io2 = __commonJS({
if (path16.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -31572,13 +31572,13 @@ var require_io2 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -86681,6 +86681,7 @@ var maximumVersion = "3.20";
var minimumVersion = "3.14";
// src/util.ts
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var BROKEN_VERSIONS = ["0.0.0-20211207"];
var GITHUB_DOTCOM_URL = "https://github.com";
var DEFAULT_DEBUG_ARTIFACT_NAME = "debug-artifacts";
@@ -87011,6 +87012,9 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
async function delay(milliseconds, opts) {
const { allowProcessExit } = opts || {};
return new Promise((resolve9) => {
@@ -88366,9 +88370,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path5.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -91526,7 +91527,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -91535,6 +91536,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

View File

@@ -18850,7 +18850,7 @@ var require_io = __commonJS({
if (path5.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -18983,13 +18983,13 @@ var require_io = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -31282,7 +31282,7 @@ var require_io2 = __commonJS({
if (path5.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -31421,13 +31421,13 @@ var require_io2 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -86463,6 +86463,7 @@ var maximumVersion = "3.20";
var minimumVersion = "3.14";
// src/util.ts
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var GITHUB_DOTCOM_URL = "https://github.com";
var MINIMUM_CGROUP_MEMORY_LIMIT_BYTES = 1024 * 1024;
function getExtraOptionsEnvParam() {
@@ -86587,6 +86588,9 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
async function delay(milliseconds, opts) {
const { allowProcessExit } = opts || {};
return new Promise((resolve4) => {
@@ -87385,9 +87389,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path2.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -88055,7 +88056,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -88064,6 +88065,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

View File

@@ -18850,7 +18850,7 @@ var require_io = __commonJS({
if (path8.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -18983,13 +18983,13 @@ var require_io = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -29985,7 +29985,7 @@ var require_io2 = __commonJS({
if (path8.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -30124,13 +30124,13 @@ var require_io2 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -86519,6 +86519,7 @@ var maximumVersion = "3.20";
var minimumVersion = "3.14";
// src/util.ts
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var BROKEN_VERSIONS = ["0.0.0-20211207"];
var GITHUB_DOTCOM_URL = "https://github.com";
var MINIMUM_CGROUP_MEMORY_LIMIT_BYTES = 1024 * 1024;
@@ -86660,6 +86661,9 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
async function delay(milliseconds, opts) {
const { allowProcessExit } = opts || {};
return new Promise((resolve4) => {
@@ -87294,9 +87298,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path3.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -89364,7 +89365,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -89373,6 +89374,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

View File

@@ -18850,7 +18850,7 @@ var require_io = __commonJS({
if (path2.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -18983,13 +18983,13 @@ var require_io = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -49888,7 +49888,7 @@ var require_io2 = __commonJS({
if (path2.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -50027,13 +50027,13 @@ var require_io2 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {

17
lib/upload-lib.js generated
View File

@@ -89381,6 +89381,7 @@ var safeDump = renamed("safeDump", "dump");
// src/util.ts
var semver = __toESM(require_semver2());
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var BROKEN_VERSIONS = ["0.0.0-20211207"];
var GITHUB_DOTCOM_URL = "https://github.com";
var MINIMUM_CGROUP_MEMORY_LIMIT_BYTES = 1024 * 1024;
@@ -89500,6 +89501,9 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
async function delay(milliseconds, opts) {
const { allowProcessExit } = opts || {};
return new Promise((resolve6) => {
@@ -90449,9 +90453,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path3.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -92080,7 +92081,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -92089,6 +92090,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

View File

@@ -18850,7 +18850,7 @@ var require_io = __commonJS({
if (path12.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -18983,13 +18983,13 @@ var require_io = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -29985,7 +29985,7 @@ var require_io2 = __commonJS({
if (path12.relative(source, newDest) === "") {
throw new Error(`'${newDest}' and '${source}' are the same file`);
}
yield copyFile(source, newDest, force);
yield copyFile2(source, newDest, force);
}
});
}
@@ -30124,13 +30124,13 @@ var require_io2 = __commonJS({
if (srcFileStat.isDirectory()) {
yield cpDirRecursive(srcFile, destFile, currentDepth, force);
} else {
yield copyFile(srcFile, destFile, force);
yield copyFile2(srcFile, destFile, force);
}
}
yield ioUtil.chmod(destDir, (yield ioUtil.stat(sourceDir)).mode);
});
}
function copyFile(srcFile, destFile, force) {
function copyFile2(srcFile, destFile, force) {
return __awaiter2(this, void 0, void 0, function* () {
if ((yield ioUtil.lstat(srcFile)).isSymbolicLink()) {
try {
@@ -89355,6 +89355,7 @@ var safeDump = renamed("safeDump", "dump");
// src/util.ts
var semver = __toESM(require_semver2());
var BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
var BROKEN_VERSIONS = ["0.0.0-20211207"];
var GITHUB_DOTCOM_URL = "https://github.com";
var MINIMUM_CGROUP_MEMORY_LIMIT_BYTES = 1024 * 1024;
@@ -89481,6 +89482,9 @@ function getCachedCodeQlVersion() {
async function codeQlVersionAtLeast(codeql, requiredVersion) {
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
async function delay(milliseconds, opts) {
const { allowProcessExit } = opts || {};
return new Promise((resolve6) => {
@@ -90244,9 +90248,6 @@ async function readBaseDatabaseOidsFile(config, logger) {
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path3.join(config.dbLocation, "base-database-oids.json");
}
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
@@ -92604,7 +92605,7 @@ ${output}`
await runCli(cmd, codeqlArgs);
}
},
async databaseBundle(databasePath, outputFilePath, databaseName) {
async databaseBundle(databasePath, outputFilePath, databaseName, alsoIncludeRelativePaths) {
const args = [
"database",
"bundle",
@@ -92613,6 +92614,14 @@ ${output}`
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"])
];
if (await this.supportsFeature("bundleSupportsIncludeOption" /* BundleSupportsIncludeOption */)) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath
])
);
}
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId) {

11
package-lock.json generated
View File

@@ -1661,6 +1661,7 @@
"resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.2.tgz",
"integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==",
"license": "MIT",
"peer": true,
"dependencies": {
"@octokit/auth-token": "^4.0.0",
"@octokit/graphql": "^7.1.0",
@@ -2487,6 +2488,7 @@
"integrity": "sha512-nm3cvFN9SqZGXjmw5bZ6cGmvJSyJPn0wU9gHAZZHDnZl2wF9PhHv78Xf06E0MaNk4zLVHL8hb2/c32XvyJOLQg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "8.53.1",
"@typescript-eslint/types": "8.53.1",
@@ -3168,6 +3170,7 @@
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"dev": true,
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -3739,6 +3742,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"caniuse-lite": "^1.0.30001669",
"electron-to-chromium": "^1.5.41",
@@ -4582,6 +4586,7 @@
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz",
"integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==",
"dev": true,
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.6.1",
@@ -4636,6 +4641,7 @@
"version": "8.3.0",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"eslint-config-prettier": "bin/cli.js"
},
@@ -4907,6 +4913,7 @@
"resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz",
"integrity": "sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==",
"dev": true,
"peer": true,
"dependencies": {
"array-includes": "^3.1.7",
"array.prototype.findlastindex": "^1.2.3",
@@ -7307,6 +7314,7 @@
"integrity": "sha512-G+YdqtITVZmOJje6QkXQWzl3fSfMxFwm1tjTyo9exhkmWSqC4Yhd1+lug++IlR2mvRVAxEDDWYkQdeSztajqgg==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"prettier": "bin/prettier.cjs"
},
@@ -8296,6 +8304,7 @@
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -8508,6 +8517,7 @@
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -8581,6 +8591,7 @@
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.17.0.tgz",
"integrity": "sha512-Drp39TXuUlD49F7ilHHCG7TTg8IkA+hxCuULdmzWYICxGXvDXmDmWEjJYZQYgf6l/TFfYNE167m7isnc3xlIEg==",
"dev": true,
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "8.17.0",
"@typescript-eslint/types": "8.17.0",

View File

@@ -147,11 +147,20 @@ export interface CodeQL {
): Promise<void>;
/**
* Run 'codeql database bundle'.
*
* @param alsoIncludeRelativePaths Additional paths that should be included in the bundle if
* supported by the version of the CodeQL CLI.
*
* These paths are relative to the database root.
*
* Older versions of the CodeQL CLI do not support including additional paths in the bundle.
* In those cases, this parameter will be ignored.
*/
databaseBundle(
databasePath: string,
outputFilePath: string,
dbName: string,
alsoIncludeRelativePaths: string[],
): Promise<void>;
/**
* Run 'codeql database run-queries'. If no `queries` are specified, then the CLI
@@ -911,6 +920,7 @@ async function getCodeQLForCmd(
databasePath: string,
outputFilePath: string,
databaseName: string,
alsoIncludeRelativePaths: string[],
): Promise<void> {
const args = [
"database",
@@ -920,6 +930,16 @@ async function getCodeQLForCmd(
`--name=${databaseName}`,
...getExtraOptionsFromEnv(["database", "bundle"]),
];
if (
await this.supportsFeature(ToolsFeature.BundleSupportsIncludeOption)
) {
args.push(
...alsoIncludeRelativePaths.flatMap((relativePath) => [
"--include",
relativePath,
]),
);
}
await new toolrunner.ToolRunner(cmd, args).exec();
},
async databaseExportDiagnostics(

View File

@@ -17,6 +17,7 @@ import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
import { Logger, withGroupAsync } from "./logging";
import {
CleanupLevel,
getBaseDatabaseOidsFilePath,
getCodeQLDatabasePath,
getErrorMessage,
isInTestMode,
@@ -98,10 +99,6 @@ async function readBaseDatabaseOidsFile(
}
}
function getBaseDatabaseOidsFilePath(config: Config): string {
return path.join(config.dbLocation, "base-database-oids.json");
}
/**
* Writes a JSON file containing the source-root-relative paths of files under
* `sourceRoot` that have changed (added, removed, or modified) from the overlay

View File

@@ -4,6 +4,7 @@ import type { VersionInfo } from "./codeql";
export enum ToolsFeature {
BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries",
BundleSupportsIncludeOption = "bundleSupportsIncludeOption",
BundleSupportsOverlay = "bundleSupportsOverlay",
DatabaseInterpretResultsSupportsSarifRunProperty = "databaseInterpretResultsSupportsSarifRunProperty",
ForceOverwrite = "forceOverwrite",

View File

@@ -16,6 +16,12 @@ import { EnvVar } from "./environment";
import { Language } from "./languages";
import { Logger } from "./logging";
/**
* The name of the file containing the base database OIDs, as stored in the
* root of the database location.
*/
const BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
/**
* Specifies bundle versions that are known to be broken
* and will not be used if found in the toolcache.
@@ -728,6 +734,10 @@ export async function codeQlVersionAtLeast(
return semver.gte((await codeql.getVersion()).version, requiredVersion);
}
export function getBaseDatabaseOidsFilePath(config: Config): string {
return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
}
// Create a bundle for the given DB, if it doesn't already exist
export async function bundleDb(
config: Config,
@@ -745,7 +755,27 @@ export async function bundleDb(
if (fs.existsSync(databaseBundlePath)) {
await fs.promises.rm(databaseBundlePath, { force: true });
}
await codeql.databaseBundle(databasePath, databaseBundlePath, dbName);
// When overlay is enabled, the base database OIDs file is included at the
// root of the database cluster. However when we bundle a database, we only
// include the per-language database. So, to ensure the base database OIDs
// file is included in the database bundle, we copy it from the cluster into
// the individual database location before bundling.
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
const additionalFiles: string[] = [];
if (fs.existsSync(baseDatabaseOidsFilePath)) {
await fsPromises.copyFile(
baseDatabaseOidsFilePath,
path.join(databasePath, BASE_DATABASE_OIDS_FILE_NAME),
);
additionalFiles.push(BASE_DATABASE_OIDS_FILE_NAME);
}
// Create the bundle, including the base database OIDs file if it exists
await codeql.databaseBundle(
databasePath,
databaseBundlePath,
dbName,
additionalFiles,
);
return databaseBundlePath;
}