Improve paths config handling for JS/Python when globs are used

src/analysis-paths.test.ts

@@ -64,11 +64,45 @@ test("nonEmptyPaths", async (t) => {
       trapCacheDownloadTime: 0,
     };
     analysisPaths.includeAndExcludeAnalysisPaths(config);
-    t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
+    t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
     t.is(process.env["LGTM_INDEX_EXCLUDE"], "path4\npath5");
     t.is(
       process.env["LGTM_INDEX_FILTERS"],
-      "include:path1\ninclude:path2\ninclude:**/path3\nexclude:path4\nexclude:path5\nexclude:path6/**",
+      "exclude:**/*\ninclude:path1\ninclude:path2\ninclude:**/path3\nexclude:path4\nexclude:path5\nexclude:path6/**",
+    );
+  });
+});
+
+test("paths only has globs", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const config = {
+      languages: [],
+      queries: {},
+      paths: ["path1/**", "**/path2"],
+      pathsIgnore: [],
+      originalUserInput: {},
+      tempDir: tmpDir,
+      codeQLCmd: "",
+      gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
+      dbLocation: path.resolve(tmpDir, "codeql_databases"),
+      packs: {},
+      debugMode: false,
+      debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
+      debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
+      augmentationProperties: {
+        injectedMlQueries: false,
+        packsInputCombines: false,
+        queriesInputCombines: false,
+      },
+      trapCaches: {},
+      trapCacheDownloadTime: 0,
+    };
+    analysisPaths.includeAndExcludeAnalysisPaths(config);
+    t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
+    t.is(process.env["LGTM_INDEX_EXCLUDE"], undefined);
+    t.is(
+      process.env["LGTM_INDEX_FILTERS"],
+      "exclude:**/*\ninclude:path1/**\ninclude:**/path2",
     );
   });
 });

src/analysis-paths.ts

@@ -48,8 +48,13 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
   // excluding and not traversing entire file subtrees.
   // It does not understand globs or double-globs because that would require it to
   // traverse the entire file tree to determine which files are matched.
-  // Any paths containing "*" are not included in these.
-  if (config.paths.length !== 0) {
+  //
+  // Since globs aren't handled by LGTM_INDEX_INCLUDE, if any of files/directories
+  // include paths contains a glob or double-glob, we fall back to not setting
+  // LGTM_INDEX_INCLUDE at all, and only use the LGTM_INDEX_FILTERS mechanism to control
+  // which files are extracted.
+  const pathsHasGlob = config.paths.some((p) => p.indexOf("*") !== -1);
+  if (config.paths.length !== 0 && pathsHasGlob === false) {
     process.env["LGTM_INDEX_INCLUDE"] = buildIncludeExcludeEnvVar(config.paths);
   }
   // If the temporary or tools directory is in the working directory ignore that too.
@@ -69,6 +74,13 @@ export function includeAndExcludeAnalysisPaths(config: configUtils.Config) {
   // extracted or ignored. It does not control which directories are traversed.
   // This does understand the glob and double-glob syntax.
   const filters: string[] = [];
+  if (pathsHasGlob) {
+    // if paths have globs we don't set LGTM_INDEX_INCLUDE and therefore by default we
+    // would extract everything. By inserting `exclude:**/*` as the first element, we
+    // need to change the default to exclude everything, meaning only the things
+    // mentioned in paths will be included.
+    filters.push("exclude:**/*");
+  }
   filters.push(...config.paths.map((p) => `include:${p}`));
   filters.push(...config.pathsIgnore.map((p) => `exclude:${p}`));
   if (filters.length !== 0) {