Add option to use ripgrep for crawling the list of files

lib/load-paths-handler.js

@@ -2,23 +2,34 @@
 
 const async = require('async')
 const fs = require('fs')
+const os = require('os')
 const path = require('path')
 const {GitRepository} = require('atom')
 const {Minimatch} = require('minimatch')
+const childProcess = require('child_process')
+const { rgPath } = require('vscode-ripgrep')
 
 const PathsChunkSize = 100
 
+// Use the unpacked path if the ripgrep binary is in asar archive.
+const realRgPath = rgPath.replace(/\bapp\.asar\b/, 'app.asar.unpacked')
+
+// Define the maximum number of concurrent crawling processes based on the number of CPUs
+// with a maximum value of 8 and minimum of 1.
+const MaxConcurrentCrawls = Math.min(Math.max(os.cpus().length - 1, 8), 1)
+
 const emittedPaths = new Set()
 
 class PathLoader {
-  constructor (rootPath, ignoreVcsIgnores, traverseSymlinkDirectories, ignoredNames) {
+  constructor (rootPath, ignoreVcsIgnores, traverseSymlinkDirectories, ignoredNames, useRipGrep) {
     this.rootPath = rootPath
     this.traverseSymlinkDirectories = traverseSymlinkDirectories
     this.ignoredNames = ignoredNames
+    this.useRipGrep = useRipGrep
     this.paths = []
     this.inodes = new Set()
     this.repo = null
-    if (ignoreVcsIgnores) {
+    if (ignoreVcsIgnores && !this.useRipGrep) {
       const repo = GitRepository.open(this.rootPath, {refreshOnWindowFocus: false})
       if ((repo && repo.relativize(path.join(this.rootPath, 'test'))) === 'test') {
         this.repo = repo
@@ -27,13 +38,53 @@ class PathLoader {
   }
 
   load (done) {
+    if (this.useRipGrep) {
+      this.loadFromRipGrep().then(done)
+
+      return
+    }
+
     this.loadPath(this.rootPath, true, () => {
       this.flushPaths()
       if (this.repo != null) this.repo.destroy()
       done()
     })
   }
 
+  async loadFromRipGrep () {
+    return new Promise((resolve) => {
+      const args = ['--files', '--hidden', '--sort', 'path']
+
+      if (this.ignoreVcsIgnores) {
+        args.push('--no-ignore')
+      }
+
+      if (this.traverseSymlinkDirectories) {
+        args.push('--follow')
+      }
+
+      for (let ignoredName of this.ignoredNames) {
+        args.push('-g', '!' + ignoredName.pattern)
+      }
+
+      let output = ''
+      const result = childProcess.spawn(realRgPath, args, {cwd: this.rootPath})
+
+      result.stdout.on('data', chunk => {
+        const files = (output + chunk).split('\n')
+        output = files.pop()
+
+        for (const file of files) {
+          this.pathLoaded(path.join(this.rootPath, file))
+        }
+      })
+      result.on('close', () => {
+        this.flushPaths()
+        resolve()
+      })
+    })
+  }
+
   isIgnored (loadedPath) {
     const relativePath = path.relative(this.rootPath, loadedPath)
     if (this.repo && this.repo.isPathIgnored(relativePath)) {
@@ -54,7 +105,7 @@ class PathLoader {
     if (this.paths.length === PathsChunkSize) {
       this.flushPaths()
     }
-    done()
+    done && done()
   }
 
   flushPaths () {
@@ -114,7 +165,7 @@ class PathLoader {
   }
 }
 
-module.exports = function (rootPaths, followSymlinks, ignoreVcsIgnores, ignores = []) {
+module.exports = function (rootPaths, followSymlinks, ignoreVcsIgnores, ignores, useRipGrep) {
   const ignoredNames = []
   for (let ignore of ignores) {
     if (ignore) {
@@ -126,14 +177,16 @@ module.exports = function (rootPaths, followSymlinks, ignoreVcsIgnores, ignores
     }
   }
 
-  async.each(
+  async.eachLimit(
     rootPaths,
+    MaxConcurrentCrawls,
     (rootPath, next) =>
       new PathLoader(
         rootPath,
         ignoreVcsIgnores,
         followSymlinks,
-        ignoredNames
+        ignoredNames,
+        useRipGrep
       ).load(next)
     ,
     this.async()

lib/path-loader.js

@@ -10,13 +10,16 @@ module.exports = {
     ignoredNames = ignoredNames.concat(atom.config.get('core.ignoredNames') || [])
     const ignoreVcsIgnores = atom.config.get('core.excludeVcsIgnoredPaths')
     const projectPaths = atom.project.getPaths().map((path) => fs.realpathSync(path))
+    const useRipGrep = atom.config.get('fuzzy-finder.useRipGrep')
 
     const task = Task.once(
       taskPath,
       projectPaths,
       followSymlinks,
       ignoreVcsIgnores,
-      ignoredNames, () => callback(results)
+      ignoredNames,
+      useRipGrep,
+      () => callback(results)
     )
 
     task.on('load-paths:paths-found',

package.json

@@ -15,9 +15,11 @@
     "minimatch": "~3.0.3",
     "temp": "~0.8.1",
     "underscore-plus": "^1.0.0",
+    "vscode-ripgrep": "^1.2.5",
     "wrench": "^1.5"
   },
   "devDependencies": {
+    "sinon": "1.17.4",
     "standard": "^10.0.3"
   },
   "standard": {
@@ -75,6 +77,11 @@
       "default": true,
       "description": "Use an alternative scoring approach which prefers run of consecutive characters, acronyms and start of words. (Experimental)"
     },
+    "useRipGrep": {
+      "type": "boolean",
+      "default": false,
+      "description": "Use the experimental `ripgrep` crawler. This will substantially speed up the indexing process on large projects."
+    },
     "prefillFromSelection": {
       "type": "boolean",
       "default": false,