Limit WalkDir to the root directories referenced by the configuration (#4850)

Summary:
When not using Watchman the Relay Compiler will iterate over all files under `root_dir`, which can possibly include the entire repository. This is unnecessary work and might slow down large repositories.

The watchman implementation only listens to files under the roots defined by the config, but this is not implemented for the directory walking implementation in `WalkDirFileSource`.  This change moves the `get_all_roots()` functionality from `WatchmanFileSource` to `Config` and shares it with `WalkDirFileSource` to make it only iterate over the folders configured as targets in Relay.

This fixes #4849

Pull Request resolved: #4850

Reviewed By: tyao1

Differential Revision: D66390485

Pulled By: captbaritone

fbshipit-source-id: cdd6daa4368419dfcc012e50f8be45440f02cdcf

Author: sverrejoh

compiler/crates/relay-compiler/src/config.rs

@@ -612,6 +612,138 @@ impl Config {
             }
         }
     }
+
+    /// Compute all root paths that we need to query. All files relevant to the
+    /// compiler should be in these directories.
+    pub fn get_all_roots(&self) -> Vec<PathBuf> {
+        let source_roots = self.get_source_roots();
+        let extra_sources_roots = self.get_generated_sources_roots();
+        let output_roots = self.get_output_dir_paths();
+        let extension_roots = self.get_extension_roots();
+        let schema_file_roots = self.get_schema_file_roots();
+        let schema_dir_roots = self.get_schema_dir_paths();
+
+        unify_roots(
+            source_roots
+                .into_iter()
+                .chain(extra_sources_roots)
+                .chain(output_roots)
+                .chain(extension_roots)
+                .chain(schema_file_roots)
+                .chain(schema_dir_roots)
+                .collect(),
+        )
+    }
+
+    /// Returns all root directories of JS source files for the config.
+    pub fn get_source_roots(&self) -> Vec<PathBuf> {
+        self.sources.keys().cloned().collect()
+    }
+
+    /// Returns all root directories of JS source files for the config.
+    pub fn get_generated_sources_roots(&self) -> Vec<PathBuf> {
+        self.generated_sources.keys().cloned().collect()
+    }
+
+    /// Returns all root directories of GraphQL schema extension files for the
+    /// config.
+    pub fn get_extension_roots(&self) -> Vec<PathBuf> {
+        self.projects
+            .values()
+            .flat_map(|project_config| project_config.schema_extensions.iter().cloned())
+            .collect()
+    }
+
+    /// Returns all output and extra artifact output directories for the config.
+    pub fn get_output_dir_paths(&self) -> Vec<PathBuf> {
+        let output_dirs = self
+            .projects
+            .values()
+            .filter_map(|project_config| project_config.output.clone());
+
+        let extra_artifact_output_dirs = self
+            .projects
+            .values()
+            .filter_map(|project_config| project_config.extra_artifacts_output.clone());
+
+        output_dirs.chain(extra_artifact_output_dirs).collect()
+    }
+
+    /// Returns all paths that contain GraphQL schema files for the config.
+    pub fn get_schema_file_paths(&self) -> Vec<PathBuf> {
+        self.projects
+            .values()
+            .filter_map(|project_config| match &project_config.schema_location {
+                SchemaLocation::File(schema_file) => Some(schema_file.clone()),
+                SchemaLocation::Directory(_) => None,
+            })
+            .collect()
+    }
+
+    /// Returns all GraphQL schema directories for the config.
+    pub fn get_schema_dir_paths(&self) -> Vec<PathBuf> {
+        self.projects
+            .values()
+            .filter_map(|project_config| match &project_config.schema_location {
+                SchemaLocation::File(_) => None,
+                SchemaLocation::Directory(schema_dir) => Some(schema_dir.clone()),
+            })
+            .collect()
+    }
+
+    /// Returns root directories that contain GraphQL schema files.
+    pub fn get_schema_file_roots(&self) -> impl Iterator<Item = PathBuf> {
+        self.get_schema_file_paths().into_iter().map(|schema_path| {
+            schema_path
+                .parent()
+                .expect("A schema in the project root directory is currently not supported.")
+                .to_owned()
+        })
+    }
+}
+
+/// Finds the roots of a set of paths. This filters any paths
+/// that are a subdirectory of other paths in the input.
+fn unify_roots(mut paths: Vec<PathBuf>) -> Vec<PathBuf> {
+    paths.sort();
+    let mut roots = Vec::new();
+    for path in paths {
+        match roots.last() {
+            Some(prev) if path.starts_with(prev) => {
+                // skip
+            }
+            _ => {
+                roots.push(path);
+            }
+        }
+    }
+    roots
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_unify_roots() {
+        assert_eq!(unify_roots(vec![]).len(), 0);
+        assert_eq!(
+            unify_roots(vec!["Apps".into(), "Libraries".into()]),
+            &[PathBuf::from("Apps"), PathBuf::from("Libraries")]
+        );
+        assert_eq!(
+            unify_roots(vec!["Apps".into(), "Apps/Foo".into()]),
+            &[PathBuf::from("Apps")]
+        );
+        assert_eq!(
+            unify_roots(vec!["Apps/Foo".into(), "Apps".into()]),
+            &[PathBuf::from("Apps")]
+        );
+        assert_eq!(
+            unify_roots(vec!["Foo".into(), "Foo2".into()]),
+            &[PathBuf::from("Foo"), PathBuf::from("Foo2"),]
+        );
+    }
 }
 
 impl fmt::Debug for Config {

compiler/crates/relay-compiler/src/file_source/walk_dir_file_source.rs

@@ -77,8 +77,11 @@ impl WalkDirFileSource {
     }
 
     fn find_files(&self) -> Vec<File> {
-        WalkDir::new(self.config.root_dir.clone())
-            .into_iter()
+        self.config
+            .get_all_roots()
+            .iter()
+            .map(|source| self.config.root_dir.join(source))
+            .flat_map(WalkDir::new)
             .filter_map(|entry| {
                 let dir_entry = entry.ok()?;
                 let relative_path = dir_entry

compiler/crates/relay-compiler/src/file_source/watchman_file_source.rs

@@ -20,7 +20,6 @@ use relay_saved_state_loader::SavedStateLoader;
 pub use watchman_client::prelude::Clock;
 use watchman_client::prelude::*;
 
-use super::watchman_query_builder::get_all_roots;
 use super::watchman_query_builder::get_watchman_expr;
 use super::FileSourceResult;
 use crate::compiler_state::CompilerState;
@@ -345,7 +344,8 @@ async fn query_file_result(
             ..Default::default()
         }
     } else {
-        let query_roots = get_all_roots(config)
+        let query_roots = config
+            .get_all_roots()
             .into_iter()
             .map(PathGeneratorElement::RecursivePath)
             .collect();

compiler/crates/relay-compiler/src/file_source/watchman_query_builder.rs

@@ -14,7 +14,6 @@ use watchman_client::prelude::*;
 
 use crate::compiler_state::ProjectSet;
 use crate::config::Config;
-use crate::config::SchemaLocation;
 
 type FnvIndexMap<K, V> = IndexMap<K, V, FnvBuildHasher>;
 
@@ -45,28 +44,28 @@ pub fn get_watchman_expr(config: &Config) -> Expr {
         expressions.push(expr_any(generated_sources_dir_exprs));
     }
 
-    let output_dir_paths = get_output_dir_paths(config);
+    let output_dir_paths = config.get_output_dir_paths();
     if !output_dir_paths.is_empty() {
         let output_dir_expr = expr_files_in_dirs(output_dir_paths);
         expressions.push(output_dir_expr);
     }
 
-    let schema_file_paths = get_schema_file_paths(config);
+    let schema_file_paths = config.get_schema_file_paths();
     if !schema_file_paths.is_empty() {
         let schema_file_expr = Expr::Name(NameTerm {
-            paths: get_schema_file_paths(config),
+            paths: config.get_schema_file_paths(),
             wholename: true,
         });
         expressions.push(schema_file_expr);
     }
 
-    let schema_dir_paths = get_schema_dir_paths(config);
+    let schema_dir_paths = config.get_schema_dir_paths();
     if !schema_dir_paths.is_empty() {
         let schema_dir_expr = expr_graphql_files_in_dirs(schema_dir_paths);
         expressions.push(schema_dir_expr);
     }
 
-    let extension_roots = get_extension_roots(config);
+    let extension_roots = config.get_extension_roots();
     if !extension_roots.is_empty() {
         let extensions_expr = expr_graphql_files_in_dirs(extension_roots);
         expressions.push(extensions_expr);
@@ -122,98 +121,6 @@ fn get_project_file_ext_expr(typegen_language: TypegenLanguage) -> Expr {
     })
 }
 
-/// Compute all root paths that we need to query Watchman with. All files
-/// relevant to the compiler should be in these directories.
-pub fn get_all_roots(config: &Config) -> Vec<PathBuf> {
-    let source_roots = get_source_roots(config);
-    let extra_sources_roots = get_generated_sources_roots(config);
-    let output_roots = get_output_dir_paths(config);
-    let extension_roots = get_extension_roots(config);
-    let schema_file_roots = get_schema_file_roots(config);
-    let schema_dir_roots = get_schema_dir_paths(config);
-    unify_roots(
-        source_roots
-            .into_iter()
-            .chain(extra_sources_roots)
-            .chain(output_roots)
-            .chain(extension_roots)
-            .chain(schema_file_roots)
-            .chain(schema_dir_roots)
-            .collect(),
-    )
-}
-
-/// Returns all root directories of JS source files for the config.
-fn get_source_roots(config: &Config) -> Vec<PathBuf> {
-    config.sources.keys().cloned().collect()
-}
-
-/// Returns all root directories of JS source files for the config.
-fn get_generated_sources_roots(config: &Config) -> Vec<PathBuf> {
-    config.generated_sources.keys().cloned().collect()
-}
-
-/// Returns all root directories of GraphQL schema extension files for the
-/// config.
-fn get_extension_roots(config: &Config) -> Vec<PathBuf> {
-    config
-        .projects
-        .values()
-        .flat_map(|project_config| project_config.schema_extensions.iter().cloned())
-        .collect()
-}
-
-/// Returns all output and extra artifact output directories for the config.
-fn get_output_dir_paths(config: &Config) -> Vec<PathBuf> {
-    let output_dirs = config
-        .projects
-        .values()
-        .filter_map(|project_config| project_config.output.clone());
-
-    let extra_artifact_output_dirs = config
-        .projects
-        .values()
-        .filter_map(|project_config| project_config.extra_artifacts_output.clone());
-
-    output_dirs.chain(extra_artifact_output_dirs).collect()
-}
-
-/// Returns all paths that contain GraphQL schema files for the config.
-fn get_schema_file_paths(config: &Config) -> Vec<PathBuf> {
-    config
-        .projects
-        .values()
-        .filter_map(|project_config| match &project_config.schema_location {
-            SchemaLocation::File(schema_file) => Some(schema_file.clone()),
-            SchemaLocation::Directory(_) => None,
-        })
-        .collect()
-}
-
-/// Returns all GraphQL schema directories for the config.
-fn get_schema_dir_paths(config: &Config) -> Vec<PathBuf> {
-    config
-        .projects
-        .values()
-        .filter_map(|project_config| match &project_config.schema_location {
-            SchemaLocation::File(_) => None,
-            SchemaLocation::Directory(schema_dir) => Some(schema_dir.clone()),
-        })
-        .collect()
-}
-
-/// Returns root directories that contain GraphQL schema files.
-fn get_schema_file_roots(config: &Config) -> impl Iterator<Item = PathBuf> {
-    get_schema_file_paths(config)
-        .into_iter()
-        .map(|schema_path| {
-            schema_path
-                .parent()
-                .expect("A schema in the project root directory is currently not supported.")
-                .to_owned()
-        })
-}
-
 fn expr_files_in_dirs(roots: Vec<PathBuf>) -> Expr {
     expr_any(
         roots
@@ -244,47 +151,3 @@ fn expr_any(expressions: Vec<Expr>) -> Expr {
         _ => Expr::Any(expressions),
     }
 }
-
-/// Finds the roots of a set of paths. This filters any paths
-/// that are a subdirectory of other paths in the input.
-fn unify_roots(mut paths: Vec<PathBuf>) -> Vec<PathBuf> {
-    paths.sort();
-    let mut roots = Vec::new();
-    for path in paths {
-        match roots.last() {
-            Some(prev) if path.starts_with(prev) => {
-                // skip
-            }
-            _ => {
-                roots.push(path);
-            }
-        }
-    }
-    roots
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_unify_roots() {
-        assert_eq!(unify_roots(vec![]).len(), 0);
-        assert_eq!(
-            unify_roots(vec!["Apps".into(), "Libraries".into()]),
-            &[PathBuf::from("Apps"), PathBuf::from("Libraries")]
-        );
-        assert_eq!(
-            unify_roots(vec!["Apps".into(), "Apps/Foo".into()]),
-            &[PathBuf::from("Apps")]
-        );
-        assert_eq!(
-            unify_roots(vec!["Apps/Foo".into(), "Apps".into()]),
-            &[PathBuf::from("Apps")]
-        );
-        assert_eq!(
-            unify_roots(vec!["Foo".into(), "Foo2".into()]),
-            &[PathBuf::from("Foo"), PathBuf::from("Foo2"),]
-        );
-    }
-}