improve files and exclude parameter handling

Author: joennlae

mypy/find_sources.py

@@ -44,7 +44,30 @@ def create_source_list(paths: Sequence[str], options: Options,
         else:
             mod = os.path.basename(path) if options.scripts_are_modules else None
             sources.append(BuildSource(path, mod, None))
-    return sources
+
+    filtered_sources: List[BuildSource] = \
+        list(filter(lambda source:
+                    source.path is None
+                    or (not matches_exclude(
+                        source.path, finder.exclude, finder.fscache, finder.verbosity >= 2)),
+                    sources))
+
+    unsupported_files: List[BuildSource] = \
+        list(filter(lambda source:
+                    source.path is not None
+                    and not source.path.endswith(PY_EXTENSIONS),
+             sources))
+
+    if unsupported_files:
+        unsupported_text = "Unsupported file type(s) where found in the source tree. "
+        unsupported_text += "This can lead to an error (only .py and .pyi are supported):"
+        print(unsupported_text)
+        for file in unsupported_files:
+            print(file.path)
+        print("Use --exclude or add exclude to the config file. More information is here: "
+        "https://mypy.readthedocs.io/en/stable/config_file.html?highlight=exclude#confval-exclude")
+
+    return filtered_sources
 
 
 def keyfunc(name: str) -> Tuple[bool, int, str]:

mypy/test/test_find_sources.py

@@ -3,6 +3,7 @@
 import shutil
 import tempfile
 import unittest
+import fnmatch
 from typing import List, Optional, Set, Tuple
 
 from mypy.find_sources import InvalidSourceList, SourceFinder, create_source_list
@@ -305,8 +306,9 @@ def test_find_sources_exclude(self) -> None:
             ("a2.b.c.d.e", "/pkg"),
             ("e", "/pkg/a1/b/c/d"),
         ]
-        assert find_sources(["/pkg/a1/b/f.py"], options, fscache) == [('f', '/pkg/a1/b')]
-        assert find_sources(["/pkg/a2/b/f.py"], options, fscache) == [('a2.b.f', '/pkg')]
+
+        assert find_sources(["/pkg/a1/b/f.py"], options, fscache) == []
+        assert find_sources(["/pkg/a2/b/f.py"], options, fscache) == []
 
         # directory name
         options.exclude = ["/a1/"]
@@ -377,3 +379,52 @@ def test_find_sources_exclude(self) -> None:
             }
             fscache = FakeFSCache(files)
             assert len(find_sources(["."], options, fscache)) == len(files)
+
+    def test_find_sources_exclude_e2e(self) -> None:
+        files_config = "/pkg/*, /src/test/"
+
+        files = {
+            "/pkg/sample.json",
+            "/pkg/test.json",
+            "/pkg/a1/__init__.py",
+            "/pkg/a1/f.py",
+            "/pkg/a1/v.py",
+            "/pkg/a2/__init__.py",
+            "/pkg/a2/b.py",
+            "/pkg/a2/a.py",
+            "/src/test/a.py",
+            "/src/test/b.py",
+            "/src/test/a/a.py",
+        }
+
+        def split_and_match_files(files: Set[str], paths: List[str]) -> List[str]:
+            # mock split_and_match_files_list config_parser.py
+            expanded_paths = []
+
+            for p in paths:
+                p = p.strip()
+                # glob uses fnmatch underneath
+                matching = fnmatch.filter(files, p)
+                print("PATH", p)
+                if matching:
+                    expanded_paths.extend(matching)
+                else:
+                    expanded_paths.append(p)
+            return expanded_paths
+
+        all_files = split_and_match_files(files, files_config.split(','))
+        del split_and_match_files
+        options = Options()
+        options.exclude = [r'(?x)(\.json$)']
+        fscache = FakeFSCache(files)
+
+        assert find_sources(all_files, options, fscache) == [
+            ('a', '/src/test/a'),
+            ('a1', '/pkg'),
+            ('a1.f', '/pkg'),
+            ('a1.v', '/pkg'),
+            ('a2', '/pkg'),
+            ('a2.a', '/pkg'),
+            ('a2.b', '/pkg'),
+            ('b', '/src/test')
+        ]

test-data/unit/cmdline.test

@@ -1130,6 +1130,9 @@ tabs.py:2: error: Incompatible return value type (got "None", expected "str")
 # coding: uft-8
 [out]
 mypy: stubgen does not support .pyd files: 'a.pyd'
+Unsupported file type(s) where found in the source tree. This can lead to an error (only .py and .pyi are supported):
+a.pyd
+Use --exclude or add exclude to the config file. More information is here: https://mypy.readthedocs.io/en/stable/config_file.html?highlight=exclude#confval-exclude
 == Return code: 2
 
 [case testDuplicateModules]