[red-knot] Support custom typeshed Markdown tests (#15683)

## Summary

- Add feature to specify a custom typeshed from within Markdown-based
  tests
- Port "builtins" unit tests from `infer.rs` to Markdown tests, part of
  #13696

## Test Plan

- Tests for the custom typeshed feature
- New Markdown tests for deleted Rust unit tests

Author: sharkdp

crates/red_knot/tests/file_watching.rs

@@ -322,7 +322,7 @@ where
         .search_paths
         .extra_paths
         .iter()
-        .chain(program_settings.search_paths.typeshed.as_ref())
+        .chain(program_settings.search_paths.custom_typeshed.as_ref())
     {
         std::fs::create_dir_all(path.as_std_path())
             .with_context(|| format!("Failed to create search path `{path}`"))?;

crates/red_knot_project/src/metadata/options.rs

@@ -92,7 +92,7 @@ impl Options {
                 .map(|path| path.absolute(project_root, system))
                 .collect(),
             src_roots,
-            typeshed: typeshed.map(|path| path.absolute(project_root, system)),
+            custom_typeshed: typeshed.map(|path| path.absolute(project_root, system)),
             site_packages: python
                 .map(|venv_path| SitePackages::Derived {
                     venv_path: venv_path.absolute(project_root, system),

crates/red_knot_python_semantic/resources/mdtest/import/builtins.md

@@ -1,8 +1,70 @@
-# Importing builtin module
+# Builtins
+
+## Importing builtin module
+
+Builtin symbols can be explicitly imported:
 
 ```py
 import builtins
 
-x = builtins.chr
-reveal_type(x)  # revealed: Literal[chr]
+reveal_type(builtins.chr)  # revealed: Literal[chr]
+```
+
+## Implicit use of builtin
+
+Or used implicitly:
+
+```py
+reveal_type(chr)  # revealed: Literal[chr]
+reveal_type(str)  # revealed: Literal[str]
+```
+
+## Builtin symbol from custom typeshed
+
+If we specify a custom typeshed, we can use the builtin symbol from it, and no longer access the
+builtins from the "actual" vendored typeshed:
+
+```toml
+[environment]
+typeshed = "/typeshed"
+```
+
+```pyi path=/typeshed/stdlib/builtins.pyi
+class Custom: ...
+
+custom_builtin: Custom
+```
+
+```pyi path=/typeshed/stdlib/typing_extensions.pyi
+def reveal_type(obj, /): ...
+```
+
+```py
+reveal_type(custom_builtin)  # revealed: Custom
+
+# error: [unresolved-reference]
+reveal_type(str)  # revealed: Unknown
+```
+
+## Unknown builtin (later defined)
+
+`foo` has a type of `Unknown` in this example, as it relies on `bar` which has not been defined at
+that point:
+
+```toml
+[environment]
+typeshed = "/typeshed"
+```
+
+```pyi path=/typeshed/stdlib/builtins.pyi
+foo = bar
+bar = 1
+```
+
+```pyi path=/typeshed/stdlib/typing_extensions.pyi
+def reveal_type(obj, /): ...
+```
+
+```py
+reveal_type(foo)  # revealed: Unknown
 ```

crates/red_knot_python_semantic/resources/mdtest/mdtest_custom_typeshed.md

@@ -0,0 +1,95 @@
+# Custom typeshed
+
+The `environment.typeshed` configuration option can be used to specify a custom typeshed directory
+for Markdown-based tests. Custom typeshed stubs can then be placed in the specified directory using
+fenced code blocks with language `pyi`, and will be used instead of the vendored copy of typeshed.
+
+A fenced code block with language `text` can be used to provide a `stdlib/VERSIONS` file in the
+custom typeshed root. If no such file is created explicitly, it will be automatically created with
+entries enabling all specified `<typeshed-root>/stdlib` files for all supported Python versions.
+
+## Basic example (auto-generated `VERSIONS` file)
+
+First, we specify `/typeshed` as the custom typeshed directory:
+
+```toml
+[environment]
+typeshed = "/typeshed"
+```
+
+We can then place custom stub files in `/typeshed/stdlib`, for example:
+
+```pyi path=/typeshed/stdlib/builtins.pyi
+class BuiltinClass: ...
+
+builtin_symbol: BuiltinClass
+```
+
+```pyi path=/typeshed/stdlib/sys/__init__.pyi
+version = "my custom Python"
+```
+
+And finally write a normal Python code block that makes use of the custom stubs:
+
+```py
+b: BuiltinClass = builtin_symbol
+
+class OtherClass: ...
+
+o: OtherClass = builtin_symbol  # error: [invalid-assignment]
+
+# Make sure that 'sys' has a proper entry in the auto-generated 'VERSIONS' file
+import sys
+```
+
+## Custom `VERSIONS` file
+
+If we want to specify a custom `VERSIONS` file, we can do so by creating a fenced code block with
+language `text`. In the following test, we set the Python version to `3.10` and then make sure that
+we can *not* import `new_module` with a version requirement of `3.11-`:
+
+```toml
+[environment]
+python-version = "3.10"
+typeshed = "/typeshed"
+```
+
+```pyi path=/typeshed/stdlib/old_module.pyi
+class OldClass: ...
+```
+
+```pyi path=/typeshed/stdlib/new_module.pyi
+class NewClass: ...
+```
+
+```text path=/typeshed/stdlib/VERSIONS
+old_module: 3.0-
+new_module: 3.11-
+```
+
+```py
+from old_module import OldClass
+
+# error: [unresolved-import] "Cannot resolve import `new_module`"
+from new_module import NewClass
+```
+
+## Using `reveal_type` with a custom typeshed
+
+When providing a custom typeshed directory, basic things like `reveal_type` will stop working
+because we rely on being able to import it from `typing_extensions`. The actual definition of
+`reveal_type` in typeshed is slightly involved (depends on generics, `TypeVar`, etc.), but a very
+simple untyped definition is enough to make `reveal_type` work in tests:
+
+```toml
+[environment]
+typeshed = "/typeshed"
+```
+
+```pyi path=/typeshed/stdlib/typing_extensions.pyi
+def reveal_type(obj, /): ...
+```
+
+```py
+reveal_type(())  # revealed: tuple[()]
+```

crates/red_knot_python_semantic/src/db.rs

@@ -136,7 +136,7 @@ pub(crate) mod tests {
         /// Target Python platform
         python_platform: PythonPlatform,
         /// Path to a custom typeshed directory
-        custom_typeshed: Option<SystemPathBuf>,
+        typeshed: Option<SystemPathBuf>,
         /// Path and content pairs for files that should be present
         files: Vec<(&'a str, &'a str)>,
     }
@@ -146,7 +146,7 @@ pub(crate) mod tests {
             Self {
                 python_version: PythonVersion::default(),
                 python_platform: PythonPlatform::default(),
-                custom_typeshed: None,
+                typeshed: None,
                 files: vec![],
             }
         }
@@ -156,11 +156,6 @@ pub(crate) mod tests {
             self
         }
 
-        pub(crate) fn with_custom_typeshed(mut self, path: &str) -> Self {
-            self.custom_typeshed = Some(SystemPathBuf::from(path));
-            self
-        }
-
         pub(crate) fn with_file(mut self, path: &'a str, content: &'a str) -> Self {
             self.files.push((path, content));
             self
@@ -176,7 +171,7 @@ pub(crate) mod tests {
                 .context("Failed to write test files")?;
 
             let mut search_paths = SearchPathSettings::new(vec![src_root]);
-            search_paths.typeshed = self.custom_typeshed;
+            search_paths.custom_typeshed = self.typeshed;
 
             Program::from_settings(
                 &db,

crates/red_knot_python_semantic/src/module_resolver/resolver.rs

@@ -169,7 +169,7 @@ impl SearchPaths {
         let SearchPathSettings {
             extra_paths,
             src_roots,
-            typeshed,
+            custom_typeshed: typeshed,
             site_packages: site_packages_paths,
         } = settings;
 
@@ -1308,7 +1308,7 @@ mod tests {
                 search_paths: SearchPathSettings {
                     extra_paths: vec![],
                     src_roots: vec![src.clone()],
-                    typeshed: Some(custom_typeshed),
+                    custom_typeshed: Some(custom_typeshed),
                     site_packages: SitePackages::Known(vec![site_packages]),
                 },
             },
@@ -1814,7 +1814,7 @@ not_a_directory
                 search_paths: SearchPathSettings {
                     extra_paths: vec![],
                     src_roots: vec![SystemPathBuf::from("/src")],
-                    typeshed: None,
+                    custom_typeshed: None,
                     site_packages: SitePackages::Known(vec![
                         venv_site_packages,
                         system_site_packages,

crates/red_knot_python_semantic/src/module_resolver/testing.rs

@@ -73,7 +73,7 @@ pub(crate) struct UnspecifiedTypeshed;
 ///
 /// For tests checking that standard-library module resolution is working
 /// correctly, you should usually create a [`MockedTypeshed`] instance
-/// and pass it to the [`TestCaseBuilder::with_custom_typeshed`] method.
+/// and pass it to the [`TestCaseBuilder::with_mocked_typeshed`] method.
 /// If you need to check something that involves the vendored typeshed stubs
 /// we include as part of the binary, you can instead use the
 /// [`TestCaseBuilder::with_vendored_typeshed`] method.
@@ -238,7 +238,7 @@ impl TestCaseBuilder<MockedTypeshed> {
                 search_paths: SearchPathSettings {
                     extra_paths: vec![],
                     src_roots: vec![src.clone()],
-                    typeshed: Some(typeshed.clone()),
+                    custom_typeshed: Some(typeshed.clone()),
                     site_packages: SitePackages::Known(vec![site_packages.clone()]),
                 },
             },

crates/red_knot_python_semantic/src/program.rs

@@ -108,7 +108,7 @@ pub struct SearchPathSettings {
     /// Optional path to a "custom typeshed" directory on disk for us to use for standard-library types.
     /// If this is not provided, we will fallback to our vendored typeshed stubs for the stdlib,
     /// bundled as a zip file in the binary
-    pub typeshed: Option<SystemPathBuf>,
+    pub custom_typeshed: Option<SystemPathBuf>,
 
     /// The path to the user's `site-packages` directory, where third-party packages from ``PyPI`` are installed.
     pub site_packages: SitePackages,
@@ -119,7 +119,7 @@ impl SearchPathSettings {
         Self {
             src_roots,
             extra_paths: vec![],
-            typeshed: None,
+            custom_typeshed: None,
             site_packages: SitePackages::Known(vec![]),
         }
     }

crates/red_knot_python_semantic/src/types/infer.rs

@@ -6003,14 +6003,13 @@ fn perform_membership_test_comparison<'db>(
 
 #[cfg(test)]
 mod tests {
-    use crate::db::tests::{setup_db, TestDb, TestDbBuilder};
+    use crate::db::tests::{setup_db, TestDb};
     use crate::semantic_index::definition::Definition;
     use crate::semantic_index::symbol::FileScopeId;
     use crate::semantic_index::{global_scope, semantic_index, symbol_table, use_def_map};
     use crate::types::check_types;
     use crate::{HasType, SemanticModel};
     use ruff_db::files::{system_path_to_file, File};
-    use ruff_db::parsed::parsed_module;
     use ruff_db::system::DbWithTestSystem;
     use ruff_db::testing::assert_function_query_was_not_run;
 
@@ -6281,56 +6280,6 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn builtin_symbol_vendored_stdlib() -> anyhow::Result<()> {
-        let mut db = setup_db();
-
-        db.write_file("/src/a.py", "c = chr")?;
-
-        assert_public_type(&db, "/src/a.py", "c", "Literal[chr]");
-
-        Ok(())
-    }
-
-    #[test]
-    fn builtin_symbol_custom_stdlib() -> anyhow::Result<()> {
-        let db = TestDbBuilder::new()
-            .with_custom_typeshed("/typeshed")
-            .with_file("/src/a.py", "c = copyright")
-            .with_file(
-                "/typeshed/stdlib/builtins.pyi",
-                "def copyright() -> None: ...",
-            )
-            .with_file("/typeshed/stdlib/VERSIONS", "builtins: 3.8-")
-            .build()?;
-
-        assert_public_type(&db, "/src/a.py", "c", "Literal[copyright]");
-
-        Ok(())
-    }
-
-    #[test]
-    fn unknown_builtin_later_defined() -> anyhow::Result<()> {
-        let db = TestDbBuilder::new()
-            .with_custom_typeshed("/typeshed")
-            .with_file("/src/a.py", "x = foo")
-            .with_file("/typeshed/stdlib/builtins.pyi", "foo = bar; bar = 1")
-            .with_file("/typeshed/stdlib/VERSIONS", "builtins: 3.8-")
-            .build()?;
-
-        assert_public_type(&db, "/src/a.py", "x", "Unknown");
-
-        Ok(())
-    }
-
-    #[test]
-    fn str_builtin() -> anyhow::Result<()> {
-        let mut db = setup_db();
-        db.write_file("/src/a.py", "x = str")?;
-        assert_public_type(&db, "/src/a.py", "x", "Literal[str]");
-        Ok(())
-    }
-
     #[test]
     fn deferred_annotation_builtin() -> anyhow::Result<()> {
         let mut db = setup_db();