Merge remote-tracking branch 'origin/main' into dcreager/nondeterminism

* origin/main:
  [ty] Improve `@override`, `@final` and Liskov checks in cases where there are multiple reachable definitions (#21767)
  [ty] Extend `invalid-explicit-override` to also cover properties decorated with `@override` that do not override anything (#21756)
  [ty] Enable LRU collection for parsed module (#21749)
  [ty] Support typevar-specialized dynamic types in generic type aliases (#21730)
  Add token based `parenthesized_ranges` implementation (#21738)
  [ty] Default-specialization of generic type aliases (#21765)
  [ty] Suppress false positives when `dataclasses.dataclass(...)(cls)` is called imperatively (#21729)
  [syntax-error] Default type parameter followed by non-default type parameter (#21657)
  new module for parsing ranged suppressions (#21441)
  [ty] `type[T]` is assignable to an inferable typevar (#21766)
  Fix syntax error false positives for `await` outside functions (#21763)
  [ty] Improve diagnostics for unsupported comparison operations (#21737)
  Move `Token`, `TokenKind` and `Tokens` to `ruff-python-ast` (#21760)
  [ty] Don't confuse multiple occurrences of `typing.Self` when binding bound methods (#21754)
  Use our org-wide Renovate preset (#21759)
  Delete `my-script.py` (#21751)
  [ty] Move `all_members`, and related types/routines, out of `ide_support.rs` (#21695)

Author: dcreager

.github/renovate.json5

@@ -2,12 +2,11 @@
   $schema: "https://docs.renovatebot.com/renovate-schema.json",
   dependencyDashboard: true,
   suppressNotifications: ["prEditedNotification"],
-  extends: ["config:recommended"],
+  extends: ["github>astral-sh/renovate-config"],
   labels: ["internal"],
   schedule: ["before 4am on Monday"],
   semanticCommits: "disabled",
   separateMajorMinor: false,
-  prHourlyLimit: 10,
   enabledManagers: ["github-actions", "pre-commit", "cargo", "pep621", "pip_requirements", "npm"],
   cargo: {
     // See https://docs.renovatebot.com/configuration-options/#rangestrategy
@@ -16,7 +15,7 @@
   pep621: {
     // The default for this package manager is to only search for `pyproject.toml` files
     // found at the repository root: https://docs.renovatebot.com/modules/manager/pep621/#file-matching
-    fileMatch: ["^(python|scripts)/.*pyproject\\.toml$"],
+    managerFilePatterns: ["^(python|scripts)/.*pyproject\\.toml$"],
   },
   pip_requirements: {
     // The default for this package manager is to run on all requirements.txt files:
@@ -34,7 +33,7 @@
   npm: {
     // The default for this package manager is to only search for `package.json` files
     // found at the repository root: https://docs.renovatebot.com/modules/manager/npm/#file-matching
-    fileMatch: ["^playground/.*package\\.json$"],
+    managerFilePatterns: ["^playground/.*package\\.json$"],
   },
   "pre-commit": {
     enabled: true,

Cargo.lock

@@ -6,7 +6,8 @@ use criterion::{
 use ruff_benchmark::{
     LARGE_DATASET, NUMPY_CTYPESLIB, NUMPY_GLOBALS, PYDANTIC_TYPES, TestCase, UNICODE_PYPINYIN,
 };
-use ruff_python_parser::{Mode, TokenKind, lexer};
+use ruff_python_ast::token::TokenKind;
+use ruff_python_parser::{Mode, lexer};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]

crates/ruff_benchmark/benches/lexer.rs

@@ -21,7 +21,11 @@ use crate::source::source_text;
 /// reflected in the changed AST offsets.
 /// The other reason is that Ruff's AST doesn't implement `Eq` which Salsa requires
 /// for determining if a query result is unchanged.
-#[salsa::tracked(returns(ref), no_eq, heap_size=ruff_memory_usage::heap_size)]
+///
+/// The LRU capacity of 200 was picked without any empirical evidence that it's optimal,
+/// instead it's a wild guess that it should be unlikely that incremental changes involve
+/// more than 200 modules. Parsed ASTs within the same revision are never evicted by Salsa.
+#[salsa::tracked(returns(ref), no_eq, heap_size=ruff_memory_usage::heap_size, lru=200)]
 pub fn parsed_module(db: &dyn Db, file: File) -> ParsedModule {
     let _span = tracing::trace_span!("parsed_module", ?file).entered();
 
@@ -92,14 +96,9 @@ impl ParsedModule {
         self.inner.store(None);
     }
 
-    /// Returns the pointer address of this [`ParsedModule`].
-    ///
-    /// The pointer uniquely identifies the module within the current Salsa revision,
-    /// regardless of whether particular [`ParsedModuleRef`] instances are garbage collected.
-    pub fn addr(&self) -> usize {
-        // Note that the outer `Arc` in `inner` is stable across garbage collection, while the inner
-        // `Arc` within the `ArcSwap` may change.
-        Arc::as_ptr(&self.inner).addr()
+    /// Returns the file to which this module belongs.
+    pub fn file(&self) -> File {
+        self.file
     }
 }
 

crates/ruff_db/src/parsed.rs

@@ -35,6 +35,7 @@ anyhow = { workspace = true }
 bitflags = { workspace = true }
 clap = { workspace = true, features = ["derive", "string"], optional = true }
 colored = { workspace = true }
+compact_str = { workspace = true }
 fern = { workspace = true }
 glob = { workspace = true }
 globset = { workspace = true }

crates/ruff_linter/Cargo.toml

@@ -17,3 +17,24 @@ def _():
 
     # Valid yield scope
     yield 3
+
+
+# await is valid in any generator, sync or async
+(await cor async for cor in f())  # ok
+(await cor for cor in f())  # ok
+
+# but not in comprehensions
+[await cor async for cor in f()]  # F704
+{await cor async for cor in f()}  # F704
+{await cor: 1 async for cor in f()}  # F704
+[await cor for cor in f()]  # F704
+{await cor for cor in f()}  # F704
+{await cor: 1 for cor in f()}  # F704
+
+# or in the iterator of an async generator, which is evaluated in the parent
+# scope
+(cor async for cor in await f())  # F704
+(await cor async for cor in [await c for c in f()])  # F704
+
+# this is also okay because the comprehension is within the generator scope
+([await c for c in cor] async for cor in f())  # ok

crates/ruff_linter/resources/test/fixtures/pyflakes/F704.py

@@ -3,3 +3,5 @@ def func():
 
 # Top-level await
 await 1
+
+([await c for c in cor] async for cor in func())  # ok

crates/ruff_linter/resources/test/fixtures/syntax_errors/await_outside_async_function.py

@@ -35,6 +35,7 @@ use ruff_python_ast::helpers::{collect_import_from_member, is_docstring_stmt, to
 use ruff_python_ast::identifier::Identifier;
 use ruff_python_ast::name::QualifiedName;
 use ruff_python_ast::str::Quote;
+use ruff_python_ast::token::Tokens;
 use ruff_python_ast::visitor::{Visitor, walk_except_handler, walk_pattern};
 use ruff_python_ast::{
     self as ast, AnyParameterRef, ArgOrKeyword, Comprehension, ElifElseClause, ExceptHandler, Expr,
@@ -48,7 +49,7 @@ use ruff_python_parser::semantic_errors::{
     SemanticSyntaxChecker, SemanticSyntaxContext, SemanticSyntaxError, SemanticSyntaxErrorKind,
 };
 use ruff_python_parser::typing::{AnnotationKind, ParsedAnnotation, parse_type_annotation};
-use ruff_python_parser::{ParseError, Parsed, Tokens};
+use ruff_python_parser::{ParseError, Parsed};
 use ruff_python_semantic::all::{DunderAllDefinition, DunderAllFlags};
 use ruff_python_semantic::analyze::{imports, typing};
 use ruff_python_semantic::{
@@ -746,6 +747,7 @@ impl SemanticSyntaxContext for Checker<'_> {
             | SemanticSyntaxErrorKind::LoadBeforeNonlocalDeclaration { .. }
             | SemanticSyntaxErrorKind::NonlocalAndGlobal(_)
             | SemanticSyntaxErrorKind::AnnotatedGlobal(_)
+            | SemanticSyntaxErrorKind::TypeParameterDefaultOrder(_)
             | SemanticSyntaxErrorKind::AnnotatedNonlocal(_) => {
                 self.semantic_errors.borrow_mut().push(error);
             }
@@ -779,6 +781,10 @@ impl SemanticSyntaxContext for Checker<'_> {
             match scope.kind {
                 ScopeKind::Class(_) => return false,
                 ScopeKind::Function(_) | ScopeKind::Lambda(_) => return true,
+                ScopeKind::Generator {
+                    kind: GeneratorKind::Generator,
+                    ..
+                } => return true,
                 ScopeKind::Generator { .. }
                 | ScopeKind::Module
                 | ScopeKind::Type
@@ -828,14 +834,19 @@ impl SemanticSyntaxContext for Checker<'_> {
         self.source_type.is_ipynb()
     }
 
-    fn in_generator_scope(&self) -> bool {
-        matches!(
-            &self.semantic.current_scope().kind,
-            ScopeKind::Generator {
-                kind: GeneratorKind::Generator,
-                ..
+    fn in_generator_context(&self) -> bool {
+        for scope in self.semantic.current_scopes() {
+            if matches!(
+                scope.kind,
+                ScopeKind::Generator {
+                    kind: GeneratorKind::Generator,
+                    ..
+                }
+            ) {
+                return true;
             }
-        )
+        }
+        false
     }
 
     fn in_loop_context(&self) -> bool {

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -1,6 +1,6 @@
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextRange};
 

crates/ruff_linter/src/checkers/logical_lines.rs

@@ -4,9 +4,9 @@ use std::path::Path;
 
 use ruff_notebook::CellOffsets;
 use ruff_python_ast::PySourceType;
+use ruff_python_ast::token::Tokens;
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::Tokens;
 
 use crate::Locator;
 use crate::directives::TodoComment;