From 60855ff4c142e0c7f0b1ae54ac82660775ee0268 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 27 Aug 2022 23:41:57 +0100 Subject: [PATCH 01/14] Play with hasattr() support --- mypy/checker.py | 79 +++++++++++++++++++++++++- mypy/checkmember.py | 8 +++ mypy/erasetype.py | 12 +++- mypy/meet.py | 15 ++++- mypy/typeops.py | 2 +- mypy/types.py | 16 +++++- test-data/unit/check-isinstance.test | 79 ++++++++++++++++++++++++++ test-data/unit/fixtures/isinstance.pyi | 1 + 8 files changed, 202 insertions(+), 10 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index be07ad69d681..9008b0f3b742 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -44,7 +44,7 @@ from mypy.join import join_types from mypy.literals import Key, literal, literal_hash from mypy.maptype import map_instance_to_supertype -from mypy.meet import is_overlapping_erased_types, is_overlapping_types +from mypy.meet import is_overlapping_erased_types, is_overlapping_types, meet_types from mypy.message_registry import ErrorMessage from mypy.messages import ( SUGGESTED_TEST_FIXTURES, @@ -114,6 +114,7 @@ ReturnStmt, StarExpr, Statement, + StrExpr, SymbolNode, SymbolTable, SymbolTableNode, @@ -175,6 +176,7 @@ AnyType, CallableType, DeletedType, + ExtraAttrs, FunctionLike, Instance, LiteralType, @@ -4697,7 +4699,7 @@ def _make_fake_typeinfo_and_full_name( return None curr_module.names[full_name] = SymbolTableNode(GDEF, info) - return Instance(info, []) + return Instance(info, [], extra_attrs=merge_extra_attrs(*instances)) def intersect_instance_callable(self, typ: Instance, callable_type: CallableType) -> Instance: """Creates a fake type that represents the intersection of an Instance and a CallableType. @@ -4724,7 +4726,7 @@ def intersect_instance_callable(self, typ: Instance, callable_type: CallableType cur_module.names[gen_name] = SymbolTableNode(GDEF, info) - return Instance(info, []) + return Instance(info, [], extra_attrs=typ.extra_attrs) def make_fake_callable(self, typ: Instance) -> Instance: """Produce a new type that makes type Callable with a generic callable type.""" @@ -5028,6 +5030,11 @@ def find_isinstance_check_helper(self, node: Expression) -> tuple[TypeMap, TypeM if literal(expr) == LITERAL_TYPE: vartype = self.lookup_type(expr) return self.conditional_callable_type_map(expr, vartype) + elif refers_to_fullname(node.callee, "builtins.hasattr"): + if len(node.args) != 2: # the error will be reported elsewhere + return {}, {} + if literal(expr) == LITERAL_TYPE and isinstance(node.args[1], StrExpr): + return self.hasattr_type_maps(expr, self.lookup_type(expr), node.args[1].value) elif isinstance(node.callee, RefExpr): if node.callee.type_guard is not None: # TODO: Follow keyword args or *args, **kwargs @@ -6207,6 +6214,52 @@ class Foo(Enum): and member_type.fallback.type == parent_type.type_object() ) + def hasattr_type_maps( + self, expr: Expression, source_type: Type, name: str + ) -> tuple[TypeMap, TypeMap]: + if self.has_valid_attribute(source_type, name): + return {expr: source_type}, None + p_source_type = get_proper_type(source_type) + if isinstance(p_source_type, Instance): + return { + expr: p_source_type.copy_with_extra_attr(name, AnyType(TypeOfAny.unannotated)) + }, {} + if isinstance(p_source_type, TupleType): + fallback = p_source_type.partial_fallback.copy_with_extra_attr( + name, AnyType(TypeOfAny.unannotated) + ) + return {expr: p_source_type.copy_modified(fallback=fallback)}, {} + if isinstance(p_source_type, CallableType): + fallback = p_source_type.fallback.copy_with_extra_attr( + name, AnyType(TypeOfAny.unannotated) + ) + return {expr: p_source_type.copy_modified(fallback=fallback)}, {} + if isinstance(p_source_type, TypeType) and isinstance(p_source_type.item, Instance): + return { + expr: TypeType.make_normalized( + p_source_type.item.copy_with_extra_attr(name, AnyType(TypeOfAny.unannotated)) + ) + }, {} + + if not isinstance(p_source_type, UnionType): + return {}, {} + return {}, {} + + def has_valid_attribute(self, typ: Type, name: str) -> bool: + with self.msg.filter_errors() as watcher: + analyze_member_access( + name, + typ, + TempNode(AnyType(TypeOfAny.special_form)), + False, + False, + False, + self.msg, + original_type=typ, + chk=self, + ) + return not watcher.has_new_errors() + class CollectArgTypes(TypeTraverserVisitor): """Collects the non-nested argument types in a set.""" @@ -7093,3 +7146,23 @@ def collapse_walrus(e: Expression) -> Expression: if isinstance(e, AssignmentExpr): return e.target return e + + +def merge_extra_attrs(l: Instance, r: Instance) -> ExtraAttrs | None: + if not l.extra_attrs and not r.extra_attrs: + return None + if l.extra_attrs: + l_attrs = l.extra_attrs.attrs.copy() + else: + l_attrs = {} + if r.extra_attrs: + r_attrs = r.extra_attrs.attrs.copy() + else: + r_attrs = {} + for name, typ in l_attrs.items(): + if name not in r_attrs: + r_attrs[name] = typ + else: + existing = r_attrs[name] + r_attrs[name] = meet_types(existing, typ) + return ExtraAttrs(r_attrs, set(), None) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index ea2544442531..e7bc05b362c0 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -540,6 +540,11 @@ def analyze_member_var_access( return AnyType(TypeOfAny.special_form) # Could not find the member. + + if itype.extra_attrs and name in itype.extra_attrs.attrs: + if not itype.extra_attrs.mod_name: + return itype.extra_attrs.attrs[name] + if mx.is_super: mx.msg.undefined_in_superclass(name, mx.context) return AnyType(TypeOfAny.from_error) @@ -858,6 +863,9 @@ def analyze_class_attribute_access( node = info.get(name) if not node: + if itype.extra_attrs and name in itype.extra_attrs.attrs: + if not itype.extra_attrs.mod_name: + return itype.extra_attrs.attrs[name] if info.fallback_to_any: return apply_class_attr_hook(mx, hook, AnyType(TypeOfAny.special_form)) return None diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 89c07186f44a..add727c04b57 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -181,18 +181,24 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: return t.copy_modified(args=[a.accept(self) for a in t.args]) -def remove_instance_last_known_values(t: Type) -> Type: - return t.accept(LastKnownValueEraser()) +def remove_instance_last_known_values(t: Type, *, keep_extra_attrs: bool = False) -> Type: + return t.accept(LastKnownValueEraser(keep_extra_attrs=keep_extra_attrs)) class LastKnownValueEraser(TypeTranslator): """Removes the Literal[...] type that may be associated with any Instance types.""" + def __init__(self, keep_extra_attrs: bool) -> None: + self.keep_extra_attrs = keep_extra_attrs + def visit_instance(self, t: Instance) -> Type: if not t.last_known_value and not t.args: return t - return t.copy_modified(args=[a.accept(self) for a in t.args], last_known_value=None) + new = t.copy_modified(args=[a.accept(self) for a in t.args], last_known_value=None) + if self.keep_extra_attrs: + new.extra_attrs = t.extra_attrs + return new def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Type aliases can't contain literal values, because they are diff --git a/mypy/meet.py b/mypy/meet.py index 1151b6ab460e..d8ea4410c547 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -3,16 +3,23 @@ from typing import Callable from mypy import join -from mypy.erasetype import erase_type +from mypy.erasetype import erase_type, remove_instance_last_known_values from mypy.maptype import map_instance_to_supertype from mypy.state import state -from mypy.subtypes import is_callable_compatible, is_equivalent, is_proper_subtype, is_subtype +from mypy.subtypes import ( + is_callable_compatible, + is_equivalent, + is_proper_subtype, + is_same_type, + is_subtype, +) from mypy.typeops import is_recursive_pair, make_simplified_union, tuple_fallback from mypy.types import ( AnyType, CallableType, DeletedType, ErasedType, + ExtraAttrs, FunctionLike, Instance, LiteralType, @@ -99,6 +106,10 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: if declared == narrowed: return original_declared + + if is_proper_subtype(narrowed, declared, ignore_promotions=True): + return remove_instance_last_known_values(original_narrowed) + if isinstance(declared, UnionType): return make_simplified_union( [narrow_declared_type(x, narrowed) for x in declared.relevant_items()] diff --git a/mypy/typeops.py b/mypy/typeops.py index 8c49b6c870ed..5fae296510e5 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -104,7 +104,7 @@ def tuple_fallback(typ: TupleType) -> Instance: raise NotImplementedError else: items.append(item) - return Instance(info, [join_type_list(items)]) + return Instance(info, [join_type_list(items)], extra_attrs=typ.partial_fallback.extra_attrs) def get_self_type(func: CallableType, default_self: Instance | TupleType) -> Type | None: diff --git a/mypy/types.py b/mypy/types.py index 9fb0ede51a68..72052a5676cf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1182,6 +1182,9 @@ def __eq__(self, other: object) -> bool: return NotImplemented return self.attrs == other.attrs and self.immutable == other.immutable + def copy(self) -> ExtraAttrs: + return ExtraAttrs(self.attrs.copy(), self.immutable.copy(), self.mod_name) + class Instance(ProperType): """An instance type of form C[T1, ..., Tn]. @@ -1224,6 +1227,7 @@ def __init__( column: int = -1, *, last_known_value: LiteralType | None = None, + extra_attrs: ExtraAttrs | None = None, ) -> None: super().__init__(line, column) self.type = typ @@ -1284,7 +1288,7 @@ def __init__( # Additional attributes defined per instance of this type. For example modules # have different attributes per instance of types.ModuleType. This is intended # to be "short lived", we don't serialize it, and even don't store as variable type. - self.extra_attrs: ExtraAttrs | None = None + self.extra_attrs = extra_attrs def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_instance(self) @@ -1354,6 +1358,16 @@ def copy_modified( new.can_be_false = self.can_be_false return new + def copy_with_extra_attr(self, name: str, typ: Type) -> Instance: + if self.extra_attrs: + existing_attrs = self.extra_attrs.copy() + else: + existing_attrs = ExtraAttrs({}, set(), None) + existing_attrs.attrs[name] = typ + new = self.copy_modified() + new.extra_attrs = existing_attrs + return new + def has_readable_member(self, name: str) -> bool: return self.type.has_readable_member(name) diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 997b22e2eb28..eb6001b6c913 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2729,3 +2729,82 @@ if type(x) is not C: reveal_type(x) # N: Revealed type is "__main__.D" else: reveal_type(x) # N: Revealed type is "__main__.C" + +[case testHasAttrExistingAttribute] +class C: + x: int +c: C +if hasattr(c, "x"): + reveal_type(c.x) # N: Revealed type is "builtins.int" +else: + reveal_type(c.x) # unreachable +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeInstance] +class B: ... +b: B +if hasattr(b, "x"): + reveal_type(b.x) # N: Revealed type is "Any" +else: + b.x # E: "B" has no attribute "x" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeFunction] + +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeClassObject] + +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeTypeType] + +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeTypeVar] + +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeChained] +class B: ... +b: B +if hasattr(b, "x"): + b.x +elif hasattr(b, "y"): + b.y +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeNested] +class A: ... +class B: ... + +x: A +if hasattr(x, "x"): + if isinstance(x, B): + x.x + +if hasattr(x, "x") and hasattr(x, "y"): + x.x + x.y + +if hasattr(x, "x"): + if hasattr(x, "y"): + x.x + x.y +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeUnion] +from typing import Union + +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeOuterUnion] +from typing import Union + +class A: ... +class B: ... +xu: Union[A, B] +if isinstance(xu, B): + if hasattr(xu, "x"): + xu.x +[builtins fixtures/isinstance.pyi] diff --git a/test-data/unit/fixtures/isinstance.pyi b/test-data/unit/fixtures/isinstance.pyi index 7f7cf501b5de..aa8bfce7fbe0 100644 --- a/test-data/unit/fixtures/isinstance.pyi +++ b/test-data/unit/fixtures/isinstance.pyi @@ -14,6 +14,7 @@ class function: pass def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass def issubclass(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass +def hasattr(x: object, name: str) -> bool: pass class int: def __add__(self, other: 'int') -> 'int': pass From b880f2ce30fbd9c3598cfd46a0a71d4a9885015a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 02:10:50 +0100 Subject: [PATCH 02/14] Make more progress --- mypy/checker.py | 77 ++++++++++++++++++---------- mypy/meet.py | 11 +--- mypy/server/objgraph.py | 6 +-- mypy/types.py | 20 +++++++- test-data/unit/check-isinstance.test | 59 +++++++++++++++++---- 5 files changed, 122 insertions(+), 51 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9008b0f3b742..d8bd67974c4b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -114,7 +114,6 @@ ReturnStmt, StarExpr, Statement, - StrExpr, SymbolNode, SymbolTable, SymbolTableNode, @@ -168,6 +167,7 @@ true_only, try_expanding_sum_type_to_union, try_getting_int_literals_from_type, + try_getting_str_literals, try_getting_str_literals_from_type, tuple_fallback, ) @@ -5033,8 +5033,9 @@ def find_isinstance_check_helper(self, node: Expression) -> tuple[TypeMap, TypeM elif refers_to_fullname(node.callee, "builtins.hasattr"): if len(node.args) != 2: # the error will be reported elsewhere return {}, {} - if literal(expr) == LITERAL_TYPE and isinstance(node.args[1], StrExpr): - return self.hasattr_type_maps(expr, self.lookup_type(expr), node.args[1].value) + attr = try_getting_str_literals(node.args[1], self.lookup_type(node.args[1])) + if literal(expr) == LITERAL_TYPE and attr and len(attr) == 1: + return self.hasattr_type_maps(expr, self.lookup_type(expr), attr[0]) elif isinstance(node.callee, RefExpr): if node.callee.type_guard is not None: # TODO: Follow keyword args or *args, **kwargs @@ -6214,37 +6215,61 @@ class Foo(Enum): and member_type.fallback.type == parent_type.type_object() ) + def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: + orig_typ = typ + typ = get_proper_type(typ) + any_type = AnyType(TypeOfAny.unannotated) + if isinstance(typ, Instance): + return typ.copy_with_extra_attr(name, any_type) + if isinstance(typ, TupleType): + fallback = typ.partial_fallback.copy_with_extra_attr(name, any_type) + return typ.copy_modified(fallback=fallback) + if isinstance(typ, CallableType): + fallback = typ.fallback.copy_with_extra_attr(name, any_type) + return typ.copy_modified(fallback=fallback) + if isinstance(typ, TypeType) and isinstance(typ.item, Instance): + return TypeType.make_normalized(self.add_any_attribute_to_type(typ.item, name)) + if isinstance(typ, TypeVarType): + return typ.copy_modified( + upper_bound=self.add_any_attribute_to_type(typ.upper_bound, name), + values=[self.add_any_attribute_to_type(v, name) for v in typ.values], + ) + if isinstance(typ, UnionType): + with_attr, without_attr = self.partition_union_by_attr(typ, name) + return make_simplified_union( + with_attr + [self.add_any_attribute_to_type(typ, name) for typ in without_attr] + ) + return orig_typ + def hasattr_type_maps( self, expr: Expression, source_type: Type, name: str ) -> tuple[TypeMap, TypeMap]: if self.has_valid_attribute(source_type, name): return {expr: source_type}, None - p_source_type = get_proper_type(source_type) - if isinstance(p_source_type, Instance): - return { - expr: p_source_type.copy_with_extra_attr(name, AnyType(TypeOfAny.unannotated)) - }, {} - if isinstance(p_source_type, TupleType): - fallback = p_source_type.partial_fallback.copy_with_extra_attr( - name, AnyType(TypeOfAny.unannotated) - ) - return {expr: p_source_type.copy_modified(fallback=fallback)}, {} - if isinstance(p_source_type, CallableType): - fallback = p_source_type.fallback.copy_with_extra_attr( - name, AnyType(TypeOfAny.unannotated) - ) - return {expr: p_source_type.copy_modified(fallback=fallback)}, {} - if isinstance(p_source_type, TypeType) and isinstance(p_source_type.item, Instance): - return { - expr: TypeType.make_normalized( - p_source_type.item.copy_with_extra_attr(name, AnyType(TypeOfAny.unannotated)) - ) - }, {} - if not isinstance(p_source_type, UnionType): - return {}, {} + source_type = get_proper_type(source_type) + if isinstance(source_type, UnionType): + _, without_attr = self.partition_union_by_attr(source_type, name) + yes_map = {expr: self.add_any_attribute_to_type(source_type, name)} + return yes_map, {expr: make_simplified_union(without_attr)} + + type_with_attr = self.add_any_attribute_to_type(source_type, name) + if type_with_attr != source_type: + return {expr: type_with_attr}, {} return {}, {} + def partition_union_by_attr( + self, source_type: UnionType, name: str + ) -> tuple[list[Type], list[Type]]: + with_attr = [] + without_attr = [] + for item in source_type.items: + if self.has_valid_attribute(item, name): + with_attr.append(item) + else: + without_attr.append(item) + return with_attr, without_attr + def has_valid_attribute(self, typ: Type, name: str) -> bool: with self.msg.filter_errors() as watcher: analyze_member_access( diff --git a/mypy/meet.py b/mypy/meet.py index d8ea4410c547..2182434fcb4a 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -6,20 +6,13 @@ from mypy.erasetype import erase_type, remove_instance_last_known_values from mypy.maptype import map_instance_to_supertype from mypy.state import state -from mypy.subtypes import ( - is_callable_compatible, - is_equivalent, - is_proper_subtype, - is_same_type, - is_subtype, -) +from mypy.subtypes import is_callable_compatible, is_equivalent, is_proper_subtype, is_subtype from mypy.typeops import is_recursive_pair, make_simplified_union, tuple_fallback from mypy.types import ( AnyType, CallableType, DeletedType, ErasedType, - ExtraAttrs, FunctionLike, Instance, LiteralType, @@ -108,7 +101,7 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: return original_declared if is_proper_subtype(narrowed, declared, ignore_promotions=True): - return remove_instance_last_known_values(original_narrowed) + return remove_instance_last_known_values(original_narrowed, keep_extra_attrs=True) if isinstance(declared, UnionType): return make_simplified_union( diff --git a/mypy/server/objgraph.py b/mypy/server/objgraph.py index f15d503f0f16..89a086b8a0ab 100644 --- a/mypy/server/objgraph.py +++ b/mypy/server/objgraph.py @@ -64,11 +64,11 @@ def get_edges(o: object) -> Iterator[tuple[object, object]]: # in closures and self pointers to other objects if hasattr(e, "__closure__"): - yield (s, "__closure__"), e.__closure__ # type: ignore + yield (s, "__closure__"), e.__closure__ if hasattr(e, "__self__"): - se = e.__self__ # type: ignore + se = e.__self__ if se is not o and se is not type(o) and hasattr(s, "__self__"): - yield s.__self__, se # type: ignore + yield s.__self__, se else: if not type(e) in TYPE_BLACKLIST: yield s, e diff --git a/mypy/types.py b/mypy/types.py index 72052a5676cf..c00d12c2ab1e 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -528,16 +528,30 @@ def new_unification_variable(old: TypeVarType) -> TypeVarType: old.column, ) + def copy_modified( + self, values: Bogus[list[Type]] = _dummy, upper_bound: Bogus[Type] = _dummy + ) -> TypeVarType: + return TypeVarType( + self.name, + self.fullname, + self.id, + self.values if values is _dummy else values, + self.upper_bound if upper_bound is _dummy else upper_bound, + self.variance, + self.line, + self.column, + ) + def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_var(self) def __hash__(self) -> int: - return hash(self.id) + return hash((self.id, self.upper_bound)) def __eq__(self, other: object) -> bool: if not isinstance(other, TypeVarType): return NotImplemented - return self.id == other.id + return self.id == other.id and self.upper_bound == other.upper_bound def serialize(self) -> JsonDict: assert not self.id.is_meta_var() @@ -1985,6 +1999,7 @@ def __hash__(self) -> int: tuple(self.arg_types), tuple(self.arg_names), tuple(self.arg_kinds), + self.fallback, ) ) @@ -1998,6 +2013,7 @@ def __eq__(self, other: object) -> bool: and self.name == other.name and self.is_type_obj() == other.is_type_obj() and self.is_ellipsis_args == other.is_ellipsis_args + and self.fallback == other.fallback ) else: return NotImplemented diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index eb6001b6c913..370f1448906f 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2750,28 +2750,44 @@ else: [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeFunction] - +def foo(x: int) -> None: ... +if hasattr(foo, "x"): + reveal_type(foo.x) # N: Revealed type is "Any" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeClassObject] - +class C: ... +if hasattr(C, "x"): + reveal_type(C.x) # N: Revealed type is "Any" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeTypeType] - +from typing import Type +class C: ... +c: Type[C] +if hasattr(c, "x"): + reveal_type(c.x) # N: Revealed type is "Any" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeTypeVar] +from typing import TypeVar +T = TypeVar("T") +def foo(x: T) -> T: + if hasattr(x, "x"): + reveal_type(x.x) # N: Revealed type is "Any" + return x + else: + return x [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeChained] class B: ... b: B if hasattr(b, "x"): - b.x + reveal_type(b.x) # N: Revealed type is "Any" elif hasattr(b, "y"): - b.y + reveal_type(b.y) # N: Revealed type is "Any" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeNested] @@ -2781,21 +2797,31 @@ class B: ... x: A if hasattr(x, "x"): if isinstance(x, B): - x.x + reveal_type(x.x) # N: Revealed type is "Any" if hasattr(x, "x") and hasattr(x, "y"): - x.x - x.y + reveal_type(x.x) # N: Revealed type is "Any" + reveal_type(x.y) # N: Revealed type is "Any" if hasattr(x, "x"): if hasattr(x, "y"): - x.x - x.y + reveal_type(x.x) # N: Revealed type is "Any" + reveal_type(x.y) # N: Revealed type is "Any" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeUnion] from typing import Union +class A: ... +class B: + x: int + +xu: Union[A, B] +if hasattr(xu, "x"): + reveal_type(xu) # N: Revealed type is "Union[__main__.B, __main__.A]" + reveal_type(xu.x) # N: Revealed type is "Union[builtins.int, Any]" +else: + reveal_type(xu) # N: Revealed type is "__main__.A" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeOuterUnion] @@ -2806,5 +2832,16 @@ class B: ... xu: Union[A, B] if isinstance(xu, B): if hasattr(xu, "x"): - xu.x + reveal_type(xu.x) # N: Revealed type is "Any" +[builtins fixtures/isinstance.pyi] + +[case testHasAttrMissingAttributeLiteral] +from typing import Final +class B: ... +b: B +ATTR: Final = "x" +if hasattr(b, ATTR): + reveal_type(b.x) # N: Revealed type is "Any" +else: + b.x # E: "B" has no attribute "x" [builtins fixtures/isinstance.pyi] From b407106b5b933cb7d7e32d3b1aa3da427e1468d6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 02:23:59 +0100 Subject: [PATCH 03/14] Fix type guard tests --- mypy/meet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mypy/meet.py b/mypy/meet.py index 2182434fcb4a..cd38dccac616 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -100,9 +100,6 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: if declared == narrowed: return original_declared - if is_proper_subtype(narrowed, declared, ignore_promotions=True): - return remove_instance_last_known_values(original_narrowed, keep_extra_attrs=True) - if isinstance(declared, UnionType): return make_simplified_union( [narrow_declared_type(x, narrowed) for x in declared.relevant_items()] @@ -118,6 +115,8 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: return make_simplified_union( [narrow_declared_type(declared, x) for x in narrowed.relevant_items()] ) + elif is_proper_subtype(narrowed, declared, ignore_promotions=True): + return remove_instance_last_known_values(original_narrowed, keep_extra_attrs=True) elif isinstance(narrowed, AnyType): return original_narrowed elif isinstance(narrowed, TypeVarType) and is_subtype(narrowed.upper_bound, declared): From 63cec166924daa59928bb977a687233470be015e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 12:49:09 +0100 Subject: [PATCH 04/14] Be more principled in some corner cases --- mypy/checker.py | 23 +------------------ mypy/erasetype.py | 12 +++------- mypy/meet.py | 16 +++++++++----- mypy/typeops.py | 33 ++++++++++++++++++++++++++-- mypy/types.py | 27 +++-------------------- test-data/unit/check-isinstance.test | 11 ++++++++-- 6 files changed, 58 insertions(+), 64 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index d8bd67974c4b..6cbdb79e4a98 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -176,7 +176,6 @@ AnyType, CallableType, DeletedType, - ExtraAttrs, FunctionLike, Instance, LiteralType, @@ -4699,7 +4698,7 @@ def _make_fake_typeinfo_and_full_name( return None curr_module.names[full_name] = SymbolTableNode(GDEF, info) - return Instance(info, [], extra_attrs=merge_extra_attrs(*instances)) + return Instance(info, [], extra_attrs=instances[0].extra_attrs or instances[1].extra_attrs) def intersect_instance_callable(self, typ: Instance, callable_type: CallableType) -> Instance: """Creates a fake type that represents the intersection of an Instance and a CallableType. @@ -7171,23 +7170,3 @@ def collapse_walrus(e: Expression) -> Expression: if isinstance(e, AssignmentExpr): return e.target return e - - -def merge_extra_attrs(l: Instance, r: Instance) -> ExtraAttrs | None: - if not l.extra_attrs and not r.extra_attrs: - return None - if l.extra_attrs: - l_attrs = l.extra_attrs.attrs.copy() - else: - l_attrs = {} - if r.extra_attrs: - r_attrs = r.extra_attrs.attrs.copy() - else: - r_attrs = {} - for name, typ in l_attrs.items(): - if name not in r_attrs: - r_attrs[name] = typ - else: - existing = r_attrs[name] - r_attrs[name] = meet_types(existing, typ) - return ExtraAttrs(r_attrs, set(), None) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index add727c04b57..89c07186f44a 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -181,24 +181,18 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: return t.copy_modified(args=[a.accept(self) for a in t.args]) -def remove_instance_last_known_values(t: Type, *, keep_extra_attrs: bool = False) -> Type: - return t.accept(LastKnownValueEraser(keep_extra_attrs=keep_extra_attrs)) +def remove_instance_last_known_values(t: Type) -> Type: + return t.accept(LastKnownValueEraser()) class LastKnownValueEraser(TypeTranslator): """Removes the Literal[...] type that may be associated with any Instance types.""" - def __init__(self, keep_extra_attrs: bool) -> None: - self.keep_extra_attrs = keep_extra_attrs - def visit_instance(self, t: Instance) -> Type: if not t.last_known_value and not t.args: return t - new = t.copy_modified(args=[a.accept(self) for a in t.args], last_known_value=None) - if self.keep_extra_attrs: - new.extra_attrs = t.extra_attrs - return new + return t.copy_modified(args=[a.accept(self) for a in t.args], last_known_value=None) def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Type aliases can't contain literal values, because they are diff --git a/mypy/meet.py b/mypy/meet.py index cd38dccac616..69d0d4b93f31 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -3,10 +3,10 @@ from typing import Callable from mypy import join -from mypy.erasetype import erase_type, remove_instance_last_known_values +from mypy.erasetype import erase_type from mypy.maptype import map_instance_to_supertype from mypy.state import state -from mypy.subtypes import is_callable_compatible, is_equivalent, is_proper_subtype, is_subtype +from mypy.subtypes import is_callable_compatible, is_equivalent, is_proper_subtype, is_subtype, is_same_type from mypy.typeops import is_recursive_pair, make_simplified_union, tuple_fallback from mypy.types import ( AnyType, @@ -61,11 +61,19 @@ def meet_types(s: Type, t: Type) -> ProperType: """Return the greatest lower bound of two types.""" if is_recursive_pair(s, t): # This case can trigger an infinite recursion, general support for this will be - # tricky so we use a trivial meet (like for protocols). + # tricky, so we use a trivial meet (like for protocols). return trivial_meet(s, t) s = get_proper_type(s) t = get_proper_type(t) + if isinstance(s, Instance) and isinstance(t, Instance) and is_same_type(s, t): + # Code in checker.py should merge any extra_items where possible, so we + # should have only one instance with extra_items here. We check this before + # the below subtype check, so that extra_attrs will not get erased. + if s.extra_attrs: + return s + return t + if not isinstance(s, UnboundType) and not isinstance(t, UnboundType): if is_proper_subtype(s, t, ignore_promotions=True): return s @@ -115,8 +123,6 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: return make_simplified_union( [narrow_declared_type(declared, x) for x in narrowed.relevant_items()] ) - elif is_proper_subtype(narrowed, declared, ignore_promotions=True): - return remove_instance_last_known_values(original_narrowed, keep_extra_attrs=True) elif isinstance(narrowed, AnyType): return original_narrowed elif isinstance(narrowed, TypeVarType) and is_subtype(narrowed.upper_bound, declared): diff --git a/mypy/typeops.py b/mypy/typeops.py index 5fae296510e5..906e126e20eb 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -56,7 +56,7 @@ UnpackType, flatten_nested_unions, get_proper_type, - get_proper_types, + get_proper_types, TypedDictType, ) from mypy.typevars import fill_typevars @@ -462,7 +462,18 @@ def make_simplified_union( ): simplified_set = try_contracting_literals_in_union(simplified_set) - return get_proper_type(UnionType.make_union(simplified_set, line, column)) + result = get_proper_type(UnionType.make_union(simplified_set, line, column)) + + # Step 4: At last, we erase any (inconsistent) extra attributes on instances. + extra_attrs_set = set() + for item in items: + item = try_getting_instance_fallback(item) + if item and item.extra_attrs: + extra_attrs_set.add(item.extra_attrs) + if len(extra_attrs_set) > 1 and isinstance(result, Instance): + result = result.copy_modified() + + return result def _remove_redundant_union_items(items: list[Type], keep_erased: bool) -> list[Type]: @@ -984,3 +995,21 @@ def separate_union_literals(t: UnionType) -> tuple[Sequence[LiteralType], Sequen union_items.append(item) return literal_items, union_items + + +def try_getting_instance_fallback(typ: Type) -> Instance | None: + """Returns the Instance fallback for this type if one exists or None.""" + typ = get_proper_type(typ) + if isinstance(typ, Instance): + return typ + elif isinstance(typ, TupleType): + return tuple_fallback(typ) + elif isinstance(typ, TypedDictType): + return typ.fallback + elif isinstance(typ, FunctionLike): + return typ.fallback + elif isinstance(typ, LiteralType): + return typ.fallback + elif isinstance(typ, TypeVarType): + return try_getting_instance_fallback(typ.upper_bound) + return None diff --git a/mypy/types.py b/mypy/types.py index c00d12c2ab1e..24134fd85849 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1205,30 +1205,9 @@ class Instance(ProperType): The list of type variables may be empty. - Several types has fallbacks to `Instance`. Why? - Because, for example `TupleTuple` is related to `builtins.tuple` instance. - And `FunctionLike` has `builtins.function` fallback. - This allows us to use types defined - in typeshed for our "special" and more precise types. - - We used to have this helper function to get a fallback from different types. - Note, that it might be incomplete, since it is not used and not updated. - It just illustrates the concept: - - def try_getting_instance_fallback(typ: ProperType) -> Optional[Instance]: - '''Returns the Instance fallback for this type if one exists or None.''' - if isinstance(typ, Instance): - return typ - elif isinstance(typ, TupleType): - return tuple_fallback(typ) - elif isinstance(typ, TypedDictType): - return typ.fallback - elif isinstance(typ, FunctionLike): - return typ.fallback - elif isinstance(typ, LiteralType): - return typ.fallback - return None - + Several types has fallbacks to `Instance`, because in Python everything is an object + and this concept is impossible to express without intersection types. We therefore use + fallbacks for all "non-special" (like UninhabitedType, ErasedType etc) types. """ __slots__ = ("type", "args", "invalid", "type_ref", "last_known_value", "_hash", "extra_attrs") diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 370f1448906f..72760756df33 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2807,6 +2807,10 @@ if hasattr(x, "x"): if hasattr(x, "y"): reveal_type(x.x) # N: Revealed type is "Any" reveal_type(x.y) # N: Revealed type is "Any" + +if hasattr(x, "x") or hasattr(x, "y"): + x.x # E: "A" has no attribute "x" + x.y # E: "A" has no attribute "y" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeUnion] @@ -2818,8 +2822,8 @@ class B: xu: Union[A, B] if hasattr(xu, "x"): - reveal_type(xu) # N: Revealed type is "Union[__main__.B, __main__.A]" - reveal_type(xu.x) # N: Revealed type is "Union[builtins.int, Any]" + reveal_type(xu) # N: Revealed type is "Union[__main__.A, __main__.B]" + reveal_type(xu.x) # N: Revealed type is "Union[Any, builtins.int]" else: reveal_type(xu) # N: Revealed type is "__main__.A" [builtins fixtures/isinstance.pyi] @@ -2833,6 +2837,9 @@ xu: Union[A, B] if isinstance(xu, B): if hasattr(xu, "x"): reveal_type(xu.x) # N: Revealed type is "Any" + +if isinstance(xu, B) and hasattr(xu, "x"): + reveal_type(xu.x) # N: Revealed type is "Any" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeLiteral] From 7b4eba2e73184e41a449ed25072d47cf0635c4ce Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 13:05:24 +0100 Subject: [PATCH 05/14] Fix lint/self-check --- mypy/checker.py | 2 +- mypy/meet.py | 8 +++++++- mypy/typeops.py | 9 +++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4eac1acbd95d..241b581cd3f5 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -44,7 +44,7 @@ from mypy.join import join_types from mypy.literals import Key, literal, literal_hash from mypy.maptype import map_instance_to_supertype -from mypy.meet import is_overlapping_erased_types, is_overlapping_types, meet_types +from mypy.meet import is_overlapping_erased_types, is_overlapping_types from mypy.message_registry import ErrorMessage from mypy.messages import ( SUGGESTED_TEST_FIXTURES, diff --git a/mypy/meet.py b/mypy/meet.py index 69d0d4b93f31..b98cd41e336a 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -6,7 +6,13 @@ from mypy.erasetype import erase_type from mypy.maptype import map_instance_to_supertype from mypy.state import state -from mypy.subtypes import is_callable_compatible, is_equivalent, is_proper_subtype, is_subtype, is_same_type +from mypy.subtypes import ( + is_callable_compatible, + is_equivalent, + is_proper_subtype, + is_same_type, + is_subtype, +) from mypy.typeops import is_recursive_pair, make_simplified_union, tuple_fallback from mypy.types import ( AnyType, diff --git a/mypy/typeops.py b/mypy/typeops.py index 5efde30e9fcc..3e7f9dbd1463 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -45,6 +45,7 @@ TupleType, Type, TypeAliasType, + TypedDictType, TypeOfAny, TypeQuery, TypeType, @@ -56,7 +57,7 @@ UnpackType, flatten_nested_unions, get_proper_type, - get_proper_types, TypedDictType, + get_proper_types, ) from mypy.typevars import fill_typevars @@ -467,9 +468,9 @@ def make_simplified_union( # Step 4: At last, we erase any (inconsistent) extra attributes on instances. extra_attrs_set = set() for item in items: - item = try_getting_instance_fallback(item) - if item and item.extra_attrs: - extra_attrs_set.add(item.extra_attrs) + instance = try_getting_instance_fallback(item) + if instance and instance.extra_attrs: + extra_attrs_set.add(instance.extra_attrs) fallback = try_getting_instance_fallback(result) if len(extra_attrs_set) > 1 and fallback: From 797255d23868545404cd59c1517b2ed8319bd1e7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 13:42:56 +0100 Subject: [PATCH 06/14] Some comments and fixes --- mypy/checker.py | 9 +++++++++ mypy/checkmember.py | 2 ++ mypy/meet.py | 10 +++++----- mypy/typeops.py | 2 +- mypy/types.py | 6 +++--- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 241b581cd3f5..0feba559ff9f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6219,6 +6219,7 @@ class Foo(Enum): ) def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: + """Inject an extra attribute with Any type using fallbacks.""" orig_typ = typ typ = get_proper_type(typ) any_type = AnyType(TypeOfAny.unannotated) @@ -6247,6 +6248,14 @@ def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: def hasattr_type_maps( self, expr: Expression, source_type: Type, name: str ) -> tuple[TypeMap, TypeMap]: + """Simple support for hasattr() checks. + + Essentially the logic is following: + * In the if branch, keep types that already has a valid attribute as is, + for other inject an attribute with `Any` type. + * In the else branch, remove types that already have a valid attribute, + while keeping the rest. + """ if self.has_valid_attribute(source_type, name): return {expr: source_type}, None diff --git a/mypy/checkmember.py b/mypy/checkmember.py index e7bc05b362c0..2b9ad8800948 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -542,6 +542,7 @@ def analyze_member_var_access( # Could not find the member. if itype.extra_attrs and name in itype.extra_attrs.attrs: + # For modules use direct symbol table lookup. if not itype.extra_attrs.mod_name: return itype.extra_attrs.attrs[name] @@ -864,6 +865,7 @@ def analyze_class_attribute_access( node = info.get(name) if not node: if itype.extra_attrs and name in itype.extra_attrs.attrs: + # For modules use direct symbol table lookup. if not itype.extra_attrs.mod_name: return itype.extra_attrs.attrs[name] if info.fallback_to_any: diff --git a/mypy/meet.py b/mypy/meet.py index b98cd41e336a..287820c5eba4 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -72,13 +72,14 @@ def meet_types(s: Type, t: Type) -> ProperType: s = get_proper_type(s) t = get_proper_type(t) - if isinstance(s, Instance) and isinstance(t, Instance) and is_same_type(s, t): + if isinstance(s, Instance) and isinstance(t, Instance) and s.type == t.type: # Code in checker.py should merge any extra_items where possible, so we # should have only one instance with extra_items here. We check this before # the below subtype check, so that extra_attrs will not get erased. - if s.extra_attrs: - return s - return t + if is_same_type(s, t) and (s.extra_attrs or t.extra_attrs): + if s.extra_attrs: + return s + return t if not isinstance(s, UnboundType) and not isinstance(t, UnboundType): if is_proper_subtype(s, t, ignore_promotions=True): @@ -113,7 +114,6 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: if declared == narrowed: return original_declared - if isinstance(declared, UnionType): return make_simplified_union( [narrow_declared_type(x, narrowed) for x in declared.relevant_items()] diff --git a/mypy/typeops.py b/mypy/typeops.py index 3e7f9dbd1463..3fc756ca4170 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -1006,7 +1006,7 @@ def try_getting_instance_fallback(typ: Type) -> Instance | None: if isinstance(typ, Instance): return typ elif isinstance(typ, TupleType): - return tuple_fallback(typ) + return typ.partial_fallback elif isinstance(typ, TypedDictType): return typ.fallback elif isinstance(typ, FunctionLike): diff --git a/mypy/types.py b/mypy/types.py index 24134fd85849..f3a864a79207 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1173,7 +1173,7 @@ class ExtraAttrs: """Summary of module attributes and types. This is used for instances of types.ModuleType, because they can have different - attributes per instance. + attributes per instance, and for type narrowing with hasattr() checks. """ def __init__( @@ -1205,7 +1205,7 @@ class Instance(ProperType): The list of type variables may be empty. - Several types has fallbacks to `Instance`, because in Python everything is an object + Several types have fallbacks to `Instance`, because in Python everything is an object and this concept is impossible to express without intersection types. We therefore use fallbacks for all "non-special" (like UninhabitedType, ErasedType etc) types. """ @@ -1280,7 +1280,7 @@ def __init__( # Additional attributes defined per instance of this type. For example modules # have different attributes per instance of types.ModuleType. This is intended - # to be "short lived", we don't serialize it, and even don't store as variable type. + # to be "short-lived", we don't serialize it, and even don't store as variable type. self.extra_attrs = extra_attrs def accept(self, visitor: TypeVisitor[T]) -> T: From 6b2002d3321dad430618c315b348d6e51278fef6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 14:05:43 +0100 Subject: [PATCH 07/14] Fix mypyc bug --- mypyc/irbuild/builder.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index cde12e2a0a75..d2442123acb7 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -45,7 +45,15 @@ UnaryExpr, Var, ) -from mypy.types import Instance, TupleType, Type, UninhabitedType, get_proper_type +from mypy.types import ( + AnyType, + Instance, + TupleType, + Type, + TypeOfAny, + UninhabitedType, + get_proper_type, +) from mypy.util import split_target from mypy.visitor import ExpressionVisitor, StatementVisitor from mypyc.common import SELF_NAME, TEMP_ATTR_NAME @@ -867,7 +875,11 @@ def get_dict_item_type(self, expr: Expression) -> RType: def _analyze_iterable_item_type(self, expr: Expression) -> Type: """Return the item type given by 'expr' in an iterable context.""" # This logic is copied from mypy's TypeChecker.analyze_iterable_item_type. - iterable = get_proper_type(self.types[expr]) + if expr not in self.types: + # Mypy thinks this is unreachable. + iterable = AnyType(TypeOfAny.from_error) + else: + iterable = get_proper_type(self.types[expr]) echk = self.graph[self.module_name].type_checker().expr_checker iterator = echk.check_method_call_by_name("__iter__", iterable, [], [], expr)[0] From 24161fb9876eb6155402446a9fc7b0c0d63f5b5e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 14:27:36 +0100 Subject: [PATCH 08/14] Tweaks/fixes --- mypy/checker.py | 4 +++- mypyc/irbuild/builder.py | 3 ++- test-data/unit/check-isinstance.test | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 0feba559ff9f..91d1acc3920b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6257,7 +6257,7 @@ def hasattr_type_maps( while keeping the rest. """ if self.has_valid_attribute(source_type, name): - return {expr: source_type}, None + return {expr: source_type}, {} source_type = get_proper_type(source_type) if isinstance(source_type, UnionType): @@ -6283,6 +6283,8 @@ def partition_union_by_attr( return with_attr, without_attr def has_valid_attribute(self, typ: Type, name: str) -> bool: + if isinstance(get_proper_type(typ), AnyType): + return False with self.msg.filter_errors() as watcher: analyze_member_access( name, diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index d2442123acb7..59bf0c6a75d7 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -48,6 +48,7 @@ from mypy.types import ( AnyType, Instance, + ProperType, TupleType, Type, TypeOfAny, @@ -877,7 +878,7 @@ def _analyze_iterable_item_type(self, expr: Expression) -> Type: # This logic is copied from mypy's TypeChecker.analyze_iterable_item_type. if expr not in self.types: # Mypy thinks this is unreachable. - iterable = AnyType(TypeOfAny.from_error) + iterable: ProperType = AnyType(TypeOfAny.from_error) else: iterable = get_proper_type(self.types[expr]) echk = self.graph[self.module_name].type_checker().expr_checker diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 72760756df33..c11a84df53bb 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2737,7 +2737,8 @@ c: C if hasattr(c, "x"): reveal_type(c.x) # N: Revealed type is "builtins.int" else: - reveal_type(c.x) # unreachable + # We don't mark this unreachable since people may check for deleted attributes + reveal_type(c.x) # N: Revealed type is "builtins.int" [builtins fixtures/isinstance.pyi] [case testHasAttrMissingAttributeInstance] From 6eb49156655f85c95a82b3d0b823f9c2bcf0b9e6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 17:30:37 +0100 Subject: [PATCH 09/14] Fix interference with deferred nodes --- mypy/checker.py | 2 ++ mypy/checkmember.py | 8 ++++++-- test-data/unit/check-isinstance.test | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 91d1acc3920b..aede7683b184 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6296,6 +6296,8 @@ def has_valid_attribute(self, typ: Type, name: str) -> bool: self.msg, original_type=typ, chk=self, + # This is not a real attribute lookup so don't mess with deferring nodes. + no_deferral=True, ) return not watcher.has_new_errors() diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 2b9ad8800948..25f22df2cd45 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -90,6 +90,7 @@ def __init__( chk: mypy.checker.TypeChecker, self_type: Type | None, module_symbol_table: SymbolTable | None = None, + no_deferral: bool = False, ) -> None: self.is_lvalue = is_lvalue self.is_super = is_super @@ -100,6 +101,7 @@ def __init__( self.msg = msg self.chk = chk self.module_symbol_table = module_symbol_table + self.no_deferral = no_deferral def named_type(self, name: str) -> Instance: return self.chk.named_type(name) @@ -124,6 +126,7 @@ def copy_modified( self.chk, self.self_type, self.module_symbol_table, + self.no_deferral, ) if messages is not None: mx.msg = messages @@ -149,6 +152,7 @@ def analyze_member_access( in_literal_context: bool = False, self_type: Type | None = None, module_symbol_table: SymbolTable | None = None, + no_deferral: bool = False, ) -> Type: """Return the type of attribute 'name' of 'typ'. @@ -183,6 +187,7 @@ def analyze_member_access( chk=chk, self_type=self_type, module_symbol_table=module_symbol_table, + no_deferral=no_deferral, ) result = _analyze_member_access(name, typ, mx, override_info) possible_literal = get_proper_type(result) @@ -540,7 +545,6 @@ def analyze_member_var_access( return AnyType(TypeOfAny.special_form) # Could not find the member. - if itype.extra_attrs and name in itype.extra_attrs.attrs: # For modules use direct symbol table lookup. if not itype.extra_attrs.mod_name: @@ -750,7 +754,7 @@ def analyze_var( else: result = expanded_signature else: - if not var.is_ready: + if not var.is_ready and not mx.no_deferral: mx.not_ready_callback(var.name, mx.context) # Implicit 'Any' type. result = AnyType(TypeOfAny.special_form) diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index c11a84df53bb..7e1f7b74c53f 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2853,3 +2853,17 @@ if hasattr(b, ATTR): else: b.x # E: "B" has no attribute "x" [builtins fixtures/isinstance.pyi] + +[case testHasAttrDeferred] +def foo() -> str: ... + +class Test: + def stream(self) -> None: + if hasattr(self, "_body"): + reveal_type(self._body) # E: Revealed type is "builtins.str" + + def body(self) -> str: + if not hasattr(self, "_body"): + self._body = foo() + return self._body +[builtins fixtures/isinstance.pyi] From 6b7c8bd221568556d47034241c8843a8545a765b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 17:32:03 +0100 Subject: [PATCH 10/14] Fix typo --- test-data/unit/check-isinstance.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 7e1f7b74c53f..1b16b4343e47 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2860,7 +2860,7 @@ def foo() -> str: ... class Test: def stream(self) -> None: if hasattr(self, "_body"): - reveal_type(self._body) # E: Revealed type is "builtins.str" + reveal_type(self._body) # N: Revealed type is "builtins.str" def body(self) -> str: if not hasattr(self, "_body"): From b9354e599d99264ae5892980cb17c96e59e3b61b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 28 Aug 2022 23:37:53 +0100 Subject: [PATCH 11/14] Add basic support for modules --- mypy/checker.py | 8 +++++++- mypy/checkexpr.py | 2 ++ mypy/meet.py | 7 ++++++- test-data/unit/check-isinstance.test | 14 ++++++++++++++ test-data/unit/fixtures/module.pyi | 1 + 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index aede7683b184..f6b82b0b99c2 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6224,7 +6224,13 @@ def add_any_attribute_to_type(self, typ: Type, name: str) -> Type: typ = get_proper_type(typ) any_type = AnyType(TypeOfAny.unannotated) if isinstance(typ, Instance): - return typ.copy_with_extra_attr(name, any_type) + result = typ.copy_with_extra_attr(name, any_type) + # For instances, we erase the possible module name, so that restrictions + # become anonymous types.ModuleType instances, allowing hasattr() to + # have effect on modules. + assert result.extra_attrs is not None + result.extra_attrs.mod_name = None + return result if isinstance(typ, TupleType): fallback = typ.partial_fallback.copy_with_extra_attr(name, any_type) return typ.copy_modified(fallback=fallback) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ad0436ada214..c2044c127a56 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -380,6 +380,8 @@ def module_type(self, node: MypyFile) -> Instance: module_attrs = {} immutable = set() for name, n in node.names.items(): + if not n.module_public: + continue if isinstance(n.node, Var) and n.node.is_final: immutable.add(name) typ = self.chk.determine_type_of_member(n) diff --git a/mypy/meet.py b/mypy/meet.py index 287820c5eba4..1da80741d70b 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -74,9 +74,14 @@ def meet_types(s: Type, t: Type) -> ProperType: if isinstance(s, Instance) and isinstance(t, Instance) and s.type == t.type: # Code in checker.py should merge any extra_items where possible, so we - # should have only one instance with extra_items here. We check this before + # should have only compatible extra_items here. We check this before # the below subtype check, so that extra_attrs will not get erased. if is_same_type(s, t) and (s.extra_attrs or t.extra_attrs): + if s.extra_attrs and t.extra_attrs: + if len(s.extra_attrs.attrs) > len(t.extra_attrs.attrs): + # Return the one that has more precise information. + return s + return t if s.extra_attrs: return s return t diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 1b16b4343e47..185aa112e325 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2867,3 +2867,17 @@ class Test: self._body = foo() return self._body [builtins fixtures/isinstance.pyi] + +[case testHasAttrModule] +import mod + +if hasattr(mod, "y"): + reveal_type(mod.y) # N: Revealed type is "Any" + reveal_type(mod.x) # N: Revealed type is "builtins.int" +else: + mod.y # E: Module has no attribute "y" + reveal_type(mod.x) # N: Revealed type is "builtins.int" + +[file mod.py] +x: int +[builtins fixtures/module.pyi] diff --git a/test-data/unit/fixtures/module.pyi b/test-data/unit/fixtures/module.pyi index 98e989e59440..47408befd5ce 100644 --- a/test-data/unit/fixtures/module.pyi +++ b/test-data/unit/fixtures/module.pyi @@ -20,3 +20,4 @@ class ellipsis: pass classmethod = object() staticmethod = object() property = object() +def hasattr(x: object, name: str) -> bool: pass From b63b67148844de0aae0d37ef947167ea67440be9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 29 Aug 2022 01:21:47 +0100 Subject: [PATCH 12/14] Add a mypyc test --- mypyc/test-data/run-generators.test | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index db658eea6504..0f2cbe152fc0 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -650,3 +650,15 @@ from testutil import run_generator yields, val = run_generator(finally_yield()) assert yields == ('x',) assert val == 'test', val + +[case testUnreachableComprehensionNoCrash] +from typing import List + +def list_comp() -> List[int]: + if True: + return [5] + return [i for i in [5]] + +[file driver.py] +from native import list_comp +assert list_comp() == [5] From e76d93baf2be8c719ceb3cc806bc6a16cd07e5cd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 29 Aug 2022 01:45:39 +0100 Subject: [PATCH 13/14] Fix merge --- mypy/types.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 742566deb0c5..fbbbb92a81d1 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -536,20 +536,6 @@ def copy_modified( self.column, ) - def copy_modified( - self, values: Bogus[list[Type]] = _dummy, upper_bound: Bogus[Type] = _dummy - ) -> TypeVarType: - return TypeVarType( - self.name, - self.fullname, - self.id, - self.values if values is _dummy else values, - self.upper_bound if upper_bound is _dummy else upper_bound, - self.variance, - self.line, - self.column, - ) - def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_type_var(self) From 73a1989d99af33639402240a24201f57f7d24865 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 29 Aug 2022 09:31:43 +0100 Subject: [PATCH 14/14] Add more test cases --- test-data/unit/check-isinstance.test | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 185aa112e325..c06802e69a69 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -2814,6 +2814,14 @@ if hasattr(x, "x") or hasattr(x, "y"): x.y # E: "A" has no attribute "y" [builtins fixtures/isinstance.pyi] +[case testHasAttrPreciseType] +class A: ... + +x: A +if hasattr(x, "a") and isinstance(x.a, int): + reveal_type(x.a) # N: Revealed type is "builtins.int" +[builtins fixtures/isinstance.pyi] + [case testHasAttrMissingAttributeUnion] from typing import Union @@ -2843,6 +2851,15 @@ if isinstance(xu, B) and hasattr(xu, "x"): reveal_type(xu.x) # N: Revealed type is "Any" [builtins fixtures/isinstance.pyi] +[case testHasAttrDoesntInterfereGetAttr] +class C: + def __getattr__(self, attr: str) -> str: ... + +c: C +if hasattr(c, "foo"): + reveal_type(c.foo) # N: Revealed type is "builtins.str" +[builtins fixtures/isinstance.pyi] + [case testHasAttrMissingAttributeLiteral] from typing import Final class B: ... @@ -2881,3 +2898,13 @@ else: [file mod.py] x: int [builtins fixtures/module.pyi] + +[case testHasAttrDoesntInterfereModuleGetAttr] +import mod + +if hasattr(mod, "y"): + reveal_type(mod.y) # N: Revealed type is "builtins.str" + +[file mod.py] +def __getattr__(attr: str) -> str: ... +[builtins fixtures/module.pyi]