From 41d8a66661c492a8b78d26301503c9f48a00bec4 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 29 Mar 2017 16:05:18 -0700 Subject: [PATCH 1/5] Make TypeQuery more general, handling nonboolean queries. Instead of TypeQuery always returning a boolean and having the strategy be an enum, the strategy is now a Callable describing how to combine partial results, and the two default strategies are plain old funcitons. To preserve the short-circuiting behavior of the previous code, this PR uses an exception. This is a pure refactor that I am using in my experimentation regarding fixing https://github.com/python/mypy/issues/1551. It should result in exactly no change to current behavior. It's separable from the other things I'm experimenting with, so I'm filing it as a separate pull request now. It enables me to rewrite the code that pulls type variables out of types as a TypeQuery. Consider waiting to merge this PR until I have some code that uses it ready for review. Or merge it now, if you think it's a pleasant cleanup instead of an ugly complication. I'm of two minds on that particular question. --- mypy/checkexpr.py | 6 +-- mypy/constraints.py | 2 +- mypy/stats.py | 2 +- mypy/types.py | 108 ++++++++++++++++++++++---------------------- 4 files changed, 59 insertions(+), 59 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ba598f166a81..739d340296db 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2332,7 +2332,7 @@ def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> Callabl return c.copy_modified(ret_type=new_ret_type) -class ArgInferSecondPassQuery(types.TypeQuery): +class ArgInferSecondPassQuery(types.TypeQuery[bool]): """Query whether an argument type should be inferred in the second pass. The result is True if the type has a type variable in a callable return @@ -2346,7 +2346,7 @@ def visit_callable_type(self, t: CallableType) -> bool: return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) -class HasTypeVarQuery(types.TypeQuery): +class HasTypeVarQuery(types.TypeQuery[bool]): """Visitor for querying whether a type has a type variable component.""" def __init__(self) -> None: super().__init__(False, types.ANY_TYPE_STRATEGY) @@ -2359,7 +2359,7 @@ def has_erased_component(t: Type) -> bool: return t is not None and t.accept(HasErasedComponentsQuery()) -class HasErasedComponentsQuery(types.TypeQuery): +class HasErasedComponentsQuery(types.TypeQuery[bool]): """Visitor for querying whether a type has an erased component.""" def __init__(self) -> None: super().__init__(False, types.ANY_TYPE_STRATEGY) diff --git a/mypy/constraints.py b/mypy/constraints.py index d6e44bea857d..e1c5f47b8f99 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -250,7 +250,7 @@ def is_complete_type(typ: Type) -> bool: return typ.accept(CompleteTypeVisitor()) -class CompleteTypeVisitor(TypeQuery): +class CompleteTypeVisitor(TypeQuery[bool]): def __init__(self) -> None: super().__init__(default=True, strategy=ALL_TYPES_STRATEGY) diff --git a/mypy/stats.py b/mypy/stats.py index 2b809a6d6267..39c954bfac90 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -226,7 +226,7 @@ def is_imprecise(t: Type) -> bool: return t.accept(HasAnyQuery()) -class HasAnyQuery(TypeQuery): +class HasAnyQuery(TypeQuery[bool]): def __init__(self) -> None: super().__init__(False, ANY_TYPE_STRATEGY) diff --git a/mypy/types.py b/mypy/types.py index 9621c0f859fa..e5a236c8a630 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -5,7 +5,7 @@ from collections import OrderedDict from typing import ( Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union, Iterable, - NamedTuple, + NamedTuple, Callable, ) import mypy.nodes @@ -1486,112 +1486,112 @@ def keywords_str(self, a: Iterable[Tuple[str, Type]]) -> str: ]) -# These constants define the method used by TypeQuery to combine multiple -# query results, e.g. for tuple types. The strategy is not used for empty -# result lists; in that case the default value takes precedence. -ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. -ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. +# Combination strategies for boolean type queries +def ANY_TYPE_STRATEGY(current: bool, accumulated: bool) -> bool: + """True if any type's result is True""" + if accumulated: + raise ShortCircuitQuery() + return current -class TypeQuery(TypeVisitor[bool]): - """Visitor for performing simple boolean queries of types. +def ALL_TYPES_STRATEGY(current: bool, accumulated: bool) -> bool: + """True if all types' results are True""" + if not accumulated: + raise ShortCircuitQuery() + return current - This class allows defining the default value for leafs to simplify the - implementation of many queries. - """ - default = False # Default result - strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). +class ShortCircuitQuery(Exception): + pass - def __init__(self, default: bool, strategy: int) -> None: - """Construct a query visitor. - Use the given default result and strategy for combining - multiple results. The strategy must be either - ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. - """ +class TypeQuery(Generic[T], TypeVisitor[T]): + """Visitor for performing queries of types. + + default is used as the query result unless a method for that type is + overridden. + + strategy is used to combine a partial result with a result for a particular + type in a series of types. + + Common use cases involve a boolean query using ANY_TYPE_STRATEGY and a + default of False or ALL_TYPES_STRATEGY and a default of True. + """ + + def __init__(self, default: T, strategy: Callable[[T, T], T]) -> None: self.default = default self.strategy = strategy - def visit_unbound_type(self, t: UnboundType) -> bool: + def visit_unbound_type(self, t: UnboundType) -> T: return self.default - def visit_type_list(self, t: TypeList) -> bool: + def visit_type_list(self, t: TypeList) -> T: return self.default - def visit_error_type(self, t: ErrorType) -> bool: + def visit_error_type(self, t: ErrorType) -> T: return self.default - def visit_any(self, t: AnyType) -> bool: + def visit_any(self, t: AnyType) -> T: return self.default - def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + def visit_uninhabited_type(self, t: UninhabitedType) -> T: return self.default - def visit_none_type(self, t: NoneTyp) -> bool: + def visit_none_type(self, t: NoneTyp) -> T: return self.default - def visit_erased_type(self, t: ErasedType) -> bool: + def visit_erased_type(self, t: ErasedType) -> T: return self.default - def visit_deleted_type(self, t: DeletedType) -> bool: + def visit_deleted_type(self, t: DeletedType) -> T: return self.default - def visit_type_var(self, t: TypeVarType) -> bool: + def visit_type_var(self, t: TypeVarType) -> T: return self.default - def visit_partial_type(self, t: PartialType) -> bool: + def visit_partial_type(self, t: PartialType) -> T: return self.default - def visit_instance(self, t: Instance) -> bool: + def visit_instance(self, t: Instance) -> T: return self.query_types(t.args) - def visit_callable_type(self, t: CallableType) -> bool: + def visit_callable_type(self, t: CallableType) -> T: # FIX generics return self.query_types(t.arg_types + [t.ret_type]) - def visit_tuple_type(self, t: TupleType) -> bool: + def visit_tuple_type(self, t: TupleType) -> T: return self.query_types(t.items) - def visit_typeddict_type(self, t: TypedDictType) -> bool: + def visit_typeddict_type(self, t: TypedDictType) -> T: return self.query_types(t.items.values()) - def visit_star_type(self, t: StarType) -> bool: + def visit_star_type(self, t: StarType) -> T: return t.type.accept(self) - def visit_union_type(self, t: UnionType) -> bool: + def visit_union_type(self, t: UnionType) -> T: return self.query_types(t.items) - def visit_overloaded(self, t: Overloaded) -> bool: + def visit_overloaded(self, t: Overloaded) -> T: return self.query_types(t.items()) - def visit_type_type(self, t: TypeType) -> bool: + def visit_type_type(self, t: TypeType) -> T: return t.item.accept(self) - def query_types(self, types: Iterable[Type]) -> bool: + def query_types(self, types: Iterable[Type]) -> T: """Perform a query for a list of types. - Use the strategy constant to combine the results. + Use the strategy to combine the results. """ if not types: # Use default result for empty list. return self.default - if self.strategy == ANY_TYPE_STRATEGY: - # Return True if at least one component is true. - res = False + res = self.default + try: for t in types: - res = res or t.accept(self) - if res: - break - return res - else: - # Return True if all components are true. - res = True - for t in types: - res = res and t.accept(self) - if not res: - break - return res + res = self.strategy(t.accept(self), res) + except ShortCircuitQuery: + pass + return res def strip_type(typ: Type) -> Type: From 139ef6a48e80091d406fbad24fbcf868f371d031 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 29 Mar 2017 16:59:37 -0700 Subject: [PATCH 2/5] Remove redundant empty check --- mypy/types.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index e5a236c8a630..13b78a34802f 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1582,9 +1582,6 @@ def query_types(self, types: Iterable[Type]) -> T: Use the strategy to combine the results. """ - if not types: - # Use default result for empty list. - return self.default res = self.default try: for t in types: From e2bd5b494e3665cfb8193aa8327a0d187dbb71b7 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 11:24:45 -0700 Subject: [PATCH 3/5] Some types in the visitor were missing recursion --- mypy/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 13b78a34802f..c15720f4c155 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1523,10 +1523,10 @@ def __init__(self, default: T, strategy: Callable[[T, T], T]) -> None: self.strategy = strategy def visit_unbound_type(self, t: UnboundType) -> T: - return self.default + return self.query_types(t.args) def visit_type_list(self, t: TypeList) -> T: - return self.default + return self.query_types(t.items) def visit_error_type(self, t: ErrorType) -> T: return self.default @@ -1550,7 +1550,7 @@ def visit_type_var(self, t: TypeVarType) -> T: return self.default def visit_partial_type(self, t: PartialType) -> T: - return self.default + return self.query_types(t.inner_types) def visit_instance(self, t: Instance) -> T: return self.query_types(t.args) From 9718f357712603c99ab697eb14a8ae7c13c486c1 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 11:45:00 -0700 Subject: [PATCH 4/5] Add ellipsis type handler to TypeQuery --- mypy/types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy/types.py b/mypy/types.py index c15720f4c155..219b4ffc1a70 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1577,6 +1577,9 @@ def visit_overloaded(self, t: Overloaded) -> T: def visit_type_type(self, t: TypeType) -> T: return t.item.accept(self) + def visit_ellipsis_type(self, t: EllipsisType) -> T: + return self.default + def query_types(self, types: Iterable[Type]) -> T: """Perform a query for a list of types. From 3d1595d2d353e9c3ca8ee47fbb5bf550371b9430 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Tue, 11 Apr 2017 19:10:14 -0700 Subject: [PATCH 5/5] Use `any` and `all` instead of fancy constants; combine an iterator at a time --- mypy/checkexpr.py | 6 ++--- mypy/constraints.py | 5 ++--- mypy/stats.py | 5 ++--- mypy/types.py | 53 ++++++++++----------------------------------- 4 files changed, 18 insertions(+), 51 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 0c73f973b402..9b430529033c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2383,7 +2383,7 @@ class ArgInferSecondPassQuery(types.TypeQuery[bool]): a type variable. """ def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) + super().__init__(any) def visit_callable_type(self, t: CallableType) -> bool: return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) @@ -2392,7 +2392,7 @@ def visit_callable_type(self, t: CallableType) -> bool: class HasTypeVarQuery(types.TypeQuery[bool]): """Visitor for querying whether a type has a type variable component.""" def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) + super().__init__(any) def visit_type_var(self, t: TypeVarType) -> bool: return True @@ -2405,7 +2405,7 @@ def has_erased_component(t: Type) -> bool: class HasErasedComponentsQuery(types.TypeQuery[bool]): """Visitor for querying whether a type has an erased component.""" def __init__(self) -> None: - super().__init__(False, types.ANY_TYPE_STRATEGY) + super().__init__(any) def visit_erased_type(self, t: ErasedType) -> bool: return True diff --git a/mypy/constraints.py b/mypy/constraints.py index e1c5f47b8f99..006e255fa9ea 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -6,8 +6,7 @@ from mypy.types import ( CallableType, Type, TypeVisitor, UnboundType, AnyType, NoneTyp, TypeVarType, Instance, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, - DeletedType, UninhabitedType, TypeType, TypeVarId, TypeQuery, ALL_TYPES_STRATEGY, - is_named_instance + DeletedType, UninhabitedType, TypeType, TypeVarId, TypeQuery, is_named_instance ) from mypy.maptype import map_instance_to_supertype from mypy import nodes @@ -252,7 +251,7 @@ def is_complete_type(typ: Type) -> bool: class CompleteTypeVisitor(TypeQuery[bool]): def __init__(self) -> None: - super().__init__(default=True, strategy=ALL_TYPES_STRATEGY) + super().__init__(all) def visit_none_type(self, t: NoneTyp) -> bool: return experiments.STRICT_OPTIONAL diff --git a/mypy/stats.py b/mypy/stats.py index 39c954bfac90..3739763069bd 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -7,8 +7,7 @@ from mypy.traverser import TraverserVisitor from mypy.types import ( - Type, AnyType, Instance, FunctionLike, TupleType, TypeVarType, - TypeQuery, ANY_TYPE_STRATEGY, CallableType + Type, AnyType, Instance, FunctionLike, TupleType, TypeVarType, TypeQuery, CallableType ) from mypy import nodes from mypy.nodes import ( @@ -228,7 +227,7 @@ def is_imprecise(t: Type) -> bool: class HasAnyQuery(TypeQuery[bool]): def __init__(self) -> None: - super().__init__(False, ANY_TYPE_STRATEGY) + super().__init__(any) def visit_any(self, t: AnyType) -> bool: return True diff --git a/mypy/types.py b/mypy/types.py index 632957ade441..e6239a944cdf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1500,40 +1500,15 @@ def keywords_str(self, a: Iterable[Tuple[str, Type]]) -> str: ]) -# Combination strategies for boolean type queries -def ANY_TYPE_STRATEGY(current: bool, accumulated: bool) -> bool: - """True if any type's result is True""" - if accumulated: - raise ShortCircuitQuery() - return current - - -def ALL_TYPES_STRATEGY(current: bool, accumulated: bool) -> bool: - """True if all types' results are True""" - if not accumulated: - raise ShortCircuitQuery() - return current - - -class ShortCircuitQuery(Exception): - pass - - class TypeQuery(Generic[T], TypeVisitor[T]): """Visitor for performing queries of types. - default is used as the query result unless a method for that type is - overridden. - - strategy is used to combine a partial result with a result for a particular - type in a series of types. + strategy is used to combine results for a series of types - Common use cases involve a boolean query using ANY_TYPE_STRATEGY and a - default of False or ALL_TYPES_STRATEGY and a default of True. + Common use cases involve a boolean query using `any` or `all` """ - def __init__(self, default: T, strategy: Callable[[T, T], T]) -> None: - self.default = default + def __init__(self, strategy: Callable[[Iterable[T]], T]) -> None: self.strategy = strategy def visit_unbound_type(self, t: UnboundType) -> T: @@ -1543,22 +1518,22 @@ def visit_type_list(self, t: TypeList) -> T: return self.query_types(t.items) def visit_any(self, t: AnyType) -> T: - return self.default + return self.strategy([]) def visit_uninhabited_type(self, t: UninhabitedType) -> T: - return self.default + return self.strategy([]) def visit_none_type(self, t: NoneTyp) -> T: - return self.default + return self.strategy([]) def visit_erased_type(self, t: ErasedType) -> T: - return self.default + return self.strategy([]) def visit_deleted_type(self, t: DeletedType) -> T: - return self.default + return self.strategy([]) def visit_type_var(self, t: TypeVarType) -> T: - return self.default + return self.strategy([]) def visit_partial_type(self, t: PartialType) -> T: return self.query_types(t.inner_types) @@ -1589,20 +1564,14 @@ def visit_type_type(self, t: TypeType) -> T: return t.item.accept(self) def visit_ellipsis_type(self, t: EllipsisType) -> T: - return self.default + return self.strategy([]) def query_types(self, types: Iterable[Type]) -> T: """Perform a query for a list of types. Use the strategy to combine the results. """ - res = self.default - try: - for t in types: - res = self.strategy(t.accept(self), res) - except ShortCircuitQuery: - pass - return res + return self.strategy(t.accept(self) for t in types) def strip_type(typ: Type) -> Type: