[ty] Implicit type aliases: Add support for Callable (#21496)

## Summary

Add support for `Callable` special forms in implicit type aliases.

## Typing conformance

Four new tests are passing

## Ecosystem impact

* All of the `invalid-type-form` errors are from libraries that use
`mypy_extensions` and do something like `Callable[[NamedArg("x", str)],
int]`.
* A handful of new false positives because we do not support generic
specializations of implicit type aliases, yet. But other
* Everything else looks like true positives or known limitations

## Test Plan

New Markdown tests.

Author: sharkdp

crates/ty_python_semantic/resources/mdtest/implicit_type_aliases.md

@@ -33,7 +33,7 @@ g(None)
 We also support unions in type aliases:
 
 ```py
-from typing_extensions import Any, Never, Literal, LiteralString, Tuple, Annotated, Optional, Union
+from typing_extensions import Any, Never, Literal, LiteralString, Tuple, Annotated, Optional, Union, Callable
 from ty_extensions import Unknown
 
 IntOrStr = int | str
@@ -68,6 +68,8 @@ IntOrOptional = int | Optional[str]
 OptionalOrInt = Optional[str] | int
 IntOrTypeOfStr = int | type[str]
 TypeOfStrOrInt = type[str] | int
+IntOrCallable = int | Callable[[str], bytes]
+CallableOrInt = Callable[[str], bytes] | int
 
 reveal_type(IntOrStr)  # revealed: types.UnionType
 reveal_type(IntOrStrOrBytes1)  # revealed: types.UnionType
@@ -101,6 +103,8 @@ reveal_type(IntOrOptional)  # revealed: types.UnionType
 reveal_type(OptionalOrInt)  # revealed: types.UnionType
 reveal_type(IntOrTypeOfStr)  # revealed: types.UnionType
 reveal_type(TypeOfStrOrInt)  # revealed: types.UnionType
+reveal_type(IntOrCallable)  # revealed: types.UnionType
+reveal_type(CallableOrInt)  # revealed: types.UnionType
 
 def _(
     int_or_str: IntOrStr,
@@ -135,6 +139,8 @@ def _(
     optional_or_int: OptionalOrInt,
     int_or_type_of_str: IntOrTypeOfStr,
     type_of_str_or_int: TypeOfStrOrInt,
+    int_or_callable: IntOrCallable,
+    callable_or_int: CallableOrInt,
 ):
     reveal_type(int_or_str)  # revealed: int | str
     reveal_type(int_or_str_or_bytes1)  # revealed: int | str | bytes
@@ -168,6 +174,8 @@ def _(
     reveal_type(optional_or_int)  # revealed: str | None | int
     reveal_type(int_or_type_of_str)  # revealed: int | type[str]
     reveal_type(type_of_str_or_int)  # revealed: type[str] | int
+    reveal_type(int_or_callable)  # revealed: int | ((str, /) -> bytes)
+    reveal_type(callable_or_int)  # revealed: ((str, /) -> bytes) | int
 ```
 
 If a type is unioned with itself in a value expression, the result is just that type. No
@@ -944,7 +952,60 @@ def _(
     reveal_type(dict_too_many_args)  # revealed: dict[Unknown, Unknown]
 ```
 
-## Stringified annotations?
+## `Callable[...]`
+
+We support implicit type aliases using `Callable[...]`:
+
+```py
+from typing import Callable, Union
+
+CallableNoArgs = Callable[[], None]
+BasicCallable = Callable[[int, str], bytes]
+GradualCallable = Callable[..., str]
+
+reveal_type(CallableNoArgs)  # revealed: GenericAlias
+reveal_type(BasicCallable)  # revealed: GenericAlias
+reveal_type(GradualCallable)  # revealed: GenericAlias
+
+def _(
+    callable_no_args: CallableNoArgs,
+    basic_callable: BasicCallable,
+    gradual_callable: GradualCallable,
+):
+    reveal_type(callable_no_args)  # revealed: () -> None
+    reveal_type(basic_callable)  # revealed: (int, str, /) -> bytes
+    reveal_type(gradual_callable)  # revealed: (...) -> str
+```
+
+Nested callables work as expected:
+
+```py
+TakesCallable = Callable[[Callable[[int], str]], bytes]
+ReturnsCallable = Callable[[int], Callable[[str], bytes]]
+
+def _(takes_callable: TakesCallable, returns_callable: ReturnsCallable):
+    reveal_type(takes_callable)  # revealed: ((int, /) -> str, /) -> bytes
+    reveal_type(returns_callable)  # revealed: (int, /) -> (str, /) -> bytes
+```
+
+Invalid uses result in diagnostics:
+
+```py
+# error: [invalid-type-form] "Special form `typing.Callable` expected exactly two arguments (parameter types and return type)"
+InvalidCallable1 = Callable[[int]]
+
+# error: [invalid-type-form] "The first argument to `Callable` must be either a list of types, ParamSpec, Concatenate, or `...`"
+InvalidCallable2 = Callable[int, str]
+
+reveal_type(InvalidCallable1)  # revealed: GenericAlias
+reveal_type(InvalidCallable2)  # revealed: GenericAlias
+
+def _(invalid_callable1: InvalidCallable1, invalid_callable2: InvalidCallable2):
+    reveal_type(invalid_callable1)  # revealed: (...) -> Unknown
+    reveal_type(invalid_callable2)  # revealed: (...) -> Unknown
+```
+
+## Stringified annotations
 
 From the [typing spec on type aliases](https://typing.python.org/en/latest/spec/aliases.html):
 
@@ -974,14 +1035,15 @@ We *do* support stringified annotations if they appear in a position where a typ
 syntactically expected:
 
 ```py
-from typing import Union, List, Dict, Annotated
+from typing import Union, List, Dict, Annotated, Callable
 
 ListOfInts1 = list["int"]
 ListOfInts2 = List["int"]
 StrOrStyle = Union[str, "Style"]
 SubclassOfStyle = type["Style"]
 DictStrToStyle = Dict[str, "Style"]
 AnnotatedStyle = Annotated["Style", "metadata"]
+CallableStyleToStyle = Callable[["Style"], "Style"]
 
 class Style: ...
 
@@ -992,13 +1054,15 @@ def _(
     subclass_of_style: SubclassOfStyle,
     dict_str_to_style: DictStrToStyle,
     annotated_style: AnnotatedStyle,
+    callable_style_to_style: CallableStyleToStyle,
 ):
     reveal_type(list_of_ints1)  # revealed: list[int]
     reveal_type(list_of_ints2)  # revealed: list[int]
     reveal_type(str_or_style)  # revealed: str | Style
     reveal_type(subclass_of_style)  # revealed: type[Style]
     reveal_type(dict_str_to_style)  # revealed: dict[str, Style]
     reveal_type(annotated_style)  # revealed: Style
+    reveal_type(callable_style_to_style)  # revealed: (Style, /) -> Style
 ```
 
 ## Recursive

crates/ty_python_semantic/src/types.rs

@@ -6747,6 +6747,7 @@ impl<'db> Type<'db> {
 
                     Ok(ty.inner(db).to_meta_type(db))
                 }
+                KnownInstanceType::Callable(callable) => Ok(Type::Callable(*callable)),
             },
 
             Type::SpecialForm(special_form) => match special_form {
@@ -7990,6 +7991,9 @@ pub enum KnownInstanceType<'db> {
     /// An instance of `typing.GenericAlias` representing a `type[...]` expression.
     TypeGenericAlias(InternedType<'db>),
 
+    /// An instance of `typing.GenericAlias` representing a `Callable[...]` expression.
+    Callable(CallableType<'db>),
+
     /// An identity callable created with `typing.NewType(name, base)`, which behaves like a
     /// subtype of `base` in type expressions. See the `struct NewType` payload for an example.
     NewType(NewType<'db>),
@@ -8029,6 +8033,9 @@ fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>(
         | KnownInstanceType::TypeGenericAlias(ty) => {
             visitor.visit_type(db, ty.inner(db));
         }
+        KnownInstanceType::Callable(callable) => {
+            visitor.visit_callable_type(db, callable);
+        }
         KnownInstanceType::NewType(newtype) => {
             if let ClassType::Generic(generic_alias) = newtype.base_class_type(db) {
                 visitor.visit_generic_alias_type(db, generic_alias);
@@ -8074,6 +8081,7 @@ impl<'db> KnownInstanceType<'db> {
             Self::Literal(ty) => Self::Literal(ty.normalized_impl(db, visitor)),
             Self::Annotated(ty) => Self::Annotated(ty.normalized_impl(db, visitor)),
             Self::TypeGenericAlias(ty) => Self::TypeGenericAlias(ty.normalized_impl(db, visitor)),
+            Self::Callable(callable) => Self::Callable(callable.normalized_impl(db, visitor)),
             Self::NewType(newtype) => Self::NewType(
                 newtype
                     .map_base_class_type(db, |class_type| class_type.normalized_impl(db, visitor)),
@@ -8096,9 +8104,10 @@ impl<'db> KnownInstanceType<'db> {
             Self::Field(_) => KnownClass::Field,
             Self::ConstraintSet(_) => KnownClass::ConstraintSet,
             Self::UnionType(_) => KnownClass::UnionType,
-            Self::Literal(_) | Self::Annotated(_) | Self::TypeGenericAlias(_) => {
-                KnownClass::GenericAlias
-            }
+            Self::Literal(_)
+            | Self::Annotated(_)
+            | Self::TypeGenericAlias(_)
+            | Self::Callable(_) => KnownClass::GenericAlias,
             Self::NewType(_) => KnownClass::NewType,
         }
     }
@@ -8184,7 +8193,9 @@ impl<'db> KnownInstanceType<'db> {
                     KnownInstanceType::Annotated(_) => {
                         f.write_str("<typing.Annotated special form>")
                     }
-                    KnownInstanceType::TypeGenericAlias(_) => f.write_str("GenericAlias"),
+                    KnownInstanceType::TypeGenericAlias(_) | KnownInstanceType::Callable(_) => {
+                        f.write_str("GenericAlias")
+                    }
                     KnownInstanceType::NewType(declaration) => {
                         write!(f, "<NewType pseudo-class '{}'>", declaration.name(self.db))
                     }

crates/ty_python_semantic/src/types/class_base.rs

@@ -174,6 +174,7 @@ impl<'db> ClassBase<'db> {
                 | KnownInstanceType::Deprecated(_)
                 | KnownInstanceType::Field(_)
                 | KnownInstanceType::ConstraintSet(_)
+                | KnownInstanceType::Callable(_)
                 | KnownInstanceType::UnionType(_)
                 | KnownInstanceType::Literal(_)
                 // A class inheriting from a newtype would make intuitive sense, but newtype

crates/ty_python_semantic/src/types/infer/builder.rs

@@ -9506,7 +9506,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
                     KnownInstanceType::UnionType(_)
                     | KnownInstanceType::Literal(_)
                     | KnownInstanceType::Annotated(_)
-                    | KnownInstanceType::TypeGenericAlias(_),
+                    | KnownInstanceType::TypeGenericAlias(_)
+                    | KnownInstanceType::Callable(_),
                 ),
                 Type::ClassLiteral(..)
                 | Type::SubclassOf(..)
@@ -9516,7 +9517,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
                     KnownInstanceType::UnionType(_)
                     | KnownInstanceType::Literal(_)
                     | KnownInstanceType::Annotated(_)
-                    | KnownInstanceType::TypeGenericAlias(_),
+                    | KnownInstanceType::TypeGenericAlias(_)
+                    | KnownInstanceType::Callable(_),
                 ),
                 ast::Operator::BitOr,
             ) if pep_604_unions_allowed() => {
@@ -10827,6 +10829,14 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
                     InternedType::new(self.db(), argument_ty),
                 ));
             }
+            Type::SpecialForm(SpecialFormType::Callable) => {
+                let callable = self
+                    .infer_callable_type(subscript)
+                    .as_callable()
+                    .expect("always returns Type::Callable");
+
+                return Type::KnownInstance(KnownInstanceType::Callable(callable));
+            }
             // `typing` special forms with a single generic argument
             Type::SpecialForm(
                 special_form @ (SpecialFormType::List

crates/ty_python_semantic/src/types/infer/builder/type_expression.rs

@@ -839,6 +839,10 @@ impl<'db> TypeInferenceBuilder<'db, '_> {
                     }
                     Type::unknown()
                 }
+                KnownInstanceType::Callable(_) => {
+                    self.infer_type_expression(slice);
+                    todo_type!("Generic specialization of typing.Callable")
+                }
                 KnownInstanceType::Annotated(_) => {
                     self.infer_type_expression(slice);
                     todo_type!("Generic specialization of typing.Annotated")
@@ -929,6 +933,58 @@ impl<'db> TypeInferenceBuilder<'db, '_> {
         ty
     }
 
+    /// Infer the type of a `Callable[...]` type expression.
+    pub(crate) fn infer_callable_type(&mut self, subscript: &ast::ExprSubscript) -> Type<'db> {
+        let db = self.db();
+
+        let arguments_slice = &*subscript.slice;
+
+        let mut arguments = match arguments_slice {
+            ast::Expr::Tuple(tuple) => Either::Left(tuple.iter()),
+            _ => {
+                self.infer_callable_parameter_types(arguments_slice);
+                Either::Right(std::iter::empty::<&ast::Expr>())
+            }
+        };
+
+        let first_argument = arguments.next();
+
+        let parameters = first_argument.and_then(|arg| self.infer_callable_parameter_types(arg));
+
+        let return_type = arguments.next().map(|arg| self.infer_type_expression(arg));
+
+        let correct_argument_number = if let Some(third_argument) = arguments.next() {
+            self.infer_type_expression(third_argument);
+            for argument in arguments {
+                self.infer_type_expression(argument);
+            }
+            false
+        } else {
+            return_type.is_some()
+        };
+
+        if !correct_argument_number {
+            report_invalid_arguments_to_callable(&self.context, subscript);
+        }
+
+        let callable_type = if let (Some(parameters), Some(return_type), true) =
+            (parameters, return_type, correct_argument_number)
+        {
+            CallableType::single(db, Signature::new(parameters, Some(return_type)))
+        } else {
+            CallableType::unknown(db)
+        };
+
+        // `Signature` / `Parameters` are not a `Type` variant, so we're storing
+        // the outer callable type on these expressions instead.
+        self.store_expression_type(arguments_slice, callable_type);
+        if let Some(first_argument) = first_argument {
+            self.store_expression_type(first_argument, callable_type);
+        }
+
+        callable_type
+    }
+
     pub(crate) fn infer_parameterized_special_form_type_expression(
         &mut self,
         subscript: &ast::ExprSubscript,
@@ -979,53 +1035,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> {
                 }
                 _ => self.infer_type_expression(arguments_slice),
             },
-            SpecialFormType::Callable => {
-                let mut arguments = match arguments_slice {
-                    ast::Expr::Tuple(tuple) => Either::Left(tuple.iter()),
-                    _ => {
-                        self.infer_callable_parameter_types(arguments_slice);
-                        Either::Right(std::iter::empty::<&ast::Expr>())
-                    }
-                };
-
-                let first_argument = arguments.next();
-
-                let parameters =
-                    first_argument.and_then(|arg| self.infer_callable_parameter_types(arg));
-
-                let return_type = arguments.next().map(|arg| self.infer_type_expression(arg));
-
-                let correct_argument_number = if let Some(third_argument) = arguments.next() {
-                    self.infer_type_expression(third_argument);
-                    for argument in arguments {
-                        self.infer_type_expression(argument);
-                    }
-                    false
-                } else {
-                    return_type.is_some()
-                };
-
-                if !correct_argument_number {
-                    report_invalid_arguments_to_callable(&self.context, subscript);
-                }
-
-                let callable_type = if let (Some(parameters), Some(return_type), true) =
-                    (parameters, return_type, correct_argument_number)
-                {
-                    CallableType::single(db, Signature::new(parameters, Some(return_type)))
-                } else {
-                    CallableType::unknown(db)
-                };
-
-                // `Signature` / `Parameters` are not a `Type` variant, so we're storing
-                // the outer callable type on these expressions instead.
-                self.store_expression_type(arguments_slice, callable_type);
-                if let Some(first_argument) = first_argument {
-                    self.store_expression_type(first_argument, callable_type);
-                }
-
-                callable_type
-            }
+            SpecialFormType::Callable => self.infer_callable_type(subscript),
 
             // `ty_extensions` special forms
             SpecialFormType::Not => {
@@ -1491,7 +1501,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> {
     ///
     /// It returns `None` if the argument is invalid i.e., not a list of types, parameter
     /// specification, `typing.Concatenate`, or `...`.
-    fn infer_callable_parameter_types(
+    pub(super) fn infer_callable_parameter_types(
         &mut self,
         parameters: &ast::Expr,
     ) -> Option<Parameters<'db>> {