Supports type parameters and type aliases (pep 695)

README.rst

@@ -80,6 +80,8 @@ This is the last major release to support Python 3.7.
 
 * Drop support for Python 3.6.
 * Add support for Python 3.12 and Python 3.13.
+* Add support for the ``type`` statement introduced in Python 3.12 (PEP-695).
+* Add support for the embedded type variable declarations in class or function introduced in Python 3.12 (PEP-695).
 * Astor is no longer a requirement starting at Python 3.9.
 * `ExtRegistrar.register_post_processor()` now supports a `priority` argument that is an int.
   Highest priority callables will be called first during post-processing.

mypy.ini

@@ -13,6 +13,9 @@ warn_unused_configs=True
 warn_unused_ignores=True
 
 plugins=mypy_zope:plugin
+exclude = (?x)(
+    ^pydoctor\/test\/testpackages\/
+  )
 
 # The following modules are currently only partially annotated:
 

pydoctor/epydoc/markup/_pyval_repr.py

@@ -40,7 +40,7 @@
 import functools
 import sys
 from inspect import signature
-from typing import Any, AnyStr, Union, Callable, Dict, Iterable, Sequence, Optional, List, Tuple, cast
+from typing import Any, AnyStr, Union, Callable, Dict, Iterable, Mapping, Sequence, Optional, List, Tuple, cast
 
 import attr
 from docutils import nodes
@@ -195,7 +195,7 @@ def __init__(self, document: nodes.document, is_complete: bool, warnings: List[s
     def to_stan(self, docstring_linker: DocstringLinker) -> Tag:
         return Tag('code')(super().to_stan(docstring_linker))
 
-def colorize_pyval(pyval: Any, linelen:Optional[int], maxlines:int, linebreakok:bool=True, refmap:Optional[Dict[str, str]]=None) -> ColorizedPyvalRepr:
+def colorize_pyval(pyval: Any, linelen:Optional[int], maxlines:int, linebreakok:bool=True, refmap:Optional[Mapping[str, str]]=None) -> ColorizedPyvalRepr:
     """
     Get a L{ColorizedPyvalRepr} instance for this piece of ast. 
 
@@ -207,7 +207,7 @@ def colorize_pyval(pyval: Any, linelen:Optional[int], maxlines:int, linebreakok:
     """
     return PyvalColorizer(linelen=linelen, maxlines=maxlines, linebreakok=linebreakok, refmap=refmap).colorize(pyval)
 
-def colorize_inline_pyval(pyval: Any, refmap:Optional[Dict[str, str]]=None) -> ColorizedPyvalRepr:
+def colorize_inline_pyval(pyval: Any, refmap:Optional[Mapping[str, str]]=None) -> ColorizedPyvalRepr:
     """
     Used to colorize type annotations and parameters default values.
     @returns: C{L{colorize_pyval}(pyval, linelen=None, linebreakok=False)}
@@ -255,12 +255,14 @@ def enc(c: str) -> str:
 def _bytes_escape(b: bytes) -> str:
     return repr(b)[2:-1]
 
+PY_312_PLUS = sys.version_info >= (3, 12)
+
 class PyvalColorizer:
     """
     Syntax highlighter for Python values.
     """
 
-    def __init__(self, linelen:Optional[int], maxlines:int, linebreakok:bool=True, refmap:Optional[Dict[str, str]]=None):
+    def __init__(self, linelen:Optional[int], maxlines:int, linebreakok:bool=True, refmap:Optional[Mapping[str, str]]=None):
         self.linelen: Optional[int] = linelen if linelen!=0 else None
         self.maxlines: Union[int, float] = maxlines if maxlines!=0 else float('inf')
         self.linebreakok = linebreakok
@@ -590,6 +592,12 @@ def _colorize_ast(self, pyval: ast.AST, state: _ColorizerState) -> None:
             else:
                 self._output('**', None, state)
             self._colorize_ast(pyval.value, state)
+        elif PY_312_PLUS and isinstance(pyval, ast.TypeVar):
+            self._colorize_ast_typevar(pyval, state)
+        elif PY_312_PLUS and isinstance(pyval, ast.TypeVarTuple):
+            self._colorize_ast_typevartuple(pyval, state)
+        elif PY_312_PLUS and isinstance(pyval, ast.ParamSpec):
+            self._colorize_ast_paramspec(pyval, state)
         else:
             self._colorize_ast_generic(pyval, state)
         assert state.stack.pop() is pyval
@@ -764,6 +772,21 @@ def _colorize_ast_re(self, node:ast.Call, state: _ColorizerState) -> None:
 
         self._output(')', self.GROUP_TAG, state)
 
+    # Python 3.12
+    def _colorize_ast_typevar(self, pyval:ast.TypeVar, state: _ColorizerState) -> None:
+        self._output(pyval.name, self.LINK_TAG, state, link=True)
+        if pyval.bound:
+            self._output(': ', self.COLON_TAG, state)
+            self._colorize_ast(pyval.bound, state)
+    # Python 3.12
+    def _colorize_ast_typevartuple(self, pyval:ast.TypeVarTuple, state: _ColorizerState) -> None:
+        self._output('*', None, state)
+        self._output(pyval.name, self.LINK_TAG, state, link=True)
+    # Python 3.12
+    def _colorize_ast_paramspec(self, pyval:ast.ParamSpec, state: _ColorizerState) -> None:
+        self._output('**', None, state)
+        self._output(pyval.name, self.LINK_TAG, state, link=True)
+
     def _colorize_ast_generic(self, pyval: ast.AST, state: _ColorizerState) -> None:
         try:
             # Always wrap the expression inside parenthesis because we can't be sure 

pydoctor/epydoc2stan.py

@@ -269,10 +269,13 @@ def __init__(self, obj: model.Documentable):
     def set_param_types_from_annotations(
             self, annotations: Mapping[str, Optional[ast.expr]]
             ) -> None:
+        """This method MUST only be called for Function instances."""
+        assert isinstance(self.obj, model.Function)
         _linker = linker._AnnotationLinker(self.obj)
         formatted_annotations = {
             name: None if value is None
-                       else ParamType(safe_to_stan(colorize_inline_pyval(value), _linker,
+                       else ParamType(safe_to_stan(colorize_inline_pyval(value, 
+                                refmap=model.gather_type_params_refs(self.obj)), _linker,
                                 self.obj, fallback=colorized_pyval_fallback, section='annotation', report=False),
                                 # don't spam the log, invalid annotation are going to be reported when the signature gets colorized
                                 origin=FieldOrigin.FROM_AST)
@@ -861,7 +864,7 @@ def format_undocumented(obj: model.Documentable) -> Tag:
     return tag
 
 
-def type2stan(obj: model.Documentable) -> Optional[Tag]:
+def type2stan(obj: model.Attribute) -> Optional[Tag]:
     """
     Get the formatted type of this attribute.
     """
@@ -874,7 +877,7 @@ def type2stan(obj: model.Documentable) -> Optional[Tag]:
         return safe_to_stan(parsed_type, _linker, obj,
             fallback=colorized_pyval_fallback, section='annotation')
 
-def get_parsed_type(obj: model.Documentable) -> Optional[ParsedDocstring]:
+def get_parsed_type(obj: model.Attribute) -> Optional[ParsedDocstring]:
     """
     Get the type of this attribute as parsed docstring.
     """
@@ -885,7 +888,7 @@ def get_parsed_type(obj: model.Documentable) -> Optional[ParsedDocstring]:
     # Only Attribute instances have the 'annotation' attribute.
     annotation: Optional[ast.expr] = getattr(obj, 'annotation', None)
     if annotation is not None:
-        return colorize_inline_pyval(annotation)
+        return colorize_inline_pyval(annotation, refmap=model.gather_type_params_refs(obj))
 
     return None
 
@@ -992,9 +995,14 @@ def _format_constant_value(obj: model.Attribute) -> Iterator["Flattenable"]:
     # yield the first row.
     yield row
 
+    refmap: Mapping[str, str] | None = None
+    if obj.kind is model.DocumentableKind.TYPE_ALIAS:
+        refmap = model.gather_type_params_refs(obj)
+
     doc = colorize_pyval(obj.value,
         linelen=obj.system.options.pyvalreprlinelen,
-        maxlines=obj.system.options.pyvalreprmaxlines)
+        maxlines=obj.system.options.pyvalreprmaxlines,
+        refmap=refmap)
 
     value_repr = safe_to_stan(doc, obj.docstring_linker, obj,
         fallback=colorized_pyval_fallback, section='rendering of constant')

pydoctor/model.py

@@ -65,6 +65,12 @@ class LineFromAst(int):
 class LineFromDocstringField(int):
     "Simple L{int} wrapper for linenumbers coming from docstrings."
 
+class AttributeValueDisplay(Enum):
+    HIDDEN = 0
+    AS_CODE_BLOCK = 1
+    # Not used yet. Default values of Attrs-like classes' attributes will one day (and maybe others).
+    #INLINE = 2
+
 class DocLocation(Enum):
     OWN_PAGE = 1
     PARENT_PAGE = 2
@@ -628,10 +634,38 @@ def _find_dunder_constructor(cls:'Class') -> Optional['Function']:
             return _init
     return None
 
+def gather_type_params_refs(ob: Class | Function | Attribute | FunctionOverload) -> Mapping[str, str]:
+    """
+    Starting with Python 3.12 classes and functions can include defintions of type variable -likes.
+    This function returns all mapping of all type variables- like defined in the context of the given object
+    to the full name of the class or function that defines it.
+
+    Type variables are not documentables so we need to treat them separately. 
+    """
+    curr = ob
+    typevar_sources: list[Class | Function | Attribute | FunctionOverload] = []
+    while True:
+        typevar_sources += [curr]
+        # For class members re-exported this will be wrong but it will be fixed 
+        # by adding a .parsed_typevars attribute alongside PR https://github.com/twisted/pydoctor/pull/723
+        # It can only be a class at the moment
+        if isinstance(curr.parent, (Module, type(None))):
+            break
+        else:
+            curr = cast(Class, curr.parent)
+
+    refmap = {t.name:o.fullName() for o in 
+              reversed(typevar_sources) for t in o.typevars or [] 
+              # the condition is only for mypy since type_params 
+              # can only be one of these three types at the moment
+              if isinstance(t, (ast.TypeVar, ast.TypeVarTuple, ast.ParamSpec))}
+    return refmap
+
 class Class(CanContainImportsDocumentable):
     kind = DocumentableKind.CLASS
     parent: CanContainImportsDocumentable
     decorators: Sequence[Tuple[str, Optional[Sequence[ast.expr]]]]
+    typevars: Sequence[ast.type_param] | None = None
 
     # set in post-processing:
     _finalbaseobjects: Optional[List[Optional['Class']]] = None 
@@ -843,6 +877,7 @@ class Function(Inheritable):
     decorators: Optional[Sequence[ast.expr]]
     signature: Optional[Signature]
     overloads: List['FunctionOverload']
+    typevars: Sequence[ast.type_param] | None = None
 
     def setup(self) -> None:
         super().setup()
@@ -857,8 +892,23 @@ class FunctionOverload:
     @note: This is not an actual documentable type. 
     """
     primary: Function
-    signature: Signature
-    decorators: Sequence[ast.expr]
+    signature: Signature = attr.ib(factory=Signature)
+    decorators: Sequence[ast.expr] = attr.ib(factory=list)
+    typevars: Sequence[ast.type_param] | None = None
+
+    @property
+    def parent(self) -> CanContainImportsDocumentable:
+        return self.primary.parent
+
+    @property
+    def module(self) -> Module:
+        return self.primary.module
+
+    def fullName(self) -> str:
+        return self.primary.fullName()
+
+    def report(self, msg:str, **kw:str | int) -> None:
+        self.primary.report(msg, **kw) # type:ignore
 
 class Attribute(Inheritable):
     kind: Optional[DocumentableKind] = DocumentableKind.ATTRIBUTE
@@ -870,10 +920,12 @@ class Attribute(Inheritable):
 
     None value means the value is not initialized at the current point of the the process. 
     """
+    typevars: Sequence[ast.type_param] | None = None
 
 # Work around the attributes of the same name within the System class.
 _ModuleT = Module
 _PackageT = Package
+_AttributeT = Attribute
 
 T = TypeVar('T')
 
@@ -906,7 +958,7 @@ def import_mod_from_file_location(module_full_name:str, path: Path) -> types.Mod
 class System:
     """A collection of related documentable objects.
 
-    PyDoctor documents collections of objects, often the contents of a
+    Pydoctor documents collections of objects, often the contents of a
     package.
     """
 
@@ -928,13 +980,6 @@ class System:
     Additional list of extensions to load alongside default extensions.
     """
 
-    show_attr_value = (DocumentableKind.CONSTANT, 
-                       DocumentableKind.TYPE_VARIABLE, 
-                       DocumentableKind.TYPE_ALIAS)
-    """
-    What kind of attributes we should display the value for?
-    """
-
     def __init__(self, options: Optional['Options'] = None):
         self.allobjects: Dict[str, Documentable] = {}
         self.rootobjects: List[_ModuleT] = []
@@ -1120,6 +1165,21 @@ def objectsOfType(self, cls: Union[Type['DocumentableT'], str]) -> Iterator['Doc
             if isinstance(o, cls):
                 yield o
 
+    # What kind of attributes we should pydoctor display the value for?
+    _show_attr_value = set((DocumentableKind.CONSTANT, 
+                    DocumentableKind.TYPE_VARIABLE, 
+                    DocumentableKind.TYPE_ALIAS))
+
+    def showAttrValue(self, ob: _AttributeT) -> AttributeValueDisplay:
+        """
+        Whether to display the value of the given attribute.
+        """
+
+        if ob.kind not in self._show_attr_value or ob.value is None:
+            return AttributeValueDisplay.HIDDEN
+        # Attribute is a constant/type alias (with a value), then display it's value
+        return AttributeValueDisplay.AS_CODE_BLOCK
+
     def privacyClass(self, ob: Documentable) -> PrivacyClass:
         ob_fullName = ob.fullName()
         cached_privacy = self._privacyClassCache.get(ob_fullName)

pydoctor/templatewriter/pages/__init__.py

@@ -51,7 +51,25 @@ def format_decorators(obj: Union[model.Function, model.Attribute, model.Function
         epydoc2stan.reportWarnings(documentable_obj, doc.warnings, section='colorize decorator')
         yield '@', stan.children, tags.br()
 
-def format_signature(func: Union[model.Function, model.FunctionOverload]) -> "Flattenable":
+def format_type_params(ob: model.Function | model.FunctionOverload | model.Class | model.Attribute) -> Iterator[Flattenable]:
+    if not ob.typevars:
+        return
+    ctx = ob.primary if isinstance(ob, model.FunctionOverload) else ob
+    _linker = linker._AnnotationLinker(ctx)
+    refmap = model.gather_type_params_refs(ob)
+    stan: list[Flattenable] = []
+    for t in ob.typevars:
+        if stan:
+            stan += [', ']
+        stan += [*epydoc2stan.safe_to_stan(
+            colorize_inline_pyval(t, refmap=refmap), _linker, ctx, 
+                    fallback=epydoc2stan.colorized_pyval_fallback, 
+                    section='rendering of type params').children]
+    yield '['
+    yield from stan
+    yield ']'
+
+def format_signature(func: Union[model.Function, model.FunctionOverload]) -> Flattenable:
     """
     Return a stan representation of a nicely-formatted source-like function signature for the given L{Function}.
     Arguments default values are linked to the appropriate objects when possible.
@@ -65,16 +83,17 @@ def format_signature(func: Union[model.Function, model.FunctionOverload]) -> "Fl
             [epydoc2stan.get_to_stan_error(e)], section='signature')
         return broken
 
-def format_class_signature(cls: model.Class) -> "Flattenable":
+def format_class_signature(cls: model.Class) -> Flattenable:
     """
     The class signature is the formatted list of bases this class extends. 
     It's not the class constructor.
     """
-    r: List["Flattenable"] = []
+    r: List["Flattenable"] = [*format_type_params(cls),]
     # the linker will only be used to resolve the generic arguments of the base classes, 
     # it won't actually resolve the base classes (see comment few lines below).
     # this is why we're using the annotation linker.
     _linker = linker._AnnotationLinker(cls)
+    base_refmap = model.gather_type_params_refs(cls)
     if cls.rawbases:
         r.append('(')
 
@@ -88,13 +107,13 @@ def format_class_signature(cls: model.Class) -> "Flattenable":
             # a class with the same name as a base class confused pydoctor and it would link 
             # to it self: https://github.com/twisted/pydoctor/issues/662
 
-            refmap = None
+            refmap = {}
             if base_obj is not None:
                 refmap = {str_base:base_obj.fullName()}
 
             # link to external class, using the colorizer here
             # to link to classes with generics (subscripts and other AST expr).
-            stan = epydoc2stan.safe_to_stan(colorize_inline_pyval(base_node, refmap=refmap), _linker, cls, 
+            stan = epydoc2stan.safe_to_stan(colorize_inline_pyval(base_node, refmap={**base_refmap, **refmap}), _linker, cls, 
                 fallback=epydoc2stan.colorized_pyval_fallback, 
                 section='rendering of class signature')
             r.extend(stan.children)
@@ -127,7 +146,10 @@ def format_function_def(func_name: str, is_async: bool,
     r.extend([
         tags.span(def_stmt, class_='py-keyword'), ' ',
         tags.span(func_name, class_='py-defname'), 
-        tags.span(format_signature(func), class_='function-signature'), ':',
+
+        tags.span([*format_type_params(func),  
+                   format_signature(func)], 
+                  class_='function-signature'), ':',
     ])
     return r