From b98186077925f32a22cf4a29862e484463f50011 Mon Sep 17 00:00:00 2001 From: udifuchs Date: Mon, 11 Aug 2025 17:08:52 -0500 Subject: [PATCH 1/5] Replace factory functions with classes. Element() and ElementTree() were factory functions until now. They are now defined as classes. With this change, we can annotate code with the public Element class, instead of using the private _Element class. For example: ``` def print_tree(tree: e.ElementTree[e.Element]) -> None: ... ``` There is a related PR that was merged into lxml: https://github.com/lxml/lxml/pull/405 In the lxml PR, _Element is a virtual subclass of Element. This means that when we create an Element it actually creates an _Element class, but the created object is also an instance of Element: ``` el = Element() reveal_type(el) # _Element isinstance(element, Element) # True isinstance(element, _Element) # True ``` --- src/lxml-stubs/_types.pyi | 26 -------- src/lxml-stubs/builder.pyi | 9 ++- src/lxml-stubs/etree/__init__.pyi | 4 +- src/lxml-stubs/etree/_element.pyi | 37 +++++++++++- src/lxml-stubs/etree/_factory_func.pyi | 84 +------------------------- src/lxml-stubs/etree/_iterparse.pyi | 3 +- src/lxml-stubs/etree/_parser.pyi | 7 +-- src/lxml-stubs/etree/_saxparser.pyi | 4 +- src/lxml-stubs/html/_element.pyi | 14 ++++- src/lxml-stubs/html/soupparser.pyi | 12 ++-- src/lxml-stubs/objectify/_factory.pyi | 3 +- src/lxml-stubs/sax.pyi | 6 +- tests/runtime/allowlist.txt | 10 +++ tests/static/test-annotations.yml | 77 +++++++++++++++++++++-- 14 files changed, 153 insertions(+), 143 deletions(-) diff --git a/src/lxml-stubs/_types.pyi b/src/lxml-stubs/_types.pyi index af10cf62..f35c6ff6 100644 --- a/src/lxml-stubs/_types.pyi +++ b/src/lxml-stubs/_types.pyi @@ -6,7 +6,6 @@ from typing import ( Any, Callable, Collection, - Generic, Iterable, Literal, Mapping, @@ -157,31 +156,6 @@ _SaxEventNames = Literal[ _ET = TypeVar("_ET", bound=_Element, default=_Element) _ET_co = TypeVar("_ET_co", bound=_Element, default=_Element, covariant=True) -class _ElementFactory(Protocol, Generic[_ET_co]): - """Element factory protocol - - This is callback protocol for `makeelement()` method of - various element objects, with following signature (which - is identical to `etree.Element()` factory): - - ```python - (_tag, attrib=..., nsmap=..., **_extra) - ``` - - The mapping in `attrib` argument and all `_extra` keyword - arguments would be merged together, with `_extra` taking - precedence over `attrib`. - """ - - def __call__( - self, - _tag: _TagName, - /, - attrib: _AttrMapping | None = None, - nsmap: _NSMapArg | None = None, - **_extra: _AttrVal, - ) -> _ET_co: ... - # HACK _TagSelector filters element type not by classes, but checks for exact # element *factory functions* instead (etree.Element() and friends). Python # typing system doesn't support such outlandish usage. Use a generic callable diff --git a/src/lxml-stubs/builder.pyi b/src/lxml-stubs/builder.pyi index b89c3085..0c34ff17 100644 --- a/src/lxml-stubs/builder.pyi +++ b/src/lxml-stubs/builder.pyi @@ -1,7 +1,6 @@ from typing import Any, Callable, Generic, Mapping, Protocol, overload from ._types import ( - _ElementFactory, _ET_co, _NSMapArg, _NSTuples, @@ -58,7 +57,7 @@ class ElementMaker(Generic[_ET_co]): namespace: str | None = None, nsmap: _NSMapArg | _NSTuples | None = None, # dict() *, - makeelement: _ElementFactory[_ET_co], + makeelement: type[_ET_co], ) -> ElementMaker[_ET_co]: ... @overload # makeelement is positional def __new__( @@ -66,7 +65,7 @@ class ElementMaker(Generic[_ET_co]): typemap: _TypeMapArg | None, namespace: str | None, nsmap: _NSMapArg | _NSTuples | None, - makeelement: _ElementFactory[_ET_co], + makeelement: type[_ET_co], ) -> ElementMaker[_ET_co]: ... @overload # makeelement is default or absent def __new__( @@ -104,7 +103,7 @@ class ElementMaker(Generic[_ET_co]): # invariance has posed some challenge to typing. We can afford # some more restriction as return value or attribute. @property - def _makeelement(self) -> _ElementFactory[_ET_co]: ... + def _makeelement(self) -> type[_ET_co]: ... @property def _namespace(self) -> str | None: ... @property @@ -112,4 +111,4 @@ class ElementMaker(Generic[_ET_co]): @property def _typemap(self) -> dict[type[Any], Callable[[_ET_co, Any], None]]: ... -E: ElementMaker +E: ElementMaker[_Element] diff --git a/src/lxml-stubs/etree/__init__.pyi b/src/lxml-stubs/etree/__init__.pyi index 0b2488c4..4f0d5bb6 100644 --- a/src/lxml-stubs/etree/__init__.pyi +++ b/src/lxml-stubs/etree/__init__.pyi @@ -26,6 +26,8 @@ from ._dtd import ( DTDValidateError as DTDValidateError, ) from ._element import ( + Element as Element, + ElementTree as ElementTree, _Attrib as _Attrib, _Comment as _Comment, _Element as _Element, @@ -36,8 +38,6 @@ from ._element import ( from ._factory_func import ( PI as PI, Comment as Comment, - Element as Element, - ElementTree as ElementTree, Entity as Entity, ProcessingInstruction as ProcessingInstruction, SubElement as SubElement, diff --git a/src/lxml-stubs/etree/_element.pyi b/src/lxml-stubs/etree/_element.pyi index f1142658..6af679b8 100644 --- a/src/lxml-stubs/etree/_element.pyi +++ b/src/lxml-stubs/etree/_element.pyi @@ -7,6 +7,7 @@ from typing import ( Iterator, Literal, Mapping, + TypeAlias, TypeVar, final, overload, @@ -40,6 +41,16 @@ class _Element: -------- - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element) """ + + def __init__( # Args identical to Element.makeelement + self, + _tag: _t._TagName, + /, + attrib: _t._AttrMapping | None = ..., + nsmap: _t._NSMapArg | None = ..., + **_extra: _t._AttrVal, + ) -> None: ... + # # Common properties # @@ -557,7 +568,7 @@ class _Element: - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.itertext) - [Possible tag values in `iter()`](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.iter) """ - makeelement: _t._ElementFactory[Self] + makeelement: type[Self] """Creates a new element associated with the same document. See Also @@ -663,6 +674,8 @@ class _Element: self, tag: _t._TagSelector | None = None, *tags: _t._TagSelector ) -> Iterator[Self]: ... +Element: TypeAlias = _Element + _ET2_co = TypeVar("_ET2_co", bound=_Element, default=_Element, covariant=True) # ET class notation is specialized, indicating the type of element @@ -673,6 +686,26 @@ _ET2_co = TypeVar("_ET2_co", bound=_Element, default=_Element, covariant=True) # It is considered harmful to support such corner case, which # adds much complexity without any benefit. class _ElementTree(Generic[_t._ET_co]): + + @overload # from element, parser ignored + def __new__(cls, element: _t._ET_co) -> _ElementTree[_t._ET_co]: ... + @overload # from file source, custom parser + def __new__( + cls, + element: None = ..., + *, + file: _t._FileReadSource, + parser: _t._DefEtreeParsers[_t._ET_co], + ) -> _ElementTree[_t._ET_co]: ... + @overload # from file source, default parser + def __new__( + cls, + element: None = ..., + *, + file: _t._FileReadSource, + parser: None = ..., + ) -> _ElementTree: ... + @property def parser(self) -> _t._DefEtreeParsers[_t._ET_co] | None: ... @property @@ -944,6 +977,8 @@ class _ElementTree(Generic[_t._ET_co]): inclusive_ns_prefixes: Iterable[str | bytes] | None = None, ) -> None: ... +ElementTree: TypeAlias = _ElementTree + # Behaves like MutableMapping but deviates a lot in details @final class _Attrib: diff --git a/src/lxml-stubs/etree/_factory_func.pyi b/src/lxml-stubs/etree/_factory_func.pyi index b417517c..3dd52b88 100644 --- a/src/lxml-stubs/etree/_factory_func.pyi +++ b/src/lxml-stubs/etree/_factory_func.pyi @@ -4,18 +4,13 @@ from .._types import ( _ET, _AttrMapping, _AttrVal, - _DefEtreeParsers, - _ElementFactory, - _ET_co, - _FileReadSource, _NSMapArg, _TagName, _TextArg, ) from ..html import HtmlElement from ..objectify import ObjectifiedElement, StringElement -from ._element import _Comment, _ElementTree, _Entity, _ProcessingInstruction -from ._parser import CustomTargetParser +from ._element import _Comment, _Entity, _ProcessingInstruction _T = TypeVar("_T") @@ -28,8 +23,6 @@ PI = ProcessingInstruction def Entity(name: _TextArg) -> _Entity: ... -Element: _ElementFactory - # SubElement is a bit more complex than expected, as it # handles other kinds of element, like HtmlElement # and ObjectifiedElement. @@ -71,78 +64,3 @@ def SubElement( nsmap: _NSMapArg | None = None, **_extra: _AttrVal, ) -> _ET: ... -@overload # from element, parser ignored -def ElementTree( - element: _ET, - *, - file: None = None, -) -> _ElementTree[_ET]: - """ElementTree wrapper class for Element objects. - - Annotation - ---------- - This overload is used when creating an ElementTree directly from a root - Element object. Other arguments are ignored in this case. - - See Also - -------- - - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) - """ - -@overload # from file source, standard parser -def ElementTree( - element: None = None, - *, - file: _FileReadSource, - parser: _DefEtreeParsers[_ET_co], -) -> _ElementTree[_ET_co]: - """ElementTree wrapper class for Element objects. - - Annotation - ---------- - This overload is used when creating an ElementTree from a file source with - user-supplied standard parser. - - See Also - -------- - - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) - """ - -@overload # from file source, custom target parser -def ElementTree( - element: None = None, - *, - file: _FileReadSource, - parser: CustomTargetParser[_T], -) -> _T: - """ElementTree wrapper class for Element objects. - - Annotation - ---------- - This overload is used when creating an ElementTree from a file source with - custom target parser. Returns the result dictated by parser target object - instead of ElementTree. - - See Also - -------- - - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) - """ - -@overload # from file source, no parser supplied -def ElementTree( - element: None = None, - *, - file: _FileReadSource, - parser: None = None, -) -> _ElementTree: - """ElementTree wrapper class for Element objects. - - Annotation - ---------- - This overload is used when creating an ElementTree from a file source - without a parser supplied. The default parser is used in this case. - - See Also - -------- - - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) - """ diff --git a/src/lxml-stubs/etree/_iterparse.pyi b/src/lxml-stubs/etree/_iterparse.pyi index 9b4221fe..b1e0b20f 100644 --- a/src/lxml-stubs/etree/_iterparse.pyi +++ b/src/lxml-stubs/etree/_iterparse.pyi @@ -3,7 +3,6 @@ from _typeshed import SupportsRead from typing import Iterable, Iterator, Literal, TypeVar, overload from .._types import ( - _ElementFactory, _ElementOrTree, _ET_co, _FilePath, @@ -181,7 +180,7 @@ class iterparse(Iterator[_T_co]): self, lookup: ElementClassLookup | None = None, ) -> None: ... - makeelement: _ElementFactory + makeelement: type[_T_co] class iterwalk(Iterator[_T_co]): """Tree walker that generates events from an existing tree as if it diff --git a/src/lxml-stubs/etree/_parser.pyi b/src/lxml-stubs/etree/_parser.pyi index acd24062..f7f35452 100644 --- a/src/lxml-stubs/etree/_parser.pyi +++ b/src/lxml-stubs/etree/_parser.pyi @@ -11,7 +11,6 @@ from typing import ( from .._types import ( _DefEtreeParsers, - _ElementFactory, _ET_co, _SaxEventNames, _TagSelector, @@ -75,7 +74,7 @@ class CustomTargetParser(Generic[_T]): """The version of the underlying XML parser.""" def copy(self) -> Self: """Create a new parser with the same configuration.""" - makeelement: _ElementFactory[_Element] + makeelement: type[_Element] """Creates a new element associated with this parser.""" @property def feed_error_log(self) -> _ListErrorLog: @@ -185,7 +184,7 @@ class XMLParser(Generic[_ET_co]): """The version of the underlying XML parser.""" def copy(self) -> Self: """Create a new parser with the same configuration.""" - makeelement: _ElementFactory[_ET_co] + makeelement: type[_ET_co] """Creates a new element associated with this parser.""" def set_element_class_lookup( self, lookup: ElementClassLookup | None = None @@ -389,7 +388,7 @@ class HTMLParser(Generic[_ET_co]): """The version of the underlying XML parser.""" def copy(self) -> Self: """Create a new parser with the same configuration.""" - makeelement: _ElementFactory[_ET_co] + makeelement: type[_ET_co] """Creates a new element associated with this parser.""" def set_element_class_lookup( self, lookup: ElementClassLookup | None = None diff --git a/src/lxml-stubs/etree/_saxparser.pyi b/src/lxml-stubs/etree/_saxparser.pyi index 113aad9b..39dfafae 100644 --- a/src/lxml-stubs/etree/_saxparser.pyi +++ b/src/lxml-stubs/etree/_saxparser.pyi @@ -1,7 +1,7 @@ from abc import abstractmethod from typing import Callable, Protocol, TypeVar -from .._types import _DefEtreeParsers, _ElementFactory +from .._types import _DefEtreeParsers from ._element import _Comment, _Element, _ProcessingInstruction from ._parser import XMLSyntaxError @@ -60,7 +60,7 @@ class TreeBuilder(ParserTarget[_Element]): def __init__( self, *, - element_factory: _ElementFactory[_Element] | None = None, + element_factory: type[_Element] | None = None, parser: _DefEtreeParsers | None = None, comment_factory: Callable[..., _Comment] | None = None, pi_factory: Callable[..., _ProcessingInstruction] | None = None, diff --git a/src/lxml-stubs/html/_element.pyi b/src/lxml-stubs/html/_element.pyi index b00bfb36..d46fbc30 100644 --- a/src/lxml-stubs/html/_element.pyi +++ b/src/lxml-stubs/html/_element.pyi @@ -11,11 +11,13 @@ from typing import ( from .. import etree from .._types import ( + _AttrMapping, _AttrName, _AttrVal, - _ElementFactory, _ElemPathArg, + _NSMapArg, _StrOnlyNSMap, + _TagName, _TagSelector, ) from ..cssselect import _CSSTransArg @@ -264,7 +266,7 @@ class HtmlElement(etree.ElementBase): *, with_tail: bool = True, ) -> Iterator[str]: ... - makeelement: _ElementFactory[HtmlElement] # pyright: ignore[reportIncompatibleVariableOverride] + makeelement: type[HtmlElement] # pyright: ignore[reportIncompatibleVariableOverride] def find( self, path: _ElemPathArg, @@ -331,4 +333,10 @@ class HtmlEntity(etree.EntityBase, HtmlElement): ... # type: ignore[misc] # py # Factory func, there is no counterpart for SubElement though # (use etree.SubElement()) # -Element: _ElementFactory[HtmlElement] +def Element( + _tag: _TagName, + /, + attrib: _AttrMapping | None = None, + nsmap: _NSMapArg | None = None, + **_extra: _AttrVal, + ) -> HtmlElement: ... diff --git a/src/lxml-stubs/html/soupparser.pyi b/src/lxml-stubs/html/soupparser.pyi index 57d6e7ed..cc88664c 100644 --- a/src/lxml-stubs/html/soupparser.pyi +++ b/src/lxml-stubs/html/soupparser.pyi @@ -6,7 +6,7 @@ from bs4.builder import TreeBuilder from bs4.element import PageElement from bs4.filter import SoupStrainer -from .._types import _ET, _ElementFactory, _FileReadSource +from .._types import _ET, _FileReadSource from ..etree import _ElementTree from . import HtmlElement @@ -70,7 +70,7 @@ def fromstring( def fromstring( data: str | bytes | IO[str] | IO[bytes], beautifulsoup: type[BeautifulSoup] | None, - makeelement: _ElementFactory[_ET], + makeelement: type[_ET], *, features: _Features | Collection[_Features] = "html.parser", builder: TreeBuilder | type[TreeBuilder] | None = None, @@ -97,7 +97,7 @@ def fromstring( data: str | bytes | IO[str] | IO[bytes], beautifulsoup: type[BeautifulSoup] | None = None, *, - makeelement: _ElementFactory[_ET], + makeelement: type[_ET], features: _Features | Collection[_Features] = "html.parser", builder: TreeBuilder | type[TreeBuilder] | None = None, parse_only: SoupStrainer | None = None, @@ -173,7 +173,7 @@ def parse( def parse( file: _FileReadSource, beautifulsoup: type[BeautifulSoup] | None, - makeelement: _ElementFactory[_ET], + makeelement: type[_ET], *, features: _Features | Collection[_Features] = "html.parser", builder: TreeBuilder | type[TreeBuilder] | None = None, @@ -199,7 +199,7 @@ def parse( # makeelement is kw file: _FileReadSource, beautifulsoup: type[BeautifulSoup] | None = None, *, - makeelement: _ElementFactory[_ET], + makeelement: type[_ET], features: _Features | Collection[_Features] = "html.parser", builder: TreeBuilder | type[TreeBuilder] | None = None, parse_only: SoupStrainer | None = None, @@ -247,7 +247,7 @@ def parse( @overload def convert_tree( beautiful_soup_tree: BeautifulSoup, - makeelement: _ElementFactory[_ET], + makeelement: type[_ET], ) -> list[_ET]: """Convert a BeautifulSoup tree to a list of Element trees. diff --git a/src/lxml-stubs/objectify/_factory.pyi b/src/lxml-stubs/objectify/_factory.pyi index d763fc0e..b7d3d977 100644 --- a/src/lxml-stubs/objectify/_factory.pyi +++ b/src/lxml-stubs/objectify/_factory.pyi @@ -8,7 +8,6 @@ from .._types import ( _AttrMapping, _AttrTuples, _AttrVal, - _ElementFactory, _NSMapArg, _TagName, ) @@ -318,7 +317,7 @@ class ElementMaker: namespace: str | None = None, nsmap: _NSMapArg | None = None, annotate: bool = True, - makeelement: _ElementFactory[_e.ObjectifiedElement] | None = None, + makeelement: type[_e.ObjectifiedElement] | None = None, ) -> None: ... # Special notes: # - Attribute values supplied as children dict will be stringified, diff --git a/src/lxml-stubs/sax.pyi b/src/lxml-stubs/sax.pyi index e05f3267..0309791a 100644 --- a/src/lxml-stubs/sax.pyi +++ b/src/lxml-stubs/sax.pyi @@ -1,7 +1,7 @@ from typing import Generic, overload from xml.sax.handler import ContentHandler -from ._types import _ET, SupportsLaxItems, Unused, _ElementFactory, _ElementOrTree +from ._types import _ET, SupportsLaxItems, Unused, _ElementOrTree from .etree import LxmlError, _ElementTree, _ProcessingInstruction class SaxError(LxmlError): ... @@ -17,10 +17,10 @@ class ElementTreeContentHandler(Generic[_ET], ContentHandler): # Not adding _get_etree(), already available as public property @overload def __new__( - cls, makeelement: _ElementFactory[_ET] + cls, makeelement: type[_ET] ) -> ElementTreeContentHandler[_ET]: ... @overload - def __new__(cls, makeelement: None = None) -> ElementTreeContentHandler: ... + def __new__(cls, makeelement: None = None) -> ElementTreeContentHandler[_ET]: ... @property def etree(self) -> _ElementTree[_ET]: ... diff --git a/tests/runtime/allowlist.txt b/tests/runtime/allowlist.txt index 43409018..f299c9d4 100644 --- a/tests/runtime/allowlist.txt +++ b/tests/runtime/allowlist.txt @@ -105,6 +105,16 @@ lxml\.objectify\.Element lxml\.objectify\.SubElement lxml\.objectify\.annotate +# Cannot inspect elements of virtual subclasses +lxml.etree.Element.* +lxml.etree.ElementTree.* + +# Generic classes do not need explicit __class_getitem__ +lxml.builder.ElementMaker.__class_getitem__ +lxml.etree.HTMLParser.__class_getitem__ +lxml.etree.XMLParser.__class_getitem__ +lxml.sax.ElementTreeContentHandler.__class_getitem__ + # Cython class __init__ clobbered by generic signature lxml\.builder\.ElementMaker\.__init__ lxml\.etree\.ETCompatXMLParser\.__init__ diff --git a/tests/static/test-annotations.yml b/tests/static/test-annotations.yml index 66189d8b..3a7acc04 100644 --- a/tests/static/test-annotations.yml +++ b/tests/static/test-annotations.yml @@ -1,5 +1,9 @@ # Test functions with lxml type annotations. -- case: annaotate_element +# There are two copies of each test: +# One with the private _Element and _ElementTree. +# One with the public Element and ElementTree. + +- case: annaotate_element_private main: | from lxml import etree as e @@ -11,7 +15,7 @@ el = e.Element("test") view_element(el) -- case: annaotate_tree +- case: annaotate_tree_private main: | from lxml import etree as e @@ -29,7 +33,7 @@ tree = e.ElementTree(el) tree_root(tree) -- case: annaotate_comment +- case: annaotate_comment_private main: | from lxml import etree as e @@ -50,7 +54,7 @@ print(comm.tag == e.Comment) check_if_element_is_comment(comm) -- case: annaotate_html +- case: annaotate_html_private main: | from lxml import etree as e from lxml import html as h @@ -63,3 +67,68 @@ def element_tree_not_html(et: e._ElementTree[e._Element]) -> None: html_tree(et) # E: Argument 1 to "html_tree" has incompatible type "_ElementTree[_Element]"; expected "_ElementTree[HtmlElement]" [arg-type] + +- case: annaotate_element_public + main: | + from lxml import etree as e + + def view_element(el: e.Element) -> None: + reveal_type(el) # N: Revealed type is "lxml.etree._element._Element" + assert isinstance(e, e.Element) + + def call_view_element() -> None: + el = e.Element("test") + view_element(el) + +- case: annaotate_tree_public + main: | + from lxml import etree as e + + def tree_root(tree: e.ElementTree[e.Element]) -> None: + el = tree.getroot() + reveal_type(el) # N: Revealed type is "lxml.etree._element._Element" + + def tree_is_generic(et: e.ElementTree) -> None: + pass + + def tree_of_int(et: e.ElementTree[int]) -> None: # E: Type argument "int" of "_ElementTree" must be a subtype of "_Element" [type-var] + pass + + def get_tree(el: e.Element) -> None: + tree = e.ElementTree(el) + tree_root(tree) + +- case: annaotate_comment_public + main: | + from lxml import etree as e + + def get_element(el: e.Element) -> None: + reveal_type(el.tag) # NR: .+ "Union\[[\w\.]+\.str, [\w\.]+\.bytes, [\w\.]+\.bytearray, [\w\.]+\.QName\]" + + def get_comment(comm: e._Comment) -> None: + reveal_type(comm) # NR: .+ "[\w\.]+\._Comment" + reveal_type(comm.tag) # NR: .+ "def \(.+\) -> [\w\.]+\._Comment" + get_element(comm) + + def check_if_element_is_comment(el: e.Element) -> bool: + # The following should not be an error. + # It is a valid check that an element is a comment. + return el.tag == e.Comment + + comm = e.Comment("comment") + print(comm.tag == e.Comment) + check_if_element_is_comment(comm) + +- case: annaotate_html_public + main: | + from lxml import etree as e + from lxml import html as h + + def element_tree(et: e.ElementTree[e.Element]) -> None: + pass + + def html_tree(et: e.ElementTree[h.HtmlElement]) -> None: + element_tree(et) + + def element_tree_not_html(et: e.ElementTree[e.Element]) -> None: + html_tree(et) # E: Argument 1 to "html_tree" has incompatible type "_ElementTree[_Element]"; expected "_ElementTree[HtmlElement]" [arg-type] From 98104aef7178e5ef1461175809a37698504a35e1 Mon Sep 17 00:00:00 2001 From: udifuchs Date: Tue, 12 Aug 2025 11:00:56 -0500 Subject: [PATCH 2/5] Update all ElementTree constructors. --- src/lxml-stubs/etree/_element.pyi | 77 +++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/src/lxml-stubs/etree/_element.pyi b/src/lxml-stubs/etree/_element.pyi index 6af679b8..bcb5d969 100644 --- a/src/lxml-stubs/etree/_element.pyi +++ b/src/lxml-stubs/etree/_element.pyi @@ -688,23 +688,84 @@ _ET2_co = TypeVar("_ET2_co", bound=_Element, default=_Element, covariant=True) class _ElementTree(Generic[_t._ET_co]): @overload # from element, parser ignored - def __new__(cls, element: _t._ET_co) -> _ElementTree[_t._ET_co]: ... - @overload # from file source, custom parser def __new__( cls, - element: None = ..., + element: _t._ET_co, + *, + file: None = None, + ) -> _ElementTree[_t._ET_co]: + """ElementTree wrapper class for Element objects. + + Annotation + ---------- + This overload is used when creating an ElementTree directly from a root + Element object. Other arguments are ignored in this case. + + See Also + -------- + - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) + """ + + @overload # from file source, standard parser + def __new__( + cls, + element: None = None, *, file: _t._FileReadSource, parser: _t._DefEtreeParsers[_t._ET_co], - ) -> _ElementTree[_t._ET_co]: ... - @overload # from file source, default parser + ) -> _ElementTree[_t._ET_co]: + """ElementTree wrapper class for Element objects. + + Annotation + ---------- + This overload is used when creating an ElementTree from a file source with + user-supplied standard parser. + + See Also + -------- + - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) + """ + + @overload # from file source, custom target parser def __new__( cls, - element: None = ..., + element: None = None, *, file: _t._FileReadSource, - parser: None = ..., - ) -> _ElementTree: ... + parser: CustomTargetParser[_t._ET_co], + ) -> _ElementTree[_t._ET_co]: + """ElementTree wrapper class for Element objects. + + Annotation + ---------- + This overload is used when creating an ElementTree from a file source with + custom target parser. Returns the result dictated by parser target object + instead of ElementTree. + + See Also + -------- + - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) + """ + + @overload # from file source, no parser supplied + def __new__( + cls, + element: None = None, + *, + file: _t._FileReadSource, + parser: None = None, + ) -> _ElementTree[_t._ET_co]: + """ElementTree wrapper class for Element objects. + + Annotation + ---------- + This overload is used when creating an ElementTree from a file source + without a parser supplied. The default parser is used in this case. + + See Also + -------- + - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) + """ @property def parser(self) -> _t._DefEtreeParsers[_t._ET_co] | None: ... From 1192d3eb833ff88e50e8c64699da190f0d16ed50 Mon Sep 17 00:00:00 2001 From: Abel Cheung Date: Wed, 13 Aug 2025 06:25:50 +0000 Subject: [PATCH 3/5] chore: refresh multiclass patch --- multi-subclass.patch | 99 ++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/multi-subclass.patch b/multi-subclass.patch index ac0d15d3..cc4aaaf6 100644 --- a/multi-subclass.patch +++ b/multi-subclass.patch @@ -8,7 +8,7 @@ 7 files changed, 63 insertions(+), 63 deletions(-) diff --git a/pyproject.toml b/pyproject.toml -index 86ed27d..57b9561 100644 +index fde935a..ab3da97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = ['pdm-backend ~= 2.4'] @@ -55,10 +55,10 @@ index 51e7c0f..f4e4072 100644 - ) -> list[_ET]: ... + ) -> list[_Element]: ... diff --git a/src/lxml-stubs/etree/_element.pyi b/src/lxml-stubs/etree/_element.pyi -index f114265..ff59370 100644 +index bcb5d96..8425502 100644 --- a/src/lxml-stubs/etree/_element.pyi +++ b/src/lxml-stubs/etree/_element.pyi -@@ -19,9 +19,9 @@ from ._parser import CustomTargetParser +@@ -20,9 +20,9 @@ from ._parser import CustomTargetParser from ._xslt import XSLTAccessControl, XSLTExtension, _Stylesheet_Param, _XSLTResultTree if sys.version_info >= (3, 11): @@ -70,7 +70,7 @@ index f114265..ff59370 100644 if sys.version_info >= (3, 13): from warnings import deprecated -@@ -170,11 +170,11 @@ class _Element: +@@ -181,11 +181,11 @@ class _Element: # def __delitem__(self, __k: int | slice) -> None: ... @overload @@ -85,7 +85,7 @@ index f114265..ff59370 100644 # An element itself can be treated as container of other elements. When used # like elem[:] = new_elem, only subelements within new_elem will be # inserted, but not new_elem itself. If there is none, the whole slice would -@@ -184,13 +184,13 @@ class _Element: +@@ -195,13 +195,13 @@ class _Element: # doesn't apply to magic methods, at least for Pylance. Thus we create # additional overload for extend() but not here. @overload @@ -102,7 +102,7 @@ index f114265..ff59370 100644 def set(self, key: _t._AttrName, value: _t._AttrVal) -> None: """Sets an element attribute. -@@ -198,7 +198,7 @@ class _Element: +@@ -209,7 +209,7 @@ class _Element: -------- - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.set) """ @@ -111,7 +111,7 @@ index f114265..ff59370 100644 """Adds a subelement to the end of this element. See Also -@@ -220,7 +220,7 @@ class _Element: +@@ -231,7 +231,7 @@ class _Element: - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.extend) """ @overload @@ -120,7 +120,7 @@ index f114265..ff59370 100644 """Extends the current children by the elements in the iterable. See Also -@@ -234,14 +234,14 @@ class _Element: +@@ -245,14 +245,14 @@ class _Element: -------- - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.clear) """ @@ -137,7 +137,7 @@ index f114265..ff59370 100644 """Removes a matching subelement. See Also -@@ -250,7 +250,7 @@ class _Element: +@@ -261,7 +261,7 @@ class _Element: """ def index( self, @@ -146,7 +146,7 @@ index f114265..ff59370 100644 start: int | None = None, stop: int | None = None, ) -> int: -@@ -308,49 +308,49 @@ class _Element: +@@ -319,49 +319,49 @@ class _Element: # # extra Element / ET methods # @@ -203,7 +203,7 @@ index f114265..ff59370 100644 """Return an ElementTree for the root node of the document that contains this element. -@@ -361,7 +361,7 @@ class _Element: +@@ -372,7 +372,7 @@ class _Element: @overload def itersiblings( self, *tags: _t._TagSelector, preceding: bool = False @@ -212,7 +212,7 @@ index f114265..ff59370 100644 """Iterate over the following or preceding siblings of this element. Annotation -@@ -380,7 +380,7 @@ class _Element: +@@ -391,7 +391,7 @@ class _Element: tag: _t._TagSelector | Iterable[_t._TagSelector] | None = None, *, preceding: bool = False, @@ -221,7 +221,7 @@ index f114265..ff59370 100644 """Iterate over the following or preceding siblings of this element. Annotation -@@ -395,7 +395,7 @@ class _Element: +@@ -406,7 +406,7 @@ class _Element: - [Possible tag values in `iter()`](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.iter) """ @overload @@ -230,7 +230,7 @@ index f114265..ff59370 100644 """Iterate over the ancestors of this element (from parent to parent). Annotation -@@ -411,7 +411,7 @@ class _Element: +@@ -422,7 +422,7 @@ class _Element: @overload def iterancestors( self, tag: _t._TagSelector | Iterable[_t._TagSelector] | None = None @@ -239,7 +239,7 @@ index f114265..ff59370 100644 """Iterate over the ancestors of this element (from parent to parent). Annotation -@@ -426,7 +426,7 @@ class _Element: +@@ -437,7 +437,7 @@ class _Element: - [Possible tag values in `iter()`](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.iter) """ @overload @@ -248,7 +248,7 @@ index f114265..ff59370 100644 """Iterate over the descendants of this element in document order. Annotation -@@ -442,7 +442,7 @@ class _Element: +@@ -453,7 +453,7 @@ class _Element: @overload def iterdescendants( self, tag: _t._TagSelector | Iterable[_t._TagSelector] | None = None @@ -257,7 +257,7 @@ index f114265..ff59370 100644 """Iterate over the descendants of this element in document order. Annotation -@@ -459,7 +459,7 @@ class _Element: +@@ -470,7 +470,7 @@ class _Element: @overload def iterchildren( self, *tags: _t._TagSelector, reversed: bool = False @@ -266,7 +266,7 @@ index f114265..ff59370 100644 """Iterate over the children of this element. Annotation -@@ -478,7 +478,7 @@ class _Element: +@@ -489,7 +489,7 @@ class _Element: tag: _t._TagSelector | Iterable[_t._TagSelector] | None = None, *, reversed: bool = False, @@ -275,7 +275,7 @@ index f114265..ff59370 100644 """Iterate over the children of this element. Annotation -@@ -493,7 +493,7 @@ class _Element: +@@ -504,7 +504,7 @@ class _Element: - [Possible tag values in `iter()`](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.iter) """ @overload @@ -284,7 +284,7 @@ index f114265..ff59370 100644 """Iterate over all elements in the subtree in document order (depth first pre-order), starting with this element. -@@ -509,7 +509,7 @@ class _Element: +@@ -520,7 +520,7 @@ class _Element: @overload def iter( self, tag: _t._TagSelector | Iterable[_t._TagSelector] | None = None @@ -293,16 +293,16 @@ index f114265..ff59370 100644 """Iterate over all elements in the subtree in document order (depth first pre-order), starting with this element. -@@ -557,7 +557,7 @@ class _Element: +@@ -568,7 +568,7 @@ class _Element: - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.itertext) - [Possible tag values in `iter()`](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Element.iter) """ -- makeelement: _t._ElementFactory[Self] -+ makeelement: _t._ElementFactory[_Element] +- makeelement: type[Self] ++ makeelement: type[_Element] """Creates a new element associated with the same document. See Also -@@ -566,7 +566,7 @@ class _Element: +@@ -577,7 +577,7 @@ class _Element: """ def find( self, path: _t._ElemPathArg, namespaces: _t._StrOnlyNSMap | None = None @@ -311,7 +311,7 @@ index f114265..ff59370 100644 """Creates a new element associated with the same document. See Also -@@ -609,7 +609,7 @@ class _Element: +@@ -620,7 +620,7 @@ class _Element: """ def findall( self, path: _t._ElemPathArg, namespaces: _t._StrOnlyNSMap | None = None @@ -320,7 +320,7 @@ index f114265..ff59370 100644 """Finds all matching subelements, by tag name or path. See Also -@@ -618,7 +618,7 @@ class _Element: +@@ -629,7 +629,7 @@ class _Element: """ def iterfind( self, path: _t._ElemPathArg, namespaces: _t._StrOnlyNSMap | None = None @@ -329,7 +329,7 @@ index f114265..ff59370 100644 """Iterates over all matching subelements, by tag name or path. See Also -@@ -646,7 +646,7 @@ class _Element: +@@ -657,7 +657,7 @@ class _Element: expr: str, *, translator: _CSSTransArg = "xml", @@ -338,7 +338,7 @@ index f114265..ff59370 100644 """Run the CSS expression on this element and its children, returning a list of the results. -@@ -655,13 +655,13 @@ class _Element: +@@ -666,13 +666,13 @@ class _Element: - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree._Entity.cssselect) """ @deprecated("Since v2.0 (2008); use list(element) or iterate over element") @@ -352,22 +352,24 @@ index f114265..ff59370 100644 - ) -> Iterator[Self]: ... + ) -> Iterator[_Element]: ... - _ET2_co = TypeVar("_ET2_co", bound=_Element, default=_Element, covariant=True) + Element: TypeAlias = _Element diff --git a/src/lxml-stubs/etree/_factory_func.pyi b/src/lxml-stubs/etree/_factory_func.pyi -index b417517..29d0261 100644 +index 3dd52b8..1dfdf67 100644 --- a/src/lxml-stubs/etree/_factory_func.pyi +++ b/src/lxml-stubs/etree/_factory_func.pyi -@@ -14,7 +14,7 @@ from .._types import ( - ) - from ..html import HtmlElement - from ..objectify import ObjectifiedElement, StringElement --from ._element import _Comment, _ElementTree, _Entity, _ProcessingInstruction -+from ._element import _Comment, _Element, _ElementTree, _Entity, _ProcessingInstruction - from ._parser import CustomTargetParser - - _T = TypeVar("_T") -@@ -64,13 +64,13 @@ def SubElement( +@@ -1,9 +1,9 @@ + from typing import TypeVar, overload + + from .._types import ( +- _ET, + _AttrMapping, + _AttrVal, ++ _Element, + _NSMapArg, + _TagName, + _TextArg, +@@ -57,10 +57,10 @@ def SubElement( ) -> HtmlElement: ... @overload def SubElement( @@ -380,14 +382,11 @@ index b417517..29d0261 100644 **_extra: _AttrVal, -) -> _ET: ... +) -> _Element: ... - @overload # from element, parser ignored - def ElementTree( - element: _ET, diff --git a/src/lxml-stubs/html/_element.pyi b/src/lxml-stubs/html/_element.pyi -index b00bfb3..8cda2b3 100644 +index d46fbc3..d327fb5 100644 --- a/src/lxml-stubs/html/_element.pyi +++ b/src/lxml-stubs/html/_element.pyi -@@ -124,7 +124,7 @@ class HtmlElement(etree.ElementBase): +@@ -126,7 +126,7 @@ class HtmlElement(etree.ElementBase): # Subclassing of _Element should not go beyond HtmlElement. For example, # while children of HtmlElement are mostly HtmlElement, FormElement never # contains FormElement as child. @@ -396,7 +395,7 @@ index b00bfb3..8cda2b3 100644 def __getitem__( self, __x: int, -@@ -134,7 +134,7 @@ class HtmlElement(etree.ElementBase): +@@ -136,7 +136,7 @@ class HtmlElement(etree.ElementBase): self, __x: slice, ) -> list[HtmlElement]: ... @@ -405,7 +404,7 @@ index b00bfb3..8cda2b3 100644 def __setitem__( self, __x: int, -@@ -150,9 +150,9 @@ class HtmlElement(etree.ElementBase): +@@ -152,9 +152,9 @@ class HtmlElement(etree.ElementBase): def __reversed__(self) -> Iterator[HtmlElement]: ... def append( # pyright: ignore[reportIncompatibleMethodOverride] self, @@ -417,7 +416,7 @@ index b00bfb3..8cda2b3 100644 @deprecated("Expects iterable of elements as value, not single element") def extend( self, -@@ -166,30 +166,30 @@ class HtmlElement(etree.ElementBase): +@@ -168,30 +168,30 @@ class HtmlElement(etree.ElementBase): def insert( # pyright: ignore[reportIncompatibleMethodOverride] self, index: int, @@ -455,7 +454,7 @@ index b00bfb3..8cda2b3 100644 ) -> None: ... def getparent(self) -> HtmlElement | None: ... def getnext(self) -> HtmlElement | None: ... -@@ -270,7 +270,7 @@ class HtmlElement(etree.ElementBase): +@@ -272,7 +272,7 @@ class HtmlElement(etree.ElementBase): path: _ElemPathArg, namespaces: _StrOnlyNSMap | None = None, ) -> HtmlElement | None: ... @@ -464,7 +463,7 @@ index b00bfb3..8cda2b3 100644 self, path: _ElemPathArg, namespaces: _StrOnlyNSMap | None = None, -@@ -280,7 +280,7 @@ class HtmlElement(etree.ElementBase): +@@ -282,7 +282,7 @@ class HtmlElement(etree.ElementBase): path: _ElemPathArg, namespaces: _StrOnlyNSMap | None = None, ) -> Iterator[HtmlElement]: ... From 24945ce47faae870e8d95fc186dc66a13efdeb00 Mon Sep 17 00:00:00 2001 From: Abel Cheung Date: Wed, 13 Aug 2025 06:39:58 +0000 Subject: [PATCH 4/5] chore: fix name import location in multisubclass patch --- multi-subclass.patch | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/multi-subclass.patch b/multi-subclass.patch index cc4aaaf6..c2547a17 100644 --- a/multi-subclass.patch +++ b/multi-subclass.patch @@ -1,11 +1,11 @@ pyproject.toml | 2 +- src/lxml-stubs/cssselect.pyi | 10 ++--- src/lxml-stubs/etree/_element.pyi | 74 +++++++++++++++++----------------- - src/lxml-stubs/etree/_factory_func.pyi | 6 +-- + src/lxml-stubs/etree/_factory_func.pyi | 7 ++-- src/lxml-stubs/html/_element.pyi | 26 ++++++------ src/lxml-stubs/objectify/_element.pyi | 6 +-- tests/runtime/_testutils/common.py | 2 +- - 7 files changed, 63 insertions(+), 63 deletions(-) + 7 files changed, 63 insertions(+), 64 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fde935a..ab3da97 100644 @@ -355,21 +355,27 @@ index bcb5d96..8425502 100644 Element: TypeAlias = _Element diff --git a/src/lxml-stubs/etree/_factory_func.pyi b/src/lxml-stubs/etree/_factory_func.pyi -index 3dd52b8..1dfdf67 100644 +index 3dd52b8..d692fb7 100644 --- a/src/lxml-stubs/etree/_factory_func.pyi +++ b/src/lxml-stubs/etree/_factory_func.pyi -@@ -1,9 +1,9 @@ +@@ -1,7 +1,6 @@ from typing import TypeVar, overload from .._types import ( - _ET, _AttrMapping, _AttrVal, -+ _Element, _NSMapArg, - _TagName, - _TextArg, -@@ -57,10 +57,10 @@ def SubElement( +@@ -10,7 +9,7 @@ from .._types import ( + ) + from ..html import HtmlElement + from ..objectify import ObjectifiedElement, StringElement +-from ._element import _Comment, _Entity, _ProcessingInstruction ++from ._element import _Comment, _Element, _Entity, _ProcessingInstruction + + _T = TypeVar("_T") + +@@ -57,10 +56,10 @@ def SubElement( ) -> HtmlElement: ... @overload def SubElement( From 2a4ea02b18e4e1988e2690829288eeb08c211917 Mon Sep 17 00:00:00 2001 From: Abel Cheung Date: Wed, 13 Aug 2025 06:47:54 +0000 Subject: [PATCH 5/5] fix: correct custom target parser support in ElementTree again --- src/lxml-stubs/etree/_element.pyi | 61 +++---------------------------- 1 file changed, 6 insertions(+), 55 deletions(-) diff --git a/src/lxml-stubs/etree/_element.pyi b/src/lxml-stubs/etree/_element.pyi index bcb5d969..2e6b62ae 100644 --- a/src/lxml-stubs/etree/_element.pyi +++ b/src/lxml-stubs/etree/_element.pyi @@ -693,19 +693,7 @@ class _ElementTree(Generic[_t._ET_co]): element: _t._ET_co, *, file: None = None, - ) -> _ElementTree[_t._ET_co]: - """ElementTree wrapper class for Element objects. - - Annotation - ---------- - This overload is used when creating an ElementTree directly from a root - Element object. Other arguments are ignored in this case. - - See Also - -------- - - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) - """ - + ) -> _ElementTree[_t._ET_co]: ... @overload # from file source, standard parser def __new__( cls, @@ -713,40 +701,15 @@ class _ElementTree(Generic[_t._ET_co]): *, file: _t._FileReadSource, parser: _t._DefEtreeParsers[_t._ET_co], - ) -> _ElementTree[_t._ET_co]: - """ElementTree wrapper class for Element objects. - - Annotation - ---------- - This overload is used when creating an ElementTree from a file source with - user-supplied standard parser. - - See Also - -------- - - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) - """ - + ) -> _ElementTree[_t._ET_co]: ... @overload # from file source, custom target parser - def __new__( + def __new__( # type: ignore[misc] cls, element: None = None, *, file: _t._FileReadSource, - parser: CustomTargetParser[_t._ET_co], - ) -> _ElementTree[_t._ET_co]: - """ElementTree wrapper class for Element objects. - - Annotation - ---------- - This overload is used when creating an ElementTree from a file source with - custom target parser. Returns the result dictated by parser target object - instead of ElementTree. - - See Also - -------- - - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) - """ - + parser: CustomTargetParser[_T], + ) -> _T: ... @overload # from file source, no parser supplied def __new__( cls, @@ -754,19 +717,7 @@ class _ElementTree(Generic[_t._ET_co]): *, file: _t._FileReadSource, parser: None = None, - ) -> _ElementTree[_t._ET_co]: - """ElementTree wrapper class for Element objects. - - Annotation - ---------- - This overload is used when creating an ElementTree from a file source - without a parser supplied. The default parser is used in this case. - - See Also - -------- - - [API Documentation](https://lxml.de/apidoc/lxml.etree.html#lxml.etree.ElementTree) - """ - + ) -> _ElementTree[_t._ET_co]: ... @property def parser(self) -> _t._DefEtreeParsers[_t._ET_co] | None: ... @property