Merge branch 'master' into win_arm64

Author: Harishmcw

.github/workflows/release.yml

@@ -78,6 +78,7 @@ jobs:
           - "cp312"
           - "cp313"
           - "cp314"
+          - "cp314t"
         exclude:
           - os: ubuntu-latest
             cibw_arch: universal2

Makefile

@@ -12,11 +12,14 @@ else
 endif
 
 compile:
-	$(PIP) install -e .
+	$(PIP) install -e . --group dev
 
 test: compile
 	$(PYTHON) -m unittest -v
 
+typecheck: compile
+	$(PYTHON) -m pyright
+
 clean:
 	find $(ROOT)/httptools/parser -name '*.c' | xargs rm -f
 	find $(ROOT)/httptools/parser -name '*.so' | xargs rm -f

httptools/__init__.py

@@ -1,6 +1,35 @@
 from . import parser
-from .parser import *  # NOQA
+from .parser import (
+    HTTPProtocol,
+    HttpRequestParser,
+    HttpResponseParser,
+    HttpParserError,
+    HttpParserCallbackError,
+    HttpParserInvalidStatusError,
+    HttpParserInvalidMethodError,
+    HttpParserInvalidURLError,
+    HttpParserUpgrade,
+    parse_url,
+)
 
-from ._version import __version__  # NOQA
+from ._version import __version__
 
-__all__ = parser.__all__ + ('__version__',)  # NOQA
+__all__ = (
+    "parser",
+    # protocol
+    "HTTPProtocol",
+    # parser
+    "HttpRequestParser",
+    "HttpResponseParser",
+    # errors
+    "HttpParserError",
+    "HttpParserCallbackError",
+    "HttpParserInvalidStatusError",
+    "HttpParserInvalidMethodError",
+    "HttpParserInvalidURLError",
+    "HttpParserUpgrade",
+    # url parser
+    "parse_url",
+    # version
+    "__version__",
+)

httptools/_version.py

@@ -10,4 +10,4 @@
 # supported platforms, publish the packages on PyPI, merge the PR
 # to the target branch, create a Git tag pointing to the commit.
 
-__version__ = '0.8.0.dev0'
+__version__ = '0.8.0'

httptools/parser/__init__.py

@@ -1,6 +1,28 @@
 from .protocol import HTTPProtocol
-from .parser import *  # NoQA
-from .errors import *  # NoQA
-from .url_parser import *  # NoQA
+from .parser import HttpRequestParser, HttpResponseParser  # NoQA
+from .errors import (
+    HttpParserError,
+    HttpParserCallbackError,
+    HttpParserInvalidStatusError,
+    HttpParserInvalidMethodError,
+    HttpParserInvalidURLError,
+    HttpParserUpgrade,
+)
+from .url_parser import parse_url
 
-__all__ = parser.__all__ + errors.__all__ + url_parser.__all__  # NoQA
+__all__ = (
+    # protocol
+    "HTTPProtocol",
+    # parser
+    "HttpRequestParser",
+    "HttpResponseParser",
+    # errors
+    "HttpParserError",
+    "HttpParserCallbackError",
+    "HttpParserInvalidStatusError",
+    "HttpParserInvalidMethodError",
+    "HttpParserInvalidURLError",
+    "HttpParserUpgrade",
+    # url_parser
+    "parse_url",
+)

httptools/parser/parser.pyi

@@ -1,57 +1,58 @@
-from typing import Union, Any
 from array import array
 from .protocol import HTTPProtocol
 
 class HttpParser:
-    def __init__(self, protocol: Union[HTTPProtocol, Any]) -> None:
-        """
-        protocol -- a Python object with the following methods
-        (all optional):
-
-          - on_message_begin()
-          - on_url(url: bytes)
-          - on_header(name: bytes, value: bytes)
-          - on_headers_complete()
-          - on_body(body: bytes)
-          - on_message_complete()
-          - on_chunk_header()
-          - on_chunk_complete()
-          - on_status(status: bytes)
+    def __init__(self, protocol: HTTPProtocol | object) -> None:
+        """The HTTP parser.
+
+        Args:
+            protocol (HTTPProtocol): Callback interface for the parser.
         """
 
+    def set_dangerous_leniencies(
+        self,
+        lenient_headers: bool | None = None,
+        lenient_chunked_length: bool | None = None,
+        lenient_keep_alive: bool | None = None,
+        lenient_transfer_encoding: bool | None = None,
+        lenient_version: bool | None = None,
+        lenient_data_after_close: bool | None = None,
+        lenient_optional_lf_after_cr: bool | None = None,
+        lenient_optional_cr_before_lf: bool | None = None,
+        lenient_optional_crlf_after_chunk: bool | None = None,
+        lenient_spaces_after_chunk_size: bool | None = None,
+    ) -> None:
+        """Set dangerous leniencies for the parser."""
+
     def get_http_version(self) -> str:
-        """Return an HTTP protocol version."""
-        ...
+        """Retrieve the HTTP protocol version e.g. "1.1"."""
 
     def should_keep_alive(self) -> bool:
-        """Return ``True`` if keep-alive mode is preferred."""
-        ...
+        """Return `True` if keep-alive mode is preferred."""
 
     def should_upgrade(self) -> bool:
-        """Return ``True`` if the parsed request is a valid Upgrade request.
+        """Return `True` if the parsed request is a valid Upgrade request.
         The method exposes a flag set just before on_headers_complete.
         Calling this method earlier will only yield `False`."""
-        ...
 
-    def feed_data(self, data: Union[bytes, bytearray, memoryview, array]) -> None:
+    def feed_data(self, data: bytes | bytearray | memoryview | array[int]) -> None:
         """Feed data to the parser.
 
-        Will eventually trigger callbacks on the ``protocol``
-        object.
+        Will eventually trigger callbacks on the ``protocol`` object.
 
         On HTTP upgrade, this method will raise an
         ``HttpParserUpgrade`` exception, with its sole argument
         set to the offset of the non-HTTP data in ``data``.
         """
 
 class HttpRequestParser(HttpParser):
-    """Used for parsing http requests from the server's side"""
+    """Used for parsing http requests from the server side."""
 
     def get_method(self) -> bytes:
-        """Return HTTP request method (GET, HEAD, etc)"""
+        """Retrieve the HTTP method of the request."""
 
 class HttpResponseParser(HttpParser):
-    """Used for parsing http requests from the client's side"""
+    """Used for parsing http responses from the client side."""
 
     def get_status_code(self) -> int:
-        """Return the status code of the HTTP response"""
+        """Retrieve the status code of the HTTP response."""

httptools/parser/protocol.py

@@ -4,12 +4,12 @@
 class HTTPProtocol(Protocol):
     """Used for providing static type-checking when parsing through the http protocol"""
 
-    def on_message_begin() -> None: ...
-    def on_url(url: bytes) -> None: ...
-    def on_header(name: bytes, value: bytes) -> None: ...
-    def on_headers_complete() -> None: ...
-    def on_body(body: bytes) -> None: ...
-    def on_message_complete() -> None: ...
-    def on_chunk_header() -> None: ...
-    def on_chunk_complete() -> None: ...
-    def on_status(status: bytes) -> None: ...
+    def on_message_begin(self) -> None: ...
+    def on_url(self, url: bytes) -> None: ...
+    def on_header(self, name: bytes, value: bytes) -> None: ...
+    def on_headers_complete(self) -> None: ...
+    def on_body(self, body: bytes) -> None: ...
+    def on_message_complete(self) -> None: ...
+    def on_chunk_header(self) -> None: ...
+    def on_chunk_complete(self) -> None: ...
+    def on_status(self, status: bytes) -> None: ...

httptools/parser/url_parser.pyi

@@ -1,4 +1,3 @@
-from typing import Union
 from array import array
 
 class URL:
@@ -10,18 +9,5 @@ class URL:
     fragment: bytes
     userinfo: bytes
 
-def parse_url(url: Union[bytes, bytearray, memoryview, array]) -> URL:
-    """Parse URL strings into a structured Python object.
-
-    Returns an instance of ``httptools.URL`` class with the
-    following attributes:
-
-      - schema: bytes
-      - host: bytes
-      - port: int
-      - path: bytes
-      - query: bytes
-      - fragment: bytes
-      - userinfo: bytes
-    """
-    ...
+def parse_url(url: bytes | bytearray | memoryview | array[int]) -> URL:
+    """Parse a URL string into a structured Python object."""

httptools/parser/url_parser.pyx

@@ -12,6 +12,8 @@ from . cimport url_cparser as uparser
 
 __all__ = ('parse_url',)
 
+DEF MAX_URL_LENGTH = (1 << 16) - 1
+
 @cython.freelist(250)
 cdef class URL:
     cdef readonly bytes schema
@@ -63,6 +65,14 @@ def parse_url(url):
 
     PyObject_GetBuffer(url, &py_buf, PyBUF_SIMPLE)
     try:
+        if py_buf.len > MAX_URL_LENGTH:
+            # http_parser stores URL field offsets/lengths as uint16_t,
+            # so URLs longer than this will cause silent truncation.
+            # See https://github.com/MagicStack/httptools/issues/142
+            raise HttpParserInvalidURLError(
+                "url is too long: url length of {} bytes exceeds the "
+                "maximum of {} bytes".format(py_buf.len, MAX_URL_LENGTH))
+
         buf_data = <char*>py_buf.buf
         res = uparser.http_parser_parse_url(buf_data, py_buf.len, 0, parsed)