diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7f46e60..fd59fae 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -37,8 +37,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools flake8 pytest - pip list + pip install --upgrade tox setuptools flake8 pytest mypy + pip freeze - name: Test with pytest run: | tox -e py -- ${{ matrix.pytest-args }} diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 318b676..a80c6ec 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -1.0.0 (unreleased) +1.0.0 IMPORTANT diff --git a/README.rst b/README.rst index b254cf8..6b8a469 100644 --- a/README.rst +++ b/README.rst @@ -26,8 +26,8 @@ Users are advised to pin their installations to "metadata_parser<{MINOR +1}" For example: -* if the current release is: `0.10.6` -* the advised pin is: `metadata_parser<0.11` +* if the current release is: `1.0.0` +* the advised pin is: `metadata_parser<1.1.0` PATCH releases will usually be bug fixes and new features that support backwards compatibility with Public Methods. Private Methods are not guaranteed to be diff --git a/setup.py b/setup.py index 54c866b..7f7f1f6 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,6 @@ long_description=long_description, classifiers=[ "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", diff --git a/src/metadata_parser/__init__.py b/src/metadata_parser/__init__.py index 4342d4a..5865822 100644 --- a/src/metadata_parser/__init__.py +++ b/src/metadata_parser/__init__.py @@ -69,7 +69,7 @@ # ============================================================================== -__VERSION__ = "1.0.0dev" +__VERSION__ = "1.0.0" # ------------------------------------------------------------------------------ diff --git a/src/metadata_parser/requests_extensions.py b/src/metadata_parser/requests_extensions.py index 26e6feb..e053cd7 100644 --- a/src/metadata_parser/requests_extensions.py +++ b/src/metadata_parser/requests_extensions.py @@ -4,18 +4,17 @@ import cgi # noqa: I202 import logging import socket +from typing import Any from typing import Optional from typing import Tuple from typing import TYPE_CHECKING # pypi -import requests from requests_toolbelt.utils.deprecated import get_encodings_from_content # local from . import config from .exceptions import AllowableError -from .utils import DummyResponse from .utils import safe_sample if TYPE_CHECKING: @@ -35,16 +34,17 @@ # peername hacks # only use for these stdlib packages # eventually will not be needed thanks to upstream changes in `requests` +_compatible_sockets: Tuple[Any, ...] try: - _compatible_sockets: Tuple = ( + _compatible_sockets = ( _socket.socket, socket._socketobject, # type: ignore[attr-defined] ) except AttributeError: - _compatible_sockets: Tuple = (_socket.socket,) # type: ignore[no-redef] + _compatible_sockets = (_socket.socket,) # type: ignore[no-redef] -def derive_encoding__hook(resp: "TYPES_RESPONSE", *args, **kwargs) -> None: +def derive_encoding__hook(resp: Any, *args, **kwargs) -> None: """ a note about `requests` @@ -58,14 +58,14 @@ def derive_encoding__hook(resp: "TYPES_RESPONSE", *args, **kwargs) -> None: servers to not follow RFC and for the default encoding to be different. """ if TYPE_CHECKING: - assert hasattr(resp, "_encoding_fallback") assert hasattr(resp, "_encoding_content") + assert hasattr(resp, "_encoding_fallback") assert hasattr(resp, "_encoding_headers") + resp._encoding_content = None resp._encoding_fallback = config.ENCODING_FALLBACK # modified version, returns `None` if no charset available resp._encoding_headers = get_encoding_from_headers(resp.headers) - resp._encoding_content = None if not resp._encoding_headers and resp.content: # html5 spec requires a meta-charset in the first 1024 bytes _sample = safe_sample(resp.content) @@ -120,7 +120,9 @@ def get_encoding_from_headers(headers: "CaseInsensitiveDict") -> Optional[str]: # ------------------------------------------------------------------------------ -def get_response_peername(resp: "TYPES_RESPONSE") -> Optional["TYPES_PEERNAME"]: +def get_response_peername( + resp: Any, +) -> Optional["TYPES_PEERNAME"]: """ used to get the peername (ip+port) data from the request if a socket is found, caches this onto the request object @@ -133,17 +135,19 @@ def get_response_peername(resp: "TYPES_RESPONSE") -> Optional["TYPES_PEERNAME"]: * _mp_peername """ - if not isinstance(resp, requests.Response) and not isinstance(resp, DummyResponse): - # raise AllowableError("Not a HTTPResponse") - log.debug("Not a supported HTTPResponse | %s", resp) - log.debug("-> received a type of: %s", type(resp)) - return None + # if not isinstance(resp, Response) and not isinstance(resp, DummyResponse): + # # raise AllowableError("Not a HTTPResponse") + # log.debug("Not a supported HTTPResponse | %s", resp) + # log.debug("-> received a type of: %s", type(resp)) + # return None if hasattr(resp, "_mp_peername"): return resp._mp_peername def _get_socket() -> Optional[socket.socket]: - if isinstance(resp, DummyResponse): + # only socket to `requests.Response` + # if not isinstance(resp, "Response"): + if not hasattr(resp, "raw"): return None i = 0 while True: @@ -168,14 +172,14 @@ def _get_socket() -> Optional[socket.socket]: pass return None + _mp_peername: Optional["TYPES_PEERNAME"] = None sock = _get_socket() - if sock: + if sock is not None: # only cache if we have a sock # we may want/need to call again - resp._mp_peername = sock.getpeername() # type: ignore [union-attr] - else: - resp._mp_peername = None # type: ignore [union-attr] - return resp._mp_peername # type: ignore [union-attr] + _mp_peername = sock.getpeername() + setattr(resp, "_mp_peername", _mp_peername) # type: ignore[union-attr] + return _mp_peername # ------------------------------------------------------------------------------ diff --git a/src/metadata_parser/typing.py b/src/metadata_parser/typing.py index 7674734..4351871 100644 --- a/src/metadata_parser/typing.py +++ b/src/metadata_parser/typing.py @@ -5,6 +5,7 @@ from typing import Optional from typing import Tuple from typing import TYPE_CHECKING +from typing import TypeVar from typing import Union # pypi @@ -13,13 +14,17 @@ if TYPE_CHECKING: from urllib.parse import ParseResult - import requests + from requests import Response - from . import DummyResponse from . import ResponseHistory + from .utils import DummyResponse + + # from requests.structures import CaseInsensitiveDict # ============================================================================== +T = TypeVar("T") + # TYPE_ENCODER = Callable[[str, Optional[str]], str] # def encode(value, strategy) TYPE_ENCODER = Callable[ [str, Optional[str]], Union[str, Dict] @@ -30,9 +35,22 @@ TYPE_URL_FETCH = Tuple[str, str, "ResponseHistory"] TYPE_URLPARSE = Callable[[str], "ParseResult"] TYPES_PEERNAME = Tuple[str, int] # (ip, port) -TYPES_RESPONSE = Union["DummyResponse", "requests.Response"] +TYPES_RESPONSE = Union["Response", "DummyResponse", T] TYPES_STRATEGY = Union[List[str], str, None] +""" +# TYPES_RESPONSE_EXTENDED = Union["TYPES_RESPONSE", "_SupportsContent", "T"] +class _SupportsContent(Protocol): + + _encoding_content: Optional[str] + _encoding_fallback: str + _encoding_headers: Optional[str] + content: str + encoding: Optional[str] + headers: "CaseInsensitiveDict" +""" + + class _UrlParserCacheable(Protocol): urlparse: TYPE_URLPARSE