Merge pull request #56 from thewebscraping/bugs/shared-extra-configs

Fix per-request header order and shared config issue (#53)

Author: thewebscraping

pyproject.toml

@@ -35,7 +35,6 @@ dependencies = [
     "idna~=3.10",
     "charset-normalizer",
     "orjson",
-    "chardet~=5.2.0",
 ]
 
 [dependency-groups]

src/tls_requests/client.py

@@ -141,25 +141,16 @@ def __init__(
     ) -> None:
         if "tls_identifier" in config:
             logger.warning(
-                "The 'tls_identifier' parameter is deprecated and will be removed in version 1.3.0. "
-                "Please use 'client_identifier' instead."
+                "The 'client_identifier' parameter is deprecated and will be removed in version 1.3.0. "
+                "Please use 'tls_identifier' instead."
             )
-            if client_identifier == DEFAULT_CLIENT_IDENTIFIER:
-                client_identifier = config.pop("tls_identifier")
-
-        if "tls_debug" in config:
-            logger.warning(
-                "The 'tls_debug' parameter is deprecated and will be removed in version 1.3.0. "
-                "Please use 'debug' instead."
-            )
-            if debug == DEFAULT_DEBUG:
-                debug = config.pop("tls_debug")
+            client_identifier = config.pop("tls_identifier")
 
         self._session = TLSClient.initialize()
         self._config = TLSConfig.from_kwargs(
             http2=http2,
             verify=verify,
-            tls_identifier=self.prepare_tls_identifier(client_identifier),
+            client_identifier=self.prepare_client_identifier(client_identifier),
             debug=debug,
             protocol_racing=protocol_racing,
             allow_http=allow_http,
@@ -297,14 +288,14 @@ def prepare_proxy(self, proxy: Optional[ProxyTypes]) -> Optional[Proxy]:
             return Proxy(str(proxy))
         raise ProxyError(f"Unsupported proxy type: {type(proxy)}")
 
-    def prepare_tls_identifier(self, identifier: Optional[IdentifierArgTypes]) -> str:
+    def prepare_client_identifier(self, identifier: Optional[IdentifierArgTypes]) -> str:
         if isinstance(identifier, str):
             return identifier
         if isinstance(identifier, TLSIdentifierRotator):
             return str(identifier.next())
         return str(DEFAULT_CLIENT_IDENTIFIER)
 
-    def prepare_config(self, request: Request, tls_identifier: str = DEFAULT_CLIENT_IDENTIFIER):
+    def prepare_config(self, request: Request, client_identifier: str = DEFAULT_CLIENT_IDENTIFIER):
         """Prepare TLS Config"""
 
         config = self.config.copy_with(
@@ -317,7 +308,7 @@ def prepare_config(self, request: Request, tls_identifier: str = DEFAULT_CLIENT_
             timeout=request.timeout,
             http2=True if self.http2 in ["auto", "http2", True, None] else False,
             verify=self.verify,
-            tls_identifier=tls_identifier,
+            client_identifier=client_identifier,
             protocol_racing=request.protocol_racing,
             allow_http=request.allow_http,
             stream_id=request.stream_id,
@@ -457,8 +448,8 @@ def _send(
     ) -> Response:
         history = [] if history is None else history
         start = start or time.perf_counter()
-        tls_identifier = self.prepare_tls_identifier(self.client_identifier)
-        config = self.prepare_config(request, tls_identifier=tls_identifier)
+        client_identifier = self.prepare_client_identifier(self.client_identifier)
+        config = self.prepare_config(request, client_identifier=client_identifier)
         response = Response.from_tls_response(
             self.session.request(config.to_dict()),
             is_byte_response=config.isByteResponse,
@@ -894,7 +885,7 @@ async def aprepare_proxy(self, proxy: Optional[ProxyTypes]) -> Optional[Proxy]:
             return Proxy(str(proxy))
         raise ProxyError(f"Unsupported proxy type: {type(proxy)}")
 
-    async def aprepare_tls_identifier(self, identifier) -> str:
+    async def aprepare_client_identifier(self, identifier) -> str:
         if isinstance(identifier, str):
             return identifier
         if isinstance(identifier, TLSIdentifierRotator):
@@ -1246,8 +1237,8 @@ async def _send(  # type: ignore[override]
     ) -> Response:
         history = [] if history is None else history
         start = start or time.perf_counter()
-        tls_identifier = await self.aprepare_tls_identifier(self.client_identifier)
-        config = self.prepare_config(request, tls_identifier=tls_identifier)
+        client_identifier = await self.aprepare_client_identifier(self.client_identifier)
+        config = self.prepare_config(request, client_identifier=client_identifier)
         response = Response.from_tls_response(
             await self.session.arequest(config.to_dict()),
             is_byte_response=config.isByteResponse,

src/tls_requests/models/encoders.py

@@ -4,7 +4,7 @@
 import os
 from io import BufferedReader, BytesIO, TextIOWrapper
 from mimetypes import guess_type
-from typing import Any, AsyncIterator, Dict, Iterator, List, Mapping, Optional, Tuple, TypeVar
+from typing import Any, AsyncIterator, Dict, Iterator, List, Mapping, Optional, Tuple, TypeVar, cast
 from urllib.parse import urlencode
 
 from ..types import BufferTypes, ByteOrStr, RequestData, RequestFiles, RequestFileValue, RequestJson
@@ -106,7 +106,7 @@ def unpack(self, value: RequestFileValue) -> Tuple[str, BufferTypes, str]:
                 if args:
                     content_type = args[0]
             else:
-                buffer = value
+                buffer = value[0]
 
         elif isinstance(value, str):
             buffer = value.encode("utf-8")
@@ -124,12 +124,12 @@ def unpack(self, value: RequestFileValue) -> Tuple[str, BufferTypes, str]:
                 buffer.close()
                 buffer = open(buffer.name, "rb")
 
-        elif not isinstance(buffer, bytes):
-            raise ValueError
-        else:
+        elif isinstance(buffer, bytes):
             buffer = BytesIO(buffer)
+        elif not hasattr(buffer, "read"):
+            raise ValueError
 
-        return str(filename or "upload"), buffer, str(content_type or "application/octet-stream")
+        return str(filename or "upload"), cast(BufferTypes, buffer), str(content_type or "application/octet-stream")
 
     def render_data(self, chunk_size: int = 65_536) -> Iterator[bytes]:
         yield from iter_buffer(self._buffer, chunk_size)

src/tls_requests/models/headers.py

@@ -115,7 +115,10 @@ def __setitem__(self, key, value) -> None:
             self._items.append((key, value))
 
     def __getitem__(self, key):
-        return self.get(key)
+        val = self.get(key)
+        if val is None:
+            raise KeyError(key)
+        return val
 
     def __delitem__(self, key):
         key = self._normalize_key(key)

src/tls_requests/models/rotators.py

@@ -24,7 +24,7 @@
 T = TypeVar("T")
 R = TypeVar("R", bound="BaseRotator")
 
-TLS_IDENTIFIER_TEMPLATES = [
+CLIENT_IDENTIFIER_TEMPLATES = [
     "chrome_120",
     "chrome_124",
     "chrome_131",
@@ -237,9 +237,20 @@ def __init__(
         self.strategy = strategy
         self._iterator: Optional[Iterator[T]] = None
         self._lock = threading.Lock()
-        self._async_lock = asyncio.Lock()
+        self._async_lock: Optional[asyncio.Lock] = None
         self._rebuild_iterator()
 
+    @property
+    def async_lock(self) -> asyncio.Lock:
+        """
+        Lazily creates and returns an `asyncio.Lock`. This ensures that the lock
+        is created within the correct event loop and avoids issues in
+        synchronous contexts or older Python versions (like 3.9).
+        """
+        if self._async_lock is None:
+            self._async_lock = asyncio.Lock()
+        return self._async_lock
+
     @classmethod
     def from_file(
         cls: type[R],
@@ -354,7 +365,7 @@ async def anext(self, *args, **kwargs) -> T:
         Raises:
             ValueError: If the rotator contains no items.
         """
-        async with self._async_lock:
+        async with self.async_lock:
             if not self.items:
                 raise ValueError("Rotator is empty.")
             if self.strategy == "random":
@@ -368,15 +379,15 @@ async def aadd(self, item: T) -> None:
         """
         Adds a new item to the rotator in a coroutine-safe manner.
         """
-        async with self._async_lock:
+        async with self.async_lock:
             self.items.append(item)
             self._rebuild_iterator()
 
     async def aremove(self, item: T) -> None:
         """
         Removes an item from the rotator in a coroutine-safe manner.
         """
-        async with self._async_lock:
+        async with self.async_lock:
             self.items = [i for i in self.items if i != item]
             self._rebuild_iterator()
 
@@ -418,7 +429,7 @@ async def amark_result(self, proxy: Proxy, success: bool, latency: Optional[floa
         """
         Coroutine-safely updates a proxy's performance statistics.
         """
-        async with self._async_lock:
+        async with self.async_lock:
             self._update_proxy_stats(proxy, success, latency)
 
     def _update_proxy_stats(self, proxy: Proxy, success: bool, latency: Optional[float] = None):
@@ -442,7 +453,7 @@ def __init__(
         items: Optional[Iterable[IdentifierTypes]] = None,
         strategy: Literal["round_robin", "random", "weighted"] = "round_robin",
     ) -> None:
-        super().__init__(items or TLS_IDENTIFIER_TEMPLATES, strategy)  # type: ignore[arg-type]
+        super().__init__(items or CLIENT_IDENTIFIER_TEMPLATES, strategy)  # type: ignore[arg-type]
 
     @classmethod
     def rebuild_item(cls, item: Any) -> Optional[IdentifierTypes]:

src/tls_requests/models/tls.py

@@ -200,7 +200,7 @@ async def _aread(cls, fn: Callable, payload: dict):
 class _BaseConfig:
     """Base configuration for TLSSession"""
 
-    _extra_kwargs: dict = field(default_factory=dict, init=False, repr=False)
+    _extra_config: dict = field(default_factory=dict, init=False, repr=False)
 
     @classmethod
     def model_fields_set(cls) -> Set[str]:
@@ -212,13 +212,13 @@ def from_kwargs(cls: type[T], **kwargs: Any) -> T:
         known_kwargs = {cls.to_camel_case(k): v for k, v in kwargs.items() if k in model_fields_set}
         extra_kwargs = {cls.to_camel_case(k): v for k, v in kwargs.items() if k not in model_fields_set}
         instance = cls(**known_kwargs)
-        instance._extra_kwargs = extra_kwargs
+        instance._extra_config = extra_kwargs
         return instance
 
     def to_dict(self) -> dict:
         data = asdict(self)
-        if hasattr(self, "_extra_kwargs"):
-            data.update(self._extra_kwargs)
+        if hasattr(self, "_extra_config"):
+            data.update(self._extra_config)
         return {k: v for k, v in data.items() if not k.startswith("_") and v is not None}
 
     def to_payload(self) -> dict:
@@ -552,13 +552,13 @@ def copy_with(
         kwargs.update(filtered_mapping)
 
         current_kwargs = asdict(self)
-        if hasattr(self, "_extra"):
-            current_kwargs.update(self._extra_kwargs)
+        if hasattr(self, "_extra_config"):
+            current_kwargs.update(self._extra_config)
 
         for k, v in kwargs.items():
             current_kwargs[k] = v
 
-        return self.__class__.from_kwargs(**current_kwargs)
+        return super().from_kwargs(**current_kwargs)
 
     @classmethod
     def from_kwargs(

src/tls_requests/models/urls.py

@@ -339,12 +339,81 @@ def _prepare(self, url: Union["URL", str, bytes]) -> ParseResult:
             url = str(url)
 
         if not isinstance(url, str):
-            raise URLError("Invalid URL: %s" % url)
+            raise URLError(f"Invalid URL: {url}")
+
+        url_to_parse = url.lstrip()
+
+        # 0. Pre-parsing: default to http if scheme is missing
+        if "://" not in url_to_parse and not url_to_parse.startswith("/") and not url_to_parse.startswith("./"):
+            # Check if it doesn't look like a potential relative URL with query/fragment
+            if not (url_to_parse.startswith("?") or url_to_parse.startswith("#")):
+                url_to_parse = f"http://{url_to_parse}"
+
+        # 1. Pre-parsing repair for raw IPv6 addresses
+        if ":" in url_to_parse:
+            # Extract authority candidate: part between scheme and path
+            if "://" in url_to_parse:
+                authority_candidate = url_to_parse.split("://", 1)[1].split("/", 1)[0].split("?", 1)[0].split("#", 1)[0]
+            else:
+                authority_candidate = url_to_parse.split("/", 1)[0].split("?", 1)[0].split("#", 1)[0]
+
+            # Extract host part (ignoring user:pass@)
+            host_candidate = authority_candidate.rsplit("@", 1)[-1]
+
+            # If it looks like IPv6 but lacks brackets
+            if host_candidate.count(":") > 1 and not (host_candidate.startswith("[") and "]" in host_candidate):
+                # Try to determine if it's IP:PORT or just IP
+                # We prioritize IP:PORT if the last part is digits
+                possible_ips = []
+                h_p, _, port = host_candidate.rpartition(":")
+                if port.isdigit() and h_p.count(":") >= 1:
+                    possible_ips.append((h_p, port))
+                possible_ips.append((host_candidate, ""))
+
+                for ip, p_val in possible_ips:
+                    try:
+                        ipaddress.IPv6Address(ip)
+                        repaired = f"[{ip}]"
+                        if p_val:
+                            repaired += f":{p_val}"
+                        url_to_parse = url_to_parse.replace(host_candidate, repaired, 1)
+                        break
+                    except ValueError:
+                        continue
 
         for attr in self.__attrs__:
             setattr(self, attr, None)
 
-        parsed = urlparse(url.lstrip())
+        # 2. Parse and Validate
+        try:
+            # First, check for malformed brackets in the string we're about to parse
+            # We strictly enforce one '[' and one ']' in the authority if any exist
+            authority = ""
+            if "://" in url_to_parse:
+                authority = url_to_parse.split("://", 1)[1].split("/", 1)[0]
+            else:
+                authority = url_to_parse.split("/", 1)[0]
+
+            if "[" in authority or "]" in authority:
+                if authority.count("[") != 1 or authority.count("]") != 1:
+                    raise ValueError("Malformed bracketed host")
+
+                start = authority.find("[")
+                end = authority.find("]")
+                if start > end:
+                    raise ValueError("Invalid bracket order")
+
+                # Content inside brackets MUST be a valid IPv6
+                ip_content = authority[start + 1 : end]
+                try:
+                    ipaddress.IPv6Address(ip_content)
+                except ValueError:
+                    raise ValueError(f"Invalid IPv6 in brackets: {ip_content}")
+
+            parsed = urlparse(url_to_parse)
+
+        except (ValueError, AttributeError) as e:
+            raise URLError(f"Invalid URL: {url}. {str(e)}") from e
 
         self.auth = parsed.username, parsed.password
         self.scheme = parsed.scheme
@@ -363,14 +432,14 @@ def _prepare(self, url: Union["URL", str, bytes]) -> ParseResult:
                 try:
                     self.host = idna.encode(hostname).decode("ascii")
                 except idna.IDNAError:
-                    raise URLError("Invalid IDNA hostname: %s" % hostname)
+                    raise URLError(f"Invalid IDNA hostname: {hostname}")
 
         self.port = ""
         try:
             if parsed.port:
                 self.port = str(parsed.port)
         except ValueError as e:
-            raise URLError("%s. port range must be 0 - 65535." % e.args[0])
+            raise URLError(f"{e.args[0]}. port range must be 0 - 65535.")
 
         self.path = parsed.path
         self.fragment = parsed.fragment
@@ -388,25 +457,28 @@ def _build(self, secure: bool = False) -> str:
         Returns:
             The final URL string.
         """
-        urls = [self.scheme, "://"]
+        scheme = self.scheme or ""
+        urls = [scheme, "://"] if scheme else []
         authority = self.netloc
         if self.username or self.password:
+            username = self.username or ""
             password = self.password or ""
             if secure:
                 password = "[secure]"
 
             authority = "@".join(
                 [
-                    ":".join([self.username, password]),
+                    ":".join([username, password]),
                     self.netloc,
                 ]
             )
 
         urls.append(authority)
+        path = self.path or ""
         if self.query:
-            urls.append("?".join([self.path, self.query]))
+            urls.append("?".join([path, self.query]))
         else:
-            urls.append(self.path)
+            urls.append(path)
 
         if self.fragment:
             urls.append("#" + self.fragment)