diff --git a/.gitignore b/.gitignore index 38c049a8..d331e522 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,10 @@ tmp/ *.so # Python temp and build files +venv/ +.venv/ +env/ + docs/html/ __pycache__/ __pypackages__/ @@ -44,6 +48,7 @@ cython_debug/ docs/autoapi cov.xml .coverage.* +oryx-build-commands.txt # Hidden files that are definitely unwanted (redundant) diff --git a/authcaptureproxy/auth_capture_proxy.py b/authcaptureproxy/auth_capture_proxy.py index 95e8a254..65cff355 100644 --- a/authcaptureproxy/auth_capture_proxy.py +++ b/authcaptureproxy/auth_capture_proxy.py @@ -65,7 +65,25 @@ def __init__( self.session_factory: Callable[[], httpx.AsyncClient] = session_factory or ( lambda: httpx.AsyncClient(verify=ssl_context) ) - self.session: httpx.AsyncClient = session if session else self.session_factory() + # NOTE: Do not instantiate httpx.AsyncClient inside the event loop. + # Some SSL initialization (e.g., load_verify_locations) is blocking and will be flagged. + # + # Keep historical behavior when NOT running inside an event loop: create a session immediately. + # When running inside an event loop, defer and create lazily + # via _ensure_session() using asyncio.to_thread(). + try: + asyncio.get_running_loop() + in_event_loop = True + except RuntimeError: + in_event_loop = False + + if session is not None: + self.session: Optional[httpx.AsyncClient] = session + elif in_event_loop: + self.session = None + else: + self.session = self.session_factory() + self._session_lock = asyncio.Lock() self._proxy_url: URL = proxy_url self._host_url: URL = host_url self._port: int = proxy_url.explicit_port if proxy_url.explicit_port else 0 # type: ignore @@ -193,15 +211,34 @@ async def reset_data(self) -> None: """ if self.session: await self.session.aclose() - self.session = self.session_factory() + self.session = None + # Reset data fields unconditionally so state is clean regardless of session outcome. self.last_resp = None self.init_query = {} self.query = {} self.data = {} self._active = False self._all_handler_active = True + await self._ensure_session() + if self.session is None: # pragma: no cover + _LOGGER.error("Internal error: HTTP session not initialized") + return + _LOGGER.debug("Proxy data reset.") + async def _ensure_session(self) -> None: + """Ensure an httpx session exists. + + httpx.AsyncClient() initialization may perform blocking SSL work + (e.g. SSLContext.load_verify_locations), so the client is created + in a background thread. + """ + if self.session is not None: + return + async with self._session_lock: + if self.session is None: + self.session = await asyncio.to_thread(self.session_factory) + def refresh_tests(self) -> None: """Refresh tests. @@ -341,6 +378,14 @@ async def all_handler(self, request: web.Request, **kwargs) -> web.Response: else: host_url = self._host_url + # Ensure the HTTP session is created off the event loop thread. + await self._ensure_session() + session = self.session + if session is None: # pragma: no cover + return await self._build_response( + text="Internal error: HTTP session not initialized", status=500 + ) + async def _process_multipart(reader: MultipartReader, writer: MultipartWriter) -> None: """Process multipart. @@ -530,15 +575,15 @@ async def _process_multipart(reader: MultipartReader, writer: MultipartWriter) - method, site, req_headers, - self.session.cookies.jar, + session.cookies.jar, ) try: if mpwriter: - resp = await getattr(self.session, method)( + resp = await getattr(session, method)( site, data=mpwriter, headers=req_headers, follow_redirects=True ) elif data: - resp = await getattr(self.session, method)( + resp = await getattr(session, method)( site, data=data, headers=req_headers, follow_redirects=True ) elif raw_body is not None: @@ -551,7 +596,7 @@ async def _process_multipart(reader: MultipartReader, writer: MultipartWriter) - # Preserve the original Content-Type for raw body requests if request.content_type and "Content-Type" not in req_headers: req_headers["Content-Type"] = request.content_type - resp = await getattr(self.session, method)( + resp = await getattr(session, method)( site, content=raw_body, headers=req_headers, follow_redirects=True ) elif json_data: @@ -559,11 +604,11 @@ async def _process_multipart(reader: MultipartReader, writer: MultipartWriter) - # remove proxy headers if req_headers.get(item): req_headers.pop(item) - resp = await getattr(self.session, method)( + resp = await getattr(session, method)( site, json=json_data, headers=req_headers, follow_redirects=True ) else: - resp = await getattr(self.session, method)( + resp = await getattr(session, method)( site, headers=req_headers, follow_redirects=True ) except httpx.ConnectError as ex: @@ -811,6 +856,7 @@ async def stop_proxy(self, delay: int = 0) -> None: if self.session: _LOGGER.debug("Closing session") await self.session.aclose() + self.session = None _LOGGER.debug("Session closed") self._active = False _LOGGER.debug("Proxy stopped") diff --git a/authcaptureproxy/cli.py b/authcaptureproxy/cli.py index 37a71c06..fe26001c 100644 --- a/authcaptureproxy/cli.py +++ b/authcaptureproxy/cli.py @@ -104,7 +104,7 @@ def test_url(resp: httpx.Response, data: Dict[Text, Any], query: Dict[Text, Any] asyncio.create_task(proxy_obj.stop_proxy(3)) # stop proxy in 3 seconds if callback_url: return URL(callback_url) # 302 redirect - return f"Successfully logged in {data.get('email')} and {data.get('password')}. Please close the window.
Post data
{json.dumps(data)}
Query Data:
{json.dumps(query)}
Cookies:
{json.dumps(list(proxy_obj.session.cookies.items()))}" + return f"Successfully logged in {data.get('email')} and {data.get('password')}. Please close the window.
Post data
{json.dumps(data)}
Query Data:
{json.dumps(query)}
Cookies:
{json.dumps(list(proxy_obj.session.cookies.items()) if proxy_obj.session else [])}" await proxy_obj.start_proxy() # add tests and modifiers after the proxy has started so that port data is available for self.access_url() @@ -119,7 +119,7 @@ def test_url(resp: httpx.Response, data: Dict[Text, Any], query: Dict[Text, Any] "autofill": partial( autofill, { - "password": "CHANGEME", + "password": "CHANGEME", # nosec }, ) } diff --git a/authcaptureproxy/examples/modifiers.py b/authcaptureproxy/examples/modifiers.py index df82a4ed..de987685 100644 --- a/authcaptureproxy/examples/modifiers.py +++ b/authcaptureproxy/examples/modifiers.py @@ -207,38 +207,50 @@ async def find_urls_bs4( # https://developer.mozilla.org/en-US/docs/Web/CSS/background-image # this currently only handles background-image as the first attribute # TODO: Rewrite regex to handle general case - pattern = r"(?<=style=[\"']background-image:url\([\"']).*(?=[\"']\))" + pattern = r"(?<=background-image:url\([\"']).*?(?=[\"']\))" attribute_value = html_tag.get(attribute) - url: Optional[URL] = URL(str(re.search(pattern, attribute_value))) - if url is not None and url not in exceptions.get(tag, []): - new_value = re.sub( - pattern, - await run_func(modifier, name="", url=url), - attribute_value, - ) - old_value = html_tag[attribute] + if not isinstance(attribute_value, str): + continue + match = re.search(pattern, attribute_value) + if not match: + continue + try: + url: Optional[URL] = URL(match.group(0)) + except (TypeError, ValueError): + url = None + if url is not None and str(url) not in exceptions.get(tag, []): + replacement = await run_func(modifier, name="", url=url) + new_value = re.sub(pattern, str(replacement), attribute_value) + old_value = html_tag.get(attribute) html_tag[attribute] = new_value - if str(old_value) != str(html_tag[attribute]): + if str(old_value) != str(html_tag.get(attribute)): _LOGGER.debug( "Modified url for style:background-image %s -> %s", url, - html_tag[attribute], + html_tag.get(attribute), ) else: - url = URL(html_tag.get(attribute)) if html_tag.get(attribute) is not None else None + raw_value = html_tag.get(attribute) + if not isinstance(raw_value, str): + url = None + else: + try: + url = URL(raw_value) # allow "" (URL("")) + except (TypeError, ValueError): + url = None if ( url is not None and not str(url).startswith("data:") and str(url) not in exceptions.get(tag, []) ): - old_value = html_tag[attribute] + old_value = html_tag.get(attribute) html_tag[attribute] = await run_func(modifier, name="", url=url) - if str(old_value) != str(html_tag[attribute]): + if str(old_value) != str(html_tag.get(attribute)): _LOGGER.debug( "Modified url for %s:%s %s -> %s", tag, attribute, url, - html_tag[attribute], + html_tag.get(attribute), ) return str(soup) diff --git a/poetry.lock b/poetry.lock index 216bca16..f647cb4e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2988,14 +2988,14 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" -version = "20.37.0" +version = "20.38.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "virtualenv-20.37.0-py3-none-any.whl", hash = "sha256:5d3951c32d57232ae3569d4de4cc256c439e045135ebf43518131175d9be435d"}, - {file = "virtualenv-20.37.0.tar.gz", hash = "sha256:6f7e2064ed470aa7418874e70b6369d53b66bcd9e9fd5389763e96b6c94ccb7c"}, + {file = "virtualenv-20.38.0-py3-none-any.whl", hash = "sha256:d6e78e5889de3a4742df2d3d44e779366325a90cf356f15621fddace82431794"}, + {file = "virtualenv-20.38.0.tar.gz", hash = "sha256:94f39b1abaea5185bf7ea5a46702b56f1d0c9aa2f41a6c2b8b0af4ddc74c10a7"}, ] [package.dependencies] diff --git a/pyproject.toml b/pyproject.toml index 60dc24c1..dec7ddb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,6 +171,7 @@ line-length = 100 target-version = ["py38"] [tool.isort] +profile = "black" line_length = 100 [tool.bandit] diff --git a/tests/__init__.py b/tests/__init__.py index 265a8faa..3d3e2dfd 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,11 +5,13 @@ Original source: https://github.com/dmyersturnbull/tyrannosaurus Copyright 2020–2021 Douglas Myers-Turnbull """ + # NOTE: If you modify this file, you should indicate your license and copyright as well. from __future__ import annotations + import logging import os -import random +import secrets import shutil import stat import tempfile @@ -20,7 +22,6 @@ from typing import Generator, Union from warnings import warn - # Keeps created temp files; turn on for debugging KEEP = False @@ -84,7 +85,7 @@ def temp_dir( Yields: The created directory as a ``pathlib.Path`` """ - path = TestResources._temp_dir / ("%0x" % random.getrandbits(64)) + path = TestResources._temp_dir / secrets.token_hex(8) if path.exists(): cls._delete_tree(path, surefire=force_delete) if copy_resource is None: diff --git a/tests/examples/test_amazon_waf.py b/tests/examples/test_amazon_waf.py index 15e83f3f..961c0647 100644 --- a/tests/examples/test_amazon_waf.py +++ b/tests/examples/test_amazon_waf.py @@ -20,9 +20,7 @@ def _make_ctx(**kwargs): """Create an InterceptContext with Amazon-like defaults.""" proxy = MagicMock() proxy._host_url = HOST_URL - proxy._build_response = MagicMock( - side_effect=lambda **kw: _async_response(kw.get("text", "")) - ) + proxy._build_response = MagicMock(side_effect=lambda **kw: _async_response(kw.get("text", ""))) defaults = dict( request=MagicMock(), proxy=proxy, @@ -47,9 +45,7 @@ async def test_on_request_amzn_host_routing(): """__amzn_host__ marker routes to correct host.""" interceptor = AmazonWAFInterceptor() req = MagicMock() - req.url = URL( - "https://192.168.1.100:8123/auth/proxy/__amzn_host__fls-eu.amazon.it/1/batch/1" - ) + req.url = URL("https://192.168.1.100:8123/auth/proxy/__amzn_host__fls-eu.amazon.it/1/batch/1") req.query_string = "" ctx = _make_ctx(request=req) @@ -63,9 +59,7 @@ async def test_on_request_amzn_host_with_query(): """__amzn_host__ routing preserves query string.""" interceptor = AmazonWAFInterceptor() req = MagicMock() - req.url = URL( - "https://192.168.1.100:8123/auth/proxy/__amzn_host__fls-eu.amazon.it/path" - ) + req.url = URL("https://192.168.1.100:8123/auth/proxy/__amzn_host__fls-eu.amazon.it/path") req.query_string = "key=value&other=1" ctx = _make_ctx(request=req) @@ -78,9 +72,7 @@ async def test_on_request_blocked_host(): """Non-Amazon host returns short_circuit error.""" interceptor = AmazonWAFInterceptor() req = MagicMock() - req.url = URL( - "https://192.168.1.100:8123/auth/proxy/__amzn_host__evil.example.com/steal" - ) + req.url = URL("https://192.168.1.100:8123/auth/proxy/__amzn_host__evil.example.com/steal") req.query_string = "" ctx = _make_ctx(request=req) @@ -157,7 +149,7 @@ async def test_on_request_data_invalid_aamation_with_totp(): proxy=proxy, site="/ap/cvf/verify", data={ - "cvf_aamation_response_token": "invalid_not_base64", + "cvf_aamation_response_token": "invalid_not_base64", # nosec B105 "cvf_aamation_error_code": "NetworkError", "cvf_captcha_captcha_action": "", }, @@ -180,7 +172,7 @@ async def test_on_request_data_invalid_aamation_no_totp(): proxy=proxy, site="/ap/cvf/verify", data={ - "cvf_aamation_response_token": "bad", + "cvf_aamation_response_token": "bad", # nosec B105 "cvf_aamation_error_code": "err", "cvf_captcha_captcha_action": "x", }, @@ -258,9 +250,9 @@ async def test_on_ajax_html_aaut_injection(): """P shim injected into /aaut/verify/cvf response.""" interceptor = AmazonWAFInterceptor() html = ( - '' + "" '' - '' + "" ) req = MagicMock() req.url = URL("https://192.168.1.100:8123/auth/proxy/aaut/verify/cvf") @@ -310,7 +302,7 @@ async def test_on_ajax_html_no_body(): async def test_on_page_html_cvf_injection(): """Submit blocker + AJAX proxy injected into CVF page.""" interceptor = AmazonWAFInterceptor() - html = 'CVF' + html = "CVF" resp = MagicMock(spec=httpx.Response) resp.url = httpx.URL("https://www.amazon.it/ap/cvf/request") ctx = _make_ctx(response=resp, text=html, content_type="text/html") @@ -329,7 +321,7 @@ async def test_on_page_html_cvf_injection(): async def test_on_page_html_non_cvf(): """Non-CVF page: text unchanged.""" interceptor = AmazonWAFInterceptor() - html = 'Signin' + html = "Signin" resp = MagicMock(spec=httpx.Response) resp.url = httpx.URL("https://www.amazon.it/ap/signin") ctx = _make_ctx(response=resp, text=html, content_type="text/html") diff --git a/tests/examples/test_modifiers.py b/tests/examples/test_modifiers.py index 49a9c399..25fa39b2 100644 --- a/tests/examples/test_modifiers.py +++ b/tests/examples/test_modifiers.py @@ -1,12 +1,13 @@ """Test modifiers.""" -from yarl import URL -from bs4 import BeautifulSoup as bs # type: ignore -import authcaptureproxy.examples.modifiers as modifiers -from typing import Text import random +from typing import Text + import pytest +from bs4 import BeautifulSoup as bs # type: ignore +from yarl import URL +import authcaptureproxy.examples.modifiers as modifiers EMPTY_URL = URL("") VALID_URL = URL("http://www.google.com") @@ -139,7 +140,7 @@ def build_random_html(size: int = 10, url: Text = HOST_URL_WITH_PATH) -> Text: for _ in range(size): tag = random.choice(list(KNOWN_URLS_ATTRS)) # nosec attribute = KNOWN_URLS_ATTRS[tag] - new_tag = soup.new_tag(tag, **{attribute: url}) + new_tag = soup.new_tag(tag, attrs={attribute: str(url)}) soup.append(new_tag) return str(soup) diff --git a/tests/test_all_handler.py b/tests/test_all_handler.py index 39de74cc..a7f6926e 100644 --- a/tests/test_all_handler.py +++ b/tests/test_all_handler.py @@ -136,7 +136,7 @@ async def on_request_data(self, ctx): proxy.interceptors = [DataModifier()] req = _make_request(method="POST") req.has_body = True - req.post = AsyncMock(return_value=CIMultiDict({"password": "test123"})) + req.post = AsyncMock(return_value=CIMultiDict({"password": "test123"})) # nosec B105 resp = _make_response() mock_session.post = AsyncMock(return_value=resp) diff --git a/tests/test_authcaptureproxy.py b/tests/test_authcaptureproxy.py index 55314ae3..16940aa5 100644 --- a/tests/test_authcaptureproxy.py +++ b/tests/test_authcaptureproxy.py @@ -5,7 +5,7 @@ """ -from unittest.mock import patch, create_autospec +from unittest.mock import create_autospec, patch from httpx import AsyncClient from pytest import fixture diff --git a/tests/test_cli.py b/tests/test_cli.py index 4bf2ad08..9d0f93ef 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,10 +3,10 @@ Copyright 2021 Alan D. Tse. """ + import pytest -from authcaptureproxy import __copyright__ -from authcaptureproxy import cli +from authcaptureproxy import __copyright__, cli class TestCli: diff --git a/tests/test_helper.py b/tests/test_helper.py index 3f911801..1692512b 100644 --- a/tests/test_helper.py +++ b/tests/test_helper.py @@ -1,4 +1,5 @@ """Test the auth_capture proxy helper.""" + from httpx import Response from multidict import MultiDict, MultiDictProxy from yarl import URL diff --git a/tests/test_interceptor.py b/tests/test_interceptor.py index a0a16f7c..af3dc24a 100644 --- a/tests/test_interceptor.py +++ b/tests/test_interceptor.py @@ -1,12 +1,12 @@ """Tests for the interceptor module.""" -import pytest from unittest.mock import MagicMock + +import pytest from yarl import URL from authcaptureproxy.interceptor import BaseInterceptor, InterceptContext - PROXY_URL = URL("https://www.proxy.com/proxy") HOST_URL = URL("https://www.host.com") @@ -71,6 +71,7 @@ def test_intercept_context_required_fields(): @pytest.mark.asyncio async def test_custom_interceptor_override(): """A custom interceptor can override individual hooks.""" + class TestInterceptor(BaseInterceptor): async def on_request(self, ctx): ctx.site = "https://custom.example.com/path" diff --git a/tests/test_regression_headers_and_json_parsing.py b/tests/test_regression_headers_and_json_parsing.py index 52822c56..c06f6acb 100644 --- a/tests/test_regression_headers_and_json_parsing.py +++ b/tests/test_regression_headers_and_json_parsing.py @@ -1,9 +1,8 @@ import asyncio - from typing import Any -import pytest -import httpx +import httpx +import pytest from aiohttp.streams import StreamReader from aiohttp.test_utils import make_mocked_request from multidict import CIMultiDict diff --git a/tests/test_stackoverflow.py b/tests/test_stackoverflow.py index 84158513..6f9443c4 100644 --- a/tests/test_stackoverflow.py +++ b/tests/test_stackoverflow.py @@ -1,4 +1,5 @@ """Test the auth_capture_proxy stackoverflow.""" + import random import string from unittest.mock import patch