pin wrangler version to fix hanging miniflare invocations (#86)

Author: whummer

aws-replicator/aws_replicator/client/auth_proxy.py

@@ -278,7 +278,7 @@ def _fix_host_and_path(self, request: Request, service_name: str):
             regex_base_domain = rf"((amazonaws\.com)|({LOCALHOST_HOSTNAME}))"
             host = request.headers.pop(HEADER_HOST_ORIGINAL, None)
             host = host or request.headers.get("Host") or ""
-            match = re.match(rf"(.+)\.s3\.{regex_base_domain}", host)
+            match = re.match(rf"(.+)\.s3\..*{regex_base_domain}", host)
             if match:
                 # prepend the bucket name (extracted from the host) to the path of the request (path-based addressing)
                 request.path = f"/{match.group(1)}{request.path}"

aws-replicator/tests/test_proxy_requests.py

@@ -65,7 +65,7 @@ def _add_header(request, **kwargs):
             url = urlparse(request.url)
             match = re.match(r"(.+)\.s3\.localhost\.localstack\.cloud", url.netloc)
             if match:
-                request.headers.add_header("host", f"{match.group(1)}.s3.amazonaws.com")
+                request.headers.add_header("host", f"{match.group(1)}.s3.us-east-1.amazonaws.com")
 
         s3_client.meta.events.register_first("before-sign.*.*", _add_header)
 

miniflare/Makefile

@@ -18,10 +18,10 @@ clean:
 	rm -rf *.egg-info/
 
 lint:              		  ## Run code linter to check code style
-	$(VENV_RUN); python -m pflake8 --show-source --ignore=E501
+	$(VENV_RUN); python -m pflake8 --show-source --ignore=E501 --exclude .venv,build
 
 format:            		  ## Run black and isort code formatter
-	$(VENV_RUN); python -m isort .; python -m black .
+	$(VENV_RUN); python -m isort .; python -m black miniflare
 
 install: venv
 	$(VENV_RUN); python -m pip install -e .[dev]

miniflare/README.md

@@ -34,6 +34,7 @@ Hello World!
 
 ## Change Log
 
+* `0.1.2`: Pin wrangler version to fix hanging miniflare invocations; fix encoding headers for invocation responses
 * `0.1.1`: Adapt for LocalStack v3.0
 * `0.1.0`: Upgrade to Miniflare 3.0
 * `0.0.1`: Initial version.

miniflare/miniflare/cloudflare_api.py

@@ -42,13 +42,19 @@ def handle_invocation(request: Request, path: str, script_name: str, port: str):
     port = SCRIPT_SERVERS[script_name].port
     response = requests.request(
         method=request.method,
-        url=f"http://localhost:{port}{request.path}",
+        url=f"http://localhost:{port}/{path}",
         data=request.get_data(),
     )
     result = Response()
     result.status_code = response.status_code
     result.set_data(response.content)
-    result.headers.update(dict(response.headers))
+    headers = dict(response.headers)
+    if headers.get('Transfer-Encoding') == 'chunked':
+        headers.pop('Transfer-Encoding')
+    if headers.get('Content-Encoding') == 'gzip':
+        headers.pop('Content-Encoding')
+    LOG.debug("Miniflare invocation response headers/body: %s / %s", headers, response.content)
+    result.headers.update(headers)
     return result
 
 
@@ -154,9 +160,11 @@ def handle_services(request: Request, account_id: str, service_name: str) -> dic
         }
     )
 
+
 def handle_standard(request: Request, account_id: str) -> dict:
     return _wrap({})
 
+
 def handle_subdomain(request: Request, account_id: str) -> dict:
     return _wrap({})
 

miniflare/miniflare/extension.py

@@ -26,8 +26,9 @@
 
 LOG = logging.getLogger(__name__)
 
-# identifier for default version installed by package installer
-DEFAULT_VERSION = "latest"
+# Identifier for default version of `wrangler` installed by package installer.
+# Note: Currently pinned to 3.1.0, as newer versions make the invocations hang in the LS container
+WRANGLER_VERSION = "3.1.0"
 
 
 class MiniflareExtension(Extension):
@@ -36,6 +37,10 @@ class MiniflareExtension(Extension):
     def update_gateway_routes(self, router: http.Router[http.RouteHandler]):
         from miniflare.config import HANDLER_PATH_MINIFLARE
 
+        logging.getLogger("miniflare").setLevel(
+            logging.DEBUG if config.DEBUG else logging.INFO
+        )
+
         LOG.info("miniflare: adding routes to activate extension")
         all_methods = ["GET", "POST", "PUT", "DELETE"]
 
@@ -83,7 +88,7 @@ def __init__(self, script: WorkerScript, port: int):
         super().__init__(port)
 
     def do_run(self):
-        root_dir = os.path.join(config.dirs.var_libs, "miniflare", DEFAULT_VERSION)
+        root_dir = os.path.join(config.dirs.var_libs, "miniflare", WRANGLER_VERSION)
         wrangler_bin = os.path.join(root_dir, "node_modules", ".bin", "wrangler")
 
         # add global aliases, and variable bindings
@@ -102,7 +107,6 @@ def do_run(self):
         cmd = [
             wrangler_bin,
             "dev",
-            "--experimental-local",
             "--port",
             str(self.port),
             script_path_final,
@@ -115,7 +119,7 @@ def do_run(self):
 
 class MiniflareInstaller(ExecutableInstaller):
     def __init__(self):
-        super().__init__("miniflare", version=DEFAULT_VERSION)
+        super().__init__("miniflare", version=WRANGLER_VERSION)
 
     def _get_install_marker_path(self, install_dir: str) -> str:
         # force re-install on every start (requires npm package + system libs like libc++)
@@ -134,5 +138,4 @@ def _install(self, target: InstallTarget) -> None:
             run(["apt", "install", "-y", "libc++-dev"])
 
         # install npm package
-        run(["npm", "install", "--prefix", target_dir, "wrangler"])
-        run(["npm", "install", "--prefix", target_dir, "@miniflare/tre"])
+        run(["npm", "install", "--prefix", target_dir, f"wrangler@{WRANGLER_VERSION}"])

miniflare/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = localstack-extension-miniflare
-version = 0.1.2
+version = 0.1.3
 summary = LocalStack Extension: Miniflare
 description = This extension makes Miniflare (dev environment for Cloudflare workers) available directly in LocalStack
 long_description = file: README.md