Support engine-registered development runs in the client (#512)

- Fix response unpacking bug in local_server -- It should have been
unwrapping the Service response like the Steamship Proxy
- Fix the HttpREPL, which depended upon the improperly unwrapped
response (see above)
- Upon `ship run local`, register the local running NGROK URL with the
Engine so that it can make async callbacks to it.

Author: eob

src/steamship/base/mime_types.py

@@ -34,6 +34,29 @@ class MimeTypes(str, Enum):
     def has_value(cls, value):
         return value in cls._value2member_map_
 
+    @classmethod
+    def is_binary(cls, value):
+        """Returns whether the mime type is likely a binary file."""
+        return value in [
+            cls.UNKNOWN,
+            cls.PDF,
+            cls.JPG,
+            cls.PNG,
+            cls.TIFF,
+            cls.GIF,
+            cls.DOC,
+            cls.PPT,
+            cls.BINARY,
+            cls.WAV,
+            cls.MP3,
+            cls.OGG_VIDEO,
+            cls.OGG_AUDIO,
+            cls.MP4_AUDIO,
+            cls.MP4_VIDEO,
+            cls.WEBM_AUDIO,
+            cls.WEBM_VIDEO,
+        ]
+
 
 class ContentEncodings:
     BASE64 = "base64"

src/steamship/cli/cli.py

@@ -9,7 +9,7 @@
 import click
 
 import steamship
-from steamship import PackageInstance, Steamship, SteamshipError
+from steamship import PackageInstance, Steamship, SteamshipError, Workspace
 from steamship.base.configuration import DEFAULT_WEB_BASE, Configuration
 from steamship.cli.create_instance import (
     config_str_to_dict,
@@ -46,6 +46,30 @@ def initialize(suppress_message: bool = False):
         click.echo(f"Steamship Python CLI version {steamship.__version__}")
 
 
+def initialize_and_get_client_and_prep_project():
+    initialize()
+    client = None
+    try:
+        client = Steamship()
+    except SteamshipError as e:
+        click.secho(e.message, fg="red")
+        click.get_current_context().abort()
+
+    user = User.current(client)
+    if path.exists("steamship.json"):
+        manifest = Manifest.load_manifest()
+    else:
+        manifest = manifest_init_wizard(client)
+        manifest.save()
+
+    if not path.exists("requirements.txt"):
+        requirements_init_wizard()
+
+    update_config_template(manifest)
+
+    return client, user, manifest
+
+
 @click.command()
 def login():
     """Log in to Steamship, creating ~/.steamship.json"""
@@ -208,9 +232,29 @@ def _run_web_interface(base_url: str) -> str:
     return web_url
 
 
-def serve_local(
+def register_locally_running_package_with_engine(
+    client: Steamship,
+    ngrok_api_url: str,
+    package_handle: str,
+    manifest: Manifest,
+    config: Optional[str] = None,
+) -> PackageInstance:
+    """Registers the locally running package with the Steamship Engine."""
+
+    # Register the Instance in the Engine
+    invocable_config, is_file = config_str_to_dict(config)
+    set_unset_params(config, invocable_config, is_file, manifest)
+    package_instance = PackageInstance.create_local_development_instance(
+        client,
+        local_development_url=ngrok_api_url,
+        package_handle=package_handle,
+        config=invocable_config,
+    )
+    return package_instance
+
+
+def serve_local(  # noqa: C901
     port: int = 8443,
-    instance_handle: Optional[str] = None,
     no_ngrok: Optional[bool] = False,
     no_repl: Optional[bool] = False,
     no_ui: Optional[bool] = False,
@@ -220,32 +264,74 @@ def serve_local(
     """Serve the invocable on localhost. Useful for debugging locally."""
     dev_logging_handler = DevelopmentLoggingHandler.init_and_take_root()
 
-    initialize()
     click.secho("Running your project...\n")
 
-    # Report the logs
+    client, user, manifest = initialize_and_get_client_and_prep_project()
+
+    if workspace:
+        workspace_obj = Workspace.get(client, handle=workspace)
+    else:
+        workspace_obj = Workspace.get(client)
+        workspace = workspace_obj.handle
+
+    # Make sure we're running a package.
+    if manifest.type != DeployableType.PACKAGE:
+        click.secho(
+            f"⚠️ Must run `ship serve local` in a folder with a Steamship Package. Found: {manifest.type}"
+        )
+        exit(-1)
+
+    # Make sure we have a package name -- this allows us to register the running copy with the engine.
+    deployer = PackageDeployer()
+    deployable = deployer.create_or_fetch_deployable(client, user, manifest)
+
+    # Report the logs output file.
     click.secho(f"📝 Log file:   {dev_logging_handler.log_filename}")
 
     # Start the NGROK connection
     ngrok_api_url = None
+    public_api_url = None
+
     if not no_ngrok:
         ngrok_api_url = _run_ngrok(port)
-        click.secho(f"🌎 Public API: {ngrok_api_url}")
+
+        # It requires a trailing slash
+        if ngrok_api_url[-1] != "/":
+            ngrok_api_url = ngrok_api_url + "/"
+
+        registered_instance = register_locally_running_package_with_engine(
+            client=client,
+            ngrok_api_url=ngrok_api_url,
+            package_handle=deployable.handle,
+            manifest=manifest,
+            config=config,
+        )
+
+        # Notes:
+        #  1. registered_instance.invocation_url is the NGROK URL, not the Steamship Proxy URL.
+        #  2. The public_api_url should still be NGROK, not the Proxy. The local server emulates the Proxy and
+        #     the Proxy blocks this kind of development traffic.
+
+        public_api_url = ngrok_api_url
+        click.secho(f"🌎 Public API: {public_api_url}")
 
     # Start the local API Server. This has to happen after NGROK because the port & url need to be plummed.
     try:
         local_api_url = _run_local_server(
             local_port=port,
-            instance_handle=instance_handle,
+            instance_handle=registered_instance.handle,
             config=config,
             workspace=workspace,
-            base_url=ngrok_api_url,
+            base_url=public_api_url,
         )
     except BaseException as e:
         click.secho("⚠️ Local API:  Unable to start local server.")
         click.secho(e)
         exit(-1)
 
+    if local_api_url[-1] != "/":
+        local_api_url = local_api_url + "/"
+
     if local_api_url:
         click.secho(f"🌎 Local API:  {local_api_url}")
     else:
@@ -254,7 +340,7 @@ def serve_local(
 
     # Start the web UI
     if not no_ui:
-        web_url = _run_web_interface(ngrok_api_url or local_api_url)
+        web_url = _run_web_interface(public_api_url or local_api_url)
         if web_url:
             click.secho(f"🌎 Web UI:     {web_url}")
 
@@ -264,7 +350,7 @@ def serve_local(
             time.sleep(1)
     else:
         click.secho("\n💬 Interactive REPL below. Type to interact.\n")
-        prompt_url = f"{local_api_url or ngrok_api_url}/prompt"
+        prompt_url = f"{local_api_url or public_api_url}prompt"
         repl = HttpREPL(prompt_url=prompt_url, dev_logging_handler=dev_logging_handler)
         repl.run()
 
@@ -337,7 +423,6 @@ def run(
     if environment == "local":
         serve_local(
             port=port,
-            instance_handle=instance_handle,
             no_ngrok=no_ngrok,
             no_repl=no_repl,
             no_ui=no_ui,
@@ -355,28 +440,10 @@ def run(
 @click.command()
 def deploy():
     """Deploy the package or plugin in this directory"""
-    initialize()
-    client = None
-    try:
-        client = Steamship()
-    except SteamshipError as e:
-        click.secho(e.message, fg="red")
-        click.get_current_context().abort()
-
-    user = User.current(client)
-    if path.exists("steamship.json"):
-        manifest = Manifest.load_manifest()
-    else:
-        manifest = manifest_init_wizard(client)
-        manifest.save()
-
-    if not path.exists("requirements.txt"):
-        requirements_init_wizard()
+    client, user, manifest = initialize_and_get_client_and_prep_project()
 
     deployable_type = manifest.type
 
-    update_config_template(manifest)
-
     deployer = None
     if deployable_type == DeployableType.PACKAGE:
         deployer = PackageDeployer()

src/steamship/cli/local_server/handler.py

@@ -1,3 +1,4 @@
+import base64
 import json
 import logging
 import re
@@ -128,8 +129,16 @@ def _do_request(self, payload: dict, http_verb: str):
                 client = self._get_client()
                 context = self._get_invocation_context(client)
 
+                # Fix for GET parameters -- in production the Proxy would have done this.
+                thepath = self.path
+                if "?" in thepath:
+                    path_parts = thepath.split("?")
+                    thepath = path_parts[0]
+                    queryargs = parse_qs(path_parts[1])
+                    payload.update(queryargs)
+
                 invocation = Invocation(
-                    http_verb=http_verb, invocation_path=self.path, arguments=payload, config=config
+                    http_verb=http_verb, invocation_path=thepath, arguments=payload, config=config
                 )
                 event = InvocableRequest(
                     client_config=client.config,
@@ -140,9 +149,31 @@ def _do_request(self, payload: dict, http_verb: str):
 
                 handler = create_safe_handler(invocable_class)
                 resp = handler(event.dict(by_alias=True), context)
-                res_str = json.dumps(resp)
-                res_bytes = bytes(res_str, "utf-8")
-                self._send_response(res_bytes, MimeTypes.JSON)
+
+                # Since the local-server handler is simulating the Steamship Proxy's behavior, we now have to unwrap the
+                # `resp.data` object if it exists. This is what the proxy would do before returning the raw values
+                # to the HTTP User. Note that this is the behavior for a Package but not a Plugin.
+
+                res_mime = MimeTypes.JSON
+                if isinstance(resp, dict):
+                    res_mime = (
+                        resp.get("http", {}).get("headers", {}).get("Content-Type", MimeTypes.JSON)
+                    )
+
+                if "data" in resp and resp.get("data") is not None:
+                    if res_mime in [MimeTypes.JSON, MimeTypes.FILE_JSON]:
+                        res_str = json.dumps(resp.get("data"))
+                        res_bytes = bytes(res_str, "utf-8")
+                    elif MimeTypes.is_binary(res_mime):
+                        res_bytes = base64.b64decode(resp.get("data"))
+                    else:
+                        res_str = resp.get("data")
+                        res_bytes = bytes(res_str, "utf-8")
+                else:
+                    res_str = json.dumps(resp)
+                    res_bytes = bytes(res_str, "utf-8")
+
+                self._send_response(res_bytes, res_mime)
             except Exception as e:
                 self._send_response(bytes(f"{e}", "utf-8"), MimeTypes.TXT)
 

src/steamship/data/package/package_instance.py

@@ -14,6 +14,10 @@
 from steamship.data.workspace import Workspace
 from steamship.utils.url import Verb
 
+LOCAL_DEVELOPMENT_VERSION_HANDLE = (
+    "local-development!"  # Special handle for a locally-running development instances.
+)
+
 
 class CreatePackageInstanceRequest(Request):
     id: str = None
@@ -26,6 +30,12 @@ class CreatePackageInstanceRequest(Request):
     config: Dict[str, Any] = None
     workspace_id: str = None
 
+    local_development_url: Optional[str] = None
+    """Special argument only intended for creating an PackageInstance bound to a local development server.
+
+    If used, the package_version_handle should be set to LOCAL_DEVELOPMENT_VERSION_HANDLE above.
+    """
+
 
 class PackageInstance(CamelModel):
     client: Client = Field(None, exclude=True)
@@ -72,6 +82,28 @@ def create(
 
         return client.post("package/instance/create", payload=req, expect=PackageInstance)
 
+    @staticmethod
+    def create_local_development_instance(
+        client: Client,
+        local_development_url: str,
+        package_id: str = None,
+        package_handle: str = None,
+        handle: str = None,
+        fetch_if_exists: bool = True,
+        config: Dict[str, Any] = None,
+    ) -> PackageInstance:
+        req = CreatePackageInstanceRequest(
+            handle=handle,
+            package_id=package_id,
+            package_handle=package_handle,
+            package_version_handle=LOCAL_DEVELOPMENT_VERSION_HANDLE,
+            fetch_if_exists=fetch_if_exists,
+            config=config,
+            local_development_url=local_development_url,
+        )
+        """Create a PackageInstance bound to a local development server."""
+        return client.post("package/instance/create", payload=req, expect=PackageInstance)
+
     def delete(self) -> PackageInstance:
         req = DeleteRequest(id=self.id)
         return self.client.post("package/instance/delete", payload=req, expect=PackageInstance)

src/steamship/utils/repl.py

@@ -246,17 +246,25 @@ def colored(text: str, color: str):
                 logging.exception(ex)
 
             if result:
-                if result.get("status", {}).get("state", None) == TaskState.failed:
-                    message = result.get("status", {}).get("status_message", None)
-                    logging.error(f"Response failed with remote error: {message or 'No message'}")
-                    if suggestion := result.get("status", {}).get("status_suggestion", None):
-                        logging.error(f"Suggestion: {suggestion}")
-                elif data := result.get("data", None):
-                    self.print_object_or_objects(data)
+                if isinstance(result, dict):
+                    if result.get("status", {}).get("state", None) == TaskState.failed:
+                        message = result.get("status", {}).get("status_message", None)
+                        logging.error(
+                            f"Response failed with remote error: {message or 'No message'}"
+                        )
+                        if suggestion := result.get("status", {}).get("status_suggestion", None):
+                            logging.error(f"Suggestion: {suggestion}")
+                    elif data := result.get("data", None):
+                        self.print_object_or_objects(data)
+                    else:
+                        logging.warning(
+                            "REPL interaction completed with empty data field in InvocableResponse."
+                        )
+                if isinstance(result, list):
+                    self.print_object_or_objects(result)
                 else:
-                    logging.warning(
-                        "REPL interaction completed with empty data field in InvocableResponse."
-                    )
+                    logging.warning("Unsure how to display result:")
+                    logging.warning(result)
             else:
                 logging.warning("REPL interaction completed with no result to print.")