Merge pull request #26 from canonical/proxy-settings

Update charm to handle PS7 proxy egress

Author: lvoytek

src/charm.py

@@ -13,11 +13,13 @@
 """
 
 import logging
+import os
+import socket
 from pathlib import Path
-from socket import getfqdn
 
 import ops
 
+import environment as env
 import importer_node as node
 import launchpad as lp
 import package_installation as pkgs
@@ -33,7 +35,7 @@
 GIT_UBUNTU_GIT_USER_NAME = "Ubuntu Git Importer"
 GIT_UBUNTU_GIT_EMAIL = "usd-importer-do-not-mail@canonical.com"
 GIT_UBUNTU_USER_HOME_DIR = "/var/local/git-ubuntu"
-GIT_UBUNTU_SOURCE_BASE_URL = "git.launchpad.net/git-ubuntu"
+GIT_UBUNTU_SOURCE_URL = "https://git.launchpad.net/git-ubuntu"
 GIT_UBUNTU_KEYRING_FOLDER = Path(__file__).parent.parent / "keyring"
 
 
@@ -138,20 +140,6 @@ def _lpuser_lp_key(self) -> str | None:
 
         return None
 
-    @property
-    def _git_ubuntu_source_url(self) -> str:
-        """Get the git-ubuntu source URL based on config.
-
-        If an SSH private key is provided, get a git+ssh URL, otherwise use https.
-
-        Returns:
-            The git-ubuntu source URL.
-        """
-        if self._lpuser_ssh_key is not None:
-            return f"git+ssh://{self._lp_username}@{GIT_UBUNTU_SOURCE_BASE_URL}"
-
-        return f"https://{GIT_UBUNTU_SOURCE_BASE_URL}"
-
     @property
     def _git_ubuntu_primary_relation(self) -> ops.Relation | None:
         """Get the peer relation that contains the primary node IP.
@@ -195,7 +183,7 @@ def _set_peer_primary_node_address(self) -> bool:
         relation = self._git_ubuntu_primary_relation
 
         if relation:
-            new_primary_address = getfqdn()
+            new_primary_address = socket.gethostbyname(socket.gethostname())
             relation.data[self.app]["primary_address"] = new_primary_address
             logger.info("Updated primary node address to %s", new_primary_address)
             return True
@@ -267,16 +255,44 @@ def _refresh_git_ubuntu_source(self) -> bool:
         """
         self.unit.status = ops.MaintenanceStatus("Refreshing git-ubuntu source.")
 
+        # Set https proxy environment variable if available.
+        https_proxy = env.get_juju_https_proxy_url()
+
+        if https_proxy != "":
+            logger.info("Using https proxy %s for git-ubuntu source refresh.", https_proxy)
+            os.environ["https_proxy"] = https_proxy
+
+        # Run clone or pull of git-ubuntu source.
         if not usr.refresh_git_ubuntu_source(
             GIT_UBUNTU_SYSTEM_USER_USERNAME,
             GIT_UBUNTU_USER_HOME_DIR,
-            self._git_ubuntu_source_url,
+            GIT_UBUNTU_SOURCE_URL,
         ):
             self.unit.status = ops.BlockedStatus("Failed to refresh git-ubuntu source.")
             return False
 
         return True
 
+    def _refresh_ssh_config(self) -> bool:
+        """Refresh the SSH config for the git-ubuntu user.
+
+        Returns:
+            True if the config was updated successfully, False otherwise.
+        """
+        self.unit.status = ops.MaintenanceStatus("Refreshing SSH config.")
+
+        if not usr.update_ssh_config(
+            GIT_UBUNTU_SYSTEM_USER_USERNAME,
+            GIT_UBUNTU_USER_HOME_DIR,
+            env.get_juju_http_proxy_url(),
+        ):
+            self.unit.status = ops.BlockedStatus(
+                "Failed to update SSH config for git-ubuntu user."
+            )
+            return False
+
+        return True
+
     def _refresh_importer_node(self) -> None:
         """Remove old and install new git-ubuntu services."""
         self.unit.status = ops.MaintenanceStatus("Refreshing git-ubuntu services.")
@@ -290,6 +306,8 @@ def _refresh_importer_node(self) -> None:
                 GIT_UBUNTU_USER_HOME_DIR,
                 GIT_UBUNTU_SYSTEM_USER_USERNAME,
                 self._controller_port,
+                env.get_juju_http_proxy_url(),
+                env.get_juju_https_proxy_url(),
             ):
                 self.unit.status = ops.BlockedStatus("Failed to install git-ubuntu services.")
                 return
@@ -309,6 +327,7 @@ def _refresh_importer_node(self) -> None:
                 self._is_publishing_active,
                 self._controller_port,
                 primary_ip,
+                env.get_juju_https_proxy_url(),
             ):
                 self.unit.status = ops.BlockedStatus("Failed to install git-ubuntu services.")
                 return
@@ -423,6 +442,7 @@ def _on_config_changed(self, _: ops.ConfigChangedEvent) -> None:
             and self._update_git_ubuntu_snap()
             and self._open_controller_port()
             and self._refresh_secret_keys()
+            and self._refresh_ssh_config()
             and self._refresh_git_ubuntu_source()
         ):
             # Initialize or re-install git-ubuntu services as needed.

src/environment.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# Copyright 2025 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""Environment information extraction."""
+
+import os
+
+
+def get_juju_http_proxy_url() -> str:
+    """Get the Juju-managed http proxy URL if available.
+
+    Returns:
+        The proxy URL or an empty string if it does not exist.
+    """
+    env = os.environ.copy()
+    http_proxy = env.get("JUJU_CHARM_HTTP_PROXY")
+
+    if http_proxy:
+        return http_proxy
+
+    return ""
+
+
+def get_juju_https_proxy_url() -> str:
+    """Get the Juju-managed https proxy URL if available.
+
+    Returns:
+        The proxy URL or an empty string if it does not exist.
+    """
+    env = os.environ.copy()
+    https_proxy = env.get("JUJU_CHARM_HTTPS_PROXY")
+
+    if https_proxy:
+        return https_proxy
+
+    return ""

src/git_ubuntu.py

@@ -178,7 +178,8 @@ def setup_poller_service(
     user: str,
     group: str,
     denylist: str,
-    proxy: str = "",
+    http_proxy: str = "",
+    https_proxy: str = "",
 ) -> bool:
     """Set up poller systemd service file.
 
@@ -187,7 +188,8 @@ def setup_poller_service(
         user: The user to run the service as.
         group: The permissions group to run the service as.
         denylist: The location of the package denylist.
-        proxy: Optional proxy url.
+        http_proxy: Optional HTTP proxy url.
+        https_proxy: Optional HTTPS proxy url.
 
     Returns:
         True if setup succeeded, False otherwise.
@@ -197,8 +199,11 @@ def setup_poller_service(
 
     environment = "PYTHONUNBUFFERED=1"
 
-    if proxy:
-        environment = f"http_proxy={proxy} " + environment
+    if http_proxy:
+        environment = f"http_proxy={http_proxy} " + environment
+
+    if https_proxy:
+        environment = f"https_proxy={https_proxy} " + environment
 
     service_string = generate_systemd_service_string(
         "git-ubuntu importer service poller",
@@ -225,6 +230,7 @@ def setup_worker_service(
     push_to_lp: bool = True,
     broker_ip: str = "127.0.0.1",
     broker_port: int = 1692,
+    https_proxy: str = "",
 ) -> bool:
     """Set up worker systemd file with designated worker name.
 
@@ -236,6 +242,7 @@ def setup_worker_service(
         push_to_lp: True if publishing repositories to Launchpad.
         broker_ip: The IP address of the broker process' node.
         broker_port: The network port that the broker provides tasks on.
+        https_proxy: Optional HTTPS proxy url.
 
     Returns:
         True if setup succeeded, False otherwise.
@@ -246,6 +253,11 @@ def setup_worker_service(
     broker_url = f"tcp://{broker_ip}:{broker_port}"
     exec_start = f"/snap/bin/git-ubuntu importer-service-worker{publish_arg} %i {broker_url}"
 
+    environment = "PYTHONUNBUFFERED=1"
+
+    if https_proxy:
+        environment = f"https_proxy={https_proxy} " + environment
+
     service_string = generate_systemd_service_string(
         "git-ubuntu importer service worker",
         user,
@@ -258,7 +270,7 @@ def setup_worker_service(
         timeout_abort_sec=600,
         watchdog_signal="SIGINT",
         private_tmp=True,
-        environment="PYTHONUNBUFFERED=1",
+        environment=environment,
         wanted_by="multi-user.target",
     )
 

src/importer_node.py

@@ -21,6 +21,7 @@ def setup_secondary_node(
     push_to_lp: bool,
     primary_port: int,
     primary_ip: str,
+    https_proxy: str = "",
 ) -> bool:
     """Set up necessary services for a worker-only git-ubuntu importer node.
 
@@ -32,6 +33,7 @@ def setup_secondary_node(
         push_to_lp: True if publishing repositories to Launchpad.
         primary_port: The network port used for worker assignments.
         primary_ip: The IP or network location of the primary node.
+        https_proxy: URL for the environment's https proxy if required.
 
     Returns:
         True if installation succeeded, False otherwise.
@@ -48,6 +50,7 @@ def setup_secondary_node(
             push_to_lp,
             primary_ip,
             primary_port,
+            https_proxy,
         ):
             logger.error("Failed to setup worker %s service.", worker_name)
             return False
@@ -59,13 +62,17 @@ def setup_primary_node(
     git_ubuntu_user_home: str,
     system_user: str,
     primary_port: int,
+    http_proxy: str = "",
+    https_proxy: str = "",
 ) -> bool:
     """Set up poller and broker services to create a primary git-ubuntu importer node.
 
     Args:
         git_ubuntu_user_home: The home directory of the git-ubuntu user.
         system_user: The user + group to run the services as.
         primary_port: The network port used for worker assignments.
+        http_proxy: URL for the environment's http proxy if required.
+        https_proxy: URL for the environment's https proxy if required.
 
     Returns:
         True if installation succeeded, False otherwise.
@@ -93,6 +100,8 @@ def setup_primary_node(
         system_user,
         system_user,
         denylist.as_posix(),
+        http_proxy,
+        https_proxy,
     ):
         logger.error("Failed to setup poller service.")
         return False

src/user_management.py

@@ -258,6 +258,61 @@ def update_launchpad_keyring_secret(user: str, home_dir: str, lp_key_data: str)
     return key_success
 
 
+def update_ssh_config(user: str, home_dir: str, http_proxy: str = "") -> bool:
+    """Create or refresh the .ssh/config file for git.launchpad.net handling.
+
+    Args:
+        user: The git-ubuntu user.
+        home_dir: The home directory for the user.
+        http_proxy: The http proxy URL if required.
+
+    Returns:
+        True if directory and file creation succeeded, False otherwise.
+    """
+    ssh_config_file = pathops.LocalPath(home_dir, ".ssh/config")
+
+    parent_dir = ssh_config_file.parent
+
+    if not _mkdir_for_user_with_error_checking(parent_dir, user, 0o700):
+        return False
+
+    config_success = False
+
+    ssh_config_content: str = (
+        "Host git.launchpad.net\n"
+        + "  HostName git.launchpad.net\n"
+        + "  Port 22\n"
+        + "  IdentityFile ~/.ssh/id\n"
+    )
+
+    # Use socat to proxy ssh data if needed.
+    if http_proxy != "":
+        proxy_base_url = http_proxy.replace("http://", "").split(":")[0]
+        proxy_port = (
+            http_proxy.replace("http://", "").split(":")[1] if ":" in http_proxy else "3128"
+        )
+        ssh_config_content += (
+            "  ProxyCommand /usr/bin/socat - "
+            + f"PROXY:{proxy_base_url}:%h:%p,proxyport={proxy_port}\n"
+        )
+
+    try:
+        ssh_config_file.write_text(
+            ssh_config_content,
+            mode=0o664,
+            user=user,
+            group=user,
+        )
+        config_success = True
+    except (FileNotFoundError, NotADirectoryError) as e:
+        logger.error("Failed to create ssh config due to directory issues: %s", str(e))
+    except LookupError as e:
+        logger.error("Failed to create ssh config due to issues with root user: %s", str(e))
+    except PermissionError as e:
+        logger.error("Failed to create ssh config due to permission issues: %s", str(e))
+    return config_success
+
+
 def set_snap_homedirs(home_dir: str) -> bool:
     """Allow snaps to run for a user with a given home directory.
 

tests/unit/test_environment.py

@@ -0,0 +1,52 @@
+# Copyright 2025 Canonical Ltd.
+# See LICENSE file for licensing details.
+#
+# Learn more about testing at: https://juju.is/docs/sdk/testing
+
+"""Unit tests for launchpad tools."""
+
+import os
+
+import environment as env
+
+
+def test_get_juju_http_proxy_url():
+    """Test getting Juju http proxy URL when not set."""
+    proxy_var_original = os.environ.get("JUJU_CHARM_HTTP_PROXY")
+
+    if proxy_var_original is not None:
+        del os.environ["JUJU_CHARM_HTTP_PROXY"]
+
+    proxy_url = env.get_juju_http_proxy_url()
+    assert proxy_url == ""
+
+    test_url = "http://proxy.internal:1234"
+    os.environ["JUJU_CHARM_HTTP_PROXY"] = test_url
+    proxy_url = env.get_juju_http_proxy_url()
+    assert proxy_url == test_url
+
+    del os.environ["JUJU_CHARM_HTTP_PROXY"]
+
+    if proxy_var_original is not None:
+        os.environ["JUJU_CHARM_HTTP_PROXY"] = proxy_var_original
+
+
+def test_get_juju_https_proxy_url():
+    """Test getting Juju https proxy URL when not set."""
+    proxy_var_original = os.environ.get("JUJU_CHARM_HTTPS_PROXY")
+
+    if proxy_var_original is not None:
+        del os.environ["JUJU_CHARM_HTTPS_PROXY"]
+
+    proxy_url = env.get_juju_https_proxy_url()
+    assert proxy_url == ""
+
+    test_url = "http://httpsproxy.internal:1234"
+    os.environ["JUJU_CHARM_HTTPS_PROXY"] = test_url
+    proxy_url = env.get_juju_https_proxy_url()
+    assert proxy_url == test_url
+
+    del os.environ["JUJU_CHARM_HTTPS_PROXY"]
+
+    if proxy_var_original is not None:
+        os.environ["JUJU_CHARM_HTTPS_PROXY"] = proxy_var_original