[DPE-9443] Otel distribution cleanup (#1532)

* Lock file maintenance

* Otel distributions cleanup

* Factor out helper and unit tests

* Downgrade amd runners to jammy

* Only run on Juju 2

---------

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

Author: dragomirp

poetry.lock

@@ -7,7 +7,7 @@ requires-poetry = ">=2.0.0"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-ops = {extras = ["tracing"], version = "^3.6.0"}
+ops = {extras = ["tracing"], version = "^3.7.0"}
 boto3 = "^1.42.74"
 pgconnstr = "^1.0.1"
 requests = "^2.32.5"
@@ -90,11 +90,6 @@ log_cli_level = "INFO"
 asyncio_mode = "auto"
 markers = ["juju2", "juju3", "juju_secrets"]
 
-# Formatting tools configuration
-[tool.black]
-line-length = 99
-target-version = ["py38"]
-
 # Linting tools configuration
 [tool.ruff]
 # preview and explicit preview are enabled for CPY001

pyproject.toml

@@ -43,7 +43,7 @@ backends:
       CONCIERGE_EXTRA_SNAPS: charmcraft
       CONCIERGE_EXTRA_DEBS: pipx
     systems:
-      - ubuntu-24.04:
+      - ubuntu-22.04:
           username: runner
     prepare: |
       systemctl disable --now unattended-upgrades.service
@@ -80,6 +80,9 @@ backends:
 
       echo "runner:$SPREAD_PASSWORD" | sudo chpasswd
 
+      # Remove the preinstalled lxd
+      sudo snap remove --purge --terminate lxd
+
       ADDRESS localhost
 
     # HACK: spread does not pass environment variables set on runner
@@ -94,7 +97,7 @@ backends:
       LANDSCAPE_ACCOUNT_NAME: '$(HOST: echo $LANDSCAPE_ACCOUNT_NAME)'
       LANDSCAPE_REGISTRATION_KEY: '$(HOST: echo $LANDSCAPE_REGISTRATION_KEY)'
     systems:
-      - ubuntu-24.04:
+      - ubuntu-22.04:
           username: runner
       - ubuntu-24.04-arm:
           username: runner

spread.yaml

@@ -19,6 +19,15 @@
 from typing import Literal, get_args
 from urllib.parse import urlparse
 
+from utils import _remove_stale_otel_sdk_packages
+
+# apply hacky patch to remove stale opentelemetry sdk packages on upgrade-charm.
+# it could be trouble if someone ever decides to implement their own tracer parallel to
+# ours and before the charm has inited. We assume they won't.
+# !!IMPORTANT!! keep all otlp imports UNDER this call.
+_remove_stale_otel_sdk_packages()
+
+# ruff: disable[E402]
 import psycopg2
 from charmlibs import snap
 from charms.data_platform_libs.v0.data_interfaces import DataPeerData, DataPeerUnitData
@@ -122,6 +131,8 @@
 from upgrade import PostgreSQLUpgrade, get_postgresql_dependencies_model
 from utils import new_password, snap_refreshed
 
+# ruff: enable[E402]
+
 logger = logging.getLogger(__name__)
 logging.getLogger("httpx").setLevel(logging.WARNING)
 logging.getLogger("httpcore").setLevel(logging.WARNING)

src/charm.py

@@ -3,9 +3,15 @@
 
 """A collection of utility functions that are used in the charm."""
 
+import logging
+import os
 import platform
 import secrets
+import shutil
 import string
+from collections import defaultdict
+
+from importlib_metadata import distributions
 
 from constants import (
     POSTGRESQL_SNAP_NAME,
@@ -47,3 +53,46 @@ def label2name(label: str) -> str:
         The converted name.
     """
     return label.rsplit("-", 1)[0] + "/" + label.rsplit("-", 1)[1]
+
+
+def _remove_stale_otel_sdk_packages():
+    """Hack to remove stale opentelemetry sdk packages from the charm's python venv.
+
+    See https://github.com/canonical/grafana-agent-operator/issues/146 and
+    https://bugs.launchpad.net/juju/+bug/2058335 for more context. This patch can be removed after
+    this juju issue is resolved and sufficient time has passed to expect most users of this library
+    have migrated to the patched version of juju.  When this patch is removed, un-ignore rule E402 for this file in the pyproject.toml (see setting
+    [tool.ruff.lint.per-file-ignores] in pyproject.toml).
+
+    This only has an effect if executed on an upgrade-charm event.
+    """
+    # all imports are local to keep this function standalone, side-effect-free, and easy to revert later
+
+    major_version = int(juju_ver.split(".")[0]) if (juju_ver := os.getenv("JUJU_VERSION")) else 3
+    if os.getenv("JUJU_DISPATCH_PATH") != "hooks/upgrade-charm" or major_version > 2:
+        return
+
+    otel_logger = logging.getLogger("charm_tracing_otel_patcher")
+    otel_logger.debug("Applying _remove_stale_otel_sdk_packages patch on charm upgrade")
+    # group by name all distributions starting with "opentelemetry_"
+    otel_distributions = defaultdict(list)
+    for distribution in distributions():
+        name = distribution._normalized_name
+        if name.startswith("opentelemetry_"):
+            otel_distributions[name].append(distribution)
+
+    otel_logger.debug(f"Found {len(otel_distributions)} opentelemetry distributions")
+
+    # If we have multiple distributions with the same name, remove any that have 0 associated files
+    for name, distributions_ in otel_distributions.items():
+        if len(distributions_) <= 1:
+            continue
+
+        otel_logger.debug(f"Package {name} has multiple ({len(distributions_)}) distributions.")
+        for distribution in distributions_:
+            if not distribution.files:  # Not None or empty list
+                path = distribution._path
+                otel_logger.info(f"Removing empty distribution of {name} at {path}.")
+                shutil.rmtree(path)
+
+    otel_logger.debug("Successfully applied _remove_stale_otel_sdk_packages patch.")

src/utils.py

@@ -2,10 +2,10 @@
 # See LICENSE file for licensing details.
 
 import re
-from unittest.mock import patch
+from unittest.mock import Mock, patch, sentinel
 
 from constants import POSTGRESQL_SNAP_NAME
-from utils import new_password, snap_refreshed
+from utils import _remove_stale_otel_sdk_packages, new_password, snap_refreshed
 
 
 def test_new_password():
@@ -34,3 +34,57 @@ def test_snap_refreshed():
     ):
         assert snap_refreshed("100") is False
         assert snap_refreshed("200") is False
+
+
+def test_remove_stale_otel_sdk_packages():
+    with (
+        patch("utils.os.getenv", return_value=None) as _getenv,
+        patch("utils.shutil") as _shutil,
+        patch("utils.distributions") as _distributions,
+    ):
+        other_dist = Mock()
+        other_dist._normalized_name = "test"
+        otel_dist = Mock()
+        otel_dist._normalized_name = "opentelemetry_test"
+        stale_otel_dist = Mock()
+        stale_otel_dist._normalized_name = "opentelemetry_test"
+        stale_otel_dist.files = []
+        stale_otel_dist._path = sentinel.path
+
+        # Not called if not upgrade hook
+        _remove_stale_otel_sdk_packages()
+
+        _distributions.assert_not_called()
+        _shutil.rmtree.assert_not_called()
+        _distributions.reset_mock()
+        _shutil.rmtree.reset_mock()
+
+        # don't execute on Juju 3
+        _getenv.side_effect = ["3.0.0", "hooks/upgrade-charm"]
+        _remove_stale_otel_sdk_packages()
+
+        _distributions.assert_not_called()
+        _shutil.rmtree.assert_not_called()
+        _distributions.reset_mock()
+        _shutil.rmtree.reset_mock()
+
+        # Upgrade hook, nothing to remove
+        _getenv.side_effect = ["2.9.53", "hooks/upgrade-charm"]
+        _distributions.return_value = [other_dist, otel_dist]
+
+        _remove_stale_otel_sdk_packages()
+
+        _distributions.assert_called_once_with()
+        _shutil.rmtree.assert_not_called()
+        _distributions.reset_mock()
+        _shutil.rmtree.reset_mock()
+
+        # Upgrade hook, duplicate otel packages
+        _getenv.side_effect = ["2.9.53", "hooks/upgrade-charm"]
+        _distributions.return_value = [other_dist, otel_dist, stale_otel_dist]
+        _remove_stale_otel_sdk_packages()
+
+        _distributions.assert_called_once_with()
+        _shutil.rmtree.assert_called_once_with(sentinel.path)
+        _distributions.reset_mock()
+        _shutil.rmtree.reset_mock()