[PR #2329/1f08cebb backport][2.24] Fix flatpak cache serialization

.github/workflows/scripts/post_before_script.sh

@@ -2,13 +2,9 @@ SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbou
 if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
   # Needed by pulp_container/tests/functional/api/test_flatpak.py:
   cmd_prefix dnf install -yq dbus-daemon flatpak
-fi
 
-# This allows flatpak to trust Pulp, but currently it breaks the trust for bindings
-# TODO: Figure out another command to fix this
-# add the copied certificates from install.sh to the container's trusted certificates list
-# if [[ "$TEST" = "azure" ]]; then
-#   cmd_prefix trust anchor /etc/pki/tls/cert.pem
-# else
-#   cmd_prefix trust anchor /etc/pulp/certs/pulp_webserver.crt
-# fi
+  # DO NOT CALL update-ca-trust, it will break the bindings TLS
+  # This copy is for the flatpak tests, flatpak uses pk11-kit which checks the source anchors
+  # to build the trust chain, it doesn't actually use the output of update-ca-trust
+  cmd_prefix cp /etc/pulp/certs/pulp_webserver.crt /etc/pki/ca-trust/source/anchors/pulp_webserver.crt
+fi

CHANGES/+flatpak-cache-key.bugfix

@@ -0,0 +1 @@
+Fixed a cache key collision in the flatpak static index that could serve incorrect results across domains.

CHANGES/+flatpak-domain-name.bugfix

@@ -0,0 +1 @@
+Fixed flatpak index response to include the domain prefix in image names when domains are enabled.

CHANGES/+flatpak-index-cache.bugfix

@@ -0,0 +1 @@
+Fixed a serialization issue with the FlatpakIndex cache.

CHANGES/+flatpak-manifest-list-filter.bugfix

@@ -0,0 +1 @@
+Fixed OS and architecture filters being silently ignored when filtering manifest lists in the flatpak index.

CHANGES/+flatpak-registry-host.bugfix

@@ -0,0 +1 @@
+Fixed the registry host in the flatpak index response when `CONTENT_ORIGIN=None`.

pulp_container/app/cache.py

@@ -89,14 +89,15 @@ def find_base_path_cached(request, cached):
 
 class FlatpakIndexStaticCache(SyncContentCache):
     def __init__(self, expires_ttl=None, auth=None):
-        updated_keys = (QUERY_KEY,)
+        updated_keys = (CacheKeys.host, QUERY_KEY)
         super().__init__(
             base_key="/index/static", expires_ttl=expires_ttl, keys=updated_keys, auth=auth
         )
 
     def make_key(self, request):
-        """Make a key composed of the request's query."""
+        """Make a key composed of the request's host and query."""
         all_keys = {
+            CacheKeys.host: request.get_host(),
             QUERY_KEY: request.query_params.urlencode(),
         }
         key = ":".join(all_keys[k] for k in self.keys)

pulp_container/app/registry_api.py

@@ -81,6 +81,7 @@
     determine_media_type,
     extract_data_from_signature,
     filter_resource,
+    get_full_path,
     has_task_completed,
     validate_manifest,
 )
@@ -578,9 +579,9 @@ def recurse_through_manifest_lists(self, tag, manifest, oss, architectures, mani
         elif manifest.media_type in (models.MEDIA_TYPE.MANIFEST_LIST, models.MEDIA_TYPE.INDEX_OCI):
             mlms = manifest.listed_manifests.through.objects.filter(image_manifest__pk=manifest.pk)
             if oss:
-                mlms.filter(os__in=oss)
+                mlms = mlms.filter(os__in=oss)
             if architectures:
-                mlms.filter(architecture__in=architectures)
+                mlms = mlms.filter(architecture__in=architectures)
             for mlm in mlms:
                 self.recurse_through_manifest_lists(
                     tag, mlm.manifest_list, oss, architectures, manifests
@@ -690,7 +691,7 @@ def get(self, request):
                     continue
                 images.append(
                     {
-                        "Tags": tagged,
+                        "Tags": list(tagged),
                         "Digest": manifest.digest,
                         "MediaType": manifest.media_type,
                         "OS": os,
@@ -699,9 +700,14 @@ def get(self, request):
                     }
                 )
             if images:
-                results.append({"Name": distribution.base_path, "Images": images})
+                results.append(
+                    {
+                        "Name": get_full_path(distribution.base_path, distribution.pulp_domain),
+                        "Images": images,
+                    }
+                )
 
-        host = settings.CONTENT_ORIGIN or request.get_host()
+        host = settings.CONTENT_ORIGIN or request.build_absolute_uri("/")
         return Response(data={"Registry": host, "Results": results})
 
 

pulp_container/app/utils.py

@@ -7,6 +7,7 @@
 from functools import partial
 
 from asgiref.sync import sync_to_async
+from django.conf import settings
 from django.db import IntegrityError
 from jsonschema import Draft7Validator, ValidationError, validate
 from pysequoia.packet import PacketPile, Tag
@@ -34,6 +35,13 @@
 log = logging.getLogger(__name__)
 
 
+def get_full_path(base_path, pulp_domain=None):
+    if settings.DOMAIN_ENABLED:
+        domain = pulp_domain or get_domain()
+        return f"{domain.name}/{base_path}"
+    return base_path
+
+
 def get_accepted_media_types(headers):
     """
     Returns a list of media types from the Accept headers.

pulp_container/tests/functional/api/test_flatpak.py

@@ -6,8 +6,6 @@
 
 from pulp_container.tests.functional.constants import REGISTRY_V2
 
-pytestmark = pytest.mark.skip(reason="TLS is broken currently. TODO: Fix")
-
 
 def run_flatpak_commands(host):
     # Install flatpak:
@@ -16,6 +14,7 @@ def run_flatpak_commands(host):
             "flatpak",
             "--user",
             "remote-add",
+            "--if-not-exists",
             "pulptest",
             "oci+" + host,
         ]
@@ -67,16 +66,21 @@ def test_flatpak_install(
     container_manifest_api,
     pulp_settings,
     bindings_cfg,
+    full_path,
 ):
     if not pulp_settings.FLATPAK_INDEX:
         pytest.skip("This test requires FLATPAK_INDEX to be enabled")
 
     image_path1 = f"{REGISTRY_V2}/pulp/oci-net.fishsoup.busyboxplatform:latest"
     registry_client.pull(image_path1)
-    local_registry.tag_and_push(image_path1, "pulptest/oci-net.fishsoup.busyboxplatform:latest")
+    local_registry.tag_and_push(
+        image_path1, full_path("pulptest/oci-net.fishsoup.busyboxplatform") + ":latest"
+    )
     image_path2 = f"{REGISTRY_V2}/pulp/oci-net.fishsoup.hello:latest"
     registry_client.pull(image_path2)
-    local_registry.tag_and_push(image_path2, "pulptest/oci-net.fishsoup.hello:latest")
+    local_registry.tag_and_push(
+        image_path2, full_path("pulptest/oci-net.fishsoup.hello") + ":latest"
+    )
     namespace = container_namespace_api.list(name="pulptest").results[0]
     add_to_cleanup(container_namespace_api, namespace.pulp_href)
 

pulp_container/tests/functional/conftest.py

@@ -501,3 +501,16 @@ def _check_manifest_arch_os_size(manifest):
         assert any(manifest.compressed_image_size > 0 for manifest in manifests)
 
     return _check_manifest_arch_os_size
+
+
+@pytest.fixture(scope="session")
+def full_path(pulp_settings):
+    def _full_path(base_path_or_distro, pulp_domain="default"):
+        if not isinstance(base_path_or_distro, str):
+            pulp_domain = base_path_or_distro.pulp_href[len(pulp_settings.API_ROOT) :].split("/")[0]
+            base_path_or_distro = base_path_or_distro.base_path
+        if pulp_settings.DOMAIN_ENABLED:
+            return f"{pulp_domain}/{base_path_or_distro}"
+        return base_path_or_distro
+
+    return _full_path