Update migrations 39-45 to perform the manifest data migration for new fields [2.26]

pulp_container/app/migrations/0039_manifest_data.py

@@ -1,15 +1,64 @@
 # Generated by Django 4.2.10 on 2024-03-05 11:22
+import json
 import warnings
 
 from django.db import migrations, models
+from django.core.files.storage import default_storage
 
+from pulp_container.constants import (
+    MEDIA_TYPE,
+)
 
-def print_warning_for_initializing_manifest_data(apps, schema_editor):
-    warnings.warn(
-        "Run 'pulpcore-manager container-handle-image-data' to move the manifests' "
-        "data from artifacts to the new 'data' database field."
-    )
+def get_content_data(artifact):
+    with default_storage.open(artifact.file.name, mode="rb") as file:
+        raw_data = file.read()
+    content_data = json.loads(raw_data)
+    return content_data, raw_data
 
+def migrate_manifest_data(apps, schema_editor):
+    """
+    Migrate the backing artifact manifest to the new 'data' field.
+    Also, initialize the 'annotations' and 'labels' fields.
+
+    'is_bootable' and 'is_flatpak' fields are not initialized since they require annotations and 
+    labels to be initialized first.
+    """
+    Manifest = apps.get_model("container", "Manifest")
+    to_update = []
+    manifests = Manifest.objects.filter(
+        contentartifact__artifact__isnull=False
+    ).distinct().prefetch_related("_artifacts").select_related("config_blob")
+    for manifest in manifests.iterator(chunk_size=1000):
+        manifest_artifact = manifest._artifacts.get()
+        manifest_data, raw_bytes_data = get_content_data(manifest_artifact)
+        manifest.data = raw_bytes_data.decode("utf-8")
+        if not manifest.annotations:
+            init_annotations(manifest, manifest_data)
+        if not manifest.labels:
+            init_labels(manifest)
+
+        to_update.append(manifest)
+        if len(to_update) > 1000:
+            Manifest.objects.bulk_update(to_update, ["data", "annotations", "labels"])
+            to_update.clear()
+    if to_update:
+        Manifest.objects.bulk_update(to_update, ["data", "annotations", "labels"])
+
+def init_annotations(manifest, manifest_data):
+    # annotations are part of OCI only
+    if manifest.media_type not in (MEDIA_TYPE.MANIFEST_OCI, MEDIA_TYPE.INDEX_OCI):
+        return
+
+    manifest.annotations = manifest_data.get("annotations", {})
+
+def init_labels(manifest):
+    if manifest.config_blob:
+        try:
+            config_artifact = manifest.config_blob._artifacts.get()
+        except models.ObjectDoesNotExist:
+            return
+        config_data, _ = get_content_data(config_artifact)
+        manifest.labels = config_data.get("config", {}).get("Labels") or {}
 
 class Migration(migrations.Migration):
 
@@ -24,7 +73,7 @@ class Migration(migrations.Migration):
             field=models.TextField(null=True),
         ),
         migrations.RunPython(
-            print_warning_for_initializing_manifest_data,
+            migrate_manifest_data,
             reverse_code=migrations.RunPython.noop,
             elidable=True,
         ),

pulp_container/app/migrations/0042_add_manifest_nature_field.py

@@ -1,7 +1,137 @@
 # Generated by Django 4.2.16 on 2024-10-21 19:14
 
+import json
 from django.db import migrations, models
 
+from pulp_container.constants import MEDIA_TYPE, MANIFEST_TYPE, MANIFEST_MEDIA_TYPES, COSIGN_MEDIA_TYPES, COSIGN_MEDIA_TYPES_MANIFEST_TYPE_MAPPING
+
+
+def migrate_manifest_nature(apps, schema_editor):
+    """
+    Populate the `type` field, and also the `is_bootable` and `is_flatpak` fields even though they are deprecated.
+    """
+    Manifest = apps.get_model("container", "Manifest")
+    to_update = []
+    # First, update the manifests that are not manifest lists or index manifests.
+    manifests = Manifest.objects.filter(data__isnull=False).exclude(media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI])
+    for manifest in manifests.iterator():
+        setattr(manifest, "json_manifest", json.loads(manifest.data))
+        if init_manifest_nature(manifest):
+            to_update.append(manifest)
+        if len(to_update) > 1000:
+            Manifest.objects.bulk_update(to_update, ["type", "is_bootable", "is_flatpak"])
+            to_update.clear()
+    if to_update:
+        Manifest.objects.bulk_update(to_update, ["type", "is_bootable", "is_flatpak"])
+        to_update.clear()
+    # Then, update the manifest lists and index manifests.
+    manifest_lists = Manifest.objects.filter(
+        data__isnull=False, media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI]
+    ).prefetch_related("listed_manifests")
+    for manifest_list in manifest_lists.iterator(chunk_size=1000):
+        setattr(manifest_list, "json_manifest", json.loads(manifest_list.data))
+        if init_manifest_list_nature(manifest_list):
+            to_update.append(manifest_list)
+        if len(to_update) > 1000:
+            Manifest.objects.bulk_update(to_update, ["type", "is_bootable", "is_flatpak"])
+            to_update.clear()
+    if to_update:
+        Manifest.objects.bulk_update(to_update, ["type", "is_bootable", "is_flatpak"])
+        to_update.clear()
+
+def init_manifest_list_nature(manifest_list):
+    updated_type = False
+    if not manifest_list.type:
+        manifest_list.type = MANIFEST_TYPE.INDEX
+        updated_type = True
+
+    for manifest in manifest_list.listed_manifests.all():
+        # it suffices just to have a single manifest of a specific nature;
+        # there is no case where the manifest is both bootable and flatpak-based
+        if manifest.type == MANIFEST_TYPE.BOOTABLE:
+            manifest_list.is_bootable = True
+            return True
+        elif manifest.type == MANIFEST_TYPE.FLATPAK:
+            manifest_list.is_flatpak = True
+            return True
+
+    return updated_type
+
+def init_manifest_nature(manifest):
+    if is_bootable_image(manifest):
+        # DEPRECATED: is_bootable is deprecated and will be removed in a future release.
+        manifest.is_bootable = True
+        manifest.type = MANIFEST_TYPE.BOOTABLE
+        return True
+    elif is_flatpak_image(manifest):
+        # DEPRECATED: is_flatpak is deprecated and will be removed in a future release.
+        manifest.is_flatpak = True
+        manifest.type = MANIFEST_TYPE.FLATPAK
+        return True
+    elif is_helm_chart(manifest):
+        manifest.type = MANIFEST_TYPE.HELM
+        return True
+    elif media_type := is_cosign(manifest):
+        manifest.type = get_cosign_type(media_type)
+        return True
+    elif is_artifact(manifest):
+        manifest.type = MANIFEST_TYPE.ARTIFACT
+        return True
+    elif is_manifest_image(manifest):
+        manifest.type = MANIFEST_TYPE.IMAGE
+        return True
+
+    return False
+
+def is_bootable_image(manifest):
+    return (
+        manifest.annotations.get("containers.bootc") == "1"
+        or manifest.labels.get("containers.bootc") == "1"
+    )
+
+def is_flatpak_image(manifest):
+    return True if manifest.labels.get("org.flatpak.ref") else False
+
+def is_manifest_image(manifest):
+    return manifest.media_type in MANIFEST_MEDIA_TYPES.IMAGE
+
+def is_cosign(manifest):
+    try:
+        # layers is not a mandatory field
+        layers = manifest.json_manifest["layers"]
+    except KeyError:
+        return False
+
+    for layer in layers:
+        if layer["mediaType"] in COSIGN_MEDIA_TYPES:
+            return layer["mediaType"]
+    return False
+
+def get_cosign_type(media_type):
+    if media_type in MEDIA_TYPE.COSIGN_SBOM:
+        return MANIFEST_TYPE.COSIGN_SBOM
+    return COSIGN_MEDIA_TYPES_MANIFEST_TYPE_MAPPING.get(media_type, MANIFEST_TYPE.UNKNOWN)
+
+def is_helm_chart(manifest):
+    try:
+        return manifest.json_manifest["config"]["mediaType"] == MEDIA_TYPE.CONFIG_BLOB_HELM
+    except KeyError:
+        return False
+
+def is_artifact(manifest):
+    # artifact is valid only for OCI spec
+    if manifest.media_type != MEDIA_TYPE.MANIFEST_OCI:
+        return False
+
+    if manifest.json_manifest.get("artifactType", None):
+        return True
+
+    manifest_config_media_type = manifest.json_manifest["config"]["mediaType"]
+    return (
+        manifest_config_media_type == MEDIA_TYPE.OCI_EMPTY_JSON
+        or manifest_config_media_type not in vars(MEDIA_TYPE).values()
+    )
+
 
 class Migration(migrations.Migration):
 
@@ -15,4 +145,5 @@ class Migration(migrations.Migration):
             name='type',
             field=models.CharField(null=True),
         ),
+        migrations.RunPython(migrate_manifest_nature, reverse_code=migrations.RunPython.noop, elidable=True),
     ]

pulp_container/app/migrations/0043_add_os_arch_image_size_manifest_fields.py

@@ -41,9 +41,5 @@ class Migration(migrations.Migration):
             name='os',
             field=models.TextField(blank=True, default=''),
         ),
-        migrations.RunPython(
-            print_warning_for_updating_manifest_fields,
-            reverse_code=migrations.RunPython.noop,
-            elidable=True,
-        ),
+        # Will update the os, architecture, and compressed_image_size in migration 45
     ]

pulp_container/app/migrations/0045_alter_manifest_compressed_image_size.py

@@ -1,7 +1,76 @@
 # Generated by Django 4.2.16 on 2025-03-06 22:37
-
+import json
+import warnings
 from django.db import migrations, models
+from django.core.files.storage import default_storage
+
+from pulp_container.constants import MEDIA_TYPE
+
+def get_content_data(artifact):
+    with default_storage.open(artifact.file.name, mode="rb") as file:
+        raw_data = file.read()
+    content_data = json.loads(raw_data)
+    return content_data, raw_data
+
+def migrate_os_arch_image_size(apps, schema_editor):
+    """
+    Populate the os, architecture, and compressed_image_size fields for the manifests.
+
+    This migration can only handle manifests within the default domain.
+    """
+    Manifest = apps.get_model("container", "Manifest")
+    to_update = []
+    manifests = Manifest.objects.filter(data__isnull=False, pulp_domain__name="default").select_related("config_blob")
+    for manifest in manifests.iterator():
+        if needs_os_arch_size_update(manifest):
+            manifest.json_manifest = json.loads(manifest.data)
+            init_architecture_and_os(manifest)
+            init_compressed_image_size(manifest)
+            to_update.append(manifest)
+
+            if len(to_update) > 1000:
+                Manifest.objects.bulk_update(to_update, ["os", "architecture", "compressed_image_size"])
+                to_update.clear()
+    if to_update:
+        Manifest.objects.bulk_update(to_update, ["os", "architecture", "compressed_image_size"])
 
+    if Manifest.objects.exclude(pulp_domain__name="default").exists():
+        warnings.warn(
+            "Run 'pulpcore-manager container-handle-image-data' to update the manifests' "
+            "os, architecture, and compressed_image_size fields."
+        )
+
+def needs_os_arch_size_update(manifest):
+    return manifest.media_type not in [MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI] and not (
+        manifest.architecture or manifest.os or manifest.compressed_image_size
+    )
+
+def init_architecture_and_os(manifest):
+    # schema1 has the architecture/os definition in the Manifest (not in the ConfigBlob)
+    # and none of these fields are required
+    if manifest.json_manifest.get("architecture", None) or manifest.json_manifest.get("os", None):
+        manifest.architecture = manifest.json_manifest.get("architecture", None)
+        manifest.os = manifest.json_manifest.get("os", None)
+    elif manifest.config_blob:
+        try:
+            config_artifact = manifest.config_blob._artifacts.get()
+        except models.ObjectDoesNotExist:
+            return
+        config_data, _ = get_content_data(config_artifact)
+        manifest.architecture = config_data.get("architecture", None)
+        manifest.os = config_data.get("os", None)
+
+def init_compressed_image_size(manifest):
+    # manifestv2 schema1 has only blobSum definition for each layer
+    if manifest.json_manifest.get("fsLayers", None):
+        manifest.compressed_image_size = 0
+        return
+
+    layers = manifest.json_manifest.get("layers")
+    compressed_size = 0
+    for layer in layers:
+        compressed_size += layer.get("size")
+    manifest.compressed_image_size = compressed_size
 
 class Migration(migrations.Migration):
 
@@ -15,4 +84,9 @@ class Migration(migrations.Migration):
             name="compressed_image_size",
             field=models.BigIntegerField(null=True),
         ),
+        migrations.RunPython(
+            migrate_os_arch_image_size,
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        ),
     ]