Merge multiple PRs in one

This combined merge takes these PRs in one: #177, #179, #182, #183
(with cherry-pick), and #187.

Author: mythi

docker/build-common-util.sh

@@ -38,7 +38,6 @@ EOF
     fi
     # lower compression levels in a PR build, to save build time
     if [ -z ${CI_ARCHIVER_MODE+x} ]; then
-        echo "ZIP_COMPRESSION_LEVEL ?= \"-1\"" >> conf/auto.conf
         echo "XZ_COMPRESSION_LEVEL ?= \"-0\"" >> conf/auto.conf
     fi
 }

docker/post-build.sh

@@ -28,6 +28,11 @@ set -u
 # create auto.conf using functions in build-common-util.sh
 auto_conf_common
 
+# post-build testing builds images but only .wic is sufficient
+# (default in IMAGE_FSTYPES). We skip compression and bmap formats
+# to optimize testing time
+echo "REFKIT_VM_IMAGE_TYPES = \"\"" >> conf/auto.conf
+
 export BUILD_ID=${CI_BUILD_ID}
 export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BUILD_ID"
 

docker/tester-exec.sh

@@ -77,13 +77,8 @@ testimg() {
   wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.bmap
   wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.xz -O - | unxz - > ${FILENAME}
   if [ ! -s ${FILENAME} ]; then
-    wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.zip
-    if [ -s ${FILENAME}.zip ]; then
-      unzip ${FILENAME}.zip
-    else
-      echo "ERROR: No file ${FILENAME}.xz or ${FILENAME}.zip found, can not continue."
+      echo "ERROR: No file ${FILENAME}.xz, can not continue."
       exit 1
-    fi
   fi
   set -e
 

meta-iot-web

@@ -15,7 +15,7 @@ REFKIT_IMAGE_COMPUTERVISION_EXTRA_FEATURES += "${@ '' if (d.getVar('IMAGE_MODE')
 # Example for customization in local.conf when building
 # refkit-image-computervision.bb:
 # IMAGE_BASENAME_pn-refkit-image-computervision = "my-refkit-image-reference"
-# REFKIT_IMAGE_COMPUTERVISION_EXTRA_INSTALL_append = "my-own-package"
-# REFKIT_IMAGE_COMPUTERVISION_EXTRA_FEATURES_append = "dev-pkgs"
+# REFKIT_IMAGE_COMPUTERVISION_EXTRA_INSTALL_append = " my-own-package"
+# REFKIT_IMAGE_COMPUTERVISION_EXTRA_FEATURES_append = " dev-pkgs"
 
 inherit refkit-image

meta-oic

@@ -5,25 +5,6 @@
 # it with the efi stub obtained from systemd-boot.
 # The layout of the image is described in a separate, customizable json file.
 
-# A layout files is built accordingly to the following example:
-#   {
-#       "gpt_initial_offset_mb": 3,          <-  Space allocated for the 1st GPT
-#       "gpt_tail_padding_mb": 3,            <-  Space allocated for the 2nd GPT
-#       "primary_uefi_boot_partition": {     <-- Name of the entry in the dictionary
-#           "name": "primary_uefi",          <-- Name of the partition in the GPT (MAX 16 ch)
-#           "uuid": 0,                       <-- UUID of the partition, 0 means random
-#           "size_mb": 30,                   <-- Size of the partition in MB
-#           "source": "${S}/hdd/boot",       <-- Directory containing the root for the partition
-#           "filesystem": "vfat",            <-- Filesystem for the partition
-#           "type": "ef00"                   <-- Type of the partition, to be used in the GPT
-#       },
-#       [...]                                Iterate partitions as needed
-#   }
-#
-#   The main rootfs partition is a special case and must be named "rootfs".
-#   This is required to identify it and pass its Partition UUID to the kernel, for booting.
-
-
 # By default, the full image is meant to fit into 4*10^9 bytes, i.e.
 # "4GB" regardless whether 1000 or 1024 is used as base. 64M are reserved
 # for potential partitioning overhead.
@@ -49,16 +30,6 @@ do_uefiapp[depends] += " \
                          ${INITRD_IMAGE}:do_image_complete \
                        "
 
-IMAGE_DEPENDS_dsk += " \
-                       gptfdisk-native:do_populate_sysroot \
-                       parted-native:do_populate_sysroot \
-                       mtools-native:do_populate_sysroot \
-                       dosfstools-native:do_populate_sysroot \
-                       dosfstools-native:do_populate_sysroot \
-                       python-native:do_populate_sysroot \
-                       bmap-tools-native:do_populate_sysroot \
-                     "
-
 # Always ensure that the INITRD_IMAGE gets added to the initramfs .cpio.
 # This needs to be done even when the actual .dsk image format is inactive,
 # because the .cpio file gets copied into the rootfs, and that rootfs
@@ -77,35 +48,7 @@ REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE ?= "deadbeef-dead-beef-dead-beefdeadbeef"
 PARTITION_TYPE_EFI = "EF00"
 PARTITION_TYPE_EFI_BACKUP = "2700"
 
-DSK_IMAGE_LAYOUT ??= ' \
-{ \
-    "gpt_initial_offset_mb": 3, \
-    "gpt_tail_padding_mb": 3, \
-    "partition_01_primary_uefi_boot": { \
-        "name": "primary_uefi", \
-        "uuid": 0, \
-        "size_mb": ${REFKIT_VFAT_MB}, \
-        "source": "${IMAGE_ROOTFS}/boot/", \
-        "filesystem": "vfat", \
-        "type": "${PARTITION_TYPE_EFI}" \
-    }, \
-    "partition_02_secondary_uefi_boot": { \
-        "name": "secondary_uefi", \
-        "uuid": 0, \
-        "size_mb": ${REFKIT_VFAT_MB}, \
-        "source": "${IMAGE_ROOTFS}/boot/", \
-        "filesystem": "vfat", \
-        "type": "${PARTITION_TYPE_EFI_BACKUP}" \
-    }, \
-    "partition_03_rootfs": { \
-        "name": "rootfs", \
-        "uuid": "${REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE}", \
-        "size_mb": 3700, \
-        "source": "${IMAGE_ROOTFS}", \
-        "filesystem": "ext4", \
-        "type": "8300" \
-    } \
-}'
+DSK_IMAGE_LAYOUT ??= '{}'
 
 inherit deploy
 
@@ -140,9 +83,10 @@ export PART_%(pnum)d_FS=%(filesystem)s
     partition_table = json.loads(layout)
 
     full_image_size_mb = partition_table["gpt_initial_offset_mb"] + \
-                         partition_table["gpt_tail_padding_mb"]
+                         partition_table["gpt_tail_padding_mb"] \
+                         if partition_table else 0
 
-    rootfs_type = None
+    rootfs_type = ''
     pnum = 0
     for key in sorted(partition_table.keys()):
         if not isinstance(partition_table[key], dict):
@@ -154,7 +98,7 @@ export PART_%(pnum)d_FS=%(filesystem)s
             partition_table[key]['uuid'] = str(uuid.uuid4())
         # Store these for the creation of the UEFI binary
         if partition_table[key]['name'] == 'rootfs':
-            rootfs_type = partition_table[key]['filesystem']
+            rootfs_type = 'rootfstype=%s ' % partition_table[key]['filesystem']
             int_part_uuid = d.getVar('INT_STORAGE_ROOTFS_PARTUUID_VALUE', True)
         else:
             int_part_uuid = partition_table[key]["uuid"]
@@ -168,8 +112,8 @@ export PART_%(pnum)d_FS=%(filesystem)s
         }
         pnum = pnum + 1
 
-    assert rootfs_type is not None
-    partition_data += "export PART_COUNT=%d\n" % pnum
+    if pnum > 0:
+        partition_data += "export PART_COUNT=%d\n" % pnum
 
     if os.path.exists(d.expand('${B}/initrd')):
         os.remove(d.expand('${B}/initrd'))
@@ -188,7 +132,7 @@ export PART_%(pnum)d_FS=%(filesystem)s
 
     def generate_app(partuuid, cmdline, suffix):
         with open(d.expand('${B}/cmdline' + suffix + '.txt'), 'w') as f:
-            f.write(d.expand('${APPEND} root=PARTUUID=%s rootfstype=%s %s' % \
+            f.write(d.expand('${APPEND} root=PARTUUID=%s %s%s' % \
                              (partuuid, rootfs_type, cmdline)))
         check_call(d.expand('objcopy ' +
                           '--add-section .osrel=${B}/machine.txt ' +
@@ -209,9 +153,10 @@ export PART_%(pnum)d_FS=%(filesystem)s
     generate_app(d.getVar('REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE', True), "installer", "")
     generate_app(d.getVar('INT_STORAGE_ROOTFS_PARTUUID_VALUE', True), "", "_internal_storage")
 
-    with open(d.expand('${B}/emmc-partitions-data'), 'w') as emmc_part_data:
-        emmc_part_data.write(partition_data)
-    shutil.copyfile(d.expand('${B}/emmc-partitions-data'), d.expand('${DEPLOYDIR}/emmc-partitions-data'))
+    if partition_data:
+        with open(d.expand('${B}/emmc-partitions-data'), 'w') as emmc_part_data:
+            emmc_part_data.write(partition_data)
+        shutil.copyfile(d.expand('${B}/emmc-partitions-data'), d.expand('${DEPLOYDIR}/emmc-partitions-data'))
 
     # The RMC database is deployed unconditionally but not read if the BIOS is in SecureBoot mode.
     # XXX: However, the check for SecureBoot is not present. The bug is tracked in
@@ -280,21 +225,3 @@ do_rootfs[depends] += '${@bb.utils.contains('IMAGE_FEATURES','secureboot','sbsig
 
 ROOTFS_POSTPROCESS_COMMAND += " ${@bb.utils.contains('IMAGE_FEATURES','secureboot','uefiapp_sign;','',d)} "
 ROOTFS_POSTPROCESS_COMMAND += " uefiapp_deploy; "
-
-# All variables explicitly passed to image-dsk.py.
-IMAGE_DSK_VARIABLES = " \
-    APPEND \
-    IMGDEPLOYDIR \
-    DSK_IMAGE_LAYOUT \
-    IMAGE_LINK_NAME \
-    IMAGE_NAME \
-    IMAGE_ROOTFS \
-    ROOTFS_TYPE \
-    REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE \
-    PARTITION_TYPE_EFI \
-    PARTITION_TYPE_EFI_BACKUP \
-    S \
-"
-
-IMAGE_CMD_dsk = "${PYTHON} ${IMAGE_DSK_BASE}/lib/image-dsk.py ${@' '.join(["'%s=%s'" % (x, d.getVar(x, True) or '') for x in d.getVar('IMAGE_DSK_VARIABLES', True).split()])}"
-IMAGE_CMD_dsk[vardeps] = "${IMAGE_DSK_VARIABLES}"

meta-refkit-computervision/recipes-image/images/refkit-image-computervision.bb

@@ -212,10 +212,8 @@ NOISO = "1"
 REFKIT_QB_DRIVE_TYPE = "/dev/vd"
 QB_DRIVE_TYPE = "${REFKIT_QB_DRIVE_TYPE}"
 
-inherit image_types_extra
-
 # Replace the default "live" (aka HDDIMG) images with whole-disk images
-REFKIT_VM_IMAGE_TYPES ?= ""
+REFKIT_VM_IMAGE_TYPES ??= ""
 IMAGE_FSTYPES_append = " ${REFKIT_VM_IMAGE_TYPES}"
 
 # unconditionally set in x86-base.inc so we just remove it to avoid

meta-refkit-core/classes/image-dsk.bbclass

@@ -107,7 +107,7 @@ BAD_RECOMMENDATIONS_append_refkit-config = " bluez5-client"
 # Prefer systemd way of creating getty@.service symlinks using
 # systemd-getty-generator (instead of the Yocto default
 # systemd-serialgetty that creates everything in do_install).
-PACKAGECONFIG_append_pn-systemd_refkit-config = "serial-getty-generator"
+PACKAGECONFIG_append_pn-systemd_refkit-config = " serial-getty-generator"
 
 # OE-core 7355a3c4f665e066925ea6327ac72365ab1f2f39 added a PACKAGECONFIG
 # to configure cairo/python3-cairo and enabled it by default. It used to