Merge branch 'feature/support_podman_update_images' into 'master'

Add support for podman and miscellaneous updates

See merge request evernym/utilities/devlab!23

Author: absltkaos

.gitlab-ci.yml

@@ -3,7 +3,7 @@ image: $CI_REGISTRY_IMAGE/devlab-build:latest
 
 .tags:
   tags:
-    - docker
+    - saas-linux-small-amd64
 
 stages:
   - docker-images
@@ -19,7 +19,7 @@ stages:
       - branches
       - merge_requests
   tags:
-    - docker
+    - saas-linux-small-amd64
   image:
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: [""]

README.md

@@ -105,7 +105,8 @@ The configuration file has the following base structure:
     "project_filter": "",
     "reprovisionable_components": [],
     "runtime_images": {},
-    "wizard_enabled": true
+    "wizard_enabled": true,
+    "disable_buildkit": false
 }
 ```
 
@@ -123,6 +124,7 @@ All Keys that are in **bold** are required to be in the config
 | reprovisionable_components | List of Strings | List of component names that need to reprovisioned (contianer stopped, and removed, then started and provisioned again) if the hosts' IP changes) |
 | runtime_images | Hash of Hashes | Defines custom images that components might be using. These images etc.. would provided by, and maintained by the project. First level key is a string of the name of the image. Structure conforms to [Runtime Image Structure](#runtime-image-structure) |
 | wizard_enabled | Boolean | Whether or not to try and execute a wizard if found in the root of the project |
+| disable_buildkit | Boolean | Whether the DOCKER_BUILDKIT env var should be set to `0` when building images |
 
 ## Component Config Structure
 The structure looks like this:

devlab

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # vim: set syntax=python et ts=4 sw=4 sts=4:
 """
 Main script for managing the devlab environment stack

devlab_bench/__init__.py

@@ -42,7 +42,8 @@
     },
     'reprovisionable_components': [],
     'runtime_images': {},
-    'paths': {}
+    'paths': {},
+    'disable_buildkit': False
 }
 LOGGING_LEVELS = {
     'debug': logging.DEBUG,

devlab_bench/actions/build.py

@@ -166,7 +166,7 @@ def action(images='*', clean=False, no_cache=False, pull=False, skip_pull_images
         image_args['docker_file'] = image_args['docker_file_full_path']
         del image_args['docker_file_full_path']
         log.debug("image: '%s' context='%s' log_output='%s' other_args='%s'", image, image_context, log_output, image_args)
-        bld_res = docker_helper_obj.build_image(image, context=image_context, log_output=log_output, network=config['network']['name'], logger=logging.getLogger('Build-{}'.format(image)), **image_args)
+        bld_res = docker_helper_obj.build_image(image, context=image_context, log_output=log_output, network=config['network']['name'], disable_build_kit=config['disable_buildkit'], logger=logging.getLogger('Build-{}'.format(image)), **image_args)
         if bld_res[0] != 0:
             log.error("Failed building image: '%s' Aborting...", image)
             abort = True

devlab_bench/helpers/command.py

@@ -175,13 +175,15 @@ def _process_output(self, max_lines=100, flush=False):
                             stdout_dangle = self._sanitize_string(self.proc.stdout.read())
                             if stdout_dangle:
                                 if self.log_output:
-                                    self.log.info(stdout_dangle)
+                                    for stdout_dangle_line in stdout_dangle.splitlines():
+                                        self.log.info(stdout_dangle_line)
                                 self.stdout.append(stdout_dangle)
                         if self.proc.stderr:
                             stderr_dangle = self._sanitize_string(self.proc.stderr.read())
                             if stderr_dangle:
                                 if self.log_output:
-                                    self.log.error(stderr_dangle)
+                                    for stderr_dangle_line in stderr_dangle.splitlines():
+                                        self.log.warning(stderr_dangle_line)
                                 self.stderr.append(stderr_dangle)
                         break
             except OSError:

devlab_bench/helpers/docker.py

@@ -35,14 +35,25 @@ def __init__(self, filter_label=None, labels=None, common_domain=None, skip_chec
             '/usr/bin/docker',
             '/usr/local/bin/docker',
             '/usr/sbin/docker',
-            '/bin/docker'
+            '/bin/docker',
+            '/opt/podman/bin/podman',
+            '/usr/bin/podman',
+            '/usr/local/bin/podman'
         )
+        self.eng_is = 'unknown'
         self.filter_label = filter_label
         self.common_domain = common_domain
         self.opt_domainname = False
         if not skip_checks:
             self._pre_check()
         self.labels = labels
+        path_check = Command(self.docker_bin_paths)._precheck()
+        if path_check[0] == 0:
+            if 'podman' in path_check[1]:
+                self.eng_is = 'podman'
+            if 'docker' in path_check[1]:
+                self.eng_is = 'docker'
+        self.log.debug("Docker engine type found is: %s", self.eng_is)
     def _pre_check(self):
         """
         Checks to make sure the script is being run as the root user, or a
@@ -60,7 +71,7 @@ def _pre_check(self):
         dchk = Command(self.docker_bin_paths, ['run', '--help'], logger=self.log, split=False).run()
         if '--domainname' in dchk[1]:
             self.opt_domainname = True
-    def build_image(self, name, tag, context, docker_file, apply_filter_label=True, build_opts=None, logger=None, network=None, **kwargs):
+    def build_image(self, name, tag, context, docker_file, apply_filter_label=True, build_opts=None, disable_buildkit=False, logger=None, network=None, **kwargs):
         """
         Build a docker image.
 
@@ -112,10 +123,11 @@ def build_image(self, name, tag, context, docker_file, apply_filter_label=True,
         ]
         if os.path.isfile(docker_file):
             with open(docker_file) as stdin:
-                if 'env' in kwargs:
-                    kwargs['env'].update({'DOCKER_BUILDKIT': "0"})
-                else:
-                    kwargs['env'] = {'DOCKER_BUILDKIT': "0"}
+                if disable_buildkit:
+                    if 'env' in kwargs:
+                        kwargs['env'].update({'DOCKER_BUILDKIT': "0"})
+                    else:
+                        kwargs['env'] = {'DOCKER_BUILDKIT': "0"}
                 cmd_ret = Command(
                     self.docker_bin_paths,
                     opts,
@@ -287,6 +299,10 @@ def get_images(self, return_all=False):
         opts = [
             'images'
         ]
+        podman_specific_prepends = [
+            'localhost/', # These are podman local images
+            'docker.io/library/' # These are docker.io hosted images
+        ]
         if self.filter_label and not return_all:
             opts.append('--filter')
             opts.append('label={}'.format(self.filter_label))
@@ -297,6 +313,21 @@ def get_images(self, return_all=False):
             opts,
             logger=self.log
         ).run()
+        if self.eng_is == 'podman':
+            if cmd_ret[0] == 0:
+                self.log.debug('Normalizing podman specific image output to match docker\'s format')
+                new_list = []
+                for cline in cmd_ret[1]:
+                    modified = False
+                    for pman_prep in podman_specific_prepends:
+                        if cline.startswith(pman_prep):
+                            new_name = cline[len(pman_prep):]
+                            new_list.append(new_name)
+                            self.log.debug("Normalized container image: '%s' -> '%s'", cline, new_name)
+                            modified = True
+                    if not modified:
+                        new_list.append(cline)
+                cmd_ret = (cmd_ret[0], new_list)
         return cmd_ret
     def get_networks(self, return_all=False):
         """

docker/base.Dockerfile

@@ -1,6 +1,6 @@
-FROM ubuntu:18.04
+FROM ubuntu:24.04
 LABEL "com.lab.devlab"="base"
-LABEL "last_modified"=1597871407
+LABEL "last_modified"=1713305264
 
 ARG APT="/usr/bin/apt-get --no-install-recommends -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef"
 ARG PACKAGES="sudo less screen tmux vim ca-certificates git curl iproute2 less gnupg2"
@@ -10,37 +10,3 @@ RUN $APT update && \
     $APT upgrade -y && \
     $APT install -y $PACKAGES && \
     rm -rf /var/lib/apt/lists/*
-
-# Provide Evernym CA
-RUN mkdir -p /usr/local/share/ca-certificates/ && \
-echo '-----BEGIN CERTIFICATE-----\n\
-MIIFJTCCAw2gAwIBAgIUMI0Z8YSLeRq8pZks40O3Dq2m8TIwDQYJKoZIhvcNAQEL\n\
-BQAwGjEYMBYGA1UEAxMPRXZlcm55bSBSb290IENBMB4XDTE3MTAxMTIwMTAxMFoX\n\
-DTQ3MTAwNDIwMTAzOVowGjEYMBYGA1UEAxMPRXZlcm55bSBSb290IENBMIICIjAN\n\
-BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1kjmtmMfLJfsqUNaco44N3brW8Vu\n\
-b02lAeEwbxc65mwfAG8kAjW7kYhI/fDXFOYXUvoa3Dg7bFeEatdIjHOahZssGM27\n\
-HsQZ4PfRhPE6HtXFszmDwXWuEekVxoyueTqL7ExnNZ+BRTXvPfm5nw1E7L3o3xHF\n\
-GSOtWFCyHfKd1LwMKzAVSjxlawEZnfk3WK3NxrC4UYMlQaDme7m3rCMfO+KBQk69\n\
-bFXsgn6/EihVeQ8T1+T8gogofzh5b4Z7kS6e6GMqotbGFg4agejkRVsIglSpaQLk\n\
-2Ztn/MP1dwgyvO4uvplB4sxZSC2FhhovlwPETmbKsnpj020+m0+YU4FPKwjroMiH\n\
-tP//YqiNKsLxtjhffW7XFToyy0qQttW5RMWnyx4MXs9Hwcy29gY1izeGMSzz3zV5\n\
-HG8JSJikuYbYiGJRVS0egovkVjja6lrVk0Q4Hm5pbw4l7LYCd6bkDLMsRaS1QnWs\n\
-9iz6XEf5SpIu1FuqHmlhj1ABehUyGIg5oC6egML3q78yk0mCW523qMFa9Kjnk871\n\
-mmXSCn3p/3DCrwWYfpcibxtVaKyJj6ISYIcl+Zu65Uzmhf+nj56x3gkNgEOva7JS\n\
-Xge+FxPxsaXBGyeSH09nNIoNmh/UucuzpNY2UyCpJuqXHtR5jaACSdsqNxG8tcDg\n\
-K9v98D/DFiShghECAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF\n\
-MAMBAf8wHQYDVR0OBBYEFOrH4oUpB94gNDNqdGG92kdVZ3qkMB8GA1UdIwQYMBaA\n\
-FOrH4oUpB94gNDNqdGG92kdVZ3qkMA0GCSqGSIb3DQEBCwUAA4ICAQCwjN3ggZ98\n\
-BXT39fKkCX3FHb0++aFcIyMKWrcZIpYrl3GoZsNKZK4QNQ+uJOP8xmqgyrCoMfch\n\
-VIGPQ0RDN/IzqCLhc/U3pDmk2hXa3xTxD3gpCQZ6Bz04KlcLfZd5jzbI741bVDyF\n\
-a1n46bEyuqV4SsNJWI/FGokJCNcZH66njBQBaQAccZ7xB9vWU9yjIYtGQDDvSm6J\n\
-SC2knrQri0vv4QLUSc1LS6AlWWSQxcCpcdO+OzIFGsf5bVmYN6J4R3COY5NyQ+yn\n\
-pOSN2NOh5h3ZrYAxm3i4Il0orVLveVcTVDGeAgZUII4YLJi/01RHGqit3aCuApSh\n\
-bzFTZ5FldFss+JX9iAhqpFDbHLgae0F3QmYEnGilt/PzO4j23QJo3FZKeruQLH7P\n\
-L9aOgN6S2+Akbbm9YTc59yzU5TZMxANwTdaYFWFqk/8nKgZiBR1l8jnWTlWnm86A\n\
-qVssH3DLKwiYrWSOHRzGuN5BmPXxxtKQJlwAXt0wJE3puUkaJSRo7CJQ3QNMoKDe\n\
-OjzXc9WvkFIXr3Eui8UTiHB/WT7N4o8hmVN404akGfWE0YNwRVfWpjGdew6g0tZi\n\
-lFnjUUk49av67um43JHcinT5NFPuleZzkjaL/D8ueOrjXQDy05rwVdgmw9pXog4B\n\
-Tw6APXtEnjfD2H8HOpOX/7ef4gWK0O1Q7A==\n\
------END CERTIFICATE-----\n' > /usr/local/share/ca-certificates/Evernym_Root_CA.crt && \
-/usr/sbin/update-ca-certificates

docker/helper.Dockerfile

@@ -1,10 +1,10 @@
 FROM devlab_base:latest
 LABEL "com.lab.devlab"="base"
-LABEL "last_modified"=1597871407
+LABEL "last_modified"=1713305264
 
 
 ARG APT="/usr/bin/apt-get --no-install-recommends -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef"
-ARG PACKAGES="docker.io python python3 python-yaml python3-yaml mysql-client netcat"
+ARG PACKAGES="docker.io python3 python3-yaml mysql-client netcat-traditional"
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN $APT update && \