Support setting the mapping for build source path between container and host mechine

* Set the path mapping with global config *.yml file
* If the mapping is not set on global config file, try to read the mapping from environment variable.

Author: JohnDing1995

apluslms_roman/backends/__init__.py

@@ -1,8 +1,11 @@
+import logging
 from collections import namedtuple
 from collections.abc import Mapping
+from pathlib import PurePosixPath
 
 from ..observer import BuildObserver
 
+logger = logging.getLogger(__name__)
 
 BACKENDS = {
     'docker': 'apluslms_roman.backends.docker.DockerBackend',
@@ -112,3 +115,18 @@ def verify(self):
 
     def version_info(self):
         pass
+
+    def get_host_path(self, original):
+        mapping = self.environment.environ.get('directory_map', {})
+        if not mapping:
+            return original
+        logger.debug("Get mapping from environment:%s", mapping)
+        path = PurePosixPath(original)
+        for container, host in mapping.items():
+            try:
+                logger.debug("Mapping:%s:%s", container, host)
+                return str(PurePosixPath(host, path.relative_to(container)))
+            except ValueError:
+                logger.error("Error when composing new path!")
+                continue
+        return ValueError("Unable to map path '%s' to any backend host path" % (path))

apluslms_roman/backends/docker.py

@@ -1,6 +1,7 @@
-import docker
-from os.path import join
+import logging
+from os.path import join, expanduser
 
+import docker
 from apluslms_yamlidator.utils.decorator import cached_property
 
 from ..utils.translation import _
@@ -12,6 +13,8 @@
 
 Mount = docker.types.Mount
 
+logger = logging.getLogger(__name__)
+
 
 class DockerBackend(Backend):
     name = 'docker'
@@ -22,14 +25,28 @@ class DockerBackend(Backend):
     @cached_property
     def _client(self):
         env = self.environment.environ
-        kwargs = {}
-        version = env.get('DOCKER_VERSION', None)
-        if version:
-            kwargs['version'] = version
-        timeout = env.get('DOCKER_TIMEOUT', None)
-        if timeout:
-            kwargs['timeout'] = timeout
-        return docker.from_env(environment=env, **kwargs)
+        params = {
+            'base_url': env.get('host'),
+            'version': env.get('version'),
+        }
+        if 'timeout' in env:
+            params['timeout'] = env['timeout']
+
+        # false values: 0, false, '', unset
+        # true values: 1, true, "yes"
+        tls_verify = bool(env.get('tls_verify', False))
+        cert_path = env.get('cert_path') or None
+        if tls_verify or cert_path:
+            if not cert_path:
+                cert_path = join(expanduser('~'), '.docker')
+            params['tls'] = docker.tls.TLSConfig(
+                client_cert=(join(cert_path, 'cert.pem'), join(cert_path, 'key.pem')),
+                ca_cert=join(cert_path, 'ca.pem'),
+                verify=tls_verify,
+                ssl_version=env.get('tls_ssl_version'),
+                assert_hostname=tls_verify and env.get('tls_assert_hostname'),
+            )
+        return docker.DockerClient(**params)
 
     def _run_opts(self, task, step):
         env = self.environment
@@ -41,16 +58,19 @@ def _run_opts(self, task, step):
             user='{}:{}'.format(env.uid, env.gid),
         )
 
+        path = self.get_host_path(task.path)
+
+        logger.debug("Final path is:%s", path)
         # mounts and workdir
         if step.mnt:
-            opts['mounts'] = [Mount(step.mnt, task.path, type='bind', read_only=False)]
+            opts['mounts'] = [Mount(step.mnt, path, type='bind', read_only=False)]
             opts['working_dir'] = step.mnt
         else:
             wpath = self.WORK_PATH
             opts['mounts'] = [
                 Mount(wpath, None, type='tmpfs', read_only=False, tmpfs_size=self.WORK_SIZE),
-                Mount(join(wpath, 'src'), task.path, type='bind', read_only=True),
-                Mount(join(wpath, 'build'), join(task.path, '_build'), type='bind', read_only=False),
+                Mount(join(wpath, 'src'), path, type='bind', read_only=True),
+                Mount(join(wpath, 'build'), join(path, '_build'), type='bind', read_only=False),
             ]
             opts['working_dir'] = wpath
 

apluslms_roman/builder.py

@@ -9,6 +9,7 @@
 from .utils.importing import import_string
 from .utils.translation import _
 
+
 class Builder:
     def __init__(self, engine, config, observer=None):
         if not isdir(config.dir):
@@ -21,7 +22,7 @@ def __init__(self, engine, config, observer=None):
 
     def get_steps(self, refs: list = None):
         steps = [BuildStep.from_config(i, step)
-            for i, step in enumerate(self.config.steps)]
+                 for i, step in enumerate(self.config.steps)]
         if refs:
             name_dict = {step.name: step for step in steps}
             refs = [int(ref) if ref.isdigit() else ref.lower() for ref in refs]
@@ -64,11 +65,9 @@ def __init__(self, backend_class=None, settings=None):
 
         name = getattr(backend_class, 'name', None) or backend_class.__name__.lower()
         env_prefix = name.upper() + '_'
-        env = {k: v for k, v in environ.items() if k.startswith(env_prefix)}
+        env = {key: value for key, value in environ.items() if key.startswith(env_prefix)}
         if settings:
-            for k, v in settings.get(name, {}).items():
-                if v is not None and v != '':
-                    env[env_prefix + k.replace('-', '_').upper()] = v
+            env.update(settings.get(name, {}))
         self._environment = Environment(getuid(), getegid(), env)
 
     @cached_property

apluslms_roman/schemas/roman_settings-v1.0.yaml

@@ -24,6 +24,10 @@ properties:
     type: object
     additionalProperties: false
     properties:
+      directory_map:
+        title: docker container-host machine path mapping
+        description: The dictornary mapping between docker and it's host
+        type: object
       host:
         title: docker host
         description: the URL to the Docker host

apluslms_roman/utils/path_mapping.py

@@ -0,0 +1,25 @@
+import json
+import logging
+import re
+from os import environ
+from apluslms_yamlidator.document import find_ml
+
+logger = logging.getLogger(__name__)
+json_re = re.compile(r'^(?:["[{]|(?:-?[1-9]\d*(?:\.\d+)?|null|true|false)$)')
+
+
+def nest_dict(flat_dict):
+    nested = {}
+    for keys, value in flat_dict.items():
+        dict_, key = find_ml(nested, keys, create_dicts=True)
+        dict_[key] = value
+    return nested
+
+
+def load_from_env(env_prefix=None, decode_json=True):
+    if decode_json:
+        decode = lambda s: json.loads(s) if json_re.match(s) is not None else s
+    else:
+        decode = lambda s: s
+    env = {key[len(env_prefix):].lower(): decode(value) for key, value in environ.items() if key.startswith(env_prefix)}
+    return nest_dict(env)

tests/utils/test_path_mapping.py

@@ -0,0 +1,64 @@
+import os
+import unittest
+from json import loads
+
+from apluslms_roman.utils.path_mapping import json_re, load_from_env
+
+test_case_loadable = (
+    'true',
+    'false',
+    'null',
+    '123',
+    '-123',
+    '3.14',
+    '-3.14',
+    '{"foo": "bar"}',
+    '[1, 2, 3]',
+    '"foo bar"'
+)
+
+test_case_not_loadable = (
+    "/foobar.py",
+    "text",
+    "yes",
+    "0123123",
+)
+
+
+class TestJsonLoadable(unittest.TestCase):
+
+    def test_loadable_not_raise(self):
+        for case in test_case_loadable:
+            with self.subTest(non_json=case):
+                loads(case)
+
+    def test_not_loadable_raise(self):
+        for case in test_case_not_loadable:
+            with self.subTest(non_json=case):
+                with self.assertRaises(ValueError, msg="Testing:{}".format(case)):
+                    loads(case)
+
+
+class TestJsonRegex(unittest.TestCase):
+
+    def test_loadable_match(self):
+        for case in test_case_loadable:
+            with self.subTest(non_json=case):
+                self.assertTrue(json_re.match(case)is not None, msg="Testing:{}".format(case))
+
+    def test_not_loadable_not_match(self):
+        for case in test_case_not_loadable:
+            with self.subTest(non_json=case):
+                self.assertFalse(json_re.match(case) is not None, msg="Testing:{}".format(case))
+
+
+class TestLoadFromEnv(unittest.TestCase):
+
+    def test_with_decode_json(self):
+        os.environ['DOCKER.FOO.BAR'] = '123'
+        self.assertEqual({'foo': {'bar': '123'}}, load_from_env('DOCKER.', False))
+
+    def test_without_decode_json(self):
+        os.environ['DOCKER.FOO.BAR'] = '123'
+        self.assertEqual({'foo': {'bar': 123}}, load_from_env('DOCKER.', True))
+