Support setting the mapping for build source path between container and host mechine

* Set the path mapping with global config *.yml file
* If the mapping is not set on global config file, try to read the mapping from environment variable.

Author: JohnDing1995

apluslms_roman/backends/__init__.py

@@ -1,19 +1,21 @@
+import logging
 from collections import namedtuple
 from collections.abc import Mapping
 
 from ..observer import BuildObserver
-
+from ..utils.path_mapping import get_host_path
 
 BACKENDS = {
     'docker': 'apluslms_roman.backends.docker.DockerBackend',
 }
 
-
 BuildTask = namedtuple('BuildTask', [
     'path',
     'steps',
 ])
 
+logger = logging.getLogger(__name__)
+
 
 def clean_image_name(image):
     if ':' not in image:
@@ -112,3 +114,9 @@ def verify(self):
 
     def version_info(self):
         pass
+
+    def remap_path(self, path):
+        map_ = self.environment.environ.get('directory_map', {})
+        logger.debug("get mapping from environment:%s", map_)
+        map_ = dict(map_) if len(map_) == 0 else map_
+        return get_host_path(path, map_)

apluslms_roman/backends/docker.py

@@ -1,6 +1,7 @@
-import docker
-from os.path import join
+import logging
+from os.path import join, expanduser
 
+import docker
 from apluslms_yamlidator.utils.decorator import cached_property
 
 from ..utils.translation import _
@@ -12,6 +13,8 @@
 
 Mount = docker.types.Mount
 
+logger = logging.getLogger(__name__)
+
 
 class DockerBackend(Backend):
     name = 'docker'
@@ -22,14 +25,28 @@ class DockerBackend(Backend):
     @cached_property
     def _client(self):
         env = self.environment.environ
-        kwargs = {}
-        version = env.get('DOCKER_VERSION', None)
-        if version:
-            kwargs['version'] = version
-        timeout = env.get('DOCKER_TIMEOUT', None)
-        if timeout:
-            kwargs['timeout'] = timeout
-        return docker.from_env(environment=env, **kwargs)
+        params = {
+            'base_url': env.get('host'),
+            'version': env.get('version'),
+        }
+        if 'timeout' in env:
+            params['timeout'] = env['timeout']
+
+        # false values: 0, false, '', unset
+        # true values: 1, true, "yes"
+        tls_verify = bool(env.get('tls_verify', False))
+        cert_path = env.get('cert_path') or None
+        if tls_verify or cert_path:
+            if not cert_path:
+                cert_path = join(expanduser('~'), '.docker')
+            params['tls'] = docker.tls.TLSConfig(
+                client_cert=(join(cert_path, 'cert.pem'), join(cert_path, 'key.pem')),
+                ca_cert=join(cert_path, 'ca.pem'),
+                verify=tls_verify,
+                ssl_version=env.get('tls_ssl_version'),
+                assert_hostname=tls_verify and env.get('tls_assert_hostname'),
+            )
+        return docker.DockerClient(**params)
 
     def _run_opts(self, task, step):
         env = self.environment
@@ -41,9 +58,12 @@ def _run_opts(self, task, step):
             user='{}:{}'.format(env.uid, env.gid),
         )
 
+        path = self.remap_path(task.path)
+
+        logger.debug("Final path is:%s", path)
         # mounts and workdir
         if step.mnt:
-            opts['mounts'] = [Mount(step.mnt, task.path, type='bind', read_only=False)]
+            opts['mounts'] = [Mount(step.mnt, path, type='bind', read_only=False)]
             opts['working_dir'] = step.mnt
         else:
             wpath = self.WORK_PATH

apluslms_roman/builder.py

@@ -9,6 +9,7 @@
 from .utils.importing import import_string
 from .utils.translation import _
 
+
 class Builder:
     def __init__(self, engine, config, observer=None):
         if not isdir(config.dir):
@@ -18,7 +19,6 @@ def __init__(self, engine, config, observer=None):
         self._engine = engine
         self._observer = observer or StreamObserver()
 
-
     def get_steps(self, refs: list = None):
         steps = [BuildStep.from_config(i, step)
             for i, step in enumerate(self.config.steps)]
@@ -33,7 +33,7 @@ def get_steps(self, refs: list = None):
     def build(self, step_refs: list = None):
         backend = self._engine.backend
         observer = self._observer
-        steps = self.get_steps(step_refs) # NOTE: may raise KeyError or IndexError
+        steps = self.get_steps(step_refs)  # NOTE: may raise KeyError or IndexError
 
         task = BuildTask(self.path, steps)
         observer.enter_prepare()
@@ -64,11 +64,9 @@ def __init__(self, backend_class=None, settings=None):
 
         name = getattr(backend_class, 'name', None) or backend_class.__name__.lower()
         env_prefix = name.upper() + '_'
-        env = {k: v for k, v in environ.items() if k.startswith(env_prefix)}
+        env = {key: value for key, value in environ.items() if key.startswith(env_prefix)}
         if settings:
-            for k, v in settings.get(name, {}).items():
-                if v is not None and v != '':
-                    env[env_prefix + k.replace('-', '_').upper()] = v
+            env.update(settings.get(name, {}))
         self._environment = Environment(getuid(), getegid(), env)
 
     @cached_property

apluslms_roman/schemas/roman_settings-v1.0.yaml

@@ -24,6 +24,10 @@ properties:
     type: object
     additionalProperties: false
     properties:
+      directory_map:
+        title: docker container-host machine path mapping
+        description: The dictornary mapping between docker and it's host
+        type: object
       host:
         title: docker host
         description: the URL to the Docker host

apluslms_roman/utils/path_mapping.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import re
+from os import environ
+from pathlib import PurePosixPath
+
+logger = logging.getLogger(__name__)
+json_re = re.compile(r'^(?:["[{]|(?:-?[1-9]\d*(?:\.\d+)?|null|true|false)$)')
+
+
+def get_host_path(original, mapping):
+    ret = original
+    orig_path = PurePosixPath(original)
+    for k, v in mapping.items():
+        try:
+            logger.debug("Mapping:%s:%s", k, v)
+            relative_path = orig_path.relative_to(k)
+            ret = PurePosixPath(v).joinpath(relative_path)
+            return str(ret)
+        except ValueError:
+            logger.warning("Error when composing new path!")
+            pass
+    return str(ret)
+
+
+def nest_dict(flat_dict, sep):
+    ret = {}
+    for key, value in flat_dict.items():
+        key_list = key.split(sep, 1)
+        if len(key_list) == 2:
+            root = key_list[0]
+            if root not in ret:
+                ret[root] = {}
+            ret[root][key_list[1]] = value
+        else:
+            ret[key] = value
+    return ret
+
+
+def load_from_env(env_prefix=None, sep=None, decode_json=True):
+    if decode_json:
+        decode = lambda s: json.loads(s) if json_re.match(s) is not None else s
+    else:
+        decode = lambda s: s
+    env = {key[len(env_prefix):].lower(): decode(value) for key, value in environ.items() if key.startswith(env_prefix)}
+    if sep is not None:
+        env = nest_dict(env, sep)
+    return env

tests/utils/test_path_mapping.py

@@ -0,0 +1,67 @@
+import os
+import unittest
+from json import loads
+
+from apluslms_roman.utils.path_mapping import json_re, load_from_env
+
+test_case_loadable = (
+    'true',
+    'false',
+    'null',
+    '123',
+    '-123',
+    '3.14',
+    '-3.14',
+    '{"foo": "bar"}',
+    '[1, 2, 3]',
+    '"foo bar"'
+)
+
+test_case_not_loadable = (
+    "/foobar.py",
+    "text",
+    "yes",
+    "0123123",
+)
+
+
+class TestJsonLoadable(unittest.TestCase):
+
+    def test_loadable_not_raise(self):
+        for case in test_case_loadable:
+            with self.subTest(non_json=case):
+                loads(case)
+
+    def test_not_loadable_raise(self):
+        for case in test_case_not_loadable:
+            with self.subTest(non_json=case):
+                with self.assertRaises(ValueError, msg="Testing:{}".format(case)):
+                    loads(case)
+
+
+class TestJsonRegex(unittest.TestCase):
+
+    def test_loadable_match(self):
+        for case in test_case_loadable:
+            with self.subTest(non_json=case):
+                self.assertTrue(json_re.match(case)is not None, msg="Testing:{}".format(case))
+
+    def test_not_loadable_not_match(self):
+        for case in test_case_not_loadable:
+            with self.subTest(non_json=case):
+                self.assertFalse(json_re.match(case) is not None, msg="Testing:{}".format(case))
+
+
+class TestLoadFromEnv(unittest.TestCase):
+
+    def test_with_decode_json(self):
+        os.environ['DOCKER_FOO_BAR'] = '123'
+        self.assertEqual({'foo': {'bar': '123'}}, load_from_env('DOCKER_', '_', False))
+
+    def test_without_decode_json(self):
+        os.environ['DOCKER_FOO_BAR'] = '123'
+        self.assertEqual({'foo': {'bar': 123}}, load_from_env('DOCKER_', '_', True))
+
+    def test_without_separation_char(self):
+        os.environ['DOCKER_FOO_BAR'] = '123'
+        self.assertEqual({'foo_bar': 123}, load_from_env('DOCKER_'))