Add logic to detect and skip already installed packages.

This mode can be enabled by a command line flag which, when set, causes
the dependencies for each feature to be checked for presence within the
active python environment and their installation skipped if present.

In practice this means the requirements files can list dependencies
exhaustively but still allow the host environment arranged by a build
process to provision packages in the host environment and have these
respected during installation. Further, if such a package is only one
of many requirements then it alone will be skipped while all other
dependencies will be correctly installed.

Author: albu-diku

mig/install/features.py

@@ -4,6 +4,7 @@
 from collections import defaultdict
 from configparser import ConfigParser
 from enum import Enum
+import importlib
 import os
 import pip
 import sys
@@ -80,27 +81,32 @@ def feature_is_enabled(self, feature_name):
 
         return self._enabled_by_feature[feature_name]
 
-    def generate_pip_args(self):
+    def generate_pip_args(self, detect_installed=False):
         """
         Create pip arguments for each enabled feature.
         """
 
         per_package_args = []
 
         for feature_name in self.list_enabled_features():
-            per_package_args.append(self.generate_pip_args_for_feature(feature_name))
+            installable_packages = self.generate_pip_args_for_feature(feature_name,
+                                        detect_installed=detect_installed)
+            if not installable_packages:
+                continue
+            per_package_args.append(installable_packages)
 
         return per_package_args
 
-    def generate_pip_args_for_feature(self, feature_name):
+    def generate_pip_args_for_feature(self, feature_name, *, detect_installed):
         """
         Create pip arguments for a particular feature.
         """
 
-        overrides = self._overrides_by_feature.get(feature_name, None)
+        overrides = self._overrides_by_feature.get(feature_name, {})
 
-        if not overrides:
-            # no overiddes detected therefore we can install
+        if not (overrides or detect_installed):
+            # no overrides detected and we do not need to check for the
+            # dependencies being already installed therefore we can install
             # by simply using the requirements file as-is
             return ['-r', self._requirements_file_by_feature[feature_name]]
 
@@ -111,11 +117,25 @@ def generate_pip_args_for_feature(self, feature_name):
         for package_name in overridden_package_names:
             package_args.append(f"{package_name}=={overrides[package_name]}")
 
+        if detect_installed:
+            packages_to_detect = self.required_package_names(feature_name)
+        else:
+            packages_to_detect = set()
+
         # add the remaining packages based on the requirements file
         for entry in self._requirements_by_feature[feature_name]:
             package_name = Features._strip_version_if_present(entry.requirement)
             if package_name in overridden_package_names:
                 continue
+
+            should_detect = package_name in packages_to_detect
+            if should_detect:
+                skip_installation = Features._is_package_present(package_name)
+            else:
+                skip_installation = False
+
+            if skip_installation:
+                continue
             package_args.append(entry.requirement)
 
         return package_args
@@ -128,6 +148,10 @@ def list_enabled_features(self, return_as=list):
         return return_as((feature_name for feature_name in self.feature_names
                                         if self._enabled_by_feature[feature_name]))
 
+    def required_package_names(self, feature_name):
+        return set((Features._strip_version_if_present(entry.requirement)
+                    for entry in self._requirements_by_feature[feature_name]))
+
     @staticmethod
     def _interpret_feature_definition(feature_name, feature_definition, requirements_dir):
         """
@@ -150,6 +174,15 @@ def _interpret_feature_definition(feature_name, feature_definition, requirements
             requirements_file=requirements_file,
         )
 
+    @staticmethod
+    def _is_package_present(package_name):
+        try:
+            importlib.util.find_spec(package_name)
+
+            return True
+        except ModuleNotFoundError as exc:
+            return False
+
     @staticmethod
     def _strip_version_if_present(requirement):
         """
@@ -270,7 +303,7 @@ def main_install(features, args, print=print, warn=warn):
         warn("no feature coniguration available; showing those enabled by default only")
         warn()
 
-    all_pip_args = features.generate_pip_args()
+    all_pip_args = features.generate_pip_args(detect_installed=args.detect_installed)
 
     if args.check:
         for pip_args in all_pip_args:
@@ -305,6 +338,7 @@ def main(argv):
     install_command = subparsers.add_parser('install')
     install_command.add_argument('-c', default=None)
     install_command.add_argument('--check', action='store_true', default=False)
+    install_command.add_argument('--detect_installed', action='store_true', default=False)
     install_command.add_argument('--dotenv', default=None, type=os.path.abspath)
     install_command.add_argument('--env', action='store_const', const=os.environ)
 

tests/data/features/detection/features.ini

@@ -0,0 +1 @@
+[EXISTS]

tests/data/features/detection/requirements/exists-requirements.txt

@@ -0,0 +1,3 @@
+# deliberately choose a package that is always installed in development
+# such that requesting detection will find it and skip its installation
+autopep8

tests/test_mig_install_features.py

@@ -148,6 +148,7 @@ def test_command_install_check(self):
         args = SimpleNamespace(
             command='install',
             check=True,
+            detect_installed=False,
             c=None,
             dotenv=os.path.join(example_dir, '.env--enable-foo'),
             env=None
@@ -161,7 +162,26 @@ def test_command_install_check(self):
             f"pip install -r {os.path.join(example_dir, 'requirements/foo-requirements.txt')}",
         ])
 
-    def test_overridden_package_version(self):
+    def test_command_install_conflicting_package_versions(self):
+        fake_print = FakePrint()
+        example_dir, features = _make_example_features_instance('detection')
+        args = SimpleNamespace(
+            command='install',
+            check=True,
+            detect_installed=True,
+            c=None,
+            dotenv=None,
+            env={
+                'ENABLE_EXISTS': 'true'
+            }
+        )
+
+        ret = args_main(args, print=fake_print, features=features)
+
+        self.assertEqual(ret, 0)
+        self.assertOutputLines(fake_print, [])
+
+    def test_command_install_overridden_package_version(self):
         fake_print = FakePrint()
         example_dir, features = _make_example_features_instance('basic',
                 overrides_supported={
@@ -172,6 +192,7 @@ def test_overridden_package_version(self):
         args = SimpleNamespace(
             command='install',
             check=True,
+            detect_installed=False,
             c=None,
             dotenv=None,
             env={
@@ -189,7 +210,6 @@ def test_overridden_package_version(self):
         ])
 
 
-
 class MigInstallFeatures_smoke(MigTestCase):
     """Unit test helper for the migrid code pointed to in class name"""