#282 Add CONAN_REMOVE_OUTDATE_PACKAGES option

README.md

@@ -1072,6 +1072,7 @@ Using **CONAN_CLANG_VERSIONS** env variable in Travis ci or Appveyor:
     - "missing": Build only missing packages.
     - "outdated": Build only missing or if the available package is not built with the current recipe. Useful to upload new configurations, e.j packages for a new compiler without
       rebuild all packages.
+- **remove_outdated_packages**: Remove all outdated packages from remote after to upload a package. Default [False]
 - **test_folder**: Custom test folder consumed by Conan create, e.j .conan/test_package
 - **conanfile**: Custom conanfile consumed by Conan create. e.j. conanfile.py
 - **config_url**: Conan config URL be installed before to build e.j https://github.com/bincrafters/conan-config.git
@@ -1218,6 +1219,7 @@ This is especially useful for CI integration.
 - **CONAN_CONFIG_ARGS**: Conan config arguments used when installing conan config
 - **CONAN_BASE_PROFILE**: Apply options, settings, etc. to this profile instead of `default`.
 - **CONAN_IGNORE_SKIP_CI**: Ignore `[skip ci]` in commit message.
+- **CONAN_REMOVE_OUTDATED_PACKAGES**: Remove all outdated packages from remote after to upload a package. Default [False]
 - **CONAN_CONANFILE**: Custom conanfile consumed by Conan create. e.j. conanfile.py
 - **CPT_TEST_FOLDER**: Custom test_package path, e.j .conan/test_package
 - **CONAN_FORCE_SELINUX**: Force docker to relabel file objects on the shared volumes

cpt/eraser.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+class Eraser(object):
+    """ Helper to connect on remove and remove outdated packages
+    """
+
+    def __init__(self, conan_api, remote_manager, auth_manager, printer, remove):
+        """ Initialize Eraser instance
+        :param conan_api: Conan API instance
+        :param remote_manager: Remote manager instance
+        :param auth_manager: Authication manager to access the remote
+        :param printer: CPT output
+        :param remove: True if should remove outdated packages from remote. Otherwise, False.
+        """
+        self.conan_api = conan_api
+        self.remote_manager = remote_manager
+        self.auth_manager = auth_manager
+        self.printer = printer
+        self.remove = remove
+
+    def remove_outdated_packages(self, reference):
+        """ Remove outdated packages from remote
+        :param reference: Package reference e.g. foo/0.1.0@user/channel
+        """
+        if not self.remote_manager or not self.remote_manager.upload_remote_name:
+            self.printer.print_message("Remove outdated skipped, no remote available")
+            return
+        remote_name = self.remote_manager.upload_remote_name
+
+        if not self.auth_manager or not self.auth_manager.credentials_ready(remote_name):
+            self.printer.print_message("Remove outdated skipped, credentials for remote '%s' not available" % remote_name)
+            return
+
+        if self.remove:
+            self.printer.print_message("Removing outdated packages for '%s'" % str(reference))
+            self.auth_manager.login(remote_name)
+            self.conan_api.remove(pattern=str(reference),
+                                  force=True,
+                                  remote_name=remote_name,
+                                  outdated=True)

cpt/packager.py

@@ -22,6 +22,7 @@
 from cpt.tools import get_bool_from_env
 from cpt.tools import split_colon_env
 from cpt.uploader import Uploader
+from cpt.eraser import Eraser
 
 
 def load_cf_class(path, conan_api):
@@ -54,6 +55,7 @@ def load_cf_class(path, conan_api):
         else:
             return conan_api.app.loader.load_basic(path)
 
+
 class PlatformInfo(object):
     """Easy mockable for testing"""
     @staticmethod
@@ -121,6 +123,7 @@ def __init__(self, username=None, channel=None, runner=None,
                  docker_shell=None,
                  pip_install=None,
                  build_policy=None,
+                 remove_outdated_packages=False,
                  always_update_conan_in_docker=False,
                  conan_api=None,
                  client_cache=None,
@@ -165,6 +168,11 @@ def __init__(self, username=None, channel=None, runner=None,
                                         default_username=self.username,
                                         skip_check_credentials=self.skip_check_credentials)
 
+        self._remove_outdated_packages = remove_outdated_packages or \
+                                        get_bool_from_env("CONAN_REMOVE_OUTDATED_PACKAGES")
+        self.eraser = Eraser(self.conan_api, self.remotes_manager, self.auth_manager, self.printer,
+                             self._remove_outdated_packages)
+
         # Upload related variables
         self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3)
 
@@ -583,6 +591,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None):
                 profile_abs_path = save_profile_to_tmp(profile_text)
                 r = CreateRunner(profile_abs_path, build.reference, self.conan_api,
                                  self.uploader,
+                                 eraser=self.eraser,
                                  exclude_vcvars_precommand=self.exclude_vcvars_precommand,
                                  build_policy=self.build_policy,
                                  runner=self.runner,
@@ -610,6 +619,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None):
                                        docker_image_skip_pull=self._docker_image_skip_pull,
                                        build_policy=self.build_policy,
                                        always_update_conan_in_docker=self._update_conan_in_docker,
+                                       remove_outdated_packages=self._remove_outdated_packages,
                                        upload=self._upload_enabled(),
                                        upload_retry=self.upload_retry,
                                        upload_only_recipe=self.upload_only_recipe,

cpt/run_in_docker.py

@@ -10,6 +10,7 @@
 from cpt.remotes import RemotesManager
 from cpt.runner import CreateRunner, unscape_env
 from cpt.uploader import Uploader
+from cpt.eraser import Eraser
 
 
 def run():
@@ -29,6 +30,9 @@ def run():
     test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER"))
     reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE"))
 
+    remove_outdated_packages = unscape_env(os.getenv("CPT_REMOVE_OUTDATED_PACKAGES"))
+    eraser = Eraser(conan_api, remotes_manager, auth_manager, printer, remove_outdated_packages)
+
     profile_text = unscape_env(os.getenv("CPT_PROFILE"))
     abs_profile_path = save_profile_to_tmp(profile_text)
     base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE"))
@@ -43,7 +47,7 @@ def run():
                    base_profile_text)
 
     upload = os.getenv("CPT_UPLOAD_ENABLED")
-    runner = CreateRunner(abs_profile_path, reference, conan_api, uploader,
+    runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, eraser=eraser,
                           build_policy=build_policy, printer=printer, upload=upload,
                           upload_only_recipe=upload_only_recipe,
                           test_folder=test_folder, config_url=config_url,

cpt/runner.py

@@ -15,7 +15,7 @@
 
 class CreateRunner(object):
 
-    def __init__(self, profile_abs_path, reference, conan_api, uploader,
+    def __init__(self, profile_abs_path, reference, conan_api, uploader, eraser,
                  exclude_vcvars_precommand=False, build_policy=None, runner=None,
                  cwd=None, printer=None, upload=False, upload_only_recipe=None,
                  test_folder=None, config_url=None, config_args=None,
@@ -25,6 +25,7 @@ def __init__(self, profile_abs_path, reference, conan_api, uploader,
         self.printer = printer or Printer()
         self._cwd = cwd or os.getcwd()
         self._uploader = uploader
+        self._eraser = eraser
         self._upload = upload
         self._conan_api = conan_api
         self._profile_abs_path = profile_abs_path
@@ -127,21 +128,23 @@ def run(self):
                             self.printer.print_rule()
                             return
                         for installed in r['installed']:
+                            str_ref = str(self._reference)
                             reference = installed["recipe"]["id"]
                             if client_version >= Version("1.10.0"):
                                 reference = ConanFileReference.loads(reference)
                                 reference = str(reference.copy_clear_rev())
-                            if ((reference == str(self._reference)) or \
+                            if ((reference == str_ref) or \
                                (reference in self._upload_dependencies) or \
                                ("all" in self._upload_dependencies)) and \
                                installed['packages']:
                                 package_id = installed['packages'][0]['id']
-                                if installed['packages'][0]["built"]:
+                                if installed['packages'][0]["built"]:                                    
                                     if self._upload_only_recipe:
                                         self._uploader.upload_recipe(reference, self._upload)
                                     else:
                                         self._uploader.upload_packages(reference,
                                                                     self._upload, package_id)
+                                    self._eraser.remove_outdated_packages(str_ref)
                                 else:
                                     self.printer.print_message("Skipping upload for %s, "
                                                                "it hasn't been built" % package_id)
@@ -154,6 +157,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference
                  docker_image_skip_update=False, build_policy=None,
                  docker_image_skip_pull=False,
                  always_update_conan_in_docker=False,
+                 remove_outdated_packages=False,
                  upload=False, upload_retry=None, upload_only_recipe=None,
                  runner=None,
                  docker_shell="", docker_conan_home="",
@@ -179,6 +183,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference
         self._build_policy = build_policy
         self._docker_image = docker_image
         self._always_update_conan_in_docker = always_update_conan_in_docker
+        self._remove_outdated_packages = remove_outdated_packages
         self._docker_image_skip_update = docker_image_skip_update
         self._docker_image_skip_pull = docker_image_skip_pull
         self._sudo_docker_command = sudo_docker_command or ""
@@ -316,6 +321,7 @@ def get_env_vars(self):
         ret["CONAN_TEMP_TEST_FOLDER"] = "1"  # test package folder to a temp one
         ret["CPT_UPLOAD_ENABLED"] = self._upload
         ret["CPT_UPLOAD_RETRY"] = self._upload_retry
+        ret["CPT_REMOVE_OUTDATED_PACKAGES"] = self._remove_outdated_packages
         ret["CPT_UPLOAD_ONLY_RECIPE"] = self._upload_only_recipe
         ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy)
         ret["CPT_TEST_FOLDER"] = escape_env(self._test_folder)

cpt/test/test_client/erase_checks_test.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+import unittest
+from parameterized import parameterized
+
+from conans.client.tools import environment_append
+from conans.test.utils.tools import TestClient, TestServer
+
+from cpt.test.test_client.tools import get_patched_multipackager
+
+
+class EraseTest(unittest.TestCase):
+
+    old_conanfile = """from conans import ConanFile
+class Pkg(ConanFile):
+    name = "lib"
+    version = "1.0"
+    options = {"shared": [True, False]}
+    default_options = "shared=False"
+
+    def build(self):
+        self.output.warn("OLD")
+"""
+
+    @parameterized.expand([
+       ("1", "assertIn"),
+       ("0", "assertNotIn")
+    ])
+    def test_remove_updated_packages_env_var(self, remove_packages, assert_func):
+        ts = TestServer(users={"user": "password"})
+        tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]})
+        tc.save({"conanfile.py": self.old_conanfile})
+        with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user",
+                                 "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user",
+                                 "CONAN_REMOVE_OUTDATED_PACKAGES": remove_packages}):
+            mulitpackager = get_patched_multipackager(tc, build_policy="missing",
+                                                      exclude_vcvars_precommand=True)
+            mulitpackager.add({}, {"shared": True})
+            mulitpackager.add({}, {"shared": False})
+            mulitpackager.run()
+            self.assertIn("Uploading package 1/2", tc.out)
+            self.assertIn("Uploading package 2/2", tc.out)
+            self.assertIn("OLD", tc.out)
+            getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/mychannel'",
+                                       tc.out)
+
+    @parameterized.expand([
+       (True, "assertIn"),
+       (False, "assertNotIn")
+    ])
+    def test_remove_updated_packages_params(self, remove_packages, assert_func):
+        ts = TestServer(users={"user": "password"})
+        tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]})
+        tc.save({"conanfile.py": self.old_conanfile})
+        with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user",
+                                 "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user"}):
+            mulitpackager = get_patched_multipackager(tc, build_policy="missing",
+                                                      exclude_vcvars_precommand=True,
+                                                      remove_outdated_packages=remove_packages)
+            mulitpackager.add({}, {"shared": True})
+            mulitpackager.add({}, {"shared": False})
+            mulitpackager.run()
+            self.assertIn("Uploading package 1/2", tc.out)
+            self.assertIn("Uploading package 2/2", tc.out)
+            self.assertIn("OLD", tc.out)
+            getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/mychannel'",
+                                       tc.out)

cpt/test/unit/eraser_test.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+import unittest
+
+from collections import namedtuple
+from conans.test.utils.tools import TestBufferConanOutput
+from cpt.eraser import Eraser
+from cpt.printer import Printer
+from cpt.test.unit.packager_test import MockConanAPI
+
+
+class AuthTest(unittest.TestCase):
+
+    def setUp(self):
+        self.conan_api = MockConanAPI()
+        self.output = TestBufferConanOutput()
+        self.printer = Printer(self.output.write)
+
+    def test_invalid_remote(self):
+        eraser = Eraser(self.conan_api, None, None, self.printer, True)
+        eraser.remove_outdated_packages("foo/0.1.0@user/channel")
+        self.assertIn("Remove outdated skipped, no remote available", self.output)
+        self.assertFalse(self.conan_api.calls)
+
+    def test_invalid_authentication(self):
+        FakeRemoteManager = namedtuple("FakeRemoteManager", "upload_remote_name")
+        remote_manager = FakeRemoteManager(upload_remote_name="default")
+        eraser = Eraser(self.conan_api, remote_manager, None, self.printer, True)
+        eraser.remove_outdated_packages("foo/0.1.0@user/channel")
+        self.assertIn("Remove outdated skipped, credentials for remote 'default' not available", self.output)
+        self.assertFalse(self.conan_api.calls)