From 2a95fc6db2746eb25d5b97f6c272f236f3d84344 Mon Sep 17 00:00:00 2001 From: Uilian Ries Date: Fri, 15 Mar 2019 18:21:33 -0300 Subject: [PATCH 1/7] #282 Add CONAN_REMOVE_OUTDATE_PACKAGES option - The new option will run conan remove -r -f --outdated after to upload a package Signed-off-by: Uilian Ries --- README.md | 2 + cpt/eraser.py | 40 ++++++++++++++ cpt/packager.py | 9 ++++ cpt/run_in_docker.py | 6 ++- cpt/runner.py | 10 +++- cpt/test/test_client/erase_checks_test.py | 66 +++++++++++++++++++++++ cpt/test/unit/eraser_test.py | 30 +++++++++++ 7 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 cpt/eraser.py create mode 100644 cpt/test/test_client/erase_checks_test.py create mode 100644 cpt/test/unit/eraser_test.py diff --git a/README.md b/README.md index 0d30009b..c99faed7 100644 --- a/README.md +++ b/README.md @@ -1032,6 +1032,7 @@ Using **CONAN_CLANG_VERSIONS** env variable in Travis ci or Appveyor: - "missing": Build only missing packages. - "outdated": Build only missing or if the available package is not built with the current recipe. Useful to upload new configurations, e.j packages for a new compiler without rebuild all packages. +- **remove_outdated_packages**: Remove all outdated packages from remote after to upload a package. Default [False] - **test_folder**: Custom test folder consumed by Conan create, e.j .conan/test_package - **config_url**: Conan config URL be installed before to build e.j https://github.com/bincrafters/conan-config.git @@ -1167,6 +1168,7 @@ This is especially useful for CI integration. - **CONAN_CONFIG_URL**: Conan config URL be installed before to build e.j https://github.com/bincrafters/conan-config.git - **CONAN_BASE_PROFILE**: Apply options, settings, etc. to this profile instead of `default`. - **CONAN_IGNORE_SKIP_CI**: Ignore `[skip ci]` in commit message. +- **CONAN_REMOVE_OUTDATED_PACKAGES**: Remove all outdated packages from remote after to upload a package. Default [False] - **CPT_TEST_FOLDER**: Custom test_package path, e.j .conan/test_package diff --git a/cpt/eraser.py b/cpt/eraser.py new file mode 100644 index 00000000..efb8855b --- /dev/null +++ b/cpt/eraser.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +class Eraser(object): + """ Helper to connect on remove and remove outdated packages + """ + + def __init__(self, conan_api, remote_manager, auth_manager, printer, remove): + """ Initialize Eraser instance + :param conan_api: Conan API instance + :param remote_manager: Remote manager instance + :param auth_manager: Authication manager to access the remote + :param printer: CPT output + :param remove: True if should remove outdated packages from remote. Otherwise, False. + """ + self.conan_api = conan_api + self.remote_manager = remote_manager + self.auth_manager = auth_manager + self.printer = printer + self.remove = remove + + def remove_outdated_packages(self, reference): + """ Remove outdated packages from remote + :param reference: Package reference e.g. foo/0.1.0@user/channel + """ + if not self.remote_manager or not self.remote_manager.upload_remote_name: + self.printer.print_message("Remove outdated skipped, no remote available") + return + remote_name = self.remote_manager.upload_remote_name + + if not self.auth_manager or not self.auth_manager.credentials_ready(remote_name): + self.printer.print_message("Remove outdated skipped, credentials for remote '%s' not available" % remote_name) + return + + if self.remove: + self.printer.print_message("Removing outdated packages for '%s'" % str(reference)) + self.auth_manager.login(remote_name) + self.conan_api.remove(pattern=str(reference), + force=True, + remote_name=remote_name, + outdated=True) diff --git a/cpt/packager.py b/cpt/packager.py index c8c7d087..5f854424 100644 --- a/cpt/packager.py +++ b/cpt/packager.py @@ -22,6 +22,7 @@ from cpt.tools import get_bool_from_env from cpt.tools import split_colon_env from cpt.uploader import Uploader +from cpt.eraser import Eraser def load_cf_class(path, conan_api): @@ -95,6 +96,7 @@ def __init__(self, username=None, channel=None, runner=None, docker_conan_home=None, pip_install=None, build_policy=None, + remove_outdated_packages=False, always_update_conan_in_docker=False, conan_api=None, client_cache=None, @@ -130,6 +132,11 @@ def __init__(self, username=None, channel=None, runner=None, default_username=self.username, skip_check_credentials=self.skip_check_credentials) + self._remove_outdated_packages = remove_outdated_packages or \ + get_bool_from_env("CONAN_REMOVE_OUTDATED_PACKAGES") + self.eraser = Eraser(self.conan_api, self.remotes_manager, self.auth_manager, self.printer, + self._remove_outdated_packages) + # Upload related variables self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3) @@ -518,6 +525,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): profile_abs_path = save_profile_to_tmp(profile_text) r = CreateRunner(profile_abs_path, build.reference, self.conan_api, self.uploader, + eraser=self.eraser, exclude_vcvars_precommand=self.exclude_vcvars_precommand, build_policy=self.build_policy, runner=self.runner, @@ -539,6 +547,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): docker_image_skip_pull=self._docker_image_skip_pull, build_policy=self.build_policy, always_update_conan_in_docker=self._update_conan_in_docker, + remove_outdated_packages=self._remove_outdated_packages, upload=self._upload_enabled(), upload_retry=self.upload_retry, runner=self.runner, diff --git a/cpt/run_in_docker.py b/cpt/run_in_docker.py index 0c4fb42b..09186015 100644 --- a/cpt/run_in_docker.py +++ b/cpt/run_in_docker.py @@ -10,6 +10,7 @@ from cpt.remotes import RemotesManager from cpt.runner import CreateRunner, unscape_env from cpt.uploader import Uploader +from cpt.eraser import Eraser def run(): @@ -29,6 +30,9 @@ def run(): test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER")) reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE")) + remove_outdated_packages = unscape_env(os.getenv("CPT_REMOVE_OUTDATED_PACKAGES")) + eraser = Eraser(conan_api, remotes_manager, auth_manager, printer, remove_outdated_packages) + profile_text = unscape_env(os.getenv("CPT_PROFILE")) abs_profile_path = save_profile_to_tmp(profile_text) base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE")) @@ -39,7 +43,7 @@ def run(): base_profile_text) upload = os.getenv("CPT_UPLOAD_ENABLED") - runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, + runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, eraser=eraser, build_policy=build_policy, printer=printer, upload=upload, test_folder=test_folder, config_url=config_url) runner.run() diff --git a/cpt/runner.py b/cpt/runner.py index b84eda53..a36c032f 100644 --- a/cpt/runner.py +++ b/cpt/runner.py @@ -13,13 +13,14 @@ class CreateRunner(object): - def __init__(self, profile_abs_path, reference, conan_api, uploader, + def __init__(self, profile_abs_path, reference, conan_api, uploader, eraser, exclude_vcvars_precommand=False, build_policy=None, runner=None, cwd=None, printer=None, upload=False, test_folder=None, config_url=None): self.printer = printer or Printer() self._cwd = cwd or os.getcwd() self._uploader = uploader + self._eraser = eraser self._upload = upload self._conan_api = conan_api self._profile_abs_path = profile_abs_path @@ -104,11 +105,13 @@ def run(self): self.printer.print_rule() return for installed in r['installed']: - if installed["recipe"]["id"] == str(self._reference): + str_ref = str(self._reference) + if installed["recipe"]["id"] == str_ref: package_id = installed['packages'][0]['id'] if installed['packages'][0]["built"]: self._uploader.upload_packages(self._reference, self._upload, package_id) + self._eraser.remove_outdated_packages(str_ref) else: self.printer.print_message("Skipping upload for %s, " "it hasn't been built" % package_id) @@ -121,6 +124,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference docker_image_skip_update=False, build_policy=None, docker_image_skip_pull=False, always_update_conan_in_docker=False, + remove_outdated_packages=False, upload=False, upload_retry=None, runner=None, docker_shell="", docker_conan_home="", @@ -137,6 +141,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference self._build_policy = build_policy self._docker_image = docker_image self._always_update_conan_in_docker = always_update_conan_in_docker + self._remove_outdated_packages = remove_outdated_packages self._docker_image_skip_update = docker_image_skip_update self._docker_image_skip_pull = docker_image_skip_pull self._sudo_docker_command = sudo_docker_command or "" @@ -256,6 +261,7 @@ def get_env_vars(self): ret["CONAN_TEMP_TEST_FOLDER"] = "1" # test package folder to a temp one ret["CPT_UPLOAD_ENABLED"] = self._upload ret["CPT_UPLOAD_RETRY"] = self._upload_retry + ret["CPT_REMOVE_OUTDATED_PACKAGES"] = self._remove_outdated_packages ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy) ret["CPT_TEST_FOLDER"] = escape_env(self._test_folder) ret["CPT_CONFIG_URL"] = escape_env(self._config_url) diff --git a/cpt/test/test_client/erase_checks_test.py b/cpt/test/test_client/erase_checks_test.py new file mode 100644 index 00000000..4d596ed5 --- /dev/null +++ b/cpt/test/test_client/erase_checks_test.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +import unittest + +from conans.client.tools import environment_append +from conans.test.utils.tools import TestClient, TestServer + +from cpt.test.test_client.tools import get_patched_multipackager + + +class EraseTest(unittest.TestCase): + + old_conanfile = """from conans import ConanFile +class Pkg(ConanFile): + name = "lib" + version = "1.0" + options = {"shared": [True, False]} + default_options = "shared=False" + + def build(self): + self.output.warn("OLD") +""" + + new_conanfile = """from conans import ConanFile +class Pkg(ConanFile): + name = "lib" + version = "1.0" + options = {"shared": [True, False], "foo": [True, False]} + default_options = "shared=False", "foo=True" + + def build(self): + self.output.warn("NEW") +""" + + def test_remove_updated_packages_env_var(self): + ts = TestServer(users={"user": "password"}) + tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]}) + tc.save({"conanfile.py": self.old_conanfile}) + with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user", + "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user", + "CONAN_REMOVE_OUTDATED_PACKAGES": "1"}): + mulitpackager = get_patched_multipackager(tc, build_policy="missing", + exclude_vcvars_precommand=True) + mulitpackager.add({}, {"shared": True}) + mulitpackager.add({}, {"shared": False}) + mulitpackager.run() + self.assertIn("Uploading package 1/2", tc.out) + self.assertIn("Uploading package 2/2", tc.out) + self.assertIn("OLD", tc.out) + self.assertIn("Removing outdated packages for 'lib/1.0@user/testing'", tc.out) + + def test_remove_updated_packages_params(self): + ts = TestServer(users={"user": "password"}) + tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]}) + tc.save({"conanfile.py": self.old_conanfile}) + with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user", + "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user"}): + mulitpackager = get_patched_multipackager(tc, build_policy="missing", + exclude_vcvars_precommand=True, + remove_outdated_packages=True) + mulitpackager.add({}, {"shared": True}) + mulitpackager.add({}, {"shared": False}) + mulitpackager.run() + self.assertIn("Uploading package 1/2", tc.out) + self.assertIn("Uploading package 2/2", tc.out) + self.assertIn("OLD", tc.out) + self.assertIn("Removing outdated packages for 'lib/1.0@user/testing'", tc.out) diff --git a/cpt/test/unit/eraser_test.py b/cpt/test/unit/eraser_test.py new file mode 100644 index 00000000..41fa0aa1 --- /dev/null +++ b/cpt/test/unit/eraser_test.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +import unittest + +from collections import namedtuple +from conans.test.utils.tools import TestBufferConanOutput +from cpt.eraser import Eraser +from cpt.printer import Printer +from cpt.test.unit.packager_test import MockConanAPI + + +class AuthTest(unittest.TestCase): + + def setUp(self): + self.conan_api = MockConanAPI() + self.output = TestBufferConanOutput() + self.printer = Printer(self.output.write) + + def test_invalid_remote(self): + eraser = Eraser(self.conan_api, None, None, self.printer, True) + eraser.remove_outdated_packages("foo/0.1.0@user/channel") + self.assertIn("Remove outdated skipped, no remote available", self.output) + self.assertFalse(self.conan_api.calls) + + def test_invalid_authentication(self): + FakeRemoteManager = namedtuple("FakeRemoteManager", "upload_remote_name") + remote_manager = FakeRemoteManager(upload_remote_name="default") + eraser = Eraser(self.conan_api, remote_manager, None, self.printer, True) + eraser.remove_outdated_packages("foo/0.1.0@user/channel") + self.assertIn("Remove outdated skipped, credentials for remote 'default' not available", self.output) + self.assertFalse(self.conan_api.calls) From f1d88d279e506c9424e2a655232a380c290de6f8 Mon Sep 17 00:00:00 2001 From: Uilian Ries Date: Fri, 15 Mar 2019 18:39:43 -0300 Subject: [PATCH 2/7] #282 Fix channel name Signed-off-by: Uilian Ries --- cpt/test/test_client/erase_checks_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpt/test/test_client/erase_checks_test.py b/cpt/test/test_client/erase_checks_test.py index 4d596ed5..8b7c00a7 100644 --- a/cpt/test/test_client/erase_checks_test.py +++ b/cpt/test/test_client/erase_checks_test.py @@ -46,7 +46,7 @@ def test_remove_updated_packages_env_var(self): self.assertIn("Uploading package 1/2", tc.out) self.assertIn("Uploading package 2/2", tc.out) self.assertIn("OLD", tc.out) - self.assertIn("Removing outdated packages for 'lib/1.0@user/testing'", tc.out) + self.assertIn("Removing outdated packages for 'lib/1.0@user/mychannel'", tc.out) def test_remove_updated_packages_params(self): ts = TestServer(users={"user": "password"}) @@ -63,4 +63,4 @@ def test_remove_updated_packages_params(self): self.assertIn("Uploading package 1/2", tc.out) self.assertIn("Uploading package 2/2", tc.out) self.assertIn("OLD", tc.out) - self.assertIn("Removing outdated packages for 'lib/1.0@user/testing'", tc.out) + self.assertIn("Removing outdated packages for 'lib/1.0@user/mychannel'", tc.out) From e8f9aa381396292f02878da338ae1b113b268041 Mon Sep 17 00:00:00 2001 From: Uilian Ries Date: Tue, 29 Oct 2019 12:28:50 -0300 Subject: [PATCH 3/7] Fix bad syntax --- cpt/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpt/runner.py b/cpt/runner.py index 738b12d6..176aef5a 100644 --- a/cpt/runner.py +++ b/cpt/runner.py @@ -128,7 +128,7 @@ def run(self): if client_version >= Version("1.10.0"): reference = ConanFileReference.loads(reference) reference = str(reference.copy_clear_rev()) - if ((reference == str_ref)) or \ + if ((reference == str_ref) or \ (reference in self._upload_dependencies) or \ ("all" in self._upload_dependencies)) and \ installed['packages']: From 8664ebf1593f85871b2692ad591ee5a2572557de Mon Sep 17 00:00:00 2001 From: Uilian Ries Date: Tue, 29 Oct 2019 14:27:02 -0300 Subject: [PATCH 4/7] Update parametrized requirement - nose_parametrized is deprecated and has been replaced by parametrized Signed-off-by: Uilian Ries --- cpt/requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpt/requirements_test.txt b/cpt/requirements_test.txt index 15475f2b..fbab192e 100644 --- a/cpt/requirements_test.txt +++ b/cpt/requirements_test.txt @@ -1,5 +1,5 @@ nose>=1.3.7 -nose_parameterized>=0.5.0, <0.6.0 +parameterized>=0.5.0, <0.8.0 mock>=1.3.0, <1.4.0 WebTest>=2.0.18, <2.1.0 tox==3.7.0 From f9deb22ecdc488d186274522e7d00adf4cea9d7e Mon Sep 17 00:00:00 2001 From: Uilian Ries Date: Tue, 29 Oct 2019 14:36:37 -0300 Subject: [PATCH 5/7] Validate when remove is false Signed-off-by: Uilian Ries --- cpt/test/test_client/erase_checks_test.py | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/cpt/test/test_client/erase_checks_test.py b/cpt/test/test_client/erase_checks_test.py index 8b7c00a7..4704e7f1 100644 --- a/cpt/test/test_client/erase_checks_test.py +++ b/cpt/test/test_client/erase_checks_test.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import unittest +from parameterized import parameterized from conans.client.tools import environment_append from conans.test.utils.tools import TestClient, TestServer @@ -20,24 +21,17 @@ def build(self): self.output.warn("OLD") """ - new_conanfile = """from conans import ConanFile -class Pkg(ConanFile): - name = "lib" - version = "1.0" - options = {"shared": [True, False], "foo": [True, False]} - default_options = "shared=False", "foo=True" - - def build(self): - self.output.warn("NEW") -""" - - def test_remove_updated_packages_env_var(self): + @parameterized.expand([ + ("1", "assertIn"), + ("0", "assertNotIn") + ]) + def test_remove_updated_packages_env_var(self, remove_packages, assert_func): ts = TestServer(users={"user": "password"}) tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]}) tc.save({"conanfile.py": self.old_conanfile}) with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user", "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user", - "CONAN_REMOVE_OUTDATED_PACKAGES": "1"}): + "CONAN_REMOVE_OUTDATED_PACKAGES": remove_packages}): mulitpackager = get_patched_multipackager(tc, build_policy="missing", exclude_vcvars_precommand=True) mulitpackager.add({}, {"shared": True}) @@ -46,9 +40,14 @@ def test_remove_updated_packages_env_var(self): self.assertIn("Uploading package 1/2", tc.out) self.assertIn("Uploading package 2/2", tc.out) self.assertIn("OLD", tc.out) - self.assertIn("Removing outdated packages for 'lib/1.0@user/mychannel'", tc.out) + getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/testing'", + tc.out) - def test_remove_updated_packages_params(self): + @parameterized.expand([ + (True, "assertIn"), + (False, "assertNotIn") + ]) + def test_remove_updated_packages_params(self, remove_packages, assert_func): ts = TestServer(users={"user": "password"}) tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]}) tc.save({"conanfile.py": self.old_conanfile}) @@ -56,11 +55,12 @@ def test_remove_updated_packages_params(self): "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user"}): mulitpackager = get_patched_multipackager(tc, build_policy="missing", exclude_vcvars_precommand=True, - remove_outdated_packages=True) + remove_outdated_packages=remove_packages) mulitpackager.add({}, {"shared": True}) mulitpackager.add({}, {"shared": False}) mulitpackager.run() self.assertIn("Uploading package 1/2", tc.out) self.assertIn("Uploading package 2/2", tc.out) self.assertIn("OLD", tc.out) - self.assertIn("Removing outdated packages for 'lib/1.0@user/mychannel'", tc.out) + getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/testing'", + tc.out) From 279c44f3420edb579c3e15063fe6cb047e599a99 Mon Sep 17 00:00:00 2001 From: Uilian Ries Date: Tue, 29 Oct 2019 14:45:58 -0300 Subject: [PATCH 6/7] Fix env vars Signed-off-by: Uilian Ries --- cpt/test/test_client/erase_checks_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpt/test/test_client/erase_checks_test.py b/cpt/test/test_client/erase_checks_test.py index 4704e7f1..f58776e9 100644 --- a/cpt/test/test_client/erase_checks_test.py +++ b/cpt/test/test_client/erase_checks_test.py @@ -40,7 +40,7 @@ def test_remove_updated_packages_env_var(self, remove_packages, assert_func): self.assertIn("Uploading package 1/2", tc.out) self.assertIn("Uploading package 2/2", tc.out) self.assertIn("OLD", tc.out) - getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/testing'", + getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/mychannel'", tc.out) @parameterized.expand([ @@ -62,5 +62,5 @@ def test_remove_updated_packages_params(self, remove_packages, assert_func): self.assertIn("Uploading package 1/2", tc.out) self.assertIn("Uploading package 2/2", tc.out) self.assertIn("OLD", tc.out) - getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/testing'", + getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/mychannel'", tc.out) From 464e571e0ead7bb8ba76281ec156ce6c9ec8d459 Mon Sep 17 00:00:00 2001 From: Uilian Ries Date: Tue, 29 Oct 2019 16:00:25 -0300 Subject: [PATCH 7/7] Support pre-conan 1.20.0 Signed-off-by: Uilian Ries --- cpt/packager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpt/packager.py b/cpt/packager.py index 2e010803..cc23dbee 100644 --- a/cpt/packager.py +++ b/cpt/packager.py @@ -50,7 +50,10 @@ def load_cf_class(path, conan_api): conan_api.create_app() remotes = conan_api.app.cache.registry.load_remotes() conan_api.app.python_requires.enable_remotes(remotes=remotes) - return conan_api.app.loader.load_class(path) + if Version(client_version) < Version("1.20.0"): + return conan_api.app.loader.load_class(path) + else: + return conan_api.app.loader.load_basic(path) class PlatformInfo(object):