From bf178e7fa3f1f0609516b5223a153650839f6d6f Mon Sep 17 00:00:00 2001 From: Ron Erez Date: Mon, 7 Jul 2025 12:44:39 +0300 Subject: [PATCH 1/2] Add direct_upload parameter to buckets add() and modify() methods - Add direct_upload parameter to Buckets.add() method with default False - Add direct_upload parameter to Buckets.modify() method with conditional assignment - Update unit tests with comprehensive coverage for direct_upload feature - Maintain backward compatibility with existing bucket types - Follow CTERA SDK coding patterns and style guidelines - All tests passing with 10/10 pylint score --- cterasdk/core/buckets.py | 9 +++- tests/ut/core/admin/test_buckets.py | 77 ++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/cterasdk/core/buckets.py b/cterasdk/core/buckets.py index ea840ed6..7198ed18 100644 --- a/cterasdk/core/buckets.py +++ b/cterasdk/core/buckets.py @@ -40,7 +40,7 @@ def get(self, name, include=None): raise ObjectNotFoundException(f'/locations/{name}') return bucket - def add(self, name, bucket, read_only=False, dedicated_to=None): + def add(self, name, bucket, read_only=False, dedicated_to=None, direct_upload=False): """ Add a Bucket @@ -48,19 +48,21 @@ def add(self, name, bucket, read_only=False, dedicated_to=None): :param cterasdk.core.types.Bucket bucket: Storage bucket to add :param bool,optional read_only: Set bucket to read-delete only, defaults to False :param str,optional dedicated_to: Name of a tenant, defaults to ``None`` + :param bool,optional direct_upload: Enable direct upload, defaults to False """ param = bucket.to_server_object() param.name = name param.readOnly = read_only param.dedicated = bool(dedicated_to) param.dedicatedPortal = self._get_tenant_base_object_ref(dedicated_to) if dedicated_to else None + param.directUpload = direct_upload logger.info('Adding bucket. %s', {'name': name, 'bucket': bucket.bucket, 'type': bucket.__class__.__name__}) response = self._core.api.add('/locations', param) logger.info('Bucket added. %s', {'name': name, 'bucket': bucket.bucket, 'type': bucket.__class__.__name__}) return response - def modify(self, current_name, new_name=None, read_only=None, dedicated_to=None, verify_ssl=None): + def modify(self, current_name, new_name=None, read_only=None, dedicated_to=None, verify_ssl=None, direct_upload=None): """ Modify a Bucket @@ -70,6 +72,7 @@ def modify(self, current_name, new_name=None, read_only=None, dedicated_to=None, :param bool,optional dedicated: Dedicate bucket to a tenant :param bool,optional verify_ssl: ``False`` to trust all certificate, ``True`` to verify. :param str,optional dedicated_to: Tenant name + :param bool,optional direct_upload: Enable direct upload """ param = self._get_entire_object(current_name) if new_name: @@ -88,6 +91,8 @@ def modify(self, current_name, new_name=None, read_only=None, dedicated_to=None, param.dedicatedPortal = self._get_tenant_base_object_ref(dedicated_to) if dedicated_to else None if verify_ssl is not None: param.trustAllCertificates = not verify_ssl + if direct_upload is not None: + param.directUpload = direct_upload logger.info("Modifying bucket. %s", {'name': current_name}) response = self._core.api.put(f'/locations/{current_name}', param) logger.info("Bucket modified. %s", {'name': current_name}) diff --git a/tests/ut/core/admin/test_buckets.py b/tests/ut/core/admin/test_buckets.py index 074ea592..dccba4b5 100644 --- a/tests/ut/core/admin/test_buckets.py +++ b/tests/ut/core/admin/test_buckets.py @@ -43,7 +43,9 @@ def test_add_bucket(self): get_multi_response = munch.Munch({'name': self._tenant_name, 'baseObjectRef': self._tenant_base_object_ref}) self._init_global_admin(get_multi_response=get_multi_response, add_response=add_response) bucket = AmazonS3(self._bucket_name, self._access_key, self._secret_key) - ret = buckets.Buckets(self._global_admin).add(self._bucket_name, bucket, read_only=True, dedicated_to=self._tenant_name) + ret = buckets.Buckets(self._global_admin).add( + self._bucket_name, bucket, read_only=True, + dedicated_to=self._tenant_name, direct_upload=True) self._global_admin.api.get_multi.assert_called_once_with(f'/portals/{self._tenant_name}', mock.ANY) expected_include = ['/' + attr for attr in portals.Portals.default + ['baseObjectRef']] actual_include = self._global_admin.api.get_multi.call_args[0][1] @@ -53,7 +55,21 @@ def test_add_bucket(self): self._global_admin.api.add.assert_called_once_with('/locations', mock.ANY) expected_param = TestCoreBuckets._customize_bucket(bucket.to_server_object(), name=self._bucket_name, readOnly=True, dedicated=True, - dedicatedPortal=self._tenant_base_object_ref, trustAllCertificates=False) + dedicatedPortal=self._tenant_base_object_ref, + trustAllCertificates=False, directUpload=True) + actual_param = self._global_admin.api.add.call_args[0][1] + self._assert_equal_objects(actual_param, expected_param) + self.assertEqual(ret, add_response) + + def test_add_bucket_default_direct_upload(self): + add_response = 'Success' + self._init_global_admin(add_response=add_response) + bucket = AmazonS3(self._bucket_name, self._access_key, self._secret_key) + ret = buckets.Buckets(self._global_admin).add(self._bucket_name, bucket) + self._global_admin.api.add.assert_called_once_with('/locations', mock.ANY) + expected_param = TestCoreBuckets._customize_bucket(bucket.to_server_object(), name=self._bucket_name, + readOnly=False, dedicated=False, + dedicatedPortal=None, trustAllCertificates=False, directUpload=False) actual_param = self._global_admin.api.add.call_args[0][1] self._assert_equal_objects(actual_param, expected_param) self.assertEqual(ret, add_response) @@ -91,6 +107,63 @@ def test_modify_bucket_not_found(self): buckets.Buckets(self._global_admin).modify(self._bucket_name) self.assertEqual(f'Bucket not found: /locations/{self._bucket_name}', str(error.exception)) + def test_modify_bucket_direct_upload(self): + put_response = 'Success' + get_response = munch.Munch({'name': self._bucket_name}) + self._init_global_admin(get_response=get_response, put_response=put_response) + ret = buckets.Buckets(self._global_admin).modify(self._bucket_name, direct_upload=True) + self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') + self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) + expected_param = TestCoreBuckets._get_bucket_param(name=self._bucket_name, directUpload=True) + actual_param = self._global_admin.api.put.call_args[0][1] + self._assert_equal_objects(actual_param, expected_param) + self.assertEqual(ret, put_response) + + def test_modify_bucket_direct_upload_false(self): + put_response = 'Success' + get_response = munch.Munch({'name': self._bucket_name}) + self._init_global_admin(get_response=get_response, put_response=put_response) + ret = buckets.Buckets(self._global_admin).modify(self._bucket_name, direct_upload=False) + self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') + self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) + expected_param = TestCoreBuckets._get_bucket_param(name=self._bucket_name, directUpload=False) + actual_param = self._global_admin.api.put.call_args[0][1] + self._assert_equal_objects(actual_param, expected_param) + self.assertEqual(ret, put_response) + + def test_modify_bucket_without_direct_upload(self): + put_response = 'Success' + get_response = munch.Munch({'name': self._bucket_name}) + self._init_global_admin(get_response=get_response, put_response=put_response) + ret = buckets.Buckets(self._global_admin).modify(self._bucket_name, new_name=self._bucket_new_name) + self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') + self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) + expected_param = TestCoreBuckets._get_bucket_param(name=self._bucket_new_name) + actual_param = self._global_admin.api.put.call_args[0][1] + self._assert_equal_objects(actual_param, expected_param) + # Verify that directUpload is not set when not provided + self.assertFalse(hasattr(actual_param, 'directUpload')) + self.assertEqual(ret, put_response) + + def test_modify_bucket_all_params_with_direct_upload(self): + put_response = 'Success' + get_response = munch.Munch({'name': self._bucket_name}) + get_multi_response = munch.Munch({'name': self._tenant_name, 'baseObjectRef': self._tenant_base_object_ref}) + self._init_global_admin(get_response=get_response, get_multi_response=get_multi_response, put_response=put_response) + ret = buckets.Buckets(self._global_admin).modify( + self._bucket_name, new_name=self._bucket_new_name, + read_only=True, dedicated_to=self._tenant_name, + verify_ssl=False, direct_upload=True) + self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') + self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) + expected_param = TestCoreBuckets._get_bucket_param( + name=self._bucket_new_name, readOnly=True, + dedicated=True, dedicatedPortal=self._tenant_base_object_ref, + trustAllCertificates=True, directUpload=True) + actual_param = self._global_admin.api.put.call_args[0][1] + self._assert_equal_objects(actual_param, expected_param) + self.assertEqual(ret, put_response) + @staticmethod def _get_bucket_param(**kwargs): param = Object() From 659a2d4e896bc722974b458f0a4f68b2480c8ae2 Mon Sep 17 00:00:00 2001 From: Saimon Michelson Date: Mon, 7 Jul 2025 18:07:38 -0400 Subject: [PATCH 2/2] reduce the amount of unit tests, change attribute name to direct, set the attribute otherwise default the bucket configuration --- cterasdk/core/buckets.py | 23 +++--- .../UserGuides/Miscellaneous/Changelog.rst | 10 +++ tests/ut/core/admin/test_buckets.py | 73 ++++--------------- 3 files changed, 35 insertions(+), 71 deletions(-) diff --git a/cterasdk/core/buckets.py b/cterasdk/core/buckets.py index 7198ed18..5004e9e3 100644 --- a/cterasdk/core/buckets.py +++ b/cterasdk/core/buckets.py @@ -40,7 +40,7 @@ def get(self, name, include=None): raise ObjectNotFoundException(f'/locations/{name}') return bucket - def add(self, name, bucket, read_only=False, dedicated_to=None, direct_upload=False): + def add(self, name, bucket, read_only=False, dedicated_to=None, direct=None): """ Add a Bucket @@ -48,21 +48,22 @@ def add(self, name, bucket, read_only=False, dedicated_to=None, direct_upload=Fa :param cterasdk.core.types.Bucket bucket: Storage bucket to add :param bool,optional read_only: Set bucket to read-delete only, defaults to False :param str,optional dedicated_to: Name of a tenant, defaults to ``None`` - :param bool,optional direct_upload: Enable direct upload, defaults to False + :param bool,optional direct: Enable CTERA Direct IO """ param = bucket.to_server_object() param.name = name param.readOnly = read_only param.dedicated = bool(dedicated_to) param.dedicatedPortal = self._get_tenant_base_object_ref(dedicated_to) if dedicated_to else None - param.directUpload = direct_upload + if direct is not None: + param.directUpload = direct - logger.info('Adding bucket. %s', {'name': name, 'bucket': bucket.bucket, 'type': bucket.__class__.__name__}) + logger.info('Adding %s bucket: %s', bucket.__class__.__name__, name) response = self._core.api.add('/locations', param) - logger.info('Bucket added. %s', {'name': name, 'bucket': bucket.bucket, 'type': bucket.__class__.__name__}) + logger.info('Bucket added: %s', name) return response - def modify(self, current_name, new_name=None, read_only=None, dedicated_to=None, verify_ssl=None, direct_upload=None): + def modify(self, current_name, new_name=None, read_only=None, dedicated_to=None, verify_ssl=None, direct=None): """ Modify a Bucket @@ -72,7 +73,7 @@ def modify(self, current_name, new_name=None, read_only=None, dedicated_to=None, :param bool,optional dedicated: Dedicate bucket to a tenant :param bool,optional verify_ssl: ``False`` to trust all certificate, ``True`` to verify. :param str,optional dedicated_to: Tenant name - :param bool,optional direct_upload: Enable direct upload + :param bool,optional direct: Set CTERA Direct IO """ param = self._get_entire_object(current_name) if new_name: @@ -91,11 +92,11 @@ def modify(self, current_name, new_name=None, read_only=None, dedicated_to=None, param.dedicatedPortal = self._get_tenant_base_object_ref(dedicated_to) if dedicated_to else None if verify_ssl is not None: param.trustAllCertificates = not verify_ssl - if direct_upload is not None: - param.directUpload = direct_upload - logger.info("Modifying bucket. %s", {'name': current_name}) + if direct is not None: + param.directUpload = direct + logger.info("Modifying bucket: %s", current_name) response = self._core.api.put(f'/locations/{current_name}', param) - logger.info("Bucket modified. %s", {'name': current_name}) + logger.info("Bucket modified: %s", current_name) return response def list_buckets(self, include=None): diff --git a/docs/source/UserGuides/Miscellaneous/Changelog.rst b/docs/source/UserGuides/Miscellaneous/Changelog.rst index 219e8e9b..73c52757 100644 --- a/docs/source/UserGuides/Miscellaneous/Changelog.rst +++ b/docs/source/UserGuides/Miscellaneous/Changelog.rst @@ -1,6 +1,16 @@ Changelog ========= +2.20.16 +------- + +Improvements +^^^^^^^^^^^^ + +* Added support for enabling or disabling Direct Mode on CTERA Portal Storage Nodes. + +Related issues and pull requests on GitHub: `#310 `_ + 2.20.15 ------- diff --git a/tests/ut/core/admin/test_buckets.py b/tests/ut/core/admin/test_buckets.py index dccba4b5..79f8f909 100644 --- a/tests/ut/core/admin/test_buckets.py +++ b/tests/ut/core/admin/test_buckets.py @@ -45,7 +45,7 @@ def test_add_bucket(self): bucket = AmazonS3(self._bucket_name, self._access_key, self._secret_key) ret = buckets.Buckets(self._global_admin).add( self._bucket_name, bucket, read_only=True, - dedicated_to=self._tenant_name, direct_upload=True) + dedicated_to=self._tenant_name, direct=True) self._global_admin.api.get_multi.assert_called_once_with(f'/portals/{self._tenant_name}', mock.ANY) expected_include = ['/' + attr for attr in portals.Portals.default + ['baseObjectRef']] actual_include = self._global_admin.api.get_multi.call_args[0][1] @@ -61,7 +61,7 @@ def test_add_bucket(self): self._assert_equal_objects(actual_param, expected_param) self.assertEqual(ret, add_response) - def test_add_bucket_default_direct_upload(self): + def test_add_bucket_default_attrs(self): add_response = 'Success' self._init_global_admin(add_response=add_response) bucket = AmazonS3(self._bucket_name, self._access_key, self._secret_key) @@ -69,7 +69,7 @@ def test_add_bucket_default_direct_upload(self): self._global_admin.api.add.assert_called_once_with('/locations', mock.ANY) expected_param = TestCoreBuckets._customize_bucket(bucket.to_server_object(), name=self._bucket_name, readOnly=False, dedicated=False, - dedicatedPortal=None, trustAllCertificates=False, directUpload=False) + dedicatedPortal=None, trustAllCertificates=False) actual_param = self._global_admin.api.add.call_args[0][1] self._assert_equal_objects(actual_param, expected_param) self.assertEqual(ret, add_response) @@ -84,6 +84,16 @@ def test_modify_bucket_name_ro_remove_dedication(self): actual_param = self._global_admin.api.put.call_args[0][1] self._assert_equal_objects(actual_param, expected_param) + def test_modify_bucket_direct_mode(self): + get_response = munch.Munch({'name': self._bucket_name, 'directUpload': False}) + self._init_global_admin(get_response=get_response) + buckets.Buckets(self._global_admin).modify(self._bucket_name, direct=True) + self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') + self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) + expected_param = TestCoreBuckets._get_bucket_param(name=self._bucket_name, directUpload=True) + actual_param = self._global_admin.api.put.call_args[0][1] + self._assert_equal_objects(actual_param, expected_param) + def test_modify_bucket_value_error(self): self._init_global_admin(get_response=None) with self.assertRaises(ValueError): @@ -107,63 +117,6 @@ def test_modify_bucket_not_found(self): buckets.Buckets(self._global_admin).modify(self._bucket_name) self.assertEqual(f'Bucket not found: /locations/{self._bucket_name}', str(error.exception)) - def test_modify_bucket_direct_upload(self): - put_response = 'Success' - get_response = munch.Munch({'name': self._bucket_name}) - self._init_global_admin(get_response=get_response, put_response=put_response) - ret = buckets.Buckets(self._global_admin).modify(self._bucket_name, direct_upload=True) - self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') - self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) - expected_param = TestCoreBuckets._get_bucket_param(name=self._bucket_name, directUpload=True) - actual_param = self._global_admin.api.put.call_args[0][1] - self._assert_equal_objects(actual_param, expected_param) - self.assertEqual(ret, put_response) - - def test_modify_bucket_direct_upload_false(self): - put_response = 'Success' - get_response = munch.Munch({'name': self._bucket_name}) - self._init_global_admin(get_response=get_response, put_response=put_response) - ret = buckets.Buckets(self._global_admin).modify(self._bucket_name, direct_upload=False) - self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') - self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) - expected_param = TestCoreBuckets._get_bucket_param(name=self._bucket_name, directUpload=False) - actual_param = self._global_admin.api.put.call_args[0][1] - self._assert_equal_objects(actual_param, expected_param) - self.assertEqual(ret, put_response) - - def test_modify_bucket_without_direct_upload(self): - put_response = 'Success' - get_response = munch.Munch({'name': self._bucket_name}) - self._init_global_admin(get_response=get_response, put_response=put_response) - ret = buckets.Buckets(self._global_admin).modify(self._bucket_name, new_name=self._bucket_new_name) - self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') - self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) - expected_param = TestCoreBuckets._get_bucket_param(name=self._bucket_new_name) - actual_param = self._global_admin.api.put.call_args[0][1] - self._assert_equal_objects(actual_param, expected_param) - # Verify that directUpload is not set when not provided - self.assertFalse(hasattr(actual_param, 'directUpload')) - self.assertEqual(ret, put_response) - - def test_modify_bucket_all_params_with_direct_upload(self): - put_response = 'Success' - get_response = munch.Munch({'name': self._bucket_name}) - get_multi_response = munch.Munch({'name': self._tenant_name, 'baseObjectRef': self._tenant_base_object_ref}) - self._init_global_admin(get_response=get_response, get_multi_response=get_multi_response, put_response=put_response) - ret = buckets.Buckets(self._global_admin).modify( - self._bucket_name, new_name=self._bucket_new_name, - read_only=True, dedicated_to=self._tenant_name, - verify_ssl=False, direct_upload=True) - self._global_admin.api.get.assert_called_once_with(f'/locations/{self._bucket_name}') - self._global_admin.api.put.assert_called_once_with(f'/locations/{self._bucket_name}', mock.ANY) - expected_param = TestCoreBuckets._get_bucket_param( - name=self._bucket_new_name, readOnly=True, - dedicated=True, dedicatedPortal=self._tenant_base_object_ref, - trustAllCertificates=True, directUpload=True) - actual_param = self._global_admin.api.put.call_args[0][1] - self._assert_equal_objects(actual_param, expected_param) - self.assertEqual(ret, put_response) - @staticmethod def _get_bucket_param(**kwargs): param = Object()