Skip to content

Commit e491a19

Browse files
odinuvodinuv
authored
Merge pull request #74 from keboola/odin-AIS-35-e
Basic configuration support
2 parents c6e3512 + 573d729 commit e491a19

11 files changed

Lines changed: 832 additions & 14 deletions

‎kbcstorage/client.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
"""
44

55
from kbcstorage.buckets import Buckets
6+
from kbcstorage.components import Components
7+
from kbcstorage.configurations import Configurations
68
from kbcstorage.workspaces import Workspaces
79
from kbcstorage.jobs import Jobs
810
from kbcstorage.tables import Tables
@@ -14,24 +16,32 @@ class Client:
1416
Storage API Client.
1517
"""
1618

17-
def __init__(self, api_domain, token):
19+
def __init__(self, api_domain, token, branch_id='default'):
1820
"""
1921
Initialise a client.
2022
2123
Args:
2224
api_domain (str): The domain on which the API sits. eg.
2325
"https://connection.keboola.com".
2426
token (str): A storage API key.
27+
branch_id (str): The ID of branch to use, use 'default' to work without branch (in main).
2528
"""
2629
self.root_url = api_domain.rstrip("/")
2730
self._token = token
31+
self._branch_id = branch_id
2832

2933
self.buckets = Buckets(self.root_url, self.token)
3034
self.files = Files(self.root_url, self.token)
3135
self.jobs = Jobs(self.root_url, self.token)
3236
self.tables = Tables(self.root_url, self.token)
3337
self.workspaces = Workspaces(self.root_url, self.token)
38+
self.components = Components(self.root_url, self.token, self.branch_id)
39+
self.configurations = Configurations(self.root_url, self.token, self.branch_id)
3440

3541
@property
3642
def token(self):
3743
return self._token
44+
45+
@property
46+
def branch_id(self):
47+
return self._branch_id

‎kbcstorage/components.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
"""
2+
Manages calls to the Storage API relating to components
3+
4+
Full documentation https://keboola.docs.apiary.io/#reference/components-and-configurations
5+
"""
6+
from kbcstorage.base import Endpoint
7+
8+
9+
class Components(Endpoint):
10+
"""
11+
Components Endpoint
12+
"""
13+
def __init__(self, root_url, token, branch_id):
14+
"""
15+
Create a Configuration endpoint.
16+
17+
Args:
18+
root_url (:obj:`str`): The base url for the API.
19+
token (:obj:`str`): A storage API key.
20+
branch_id (str): The ID of branch to use, use 'default' to work without branch (in main).
21+
"""
22+
super().__init__(root_url, f"branch/{branch_id}/components", token)
23+
24+
def list(self, include=None):
25+
"""
26+
List all components (and optionally configurations) in a project.
27+
28+
Args:
29+
include (list): Properties to list (configuration, rows, state)
30+
Returns:
31+
response_body: The parsed json from the HTTP response.
32+
33+
Raises:
34+
requests.HTTPError: If the API request fails.
35+
"""
36+
params = {'include': ',' . join(include)} if include else {}
37+
return self._get(self.base_url, params=params)

‎kbcstorage/configurations.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
"""
2+
Manages calls to the Storage API relating to configurations
3+
4+
Full documentation https://keboola.docs.apiary.io/#reference/components-and-configurations
5+
"""
6+
from kbcstorage.base import Endpoint
7+
8+
9+
class Configurations(Endpoint):
10+
"""
11+
Configurations Endpoint
12+
"""
13+
14+
def __init__(self, root_url, token, branch_id):
15+
"""
16+
Create a Component endpoint.
17+
18+
Args:
19+
root_url (:obj:`str`): The base url for the API.
20+
token (:obj:`str`): A storage API key.
21+
branch_id (str): The ID of branch to use, use 'default' to work without branch (in main).
22+
"""
23+
super().__init__(root_url, f"branch/{branch_id}/components", token)
24+
25+
def detail(self, component_id, configuration_id):
26+
"""
27+
Retrieves information about a given configuration.
28+
29+
Args:
30+
component_id (str): The id of the component.
31+
configuration_id (str): The id of the configuration.
32+
33+
Returns:
34+
response_body: The parsed json from the HTTP response.
35+
36+
Raises:
37+
requests.HTTPError: If the API request fails.
38+
"""
39+
if not isinstance(component_id, str) or component_id == '':
40+
raise ValueError("Invalid component_id '{}'.".format(component_id))
41+
if not isinstance(configuration_id, str) or configuration_id == '':
42+
raise ValueError("Invalid component_id '{}'.".format(configuration_id))
43+
url = '{}/{}/configs/{}'.format(self.base_url, component_id, configuration_id)
44+
return self._get(url)
45+
46+
def delete(self, component_id, configuration_id):
47+
"""
48+
Deletes the configuration.
49+
50+
Args:
51+
component_id (str): The id of the component.
52+
configuration_id (str): The id of the configuration.
53+
54+
Raises:
55+
requests.HTTPError: If the API request fails.
56+
"""
57+
if not isinstance(component_id, str) or component_id == '':
58+
raise ValueError("Invalid component_id '{}'.".format(component_id))
59+
if not isinstance(configuration_id, str) or configuration_id == '':
60+
raise ValueError("Invalid component_id '{}'.".format(configuration_id))
61+
url = '{}/{}/configs/{}'.format(self.base_url, component_id, configuration_id)
62+
self._delete(url)
63+
64+
def list(self, component_id):
65+
"""
66+
Lists configurations of the given component.
67+
68+
Args:
69+
component_id (str): The id of the component.
70+
71+
Raises:
72+
requests.HTTPError: If the API request fails.
73+
"""
74+
if not isinstance(component_id, str) or component_id == '':
75+
raise ValueError("Invalid component_id '{}'.".format(component_id))
76+
url = '{}/{}/configs'.format(self.base_url, component_id)
77+
return self._get(url)
78+
79+
def create(self, component_id, name, description='', configuration=None, state=None, change_description='',
80+
is_disabled=False, configuration_id=None):
81+
"""
82+
Create a new configuration.
83+
84+
Args:
85+
component_id (str): ID of the component to create configuration for.
86+
name (str): Name of the configuration visible to end-user.
87+
description (str): Optional configuration description
88+
configuration (dict): Actual configuration parameters
89+
state (dict): Optional state parameters
90+
changeDescription (str): Optional change description
91+
is_disabled (bool): Optional flag to disable the configuration, default False
92+
configuration_id (str): Optional configuration ID, if not specified, new ID is generated
93+
Returns:
94+
response_body: The parsed json from the HTTP response.
95+
96+
Raises:
97+
requests.HTTPError: If the API request fails.
98+
"""
99+
if not isinstance(component_id, str) or component_id == '':
100+
raise ValueError("Invalid component_id '{}'.".format(component_id))
101+
if state is None:
102+
state = {}
103+
if configuration is None:
104+
configuration = {}
105+
body = {
106+
'name': name,
107+
'description': description,
108+
'configuration': configuration,
109+
'state': state,
110+
'changeDescription': change_description,
111+
'isDisabled': is_disabled
112+
}
113+
if configuration_id:
114+
body['id'] = configuration_id
115+
url = '{}/{}/configs'.format(self.base_url, component_id)
116+
return self._post(url, data=body)

‎tests/base_test_case.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55

66
class BaseTestCase(unittest.TestCase):
7+
TEST_COMPONENT_NAME = 'keboola.runner-config-test'
78

89
@classmethod
910
def setUpClass(cls) -> None:

‎tests/functional/test_components.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import os
2+
from requests import exceptions
3+
from kbcstorage.components import Components
4+
from kbcstorage.configurations import Configurations
5+
from tests.base_test_case import BaseTestCase
6+
7+
8+
class TestEndpoint(BaseTestCase):
9+
def setUp(self):
10+
self.components = Components(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN'), 'default')
11+
self.configurations = Configurations(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN'), 'default')
12+
self.configurations.create(self.TEST_COMPONENT_NAME, 'test_components')
13+
14+
def tearDown(self):
15+
try:
16+
for configuration in self.configurations.list(self.TEST_COMPONENT_NAME):
17+
self.configurations.delete(self.TEST_COMPONENT_NAME, configuration['id'])
18+
except exceptions.HTTPError as e:
19+
if e.response.status_code != 404:
20+
raise
21+
22+
def testListComponents(self):
23+
components = self.components.list()
24+
self.assertTrue(len(components) > 0)
25+
for component in components:
26+
with self.subTest():
27+
self.assertTrue('id' in component)
28+
self.assertTrue('name' in component)
29+
self.assertTrue('type' in component)
30+
self.assertTrue('uri' in component)
31+
32+
with self.subTest():
33+
for configuration in component['configurations']:
34+
self.assertTrue('id' in configuration)
35+
self.assertTrue('name' in configuration)
36+
self.assertTrue('description' in configuration)
37+
self.assertFalse('configuration' in configuration)
38+
self.assertFalse('rows' in configuration)
39+
self.assertFalse('state' in configuration)
40+
41+
def testListComponentsIncludeConfigurations(self):
42+
components = self.components.list(include=['configuration', 'rows', 'state'])
43+
self.assertTrue(len(components) > 0)
44+
for component in components:
45+
with self.subTest():
46+
self.assertTrue('id' in component)
47+
self.assertTrue('name' in component)
48+
self.assertTrue('type' in component)
49+
self.assertTrue('uri' in component)
50+
51+
with self.subTest():
52+
self.assertTrue('configurations' in component)
53+
for configuration in component['configurations']:
54+
self.assertTrue('id' in configuration)
55+
self.assertTrue('name' in configuration)
56+
self.assertTrue('description' in configuration)
57+
self.assertTrue('configuration' in configuration)
58+
self.assertTrue('rows' in configuration)
59+
self.assertTrue('state' in configuration)

‎tests/functional/test_configurations.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import os
2+
from requests import exceptions
3+
from kbcstorage.configurations import Configurations
4+
from tests.base_test_case import BaseTestCase
5+
6+
7+
class TestEndpoint(BaseTestCase):
8+
def setUp(self):
9+
self.configurations = Configurations(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN'), 'default')
10+
11+
def tearDown(self):
12+
try:
13+
for configuration in self.configurations.list(self.TEST_COMPONENT_NAME):
14+
self.configurations.delete(self.TEST_COMPONENT_NAME, configuration['id'])
15+
except exceptions.HTTPError as e:
16+
if e.response.status_code != 404:
17+
raise
18+
19+
def testCreateConfiguration(self):
20+
configuration = self.configurations.create(self.TEST_COMPONENT_NAME, 'test_create_configuration')
21+
self.assertTrue('id' in configuration)
22+
self.assertTrue('name' in configuration)
23+
self.assertTrue('description' in configuration)
24+
self.assertTrue('configuration' in configuration)
25+
26+
def testDeleteConfiguration(self):
27+
configuration = self.configurations.create(self.TEST_COMPONENT_NAME, 'test_delete_configuration')
28+
configuration = self.configurations.detail(self.TEST_COMPONENT_NAME, configuration['id'])
29+
self.assertTrue('id' in configuration)
30+
self.assertTrue('name' in configuration)
31+
self.assertEqual(configuration['name'], 'test_delete_configuration')
32+
self.assertTrue('description' in configuration)
33+
self.assertTrue('configuration' in configuration)
34+
35+
self.configurations.delete(self.TEST_COMPONENT_NAME, configuration['id'])
36+
with self.assertRaises(exceptions.HTTPError):
37+
self.configurations.detail(self.TEST_COMPONENT_NAME, configuration['id'])
38+
39+
def testListConfigurations(self):
40+
self.configurations.create(self.TEST_COMPONENT_NAME, 'test_list_configurations')
41+
configurations = self.configurations.list(self.TEST_COMPONENT_NAME)
42+
self.assertTrue(len(configurations) > 0)
43+
for configuration in configurations:
44+
with self.subTest():
45+
self.assertTrue('id' in configuration)
46+
self.assertTrue('name' in configuration)
47+
self.assertTrue('description' in configuration)
48+
self.assertTrue('configuration' in configuration)
49+
50+
with self.subTest():
51+
with self.assertRaises(exceptions.HTTPError):
52+
configurations = self.configurations.list('non-existent-component')

‎tests/functional/test_workspaces.py‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
import csv
22
import os
33
import tempfile
4-
import warnings
54
from requests import exceptions
65
from kbcstorage.buckets import Buckets
76
from kbcstorage.jobs import Jobs
8-
from kbcstorage.files import Files
97
from kbcstorage.tables import Tables
108
from kbcstorage.workspaces import Workspaces
119
from tests.base_test_case import BaseTestCase
@@ -17,23 +15,12 @@ def setUp(self):
1715
self.buckets = Buckets(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN'))
1816
self.jobs = Jobs(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN'))
1917
self.tables = Tables(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN'))
20-
self.files = Files(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN'))
21-
try:
22-
file_list = self.files.list(tags=['sapi-client-python-tests'])
23-
for file in file_list:
24-
self.files.delete(file['id'])
25-
except exceptions.HTTPError as e:
26-
if e.response.status_code != 404:
27-
raise
2818
try:
2919
self.buckets.delete('in.c-py-test-buckets', force=True)
3020
except exceptions.HTTPError as e:
3121
if e.response.status_code != 404:
3222
raise
3323

34-
# https://github.com/boto/boto3/issues/454
35-
warnings.simplefilter("ignore", ResourceWarning)
36-
3724
def tearDown(self):
3825
try:
3926
if hasattr(self, 'workspace_id'):

0 commit comments

Comments
 (0)