Skip to content

Commit 6d5d3ca

Browse files
tomasfejfartomasfejfar
authored
Merge pull request #79 from keboola/KAB-46-prepare-structure-for-metastore-metadata-to-be-stored-with-storage-objects
KAB-46 prepare structure for metastore metadata to be stored with storage objects
2 parents 72a13ff + 859aeb5 commit 6d5d3ca

7 files changed

Lines changed: 337 additions & 3 deletions

File tree

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,6 @@ services:
1212
<<: *ci
1313
tty: true
1414
stdin_open: true
15-
command: bash
15+
entrypoint: bash
1616
volumes:
1717
- .:/code

‎kbcstorage/configurations.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@
33
44
Full documentation https://keboola.docs.apiary.io/#reference/components-and-configurations
55
"""
6+
import json
67
from kbcstorage.base import Endpoint
8+
from kbcstorage.configurations_metadata import ConfigurationsMetadata
79

810

911
class Configurations(Endpoint):
@@ -21,6 +23,7 @@ def __init__(self, root_url, token, branch_id):
2123
branch_id (str): The ID of branch to use, use 'default' to work without branch (in main).
2224
"""
2325
super().__init__(root_url, f"branch/{branch_id}/components", token)
26+
self.metadata = ConfigurationsMetadata(root_url, token, branch_id)
2427

2528
def detail(self, component_id, configuration_id):
2629
"""
@@ -111,6 +114,6 @@ def create(self, component_id, name, description='', configuration=None, state=N
111114
'isDisabled': is_disabled
112115
}
113116
if configuration_id:
114-
body['id'] = configuration_id
117+
body['configurationId'] = configuration_id
115118
url = '{}/{}/configs'.format(self.base_url, component_id)
116-
return self._post(url, data=body)
119+
return self._post(url, data=json.dumps(body), headers={'Content-Type': 'application/json'})

‎kbcstorage/configurations_metadata.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
"""
2+
Manages calls to the Storage API relating to configurations metadata
3+
4+
Full documentation https://keboola.docs.apiary.io/#reference/metadata/components-configurations-metadata/
5+
"""
6+
import json
7+
from kbcstorage.base import Endpoint
8+
9+
10+
class ConfigurationsMetadata(Endpoint):
11+
"""
12+
Configurations metadata Endpoint
13+
"""
14+
15+
def __init__(self, root_url, token, branch_id):
16+
"""
17+
Create a Component metadata endpoint.
18+
19+
Args:
20+
root_url (:obj:`str`): The base url for the API.
21+
token (:obj:`str`): A storage API key.
22+
branch_id (str): The ID of branch to use, use 'default' to work without branch (in main).
23+
"""
24+
super().__init__(root_url, f"branch/{branch_id}/components", token)
25+
26+
def delete(self, component_id, configuration_id, metadata_id):
27+
"""
28+
Deletes the configuration metadata identified by ``metadata_id``.
29+
30+
Args:
31+
component_id (str): The id of the component.
32+
configuration_id (str): The id of the configuration.
33+
metadata_id (str): The id of the metadata (not key!).
34+
35+
Raises:
36+
requests.HTTPError: If the API request fails.
37+
ValueError: If the component_id/configuration_id/metadata_id is not a string or is empty.
38+
"""
39+
if not isinstance(component_id, str) or component_id == '':
40+
raise ValueError("Invalid component_id '{}'.".format(component_id))
41+
if not isinstance(configuration_id, str) or configuration_id == '':
42+
raise ValueError("Invalid configuration_id '{}'.".format(configuration_id))
43+
if not isinstance(metadata_id, str) or metadata_id == '':
44+
raise ValueError("Invalid metadata_id '{}'.".format(metadata_id))
45+
url = '{}/{}/configs/{}/metadata/{}'.format(self.base_url, component_id, configuration_id, metadata_id)
46+
self._delete(url)
47+
48+
def list(self, component_id, configuration_id):
49+
"""
50+
Lists metadata for a given component configuration.
51+
52+
Args:
53+
component_id (str): The id of the component.
54+
configuration_id (str): The id of the configuration.
55+
56+
Raises:
57+
requests.HTTPError: If the API request fails.
58+
ValueError: If the component_id/configuration_id is not a string or is empty.
59+
"""
60+
if not isinstance(component_id, str) or component_id == '':
61+
raise ValueError("Invalid component_id '{}'.".format(component_id))
62+
if not isinstance(configuration_id, str) or configuration_id == '':
63+
raise ValueError("Invalid configuration_id '{}'.".format(configuration_id))
64+
url = '{}/{}/configs/{}/metadata'.format(self.base_url, component_id, configuration_id)
65+
return self._get(url)
66+
67+
def create(self, component_id, configuration_id, provider, metadata):
68+
"""
69+
Writes metadata for a given component configuration.
70+
71+
Args:
72+
component_id (str): The id of the component.
73+
configuration (str): The id of the configuration.
74+
provider (str): The provider of the configuration (currently ignored and "user" is sent).
75+
metadata (list): A list of metadata items. Item is a dictionary with 'key' and 'value' keys.
76+
77+
Returns:
78+
response_body: The parsed json from the HTTP response.
79+
80+
Raises:
81+
requests.HTTPError: If the API request fails.
82+
ValueError: If the component_id/configuration_id is not a string or is empty.
83+
ValueError: If the metadata is not a list.
84+
ValueError: If the metadata item is not a dictionary.
85+
"""
86+
if not isinstance(component_id, str) or component_id == '':
87+
raise ValueError("Invalid component_id '{}'.".format(component_id))
88+
if not isinstance(configuration_id, str) or configuration_id == '':
89+
raise ValueError("Invalid component_id '{}'.".format(configuration_id))
90+
url = '{}/{}/configs/{}/metadata'.format(self.base_url, component_id, configuration_id)
91+
if not isinstance(metadata, list):
92+
raise ValueError("Metadata must be a list '{}'.".format(metadata))
93+
for metadataItem in metadata:
94+
if not isinstance(metadataItem, dict):
95+
raise ValueError("Metadata item must be a dictionary '{}'.".format(metadataItem))
96+
97+
headers = {
98+
'Content-Type': 'application/json',
99+
}
100+
data = {
101+
# 'provider': provider, # not yet implemented
102+
'metadata': metadata
103+
}
104+
return self._post(url, data=json.dumps(data), headers=headers)

‎kbcstorage/tables.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from kbcstorage.base import Endpoint
1212
from kbcstorage.files import Files
1313
from kbcstorage.jobs import Jobs
14+
from kbcstorage.tables_metadata import TablesMetadata
1415

1516

1617
class Tables(Endpoint):
@@ -26,6 +27,7 @@ def __init__(self, root_url, token):
2627
token (:obj:`str`): A storage API key.
2728
"""
2829
super().__init__(root_url, 'tables', token)
30+
self.metadata = TablesMetadata(root_url, token)
2931

3032
def list(self, include=None):
3133
"""

‎kbcstorage/tables_metadata.py‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
"""
2+
Manages calls to the Storage API relating to table metadatas
3+
4+
Full documentation `here`.
5+
6+
.. _here:
7+
http://docs.keboola.apiary.io/#reference/metadata/table-metadata
8+
"""
9+
import json
10+
from kbcstorage.base import Endpoint
11+
12+
13+
class TablesMetadata(Endpoint):
14+
"""
15+
Tables Metadata Endpoint
16+
"""
17+
def __init__(self, root_url, token):
18+
"""
19+
Create a Tables metadata endpoint.
20+
21+
Args:
22+
root_url (:obj:`str`): The base url for the API.
23+
token (:obj:`str`): A storage API key.
24+
"""
25+
super().__init__(root_url, 'tables', token)
26+
27+
def list(self, table_id):
28+
"""
29+
List all metadata for table
30+
31+
Args:
32+
table_id (str): Table id
33+
34+
Returns:
35+
response_body: The parsed json from the HTTP response.
36+
37+
Raises:
38+
requests.HTTPError: If the API request fails.
39+
ValueError: If the table_id is not a string or is empty.
40+
"""
41+
if not isinstance(table_id, str) or table_id == '':
42+
raise ValueError("Invalid table_id '{}'.".format(table_id))
43+
44+
url = '{}/{}/metadata'.format(self.base_url, table_id)
45+
46+
return self._get(url)
47+
48+
def delete(self, table_id, metadata_id):
49+
"""
50+
Delete a table metadata referenced by ``metadata_id``.
51+
52+
Args:
53+
table_id (str): The id of the table.
54+
metadata_id (str): The id of the table metdata entry to be deleted.
55+
56+
Raises:
57+
requests.HTTPError: If the API request fails.
58+
ValueError: If the table_id/metadata_id is not a string or is empty.
59+
"""
60+
if not isinstance(table_id, str) or table_id == '':
61+
raise ValueError("Invalid table_id '{}'.".format(table_id))
62+
if not isinstance(metadata_id, str) or metadata_id == '':
63+
raise ValueError("Invalid metadata_id '{}'.".format(metadata_id))
64+
65+
url = '{}/{}/metadata/{}'.format(self.base_url, table_id, metadata_id)
66+
67+
self._delete(url)
68+
69+
def create(self, table_id, provider, metadata, columns_metadata):
70+
"""
71+
Post metadata to a table.
72+
73+
Args:
74+
table_id (str): Table id
75+
provider (str): Provider of the metadata
76+
metadata (list): List of metadata dictionaries with 'key' and 'value'
77+
columns_metadata (dict): Dictionary with lists of metadata dictionaries with 'key', 'value', 'columnName'.
78+
79+
Returns:
80+
response_body: The parsed json from the HTTP response.
81+
82+
Raises:
83+
requests.HTTPError: If the API request fails.
84+
ValueError: If the table_id is not a string or is empty.
85+
ValueError: If the provider is not a string or is empty.
86+
ValueError: If the metadata is not a list.
87+
ValueError: If the columns_metadata is not a list
88+
"""
89+
if not isinstance(table_id, str) or table_id == '':
90+
raise ValueError("Invalid table_id '{}'.".format(table_id))
91+
if not isinstance(provider, str) or provider == '':
92+
raise ValueError("Invalid provider '{}'.".format(provider))
93+
if not isinstance(metadata, list):
94+
raise ValueError("Invalid metadata '{}'.".format(metadata))
95+
if not isinstance(columns_metadata, list):
96+
raise ValueError("Invalid columns_metadata '{}'.".format(columns_metadata))
97+
98+
url = '{}/{}/metadata'.format(self.base_url, table_id)
99+
headers = {
100+
'Content-Type': 'application/json',
101+
}
102+
data = {
103+
"provider": provider,
104+
"metadata": metadata,
105+
"columnsMetadata": columns_metadata
106+
}
107+
return self._post(url, data=json.dumps(data), headers=headers)

‎tests/functional/test_configurations.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,56 @@ def testListConfigurations(self):
5050
with self.subTest():
5151
with self.assertRaises(exceptions.HTTPError):
5252
configurations = self.configurations.list('non-existent-component')
53+
54+
def testConfigurationMetadata(self):
55+
self.configurations.create(
56+
component_id=self.TEST_COMPONENT_NAME,
57+
configuration_id='test_configuration_metadata',
58+
name='test_configuration_metadata',
59+
)
60+
metadataPayload = [
61+
{
62+
'key': 'testConfigurationMetadata',
63+
'value': 'success',
64+
}
65+
]
66+
metadataList = self.configurations.metadata.create(
67+
component_id=self.TEST_COMPONENT_NAME,
68+
configuration_id='test_configuration_metadata',
69+
provider='test',
70+
metadata=metadataPayload,
71+
)
72+
73+
with (self.subTest('assert metadata create response')):
74+
self.assertEqual(1, len(metadataList))
75+
metadataItem = metadataList[0]
76+
self.assertTrue('id' in metadataItem)
77+
# self.assertTrue('provider' in metadata) not yet
78+
self.assertTrue('key' in metadataItem)
79+
self.assertTrue('value' in metadataItem)
80+
81+
metadataList = self.configurations.metadata.list(
82+
component_id=self.TEST_COMPONENT_NAME,
83+
configuration_id='test_configuration_metadata'
84+
)
85+
86+
with (self.subTest('assert metadata list response')):
87+
self.assertTrue(len(metadataList) > 0)
88+
for metadataList in metadataList:
89+
self.assertTrue('id' in metadataList)
90+
# self.assertTrue('provider' in metadata) not yet
91+
self.assertTrue('key' in metadataList)
92+
self.assertTrue('value' in metadataList)
93+
94+
self.configurations.metadata.delete(
95+
component_id=self.TEST_COMPONENT_NAME,
96+
configuration_id='test_configuration_metadata',
97+
metadata_id=metadataList['id']
98+
)
99+
metadataList = self.configurations.metadata.list(
100+
component_id=self.TEST_COMPONENT_NAME,
101+
configuration_id='test_configuration_metadata'
102+
)
103+
104+
with (self.subTest('assert metadata delete means metadata no longer in list')):
105+
self.assertTrue(len(metadataList) == 0)

‎tests/functional/test_tables.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,3 +327,68 @@ def test_table_columns(self):
327327
with open(local_path, mode='rt') as file:
328328
lines = file.readlines()
329329
self.assertEqual(['"col3","col2"\n', '"king","pong"\n'], sorted(lines))
330+
331+
def test_table_with_metadata(self):
332+
file, path = tempfile.mkstemp(prefix='sapi-test')
333+
with open(path, 'w') as csv_file:
334+
writer = csv.DictWriter(csv_file, fieldnames=['col1', 'col2'],
335+
lineterminator='\n', delimiter=',',
336+
quotechar='"')
337+
writer.writeheader()
338+
writer.writerow({'col1': 'ping', 'col2': 'pong'})
339+
os.close(file)
340+
table_id = self.tables.create(name='some-table', file_path=path,
341+
bucket_id='in.c-py-test-tables')
342+
343+
self.tables.metadata.create(
344+
table_id=table_id,
345+
provider='test',
346+
metadata=[{
347+
'key': 'test_table_with_metadata',
348+
'value': 'success'
349+
}],
350+
columns_metadata=[
351+
[
352+
{
353+
'key': 'test_column_with_metadata',
354+
'value': 'success',
355+
'columnName': 'col1'
356+
}
357+
]
358+
]
359+
)
360+
361+
table_info = self.tables.detail(table_id)
362+
with self.subTest("Test metadata key in response"):
363+
self.assertIn('metadata', table_info)
364+
with self.subTest("Test metadata structure"):
365+
self.assertEqual(1, len(table_info['metadata']))
366+
self.assertIn('id', table_info['metadata'][0])
367+
self.assertEqual('test_table_with_metadata', table_info['metadata'][0]['key'])
368+
self.assertEqual('test', table_info['metadata'][0]['provider'])
369+
self.assertIn('timestamp', table_info['metadata'][0])
370+
self.assertEqual('success', table_info['metadata'][0]['value'])
371+
with self.subTest('Test columns metadata key in response'):
372+
self.assertIn('columnMetadata', table_info)
373+
with self.subTest('Test columns metadata structure'):
374+
self.assertIn('col1', table_info['columnMetadata'])
375+
self.assertEqual(1, len(table_info['columnMetadata']['col1']))
376+
self.assertIn('id', table_info['columnMetadata']['col1'][0])
377+
self.assertEqual('test_column_with_metadata', table_info['columnMetadata']['col1'][0]['key'])
378+
self.assertEqual('test', table_info['columnMetadata']['col1'][0]['provider'])
379+
self.assertIn('timestamp', table_info['columnMetadata']['col1'][0])
380+
self.assertEqual('success', table_info['columnMetadata']['col1'][0]['value'])
381+
382+
listedMetadata = self.tables.metadata.list(table_id=table_id)
383+
384+
with self.subTest("Test metadata key in list response"):
385+
self.assertEqual(1, len(listedMetadata))
386+
self.assertEqual('test_table_with_metadata', listedMetadata[0]['key'])
387+
self.assertEqual('test', listedMetadata[0]['provider'])
388+
self.assertEqual('success', listedMetadata[0]['value'])
389+
390+
self.tables.metadata.delete(table_id=table_id, metadata_id=listedMetadata[0]['id'])
391+
392+
listedMetadata = self.tables.metadata.list(table_id=table_id)
393+
with self.subTest('Test metadata can was deleted'):
394+
self.assertEqual(0, len(listedMetadata))

0 commit comments

Comments
 (0)