Merge pull request #28 from keboola/dev

A completely new version

Author: pivnicek

.travis.yml

@@ -5,8 +5,8 @@ services:
 before_script:
 - docker-compose build sapi-python-client
 script:
-- docker-compose run --rm --entrypoint=flake8 sapi-python-client || true
-- docker-compose run --rm -e KBC_TEST_TOKEN sapi-python-client -m unittest discover
+- docker-compose run --rm --entrypoint=flake8 sapi-python-client
+- docker-compose run --rm -e KBC_TEST_TOKEN -e KBC_TEST_API_URL sapi-python-client -m unittest discover
 after_success:
 - docker images
 deploy:

Dockerfile

@@ -2,6 +2,6 @@ FROM python:3.6
 
 WORKDIR /code
 COPY . /code/
-RUN pip3 install --no-cache-dir flake8
+RUN pip3 install --no-cache-dir flake8 responses
 RUN python setup.py install
 ENTRYPOINT ["python"]

README.md

@@ -1,10 +1,12 @@
 [![Build Status](https://travis-ci.org/keboola/sapi-python-client.svg?branch=master)](https://travis-ci.org/keboola/sapi-python-client)
 
 # Python client for the Keboola Storage API
+Client for using [Keboola Connection Storage API](http://docs.keboola.apiary.io/). This API client provides client methods to get data from KBC and store data in KBC. The endpoints 
+for working with buckets, tables and workspaces are covered.
 
 ## Install
 
-`$ pip install git+https://github.com/keboola/sapi-python-client.git`
+`$ pip3 install git+https://github.com/keboola/sapi-python-client.git`
 
 or 
 
@@ -15,24 +17,26 @@ $ python setup.py install
 
 ## Usage 
 ```
-from kbcstorage.client import Client
+from kbcstorage.tables import Tables
+from kbcstorage.buckets import Buckets
 
-cl = Client("MY_KBC_TOKEN")
+tables = Tables('https://connection.keboola.com', 'your-token')
 
 # get table data into local file
-cl.get_table_data("in.c-myBucket.myTable", "local_file_name.csv')
+tables.export_to_file(table_id='in.c-demo.some-table', path_name='/data/')
 
 # save data
-cl.save_table("tableName", "in.c-myBucket", "csv_I_want_to_store.csv")
+tables.create(name='some-table-2', bucket_id='in.c-demo', file_path='/data/some-table')
 
 # list buckets
-cl.list_buckets()
+buckets = Buckets('https://connection.keboola.com', 'your-token')
+buckets.list()
 
 # list bucket tables
-cl.list_tables(bucketId)
+buckets.list_tables('in.c-demo')
 
 # get table info
-cl.get_table(tableId)
+tables.detail('in.c-demo')
 
 ```
 
@@ -43,6 +47,19 @@ Docker image with pre-installed library is also available, run it via:
 docker run -i -t quay.io/keboola/sapi-python-client
 ```
 
+## Tests
+
+```bash
+$ git clone https://github.com/keboola/sapi-python-client.git && cd sapi-python-client
+$ python setup.py test
+```
+
+or 
+
+```bash
+$ docker-compose run --rm -e KBC_TEST_TOKEN -e KBC_TEST_API_URL sapi-python-client -m unittest discover
+```
+
 Under development -- all contributions very welcome :)
 
 Kickstarted via https://gist.github.com/Halama/6006960 

kbcstorage/base.py

@@ -0,0 +1,110 @@
+"""
+Base classes for constructing the client.
+
+Primarily exposes a base Endpoint class which deduplicates functionality across
+various endpoints, such as tables, workspaces, jobs, etc. as described in the
+`Storage API documentation`.
+
+
+.. _Storage API documentation:
+    http://docs.keboola.apiary.io/
+"""
+import requests
+
+
+class Endpoint:
+    """
+    Base class for implementing a single endpoint related to a single entities
+    as described in the Storage API.
+
+    Attributes:
+        base_url (str): The base URL for this endpoint.
+        token (str): A key for the Storage API.
+    """
+    def __init__(self, root_url, path_component, token):
+        """
+        Create an endpoint.
+
+        Args
+            root_url (str): Root url of API. eg.
+                "https://connection.keboola.com/v2/storage/"
+            path_component (str): The section of the path specific to the
+                endpoint. eg. "buckets"
+            token (str): A key for the Storage API. Can be found in the storage
+                console.
+        """
+        self.root_url = root_url
+        self.base_url = '{}/v2/storage/{}'.format(root_url.strip('/'),
+                                                  path_component.strip('/'))
+        self.token = token
+
+    def get(self, *args, **kwargs):
+        """
+        Construct a requests GET call with args and kwargs and process the
+        results.
+
+        Args:
+            *args: Positional arguments to pass to the get request.
+            **kwargs: Key word arguments to pass to the get request.
+
+        Returns:
+            body: Response body parsed from json.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+        """
+        r = requests.get(*args, **kwargs)
+        try:
+            r.raise_for_status()
+        except requests.HTTPError:
+            # Handle different error codes
+            raise
+        else:
+            return r.json()
+
+    def post(self, *args, **kwargs):
+        """
+        Construct a requests POST call with args and kwargs and process the
+        results.
+
+        Args:
+            *args: Positional arguments to pass to the post request.
+            **kwargs: Key word arguments to pass to the post request.
+
+        Returns:
+            body: Response body parsed from json.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+        """
+        r = requests.post(*args, **kwargs)
+        try:
+            r.raise_for_status()
+        except requests.HTTPError:
+            # Handle different error codes
+            raise
+        else:
+            return r.json()
+
+    def delete(self, *args, **kwargs):
+        """
+        Construct a requests DELETE call with args and kwargs and process the
+        result
+
+        Args:
+            *args: Positional arguments to pass to the delete request.
+            **kwargs: Key word arguments to pass to the delete request.
+
+        Returns:
+            body: Response body parsed from json.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+        """
+        r = requests.delete(*args, **kwargs)
+        try:
+            r.raise_for_status()
+        except requests.HTTPError:
+            # Handle different error codes
+            raise
+        # Should delete return something on success?

kbcstorage/buckets.py

@@ -0,0 +1,164 @@
+"""
+Manages calls to the Storage API relating to buckets
+
+Full documentation `here`.
+
+.. _here:
+    http://docs.keboola.apiary.io/#reference/buckets/
+"""
+from kbcstorage.base import Endpoint
+
+
+class Buckets(Endpoint):
+    """
+    Buckets Endpoint
+    """
+    def __init__(self, root_url, token):
+        """
+        Create a Buckets endpoint.
+
+        Args:
+            root_url (:obj:`str`): The base url for the API.
+            token (:obj:`str`): A storage API key.
+        """
+        super().__init__(root_url, 'buckets', token)
+
+    def list(self):
+        """
+        List all buckets in project.
+
+        Returns:
+            response_body: The parsed json from the HTTP response.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+        """
+        headers = {'X-StorageApi-Token': self.token}
+
+        return self.get(self.base_url, headers=headers)
+
+    def list_tables(self, bucket_id, include=None):
+        """
+        List all tables in a bucket.
+
+        Args:
+            bucket_id (str): Id of the bucket
+            include (list): Properties to list (attributes, columns)
+        Returns:
+            response_body: The parsed json from the HTTP response.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+        """
+        headers = {'X-StorageApi-Token': self.token}
+
+        url = '{}/{}/tables'.format(self.base_url, bucket_id)
+        params = {}
+        if include is not None and isinstance(include, list):
+            params['include'] = ','.join(include)
+        return self.get(url, headers=headers, params=params)
+
+    def detail(self, bucket_id):
+        """
+        Retrieves information about a given bucket.
+
+        Args:
+            bucket_id (str): The id of the bucket.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+        """
+        url = '{}/{}'.format(self.base_url, bucket_id)
+        headers = {'X-StorageApi-Token': self.token}
+
+        return self.get(url, headers=headers)
+
+    def create(self, name, stage='in', description='', backend=None):
+        """
+        Create a new bucket.
+
+        Args:
+            name (str): The new bucket name (only alphanumeric and underscores)
+            stage (str): The new bucket stage. Can be one of ``in`` or ``out``.
+                Default ``in``.
+            description (str): The new bucket description.
+            backend (str): The new bucket backend. Cand be one of
+                ``snowflake``, ``redshift`` or ``mysql``. Default determined by
+                project settings.
+        Returns:
+            response_body: The parsed json from the HTTP response.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+        """
+        # Separating create and link into two distinct functions...
+        headers = {
+            'X-StorageApi-Token': self.token,
+            'Content-Type': 'application/x-www-form-urlencoded'
+        }
+        # Need to check args...
+        body = {
+            'name': name,
+            'stage': stage,
+            'description': description,
+            'backend': backend
+        }
+
+        return self.post(self.base_url, headers=headers, data=body)
+
+    def delete(self, bucket_id, force=False):
+        """
+        Delete a bucket referenced by ``bucket_id``.
+
+        By default, only empty buckets without dependencies (aliases etc) can
+        be deleted. The optional ``force`` parameter allows for the deletion
+        of non-empty buckets.
+
+        Args:
+            bucket_id (str): The id of the bucket to be deleted.
+            force (bool): If ``True``, deletes the bucket even if it is not
+                empty. Default ``False``.
+        """
+        # How does the API handle it when force == False and the bucket is non-
+        # empty?
+        url = '{}/{}'.format(self.base_url, bucket_id)
+        headers = {'X-StorageApi-Token': self.token}
+        params = {'force': force}
+        super().delete(url, headers=headers, params=params)
+
+    def link(self, *args, **kwargs):
+        """
+        **Not implemented**
+
+        Link an existing bucket from another project.
+
+        Creates a new bucket which contains the contents of a shared bucket in
+        a source project. Linking a bucket from another project is only
+        possible if it has been enabled in the project.
+        """
+        raise NotImplementedError
+
+    def share(self, *args, **kwargs):
+        """
+        **Not implemented**
+
+        Enable sharing of a bucket.
+
+        The bucket will be shared to the entire organisation to which the
+        project belongs. It may then be shared to any project of that
+        organization. This operation is only available to administrator tokens.
+        """
+        raise NotImplementedError
+
+    def unshare(self, *args, **kwargs):
+        """
+        **Not implemented**
+
+        Stop sharing a bucket.
+
+        The bucket must not be linked to other projects. To unshare an already
+        linked bucket, the links must first be deleted - use ``delete`` on the
+        bucket in the linking project. This operation is only available for
+        administrator tokens.
+        """
+        raise NotImplementedError