Merge branch 'dev' of https://github.com/datajoint/datajoint-python into refactor-errors

Author: dimitri-yatsenko

.gitignore

@@ -18,3 +18,8 @@ build/
 ./tests/.coverage
 ./tests/dj-store/*
 *.log
+*.env
+LNX-docker-compose.yml
+notebooks/*
+test.dockerfile
+__main__.py

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 dist: xenial
 language: python
 env:
-  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASS="" S3_ENDPOINT="127.0.0.1:9000" S3_ACCESS_KEY="datajoint" S3_SECRET_KEY="datajoint" S3_BUCKET="datajoint-test"
+  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASS="" S3_ACCESS_KEY="datajoint" S3_SECRET_KEY="datajoint"
 python:
   - "3.4"
   - "3.5"
@@ -16,8 +16,6 @@ before_install:
   - docker pull minio/minio
   - docker run -d -p 9000:9000 -e "MINIO_ACCESS_KEY=$S3_ACCESS_KEY" -e "MINIO_SECRET_KEY=$S3_SECRET_KEY" minio/minio server /data
   - sleep 120
-  - docker pull minio/mc
-  - docker run --network="host" --entrypoint=/bin/sh minio/mc -c "mc config host add dj-s3 http://$S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY;mc mb $S3_BUCKET;mc policy download $S3_BUCKET;exit 0;"
   - sudo apt-get install -y libblas-dev liblapack-dev libatlas-dev gfortran
   - sudo apt-get install -y graphviz graphviz-dev pkg-config
   - mysql --version 

CHANGELOG.md

@@ -1,18 +1,28 @@
 ## Release notes
 
-### 0.12.0 -- August 15, 2019 (planned)
-
-* Configurable blob storage (#497, #532, #475)
-* Support file attachments: (#480, #532, #475)
-* Support for filepath datatype: (#481, 603)
-* Expand support of blob serialization (fixed #572, #520, #427, #392, #244)
-* Add support for UUID attributes (#562, #567)
-* Support of ellipsis in `proj`:  `query_expression.proj(.., '-movie')` (#499)
-* `dj.conn()` accepts a `port` keyword argument (#563, #571)
-* `query_expr.fetch("KEY", as_dict=False)` returns results as `np.recarray`(#414, #574)
-* `dj.ERD` is now called `dj.Diagram` (#255, #565)
-* `dj.Diagram` underlines "distinguished" classes (#378, #557)
-* Bugfixes: #629, #633
+### 0.12.0 -- Aug 23, 2019
+* Support TLS/SSL connections PR620
+* Convert numpy array from python object to appropriate data type if all elements are of the same type (#587) PR #608
+* Remove expression requirement to have additional attributes (#604) PR #604
+* Support for filepath datatype (#481) PR #603
+* Avoid creating multiple copies of attachments and return a dictionary array when specifying `as_dict=True` (#592, #595) PR #593
+* Support of ellipsis in `proj`:  `query_expression.proj(.., '-movie')` (#499) PR #578
+* Expand support of blob serialization (#572, #520, #427, #392, #244, #594) PR #577
+* Support for alter (#110) PR #573
+* Support for `conda install datajoint` via `conda-forge` channel (#293)
+* `dj.conn()` accepts a `port` keyword argument (#563) PR #571
+* Support for UUID datatype (#562) PR #567
+* `query_expr.fetch("KEY", as_dict=False)` returns results as `np.recarray`(#414) PR #574
+* `dj.ERD` is now called `dj.Diagram` (#255, #546) PR #565
+* `dj.Diagram` underlines "distinguished" classes (#378) PR #557
+* Accept alias for supported MySQL datatypes (#544) PR #545
+* Support for pandas and order by "KEY" (#459, #537, #538, #541) PR #534
+* Support file attachment datatype and configurable blob storage (#467, #475, #480, #497) PR #532
+* Increase default display rows (#523) PR #526
+* Bugfixes (#521, #205, #279, #570, #581, #597, #596, #618, #633)
+
+### 0.11.3 -- Jul 26, 2019
+* Fix incompatibility with pyparsing 2.4.1 (#629) PR #631
 
 ### 0.11.2 -- July 25, 2019
 * Fix #628 - incompatibility with pyparsing 2.4.1

datajoint/errors.py

@@ -66,7 +66,14 @@ class IntegrityError(QueryError):
     An integrity error triggered by foreign key constraints
     """
 
+
 class MissingAttributeError(QueryError):
     """
     An error arising when a required attribute value is not provided in INSERT
     """
+
+
+class MissingExternalFile(DataJointError):
+    """
+    Error raised when an external file managed by DataJoint is no longer accessible
+    """

datajoint/external.py

@@ -2,14 +2,15 @@
 import itertools
 from collections import Mapping
 from .settings import config
-from .errors import DataJointError
+from .errors import DataJointError, MissingExternalFile
 from .hash import uuid_from_buffer, uuid_from_file
 from .table import Table
 from .declare import EXTERNAL_TABLE_ROOT
 from . import s3 
 from .utils import safe_write, safe_copy
 
 CACHE_SUBFOLDING = (2, 2)   # (2, 2) means  "0123456789abcd" will be saved as "01/23/0123456789abcd"
+SUPPORT_MIGRATED_BLOBS = True   # support blobs migrated from datajoint 0.11.*
 
 
 def subfold(name, folds):
@@ -124,11 +125,21 @@ def fput(self, local_filepath):
     def peek(self, blob_hash, bytes_to_peek=120):
         return self.get(blob_hash, size=bytes_to_peek)
 
-    def get(self, blob_hash, size=-1):
+    def get(self, blob_hash, *, size=-1):
         """
         get an object from external store.
         :param size: max number of bytes to retrieve. If size<0, retrieve entire blob
+        :param explicit_path: if given, then use it as relative path rather than the path derived from
         """
+
+        def read_file(filepath, size):
+            try:
+                with open(filepath, 'rb') as f:
+                    blob = f.read(size)
+            except FileNotFoundError:
+                raise MissingExternalFile('Lost access to external blob %s.' % full_path) from None
+            return blob
+
         if blob_hash is None:
             return None
 
@@ -154,18 +165,33 @@ def get(self, blob_hash, size=-1):
                 subfolders = os.path.join(*subfold(blob_hash.hex, self.spec['subfolding']))
                 full_path = os.path.join(self.spec['location'], self.database, subfolders, blob_hash.hex)
                 try:
-                    with open(full_path, 'rb') as f:
-                        blob = f.read(size)
-                except FileNotFoundError:
-                    raise DataJointError('Lost access to external blob %s.' % full_path) from None
+                    blob = read_file(full_path, size)
+                except MissingExternalFile:
+                    if not SUPPORT_MIGRATED_BLOBS:
+                        raise
+                    # migrated blobs from 0.11
+                    relative_filepath, contents_hash = (self & {'hash': blob_hash}).fetch1(
+                        'filepath', 'contents_hash')
+                    if relative_filepath is None:
+                        raise
+                    blob = read_file(os.path.join(self.spec['location'], relative_filepath))
                 else:
                     if size > 0:
                         blob_size = os.path.getsize(full_path)
             elif self.spec['protocol'] == 's3':
                 full_path = '/'.join(
                     (self.database,) + subfold(blob_hash.hex, self.spec['subfolding']) + (blob_hash.hex,))
                 if size < 0:
-                    blob = self.s3.get(full_path)
+                    try:
+                        blob = self.s3.get(full_path)
+                    except MissingExternalFile:
+                        if not SUPPORT_MIGRATED_BLOBS:
+                            raise
+                        relative_filepath, contents_hash = (self & {'hash': blob_hash}).fetch1(
+                            'filepath', 'contents_hash')
+                        if relative_filepath is None:
+                            raise
+                        blob = self.s3.get(relative_filepath)
                 else:
                     blob = self.s3.partial_get(full_path, 0, size)
                     blob_size = self.s3.get_size(full_path)

datajoint/heading.py

@@ -4,7 +4,7 @@
 import re
 import logging
 from .errors import DataJointError
-from .declare import UUID_DATA_TYPE, CUSTOM_TYPES, TYPE_PATTERN, EXTERNAL_TYPES, SERIALIZED_TYPES
+from .declare import UUID_DATA_TYPE, CUSTOM_TYPES, TYPE_PATTERN, EXTERNAL_TYPES
 from .utils import OrderedDict
 
 
@@ -225,11 +225,13 @@ def init_from_database(self, conn, database, table_name):
                 try:
                     category = next(c for c in CUSTOM_TYPES if TYPE_PATTERN[c].match(attr['type']))
                 except StopIteration:
+                    if attr['type'].startswith('external'):
+                        raise DataJointError('Legacy datatype `{type}`.'.format(**attr)) from None
                     raise DataJointError('Unknown attribute type `{type}`'.format(**attr)) from None
                 attr.update(
                     is_attachment=category in ('INTERNAL_ATTACH', 'EXTERNAL_ATTACH'),
                     is_filepath=category == 'FILEPATH',
-                    is_blob=category in ('INTERNAL_BLOB', 'EXTERNAL_BLOB'), # INTERNAL_BLOB is not a custom type but is included for completeness
+                    is_blob=category in ('INTERNAL_BLOB', 'EXTERNAL_BLOB'),  # INTERNAL_BLOB is not a custom type but is included for completeness
                     uuid=category == 'UUID',
                     is_external=category in EXTERNAL_TYPES,
                     store=attr['type'].split('@')[1] if category in EXTERNAL_TYPES else None)

datajoint/s3.py

@@ -6,7 +6,7 @@
 import warnings
 import uuid
 import os
-
+from . import errors
 
 class Folder:
     """
@@ -29,7 +29,10 @@ def fput(self, relative_name, local_file, **meta):
             self.bucket, '/'.join((self.remote_path, relative_name)), local_file, metadata=meta or None)
 
     def get(self, relative_name):
-        return self.client.get_object(self.bucket, '/'.join((self.remote_path, relative_name))).data
+        try:
+            return self.client.get_object(self.bucket, '/'.join((self.remote_path, relative_name))).data
+        except minio.error.NoSuchKey:
+            raise errors.MissingExternalFile from None
 
     def fget(self, relative_name, local_filepath):
         """get file from object name to local filepath"""
@@ -48,13 +51,13 @@ def partial_get(self, relative_name, offset, size):
             return self.client.get_partial_object(
                 self.bucket, '/'.join((self.remote_path, relative_name)), offset, size).data
         except minio.error.NoSuchKey:
-            return None
+            raise errors.MissingExternalFile from None
 
     def get_size(self, relative_name):
         try:
             return self.client.stat_object(self.bucket, '/'.join((self.remote_path, relative_name))).size
         except minio.error.NoSuchKey:
-            return None
+            raise errors.MissingExternalFile from None
 
     def list_objects(self, folder=''):
         return self.client.list_objects(self.bucket, '/'.join((self.remote_path, folder, '')), recursive=True)

datajoint/utils.py

@@ -95,3 +95,19 @@ def safe_copy(src, dest, overwrite=False):
         temp_file = dest + '.copying'
         shutil.copyfile(src, temp_file)
         os.rename(temp_file, dest)
+
+
+def parse_sql(filepath):
+    DELIMITER = ';'
+    statement = []
+    with open(filepath, 'rt') as f:
+        for line in f:
+            line = line.strip()
+            if not line.startswith('--') and len(line) > 1:
+                if line.startswith('DELIMITER'):
+                    DELIMITER = line.split()[1]
+                else:
+                    statement.append(line)
+                    if line.endswith(DELIMITER):
+                        yield ' '.join(statement)
+                        statement = []

tests/__init__.py

@@ -9,8 +9,15 @@
 from os import environ, remove
 import datajoint as dj
 from distutils.version import LooseVersion
+import os
+from pathlib import Path
+from minio import Minio
+import urllib3
+import certifi
+import shutil
+from datajoint.utils import parse_sql
 
-__author__ = 'Edgar Walker, Fabian Sinz, Dimitri Yatsenko'
+__author__ = 'Edgar Walker, Fabian Sinz, Dimitri Yatsenko, Raphael Guzman'
 
 # turn on verbose logging
 logging.basicConfig(level=logging.DEBUG)
@@ -61,7 +68,7 @@
     conn_root.query(
         "GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';")
 else:
-    # grant permissions. For mysql5.6/5.7 this also automatically creates user 
+    # grant permissions. For mysql5.6/5.7 this also automatically creates user
     # if not exists
     conn_root.query("""
         GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%'
@@ -76,6 +83,25 @@
         REQUIRE SSL;
         """)
 
+# Initialize httpClient with relevant timeout.
+httpClient = urllib3.PoolManager(
+            timeout=30,
+            cert_reqs='CERT_REQUIRED',
+            ca_certs=certifi.where(),
+            retries=urllib3.Retry(
+                total=3,
+                backoff_factor=0.2,
+                status_forcelist=[500, 502, 503, 504]
+            )
+        )
+
+# Initialize minioClient with an endpoint and access/secret keys.
+minioClient = Minio(S3_CONN_INFO['endpoint'],
+                    access_key=S3_CONN_INFO['access_key'],
+                    secret_key=S3_CONN_INFO['secret_key'],
+                    secure=False,
+                    http_client=httpClient)
+
 
 def setup_package():
     """
@@ -84,6 +110,44 @@ def setup_package():
     """
     dj.config['safemode'] = False
 
+    # Add old MySQL
+    source = os.path.dirname(os.path.realpath(__file__)) + \
+        "/external-legacy-data"
+    db_name = "djtest_blob_migrate"
+    db_file = "v0_11.sql"
+    conn_root.query("""
+        CREATE DATABASE {};
+        """.format(db_name))
+
+    statements = parse_sql('{}/{}'.format(source, db_file))
+    for s in statements:
+        conn_root.query(s)
+
+    # Add old S3
+    source = os.path.dirname(os.path.realpath(__file__)) + \
+        "/external-legacy-data/s3"
+    bucket = "migrate-test"
+    region = "us-east-1"
+    minioClient.make_bucket(bucket, location=region)
+
+    pathlist = Path(source).glob('**/*')
+    for path in pathlist:
+        if os.path.isfile(str(path)) and ".sql" not in str(path):
+            minioClient.fput_object(
+                    bucket, os.path.relpath(
+                        str(path),
+                        '{}/{}'.format(source, bucket)
+                        ), str(path))
+
+    # Add S3
+    minioClient.make_bucket("datajoint-test", location=region)
+
+    # Add old File Content
+    shutil.copytree(
+            os.path.dirname(os.path.realpath(__file__)) +
+            "/external-legacy-data/file/temp",
+            os.path.expanduser('~/temp'))
+
 
 def teardown_package():
     """
@@ -99,3 +163,21 @@ def teardown_package():
         conn.query('DROP DATABASE `{}`'.format(db[0]))
     conn.query('SET FOREIGN_KEY_CHECKS=1')
     remove("dj_local_conf.json")
+
+    # Remove old S3
+    bucket = "migrate-test"
+    objs = list(minioClient.list_objects_v2(
+            bucket, recursive=True))
+    objs = [minioClient.remove_object(bucket,
+            o.object_name.encode('utf-8')) for o in objs]
+    minioClient.remove_bucket(bucket)
+
+    # Remove S3
+    bucket = "datajoint-test"
+    objs = list(minioClient.list_objects_v2(bucket, recursive=True))
+    objs = [minioClient.remove_object(bucket,
+            o.object_name.encode('utf-8')) for o in objs]
+    minioClient.remove_bucket(bucket)
+
+    # Remove old File Content
+    shutil.rmtree(os.path.expanduser('~/temp'))