feat: upgrade to 0.0.7 (#206)

Author: jackye1995

README.md

@@ -1,7 +1,5 @@
 # Lance Namespace
 
-![logo](./docs/src/logo/wide.png)
-
 **Lance Namespace** is an open specification on top of the storage-based Lance data format
 to standardize access to a collection of Lance tables (a.k.a. Lance datasets).
 It describes how a metadata service like Apache Hive MetaStore (HMS),

python/Makefile

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = 0.0.6
+VERSION = 0.0.7
 
 .PHONY: clean-urllib3-client
 clean-urllib3-client:
@@ -38,7 +38,7 @@ build-urllib3-client: gen-urllib3-client
 .PHONY: build-ns
 build-ns: build-urllib3-client
 	cd lance_namespace; \
-		uv pip install ".[test]"
+		uv pip install ".[glue,hive2,test]"
 		uv run pytest
 
 .PHONY: clean

python/lance_namespace/README.md

@@ -1,11 +1,10 @@
 [project]
 name = "lance-namespace"
-version = "0.0.6"
+version = "0.0.7"
 description = "Python client for Lance Namespace API"
-readme = "README.md"
-authors = [
-    { name = "Jack Ye", email = "yezhaoqin@gmail.com" }
-]
+readme = "../../README.md"
+license = {file = "../../LICENSE"}
+authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
 requires-python = ">=3.9"
 dependencies = [
     "lance-namespace-urllib3-client",

python/lance_namespace/pyproject.toml

@@ -133,7 +133,14 @@ def __init__(self, **properties):
             )
 
         self.config = GlueNamespaceConfig(properties)
-        self.glue = self._initialize_glue_client()
+        self._glue = None  # Lazy initialization to support pickling
+    
+    @property
+    def glue(self):
+        """Get the Glue client, initializing it if necessary."""
+        if self._glue is None:
+            self._glue = self._initialize_glue_client()
+        return self._glue
 
     def _initialize_glue_client(self):
         """Initialize the AWS Glue client."""
@@ -634,6 +641,18 @@ def _convert_pyarrow_type_to_glue_type(self, arrow_type: pa.DataType) -> str:
             # Default to string for unknown types
             return 'string'
 
+    def __getstate__(self):
+        """Prepare instance for pickling by excluding unpickleable objects."""
+        state = self.__dict__.copy()
+        # Remove the unpickleable Glue client
+        state['_glue'] = None
+        return state
+    
+    def __setstate__(self, state):
+        """Restore instance from pickled state."""
+        self.__dict__.update(state)
+        # The Glue client will be re-initialized lazily via the property
+    
 
 
 class GlueNamespaceConfig:
@@ -663,6 +682,9 @@ def __init__(self, properties: Optional[Dict[str, str]] = None):
         if properties is None:
             properties = {}
 
+        # Store raw properties for pickling support
+        self._properties = properties.copy()
+        
         self._catalog_id = properties.get(self.CATALOG_ID)
         self._endpoint = properties.get(self.ENDPOINT)
         self._region = properties.get(self.REGION)
@@ -733,4 +755,9 @@ def root(self) -> Optional[str]:
     @property
     def storage_options(self) -> Dict[str, str]:
         """Get the storage configuration properties."""
-        return self._storage_options.copy()
+        return self._storage_options.copy()
+    
+    @property
+    def properties(self) -> Dict[str, str]:
+        """Get the raw properties dictionary."""
+        return self._properties.copy()

python/lance_namespace/src/lance_namespace/glue.py

@@ -203,8 +203,18 @@ def __init__(self, **properties):
         # Extract storage properties
         self.storage_properties = {k[8:]: v for k, v in properties.items() if k.startswith("storage.")}
 
-        # Create client
-        self._client = HiveMetastoreClient(self.uri, self.ugi)
+        # Store properties for pickling support
+        self._properties = properties.copy()
+        
+        # Lazy initialization to support pickling
+        self._client = None
+    
+    @property
+    def client(self):
+        """Get the Hive client, initializing it if necessary."""
+        if self._client is None:
+            self._client = HiveMetastoreClient(self.uri, self.ugi)
+        return self._client
 
     def _normalize_identifier(self, identifier: List[str]) -> tuple:
         """Normalize identifier to (database, table) tuple."""
@@ -231,7 +241,7 @@ def list_namespaces(self, request: ListNamespacesRequest) -> ListNamespacesRespo
                 # Non-root namespaces don't have children in Hive2
                 return ListNamespacesResponse(namespaces=[])
 
-            with self._client as client:
+            with self.client as client:
                 databases = client.get_all_databases()
                 # Return just database names as strings (excluding default)
                 namespaces = [db for db in databases if db != "default"]
@@ -259,7 +269,7 @@ def describe_namespace(self, request: DescribeNamespaceRequest) -> DescribeNames
 
             database_name = request.id[0]
 
-            with self._client as client:
+            with self.client as client:
                 database = client.get_database(database_name)
 
                 properties = {}
@@ -309,7 +319,7 @@ def create_namespace(self, request: CreateNamespaceRequest) -> CreateNamespaceRe
                 if k not in ["comment", "owner", "location"]
             }
 
-            with self._client as client:
+            with self.client as client:
                 client.create_database(database)
 
             return CreateNamespaceResponse()
@@ -331,7 +341,7 @@ def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse
 
             database_name = request.id[0]
 
-            with self._client as client:
+            with self.client as client:
                 # Check if database is empty
                 tables = client.get_all_tables(database_name)
                 cascade = request.behavior == "CASCADE" if request.behavior else False
@@ -360,7 +370,7 @@ def namespace_exists(self, request: NamespaceExistsRequest) -> None:
 
             database_name = request.id[0]
 
-            with self._client as client:
+            with self.client as client:
                 client.get_database(database_name)
         except Exception as e:
             if NoSuchObjectException and isinstance(e, NoSuchObjectException):
@@ -380,7 +390,7 @@ def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
 
             database_name = request.id[0]
 
-            with self._client as client:
+            with self.client as client:
                 table_names = client.get_all_tables(database_name)
 
                 # Filter for Lance tables if needed
@@ -410,7 +420,7 @@ def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse
         try:
             database, table_name = self._normalize_identifier(request.id)
 
-            with self._client as client:
+            with self.client as client:
                 table = client.get_table(database, table_name)
 
                 # Check if it's a Lance table (case insensitive)
@@ -531,7 +541,7 @@ def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse
                     if k not in [TABLE_TYPE_KEY, MANAGED_BY_KEY, VERSION_KEY]:
                         hive_table.parameters[k] = v
 
-            with self._client as client:
+            with self.client as client:
                 client.create_table(hive_table)
 
             return RegisterTableResponse(
@@ -549,7 +559,7 @@ def table_exists(self, request: TableExistsRequest) -> None:
         try:
             database, table_name = self._normalize_identifier(request.id)
 
-            with self._client as client:
+            with self.client as client:
                 table = client.get_table(database, table_name)
 
                 # Check if it's a Lance table (case insensitive)
@@ -569,7 +579,7 @@ def drop_table(self, request: DropTableRequest) -> DropTableResponse:
         try:
             database, table_name = self._normalize_identifier(request.id)
 
-            with self._client as client:
+            with self.client as client:
                 # Get table to check if it's a Lance table
                 table = client.get_table(database, table_name)
 
@@ -595,7 +605,7 @@ def deregister_table(self, request: DeregisterTableRequest) -> DeregisterTableRe
         try:
             database, table_name = self._normalize_identifier(request.id)
 
-            with self._client as client:
+            with self.client as client:
                 # Get table to check if it's a Lance table
                 table = client.get_table(database, table_name)
 
@@ -710,4 +720,16 @@ def _pyarrow_type_to_hive_type(self, dtype: pa.DataType) -> str:
                 field_strs.append(f"{field.name}:{field_type}")
             return f"struct<{','.join(field_strs)}>"
         else:
-            return "string"  # Default to string for unknown types
+            return "string"  # Default to string for unknown types
+    
+    def __getstate__(self):
+        """Prepare instance for pickling by excluding unpickleable objects."""
+        state = self.__dict__.copy()
+        # Remove the unpickleable Hive client
+        state['_client'] = None
+        return state
+    
+    def __setstate__(self, state):
+        """Restore instance from pickled state."""
+        self.__dict__.update(state)
+        # The Hive client will be re-initialized lazily via the property

python/lance_namespace/src/lance_namespace/hive.py

@@ -594,3 +594,41 @@ def test_pyarrow_schema_to_glue_columns(self, glue_namespace):
         assert columns[2] == {'Name': 'scores', 'Type': 'array<float>'}
         assert columns[3] == {'Name': 'metadata', 'Type': 'struct<created:timestamp,version:int>'}
 
+    def test_pickle_support(self, mock_boto3):
+        """Test that GlueNamespace can be pickled and unpickled for Ray compatibility."""
+        import pickle
+        
+        # Create a GlueNamespace instance
+        properties = {
+            'region': 'us-east-1',
+            'catalog_id': '123456789012',
+            'endpoint': 'https://glue.example.com',
+            'storage.access_key_id': 'test-key',
+            'storage.secret_access_key': 'test-secret'
+        }
+        namespace = GlueNamespace(**properties)
+        
+        # Test pickling
+        pickled = pickle.dumps(namespace)
+        assert pickled is not None
+        
+        # Test unpickling
+        restored = pickle.loads(pickled)
+        assert isinstance(restored, GlueNamespace)
+        
+        # Verify configuration is preserved
+        assert restored.config.region == 'us-east-1'
+        assert restored.config.catalog_id == '123456789012'
+        assert restored.config.endpoint == 'https://glue.example.com'
+        assert restored.config.storage_options['access_key_id'] == 'test-key'
+        assert restored.config.storage_options['secret_access_key'] == 'test-secret'
+        
+        # Verify glue client is None after unpickling (will be lazily initialized)
+        assert restored._glue is None
+        
+        # Test that glue client can be re-initialized after unpickling
+        # This will create a new mock client when accessed
+        client = restored.glue
+        assert client is not None
+        assert restored._glue is not None
+    

python/lance_namespace/tests/test_glue.py

@@ -62,6 +62,12 @@ def test_initialization(self):
                 assert namespace.uri == "thrift://localhost:9083"
                 assert namespace.root == "/tmp/warehouse"
                 assert namespace.ugi == "user:group1,group2"
+                
+                # Client should not be initialized yet (lazy loading)
+                mock_client.assert_not_called()
+                
+                # Access the client property to trigger initialization
+                _ = namespace.client
                 mock_client.assert_called_once_with("thrift://localhost:9083", "user:group1,group2")
 
     def test_initialization_without_hive_deps(self):
@@ -428,4 +434,47 @@ def test_root_namespace_operations(self, hive_namespace):
         # Test drop_namespace for root (should fail)
         request = DropNamespaceRequest(id=[])
         with pytest.raises(ValueError, match="Cannot drop root namespace"):
-            hive_namespace.drop_namespace(request)
+            hive_namespace.drop_namespace(request)
+    
+    def test_pickle_support(self):
+        """Test that Hive2Namespace can be pickled and unpickled for Ray compatibility."""
+        import pickle
+        
+        with patch("lance_namespace.hive.HIVE_AVAILABLE", True):
+            with patch("lance_namespace.hive.HiveMetastoreClient"):
+                # Create a Hive2Namespace instance
+                namespace = connect("hive2", {
+                    "uri": "thrift://localhost:9083",
+                    "root": "/tmp/warehouse",
+                    "ugi": "user:group1,group2",
+                    "client.pool-size": "5",
+                    "storage.access_key_id": "test-key",
+                    "storage.secret_access_key": "test-secret"
+                })
+                
+                # Test pickling
+                pickled = pickle.dumps(namespace)
+                assert pickled is not None
+                
+                # Test unpickling
+                restored = pickle.loads(pickled)
+                assert isinstance(restored, namespace.__class__)
+                
+                # Verify configuration is preserved
+                assert restored.uri == "thrift://localhost:9083"
+                assert restored.root == "/tmp/warehouse"
+                assert restored.ugi == "user:group1,group2"
+                assert restored.pool_size == 5
+                assert restored.storage_properties["access_key_id"] == "test-key"
+                assert restored.storage_properties["secret_access_key"] == "test-secret"
+                
+                # Verify client is None after unpickling (will be lazily initialized)
+                assert restored._client is None
+                
+                # Test that client can be re-initialized after unpickling
+                with patch("lance_namespace.hive.HiveMetastoreClient") as mock_client:
+                    # This will create a new mock client when accessed
+                    client = restored.client
+                    assert client is not None
+                    assert restored._client is not None
+                    mock_client.assert_called_once_with("thrift://localhost:9083", "user:group1,group2")

python/lance_namespace/tests/test_hive.py

@@ -13,7 +13,7 @@ See https://lancedb.github.io/lance-namespace/spec/impls/rest for more details.
 This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
 - API version: 1.0.0
-- Package version: 0.0.6
+- Package version: 0.0.7
 - Generator version: 7.12.0
 - Build package: org.openapitools.codegen.languages.PythonClientCodegen
 

python/lance_namespace_urllib3_client/README.md

@@ -14,7 +14,7 @@
 """  # noqa: E501
 
 
-__version__ = "0.0.6"
+__version__ = "0.0.7"
 
 # import apis into sdk package
 from lance_namespace_urllib3_client.api.data_api import DataApi