diff --git a/geneva_examples/core/config.py b/geneva_examples/core/config.py index c1f1ca0..dcedb84 100644 --- a/geneva_examples/core/config.py +++ b/geneva_examples/core/config.py @@ -62,6 +62,8 @@ class Config: s3_endpoint: str | None = None s3_region: str | None = None aws_allow_http: bool = False + azure_account_name: str | None = None + azure_account_key: str | None = None hf_token: str | None = None @property @@ -70,7 +72,18 @@ def is_local(self) -> bool: return self.mode == "local" def storage_options(self) -> dict[str, str] | None: - """Build S3 ``storage_options``; ``None`` unless all four creds present.""" + """Build object-store ``storage_options`` for the data plane. + + Azure (account name + key) takes precedence; else S3-compatible when all + four creds are present; else ``None``. Enterprise clients write Lance + data fragments directly to object storage, so the account-less + ``az://`` root URI needs the account named here. + """ + if self.azure_account_name and self.azure_account_key: + return { + "azure_storage_account_name": self.azure_account_name, + "azure_storage_account_key": self.azure_account_key, + } if not ( self.s3_access_key and self.s3_secret_key @@ -155,5 +168,7 @@ def load_config( s3_endpoint=data.get("s3_endpoint"), s3_region=data.get("s3_region"), aws_allow_http=_as_bool(data.get("aws_allow_http")), + azure_account_name=data.get("azure_account_name"), + azure_account_key=data.get("azure_account_key"), hf_token=data.get("hf_token") or None, ) diff --git a/pyproject.toml b/pyproject.toml index f65a63b..f1fdb32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,13 +7,15 @@ dependencies = [ "typer>=0.24.0", "pyyaml>=6.0", # Pin the LanceDB stack to the versions the deployed cluster was built - # against (client/server must match or declare_table 500s). pylance 8.0.x - # lives on the lance-format index (see [tool.uv.sources]). pyarrow/numpy are - # left to geneva's own constraints. geneva 0.13.0b17+ adds source_task_size - # to chunker-view refresh; b18 requires pylance>=8.0.0b11. - "geneva==0.13.0b18", - "lancedb==0.33.1b2", - "pylance==8.0.0b16", + # against (client/server must match or declare_table/backfill_column 500s). + # pylance 9.0.x lives on the lance-format index (see [tool.uv.sources]). + # pyarrow/numpy are left to geneva's own constraints. This set matches the + # data plane's geneva 0.13.0b30 build (see sophon lancedb_integtest pins): + # older pylance (8.x)/lancedb (0.33) spoke a different namespace API, so + # add_columns would succeed but backfill_column returned "column not found". + "geneva==0.13.0b30", + "lancedb==0.34.0b5", + "pylance>=9.0.0b15", "pyarrow", "numpy", "pillow>=10", diff --git a/uv.lock b/uv.lock index 126e916..14261b3 100644 --- a/uv.lock +++ b/uv.lock @@ -1009,7 +1009,7 @@ wheels = [ [[package]] name = "geneva" -version = "0.13.0b18" +version = "0.13.0b30" source = { registry = "https://pypi.fury.io/lancedb/" } dependencies = [ { name = "aiohttp" }, @@ -1028,6 +1028,8 @@ dependencies = [ { name = "more-itertools" }, { name = "multiprocess" }, { name = "numpy" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-sdk" }, { name = "overrides" }, { name = "pip" }, { name = "pylance" }, @@ -1042,9 +1044,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://pypi.fury.io/lancedb/-/ver_eAtrP/geneva-0.13.0b18.tar.gz", hash = "sha256:b2e3e2992838c89c6df7d6652b7b1f3739d0d5268dc38188259aa54f5a55c02e" } +sdist = { url = "https://pypi.fury.io/lancedb/-/ver_qCpzm/geneva-0.13.0b30.tar.gz", hash = "sha256:fcad7510b85c37ac0863011743a5d5e491c8720936620b0470d4251aaa11e986" } wheels = [ - { url = "https://pypi.fury.io/lancedb/-/ver_jLkeq/geneva-0.13.0b18-py3-none-any.whl", hash = "sha256:08c0a9c464cc72e848a580be8d6e515b703ee714b9c13396d6244380eeadab03" }, + { url = "https://pypi.fury.io/lancedb/-/ver_2dXfhD/geneva-0.13.0b30-py3-none-any.whl", hash = "sha256:ce4145424983af449a3e1f885a49ff09110f4812d6a9de0c6d41950f6c391405" }, ] [[package]] @@ -1087,16 +1089,16 @@ requires-dist = [ { name = "av", specifier = ">=13" }, { name = "controlnet-aux", specifier = ">=0.0.7" }, { name = "datasets", specifier = ">=3.0.0" }, - { name = "geneva", specifier = "==0.13.0b18", index = "https://pypi.fury.io/lancedb/" }, + { name = "geneva", specifier = "==0.13.0b30", index = "https://pypi.fury.io/lancedb/" }, { name = "gradio", specifier = ">=6" }, { name = "huggingface-hub", specifier = ">=0.24" }, - { name = "lancedb", specifier = "==0.33.1b2", index = "https://pypi.fury.io/lancedb/" }, + { name = "lancedb", specifier = "==0.34.0b5", index = "https://pypi.fury.io/lancedb/" }, { name = "langchain-text-splitters", specifier = ">=0.3,<0.4" }, { name = "numpy" }, { name = "open-clip-torch", specifier = ">=2.24" }, { name = "pillow", specifier = ">=10" }, { name = "pyarrow" }, - { name = "pylance", specifier = "==8.0.0b16", index = "https://pypi.fury.io/lance-format/" }, + { name = "pylance", specifier = ">=9.0.0b15", index = "https://pypi.fury.io/lance-format/" }, { name = "pypdf", specifier = ">=5,<6" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "textual", specifier = ">=1.0.0" }, @@ -1523,7 +1525,7 @@ wheels = [ [[package]] name = "lancedb" -version = "0.33.1b2" +version = "0.34.0b5" source = { registry = "https://pypi.fury.io/lancedb/" } dependencies = [ { name = "deprecation" }, @@ -1535,10 +1537,10 @@ dependencies = [ { name = "tqdm" }, ] wheels = [ - { url = "https://pypi.fury.io/lancedb/-/ver_1Nlsr0/lancedb-0.33.1b2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec87aee4fcb644ec84c78cd3f7029b3bf0d412f98cf12374f682c68f14265fef" }, - { url = "https://pypi.fury.io/lancedb/-/ver_1JjYh/lancedb-0.33.1b2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:da5734098ebf16465e850ea661014fe0992f75234aa5a23f04ec95787bd05570" }, - { url = "https://pypi.fury.io/lancedb/-/ver_1a3Ksg/lancedb-0.33.1b2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:22cde3613d74d93aa27b7f426453017538d03a36c6a1ddc6c9bf41944953079c" }, - { url = "https://pypi.fury.io/lancedb/-/ver_1FdgZt/lancedb-0.33.1b2-cp39-abi3-win_amd64.whl", hash = "sha256:db1de7ddb72916acf3ab66bfa1bfc8f32f22230716ef60fc7a88df3bf41a0572" }, + { url = "https://pypi.fury.io/lancedb/-/ver_yLvTX/lancedb-0.34.0b5-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:6095cc77b2f0a4612299d24269a983d1e037c8e44babaedc22ee09f5a3002899" }, + { url = "https://pypi.fury.io/lancedb/-/ver_16c6NC/lancedb-0.34.0b5-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:18ebc85e47e98497e6f0728007742ab708a4a63d6852fb91f5876d8553f6c318" }, + { url = "https://pypi.fury.io/lancedb/-/ver_1GrjtR/lancedb-0.34.0b5-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b9fb6b671c9d0b8e05df536a59dc2a77107b93df0852c951ca3d87e79e8c3641" }, + { url = "https://pypi.fury.io/lancedb/-/ver_2dRU7w/lancedb-0.34.0b5-cp39-abi3-win_amd64.whl", hash = "sha256:119ede8ebf0d5136d5d1f9c365875006027a04f57fc72f38a2fa4e5870c45429" }, ] [[package]] @@ -2227,6 +2229,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/83/6dba32b85f31868400440dc7ad2ca1eab94cbbf3a7b0459ed39f8311a9e2/opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd", size = 61912, upload-time = "2026-06-24T15:19:35.434Z" }, ] +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/c1/e8098490ab15abf116dcaf9fa89ededcb35547c7d08d4b5a62f573dc1e63/opentelemetry_exporter_otlp_proto_common-1.43.0.tar.gz", hash = "sha256:c4e32ba6d6b13bdb2b8f6764c4fd28d00192826561aa04f6d14eedfce7ac076f", size = 20197, upload-time = "2026-06-24T15:20:00.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/b2/41ebc74ae1d5859901f1b69305de58724bf043381103d6ef413521cbc35a/opentelemetry_exporter_otlp_proto_common-1.43.0-py3-none-any.whl", hash = "sha256:123c3f9cc87218562490c63b36f497bf3a722faf174a515d1443f31ababa6264", size = 17048, upload-time = "2026-06-24T15:19:41.264Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/1d/6336453716ca0a240d4417d19e6d5b77a5e7163e5670ec4f7ec4d3ede7bf/opentelemetry_exporter_otlp_proto_grpc-1.43.0.tar.gz", hash = "sha256:1b3e0627daa9bc21884d4a13946807c255eb558bfe5bdd543dffb6f4c9faee0d", size = 27213, upload-time = "2026-06-24T15:20:00.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/74/2700b5d5c946bf2dba87073fce3dfc198c46bc92ea3d5693f54bc51c90b1/opentelemetry_exporter_otlp_proto_grpc-1.43.0-py3-none-any.whl", hash = "sha256:6a10d1feacffffda19acacbf277b736094b1e2f4dbb98c90ccb2c6e1962e2ec6", size = 19626, upload-time = "2026-06-24T15:19:42.233Z" }, +] + [[package]] name = "opentelemetry-exporter-prometheus" version = "0.64b0" @@ -2846,7 +2878,7 @@ wheels = [ [[package]] name = "pylance" -version = "8.0.0b16" +version = "9.0.0b19" source = { registry = "https://pypi.fury.io/lance-format/" } dependencies = [ { name = "lance-namespace" }, @@ -2854,12 +2886,11 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://pypi.fury.io/lance-format/-/ver_2bOHvs/pylance-8.0.0b16-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:99cea3f3a7e1d5f77d0cfb7abb0ffe76b722ae222e28fe65b7c485df0383f0a2" }, - { url = "https://pypi.fury.io/lance-format/-/ver_QIfth/pylance-8.0.0b16-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5de2ea50234df274dcbdad0b940799a209cfbb52af75df1e01ba0cae6c0ddc5e" }, - { url = "https://pypi.fury.io/lance-format/-/ver_anpUU/pylance-8.0.0b16-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9256cfba538e4a600c8ea1ec83611e61109a88e641fb76b38b288f6189ded827" }, - { url = "https://pypi.fury.io/lance-format/-/ver_TCipG/pylance-8.0.0b16-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ab48ab9ea9f6b7a8282b735ae6126cff2d3b30e3bcb4a7cde11868e375e39fea" }, - { url = "https://pypi.fury.io/lance-format/-/ver_FvQwv/pylance-8.0.0b16-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:81c97a6f0a01e575979ce97687f0b360a492b5d0f95c0ace922c916fe9b35ab7" }, - { url = "https://pypi.fury.io/lance-format/-/ver_zEs7m/pylance-8.0.0b16-cp39-abi3-win_amd64.whl", hash = "sha256:d05ba3e657f22f04812006151e99cc3f5687c2900b9535f2ae5a7312d913391e" }, + { url = "https://pypi.fury.io/lance-format/-/ver_2Jumm/pylance-9.0.0b19-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2aaf94bcbc48384ee2b93c0796563820019f7404d6f74085287c16841454df60" }, + { url = "https://pypi.fury.io/lance-format/-/ver_21PsxY/pylance-9.0.0b19-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae7b684bf9bd1108946695f1db26260892a19816a745c2345dff377e22d9be95" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1T1yPc/pylance-9.0.0b19-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ecde0610850271bea8380bdd87e6facfb0bc4b3aca64bea9ae07df79894577f" }, + { url = "https://pypi.fury.io/lance-format/-/ver_v5hPS/pylance-9.0.0b19-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f1740df890afbc5c931958eb11548a065c11144a908394ae00ea25f2985bc82" }, + { url = "https://pypi.fury.io/lance-format/-/ver_2j0pZT/pylance-9.0.0b19-cp310-abi3-win_amd64.whl", hash = "sha256:18de6f682372c70484ca41380f7a4571983447ab46ddc034477c7af2d2baae1d" }, ] [[package]]