Merge pull request #2 from lancedb/load-package-specs-from-env

udfs: resolve geneva/lancedb/pylance pins from installed versions

Author: justinrmiller

geneva_examples/core/package_specs.py

@@ -0,0 +1,30 @@
+"""Resolve remote-runtime package pins from the installed environment.
+
+The Geneva remote runtime installs a pinned set of pip packages into each
+worker's env. Rather than hardcoding versions in every UDF module — where they
+silently drift from what's actually installed locally — we read the installed
+version of each package and emit an exact ``name==X.Y.Z`` spec. This keeps the
+remote workers on the same versions the client resolved (via ``uv.lock``).
+
+An environment variable override (``{PACKAGE}_PACKAGE_SPEC``, e.g.
+``GENEVA_PACKAGE_SPEC``) still wins verbatim, so a different build can be
+targeted on the workers without touching code.
+"""
+
+from __future__ import annotations
+
+import os
+from importlib.metadata import version
+
+
+def package_spec(package: str, *, env_var: str | None = None) -> str:
+    """Return ``package==<installed version>``, or an env override if set.
+
+    ``env_var`` defaults to ``{PACKAGE}_PACKAGE_SPEC`` (uppercased package name).
+    The override is returned exactly as given — it need not be an ``==`` pin.
+    """
+    env_var = env_var or f"{package.upper()}_PACKAGE_SPEC"
+    override = os.environ.get(env_var)
+    if override:
+        return override
+    return f"{package}=={version(package)}"

geneva_examples/udfs/blip.py

@@ -5,10 +5,14 @@
 import os
 import uuid
 
+from geneva_examples.core.package_specs import package_spec
+
 # Geneva remote runtime package pins (env-overridable for targeting other builds).
-GENEVA_PACKAGE_SPEC = os.environ.get("GENEVA_PACKAGE_SPEC", "geneva==0.13.0b18")
-LANCEDB_PACKAGE_SPEC = os.environ.get("LANCEDB_PACKAGE_SPEC", "lancedb==0.33.1b2")
-PYLANCE_PACKAGE_SPEC = os.environ.get("PYLANCE_PACKAGE_SPEC", "pylance==8.0.0b16")
+# geneva/lancedb/pylance track the installed versions so the workers match the
+# client's locked env; the rest stay exact-pinned for reproducible worker builds.
+GENEVA_PACKAGE_SPEC = package_spec("geneva")
+LANCEDB_PACKAGE_SPEC = package_spec("lancedb")
+PYLANCE_PACKAGE_SPEC = package_spec("pylance")
 PYARROW_PACKAGE_SPEC = os.environ.get("PYARROW_PACKAGE_SPEC", "pyarrow==23.0.0")
 PILLOW_PACKAGE_SPEC = os.environ.get("PILLOW_PACKAGE_SPEC", "pillow==12.2.0")
 TORCH_PACKAGE_SPEC = os.environ.get("TORCH_PACKAGE_SPEC", "torch==2.12.0")

geneva_examples/udfs/chunkers.py

@@ -19,10 +19,14 @@
 import os
 import uuid
 
+from geneva_examples.core.package_specs import package_spec
+
 # Geneva remote runtime package pins (env-overridable for targeting other builds).
-GENEVA_PACKAGE_SPEC = os.environ.get("GENEVA_PACKAGE_SPEC", "geneva==0.13.0b18")
-LANCEDB_PACKAGE_SPEC = os.environ.get("LANCEDB_PACKAGE_SPEC", "lancedb==0.33.1b2")
-PYLANCE_PACKAGE_SPEC = os.environ.get("PYLANCE_PACKAGE_SPEC", "pylance==8.0.0b16")
+# geneva/lancedb/pylance track the installed versions so the workers match the
+# client's locked env; the rest stay exact-pinned for reproducible worker builds.
+GENEVA_PACKAGE_SPEC = package_spec("geneva")
+LANCEDB_PACKAGE_SPEC = package_spec("lancedb")
+PYLANCE_PACKAGE_SPEC = package_spec("pylance")
 PYARROW_PACKAGE_SPEC = os.environ.get("PYARROW_PACKAGE_SPEC", "pyarrow==23.0.0")
 PILLOW_PACKAGE_SPEC = os.environ.get("PILLOW_PACKAGE_SPEC", "pillow==12.2.0")
 AV_PACKAGE_SPEC = os.environ.get("AV_PACKAGE_SPEC", "av>=12,<14")

geneva_examples/udfs/clip.py

@@ -6,10 +6,14 @@
 import uuid
 from collections.abc import Callable
 
+from geneva_examples.core.package_specs import package_spec
+
 # Geneva remote runtime package pins (env-overridable for targeting other builds).
-GENEVA_PACKAGE_SPEC = os.environ.get("GENEVA_PACKAGE_SPEC", "geneva==0.13.0b18")
-LANCEDB_PACKAGE_SPEC = os.environ.get("LANCEDB_PACKAGE_SPEC", "lancedb==0.33.1b2")
-PYLANCE_PACKAGE_SPEC = os.environ.get("PYLANCE_PACKAGE_SPEC", "pylance==8.0.0b16")
+# geneva/lancedb/pylance track the installed versions so the workers match the
+# client's locked env; the rest stay exact-pinned for reproducible worker builds.
+GENEVA_PACKAGE_SPEC = package_spec("geneva")
+LANCEDB_PACKAGE_SPEC = package_spec("lancedb")
+PYLANCE_PACKAGE_SPEC = package_spec("pylance")
 PYARROW_PACKAGE_SPEC = os.environ.get("PYARROW_PACKAGE_SPEC", "pyarrow==23.0.0")
 PILLOW_PACKAGE_SPEC = os.environ.get("PILLOW_PACKAGE_SPEC", "pillow==12.2.0")
 NUMPY_PACKAGE_SPEC = os.environ.get("NUMPY_PACKAGE_SPEC", "numpy==2.4.6")

geneva_examples/udfs/imageinfo.py

@@ -5,10 +5,14 @@
 import os
 import uuid
 
+from geneva_examples.core.package_specs import package_spec
+
 # Geneva remote runtime package pins (env-overridable for targeting other builds).
-GENEVA_PACKAGE_SPEC = os.environ.get("GENEVA_PACKAGE_SPEC", "geneva==0.13.0b18")
-LANCEDB_PACKAGE_SPEC = os.environ.get("LANCEDB_PACKAGE_SPEC", "lancedb==0.33.1b2")
-PYLANCE_PACKAGE_SPEC = os.environ.get("PYLANCE_PACKAGE_SPEC", "pylance==8.0.0b16")
+# geneva/lancedb/pylance track the installed versions so the workers match the
+# client's locked env; the rest stay exact-pinned for reproducible worker builds.
+GENEVA_PACKAGE_SPEC = package_spec("geneva")
+LANCEDB_PACKAGE_SPEC = package_spec("lancedb")
+PYLANCE_PACKAGE_SPEC = package_spec("pylance")
 PYARROW_PACKAGE_SPEC = os.environ.get("PYARROW_PACKAGE_SPEC", "pyarrow==23.0.0")
 PILLOW_PACKAGE_SPEC = os.environ.get("PILLOW_PACKAGE_SPEC", "pillow==12.2.0")
 

geneva_examples/udfs/openpose.py

@@ -5,10 +5,14 @@
 import os
 import uuid
 
+from geneva_examples.core.package_specs import package_spec
+
 # Geneva remote runtime package pins (env-overridable for targeting other builds).
-GENEVA_PACKAGE_SPEC = os.environ.get("GENEVA_PACKAGE_SPEC", "geneva==0.13.0b18")
-LANCEDB_PACKAGE_SPEC = os.environ.get("LANCEDB_PACKAGE_SPEC", "lancedb==0.33.1b2")
-PYLANCE_PACKAGE_SPEC = os.environ.get("PYLANCE_PACKAGE_SPEC", "pylance==8.0.0b16")
+# geneva/lancedb/pylance track the installed versions so the workers match the
+# client's locked env; the rest stay exact-pinned for reproducible worker builds.
+GENEVA_PACKAGE_SPEC = package_spec("geneva")
+LANCEDB_PACKAGE_SPEC = package_spec("lancedb")
+PYLANCE_PACKAGE_SPEC = package_spec("pylance")
 PYARROW_PACKAGE_SPEC = os.environ.get("PYARROW_PACKAGE_SPEC", "pyarrow==23.0.0")
 PILLOW_PACKAGE_SPEC = os.environ.get("PILLOW_PACKAGE_SPEC", "pillow==12.2.0")
 NUMPY_PACKAGE_SPEC = os.environ.get("NUMPY_PACKAGE_SPEC", "numpy==2.4.6")