Skip to content

Commit aed95df

Browse files
kesmit13claude
kesmit13
and
claude
committed
Replace top-level optional imports with lazy import helpers
Heavy optional dependencies (numpy, pandas, polars, pyarrow) were imported at module load time, causing failures in WASM environments where these packages may not be available. This adds a lazy import utility module and converts all eager try/except import patterns to use cached lazy accessors. Type maps in dtypes.py are also converted from module-level dicts to lru_cached factory functions. The pandas DataFrame isinstance check in connection.py is replaced with a duck-type hasattr check to avoid importing pandas at module scope. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 5d1e9e6 commit aed95df

File tree

9 files changed

+220
-162
lines changed

9 files changed

+220
-162
lines changed

‎singlestoredb/connection.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,6 @@
2525
from urllib.parse import urlparse
2626

2727
import sqlparams
28-
try:
29-
from pandas import DataFrame
30-
except ImportError:
31-
class DataFrame(object): # type: ignore
32-
def itertuples(self, *args: Any, **kwargs: Any) -> None:
33-
pass
3428

3529
from . import auth
3630
from . import exceptions
@@ -1175,7 +1169,7 @@ def _iquery(
11751169
out = list(cur.fetchall())
11761170
if not out:
11771171
return []
1178-
if isinstance(out, DataFrame):
1172+
if hasattr(out, 'to_dict') and callable(getattr(out, 'to_dict')):
11791173
out = out.to_dict(orient='records')
11801174
elif isinstance(out[0], (tuple, list)):
11811175
if cur.description:

‎singlestoredb/converters.py‎

Lines changed: 43 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,7 @@
2626
except (AttributeError, ImportError):
2727
has_pygeos = False
2828

29-
try:
30-
import numpy
31-
has_numpy = True
32-
except ImportError:
33-
has_numpy = False
29+
from .utils._lazy_import import get_numpy
3430

3531
try:
3632
import bson
@@ -563,8 +559,9 @@ def float32_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
563559
if x is None:
564560
return None
565561

566-
if has_numpy:
567-
return numpy.array(json_loads(x), dtype=numpy.float32)
562+
np = get_numpy()
563+
if np is not None:
564+
return np.array(json_loads(x), dtype=np.float32)
568565

569566
return map(float, json_loads(x))
570567

@@ -591,8 +588,9 @@ def float32_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
591588
if x is None:
592589
return None
593590

594-
if has_numpy:
595-
return numpy.frombuffer(x, dtype=numpy.float32)
591+
np = get_numpy()
592+
if np is not None:
593+
return np.frombuffer(x, dtype=np.float32)
596594

597595
return struct.unpack(f'<{len(x)//4}f', x)
598596

@@ -619,8 +617,9 @@ def float16_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
619617
if x is None:
620618
return None
621619

622-
if has_numpy:
623-
return numpy.array(json_loads(x), dtype=numpy.float16)
620+
np = get_numpy()
621+
if np is not None:
622+
return np.array(json_loads(x), dtype=np.float16)
624623

625624
return map(float, json_loads(x))
626625

@@ -647,8 +646,9 @@ def float16_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
647646
if x is None:
648647
return None
649648

650-
if has_numpy:
651-
return numpy.frombuffer(x, dtype=numpy.float16)
649+
np = get_numpy()
650+
if np is not None:
651+
return np.frombuffer(x, dtype=np.float16)
652652

653653
return struct.unpack(f'<{len(x)//2}e', x)
654654

@@ -675,8 +675,9 @@ def float64_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
675675
if x is None:
676676
return None
677677

678-
if has_numpy:
679-
return numpy.array(json_loads(x), dtype=numpy.float64)
678+
np = get_numpy()
679+
if np is not None:
680+
return np.array(json_loads(x), dtype=np.float64)
680681

681682
return map(float, json_loads(x))
682683

@@ -703,8 +704,9 @@ def float64_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
703704
if x is None:
704705
return None
705706

706-
if has_numpy:
707-
return numpy.frombuffer(x, dtype=numpy.float64)
707+
np = get_numpy()
708+
if np is not None:
709+
return np.frombuffer(x, dtype=np.float64)
708710

709711
return struct.unpack(f'<{len(x)//8}d', x)
710712

@@ -731,8 +733,9 @@ def int8_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
731733
if x is None:
732734
return None
733735

734-
if has_numpy:
735-
return numpy.array(json_loads(x), dtype=numpy.int8)
736+
np = get_numpy()
737+
if np is not None:
738+
return np.array(json_loads(x), dtype=np.int8)
736739

737740
return map(int, json_loads(x))
738741

@@ -759,8 +762,9 @@ def int8_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
759762
if x is None:
760763
return None
761764

762-
if has_numpy:
763-
return numpy.frombuffer(x, dtype=numpy.int8)
765+
np = get_numpy()
766+
if np is not None:
767+
return np.frombuffer(x, dtype=np.int8)
764768

765769
return struct.unpack(f'<{len(x)}b', x)
766770

@@ -787,8 +791,9 @@ def int16_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
787791
if x is None:
788792
return None
789793

790-
if has_numpy:
791-
return numpy.array(json_loads(x), dtype=numpy.int16)
794+
np = get_numpy()
795+
if np is not None:
796+
return np.array(json_loads(x), dtype=np.int16)
792797

793798
return map(int, json_loads(x))
794799

@@ -815,8 +820,9 @@ def int16_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
815820
if x is None:
816821
return None
817822

818-
if has_numpy:
819-
return numpy.frombuffer(x, dtype=numpy.int16)
823+
np = get_numpy()
824+
if np is not None:
825+
return np.frombuffer(x, dtype=np.int16)
820826

821827
return struct.unpack(f'<{len(x)//2}h', x)
822828

@@ -843,8 +849,9 @@ def int32_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
843849
if x is None:
844850
return None
845851

846-
if has_numpy:
847-
return numpy.array(json_loads(x), dtype=numpy.int32)
852+
np = get_numpy()
853+
if np is not None:
854+
return np.array(json_loads(x), dtype=np.int32)
848855

849856
return map(int, json_loads(x))
850857

@@ -871,8 +878,9 @@ def int32_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
871878
if x is None:
872879
return None
873880

874-
if has_numpy:
875-
return numpy.frombuffer(x, dtype=numpy.int32)
881+
np = get_numpy()
882+
if np is not None:
883+
return np.frombuffer(x, dtype=np.int32)
876884

877885
return struct.unpack(f'<{len(x)//4}l', x)
878886

@@ -899,8 +907,9 @@ def int64_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
899907
if x is None:
900908
return None
901909

902-
if has_numpy:
903-
return numpy.array(json_loads(x), dtype=numpy.int64)
910+
np = get_numpy()
911+
if np is not None:
912+
return np.array(json_loads(x), dtype=np.int64)
904913

905914
return map(int, json_loads(x))
906915

@@ -928,8 +937,9 @@ def int64_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
928937
return None
929938

930939
# Bytes
931-
if has_numpy:
932-
return numpy.frombuffer(x, dtype=numpy.int64)
940+
np = get_numpy()
941+
if np is not None:
942+
return np.frombuffer(x, dtype=np.int64)
933943

934944
return struct.unpack(f'<{len(x)//8}l', x)
935945

‎singlestoredb/functions/dtypes.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,9 @@
1111
from ..converters import converters
1212
from ..mysql.converters import escape_item # type: ignore
1313
from ..utils.dtypes import DEFAULT_VALUES # noqa
14-
from ..utils.dtypes import NUMPY_TYPE_MAP # noqa
15-
from ..utils.dtypes import PANDAS_TYPE_MAP # noqa
16-
from ..utils.dtypes import POLARS_TYPE_MAP # noqa
17-
from ..utils.dtypes import PYARROW_TYPE_MAP # noqa
14+
from ..utils.dtypes import get_numpy_type_map # noqa
15+
from ..utils.dtypes import get_polars_type_map # noqa
16+
from ..utils.dtypes import get_pyarrow_type_map # noqa
1817

1918

2019
DataType = Union[str, Callable[..., Any]]

‎singlestoredb/functions/ext/json.py‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,9 @@
77
from typing import TYPE_CHECKING
88

99
from ..dtypes import DEFAULT_VALUES
10-
from ..dtypes import NUMPY_TYPE_MAP
11-
from ..dtypes import PANDAS_TYPE_MAP
12-
from ..dtypes import POLARS_TYPE_MAP
13-
from ..dtypes import PYARROW_TYPE_MAP
10+
from ..dtypes import get_numpy_type_map
11+
from ..dtypes import get_polars_type_map
12+
from ..dtypes import get_pyarrow_type_map
1413
from ..dtypes import PYTHON_CONVERTERS
1514

1615
if TYPE_CHECKING:
@@ -140,7 +139,7 @@ def load_pandas(
140139
(
141140
pd.Series(
142141
data, index=index, name=spec[0],
143-
dtype=PANDAS_TYPE_MAP[spec[1]],
142+
dtype=get_numpy_type_map()[spec[1]],
144143
),
145144
pd.Series(mask, index=index, dtype=np.longlong),
146145
)
@@ -172,7 +171,7 @@ def load_polars(
172171
return pl.Series(None, row_ids, dtype=pl.Int64), \
173172
[
174173
(
175-
pl.Series(spec[0], data, dtype=POLARS_TYPE_MAP[spec[1]]),
174+
pl.Series(spec[0], data, dtype=get_polars_type_map()[spec[1]]),
176175
pl.Series(None, mask, dtype=pl.Boolean),
177176
)
178177
for (data, mask), spec in zip(cols, colspec)
@@ -203,7 +202,7 @@ def load_numpy(
203202
return np.asarray(row_ids, dtype=np.longlong), \
204203
[
205204
(
206-
np.asarray(data, dtype=NUMPY_TYPE_MAP[spec[1]]), # type: ignore
205+
np.asarray(data, dtype=get_numpy_type_map()[spec[1]]), # type: ignore
207206
np.asarray(mask, dtype=np.bool_), # type: ignore
208207
)
209208
for (data, mask), spec in zip(cols, colspec)
@@ -235,7 +234,7 @@ def load_arrow(
235234
[
236235
(
237236
pa.array(
238-
data, type=PYARROW_TYPE_MAP[dtype],
237+
data, type=get_pyarrow_type_map()[dtype],
239238
mask=pa.array(mask, type=pa.bool_()),
240239
),
241240
pa.array(mask, type=pa.bool_()),

‎singlestoredb/functions/ext/rowdat_1.py‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,9 @@
1212
from ...config import get_option
1313
from ...mysql.constants import FIELD_TYPE as ft
1414
from ..dtypes import DEFAULT_VALUES
15-
from ..dtypes import NUMPY_TYPE_MAP
16-
from ..dtypes import PANDAS_TYPE_MAP
17-
from ..dtypes import POLARS_TYPE_MAP
18-
from ..dtypes import PYARROW_TYPE_MAP
15+
from ..dtypes import get_numpy_type_map
16+
from ..dtypes import get_polars_type_map
17+
from ..dtypes import get_pyarrow_type_map
1918

2019
if TYPE_CHECKING:
2120
try:
@@ -212,7 +211,7 @@ def _load_pandas(
212211
index = pd.Series(row_ids)
213212
return pd.Series(row_ids, dtype=np.int64), [
214213
(
215-
pd.Series(data, index=index, name=name, dtype=PANDAS_TYPE_MAP[dtype]),
214+
pd.Series(data, index=index, name=name, dtype=get_numpy_type_map()[dtype]),
216215
pd.Series(mask, index=index, dtype=np.bool_),
217216
)
218217
for (data, mask), (name, dtype) in zip(cols, colspec)
@@ -247,7 +246,7 @@ def _load_polars(
247246
return pl.Series(None, row_ids, dtype=pl.Int64), \
248247
[
249248
(
250-
pl.Series(name=name, values=data, dtype=POLARS_TYPE_MAP[dtype]),
249+
pl.Series(name=name, values=data, dtype=get_polars_type_map()[dtype]),
251250
pl.Series(values=mask, dtype=pl.Boolean),
252251
)
253252
for (data, mask), (name, dtype) in zip(cols, colspec)
@@ -282,7 +281,7 @@ def _load_numpy(
282281
return np.asarray(row_ids, dtype=np.int64), \
283282
[
284283
(
285-
np.asarray(data, dtype=NUMPY_TYPE_MAP[dtype]), # type: ignore
284+
np.asarray(data, dtype=get_numpy_type_map()[dtype]), # type: ignore
286285
np.asarray(mask, dtype=np.bool_), # type: ignore
287286
)
288287
for (data, mask), (name, dtype) in zip(cols, colspec)
@@ -318,7 +317,7 @@ def _load_arrow(
318317
[
319318
(
320319
pa.array(
321-
data, type=PYARROW_TYPE_MAP[dtype],
320+
data, type=get_pyarrow_type_map()[dtype],
322321
mask=pa.array(mask, type=pa.bool_()),
323322
),
324323
pa.array(mask, type=pa.bool_()),
@@ -565,7 +564,7 @@ def _load_pandas_accel(
565564
numpy_ids, numpy_cols = _singlestoredb_accel.load_rowdat_1_numpy(colspec, data)
566565
cols = [
567566
(
568-
pd.Series(data, name=name, dtype=PANDAS_TYPE_MAP[dtype]),
567+
pd.Series(data, name=name, dtype=get_numpy_type_map()[dtype]),
569568
pd.Series(mask, dtype=np.bool_),
570569
)
571570
for (name, dtype), (data, mask) in zip(colspec, numpy_cols)
@@ -610,7 +609,7 @@ def _load_polars_accel(
610609
pl.Series(
611610
name=name, values=data.tolist()
612611
if dtype in string_types or dtype in binary_types else data,
613-
dtype=POLARS_TYPE_MAP[dtype],
612+
dtype=get_polars_type_map()[dtype],
614613
),
615614
pl.Series(values=mask, dtype=pl.Boolean),
616615
)
@@ -653,7 +652,7 @@ def _load_arrow_accel(
653652
numpy_ids, numpy_cols = _singlestoredb_accel.load_rowdat_1_numpy(colspec, data)
654653
cols = [
655654
(
656-
pa.array(data, type=PYARROW_TYPE_MAP[dtype], mask=mask),
655+
pa.array(data, type=get_pyarrow_type_map()[dtype], mask=mask),
657656
pa.array(mask, type=pa.bool_()),
658657
)
659658
for (data, mask), (name, dtype) in zip(numpy_cols, colspec)

0 commit comments

Comments
 (0)