Skip to content

Commit 84e7794

Browse files
committed
Prevent threaded Dask workers in DistRDF backend
1 parent 681b003 commit 84e7794

1 file changed

Lines changed: 11 additions & 0 deletions

File tree

  • bindings/distrdf/python/DistRDF/Backends/Dask

bindings/distrdf/python/DistRDF/Backends/Dask/Backend.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,17 @@ def __init__(self, daskclient: Optional[Client] = None):
105105
# N is the number of cores on the local machine.
106106
self.client = (daskclient if daskclient is not None else
107107
Client(LocalCluster(n_workers=os.cpu_count(), threads_per_worker=1, processes=True)))
108+
109+
workers = self.client.scheduler_info()["workers"]
110+
111+
for worker in workers.values():
112+
threads = worker.get("nthreads", 1)
113+
114+
if threads > 1:
115+
raise RuntimeError(
116+
"DistRDF with Dask does not support threaded workers. "
117+
"Please use processes=True and threads_per_worker=1."
118+
)
108119

109120
def optimize_npartitions(self) -> int:
110121
"""

0 commit comments

Comments
 (0)