Skip to content

Commit d307601

Browse files
committed
Prevent threaded Dask workers in DistRDF backend
1 parent 9a7f3e7 commit d307601

1 file changed

Lines changed: 11 additions & 0 deletions

File tree

  • bindings/distrdf/python/DistRDF/Backends/Dask

bindings/distrdf/python/DistRDF/Backends/Dask/Backend.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,17 @@ def __init__(self, daskclient: Optional[Client] = None):
104104
# N is the number of cores on the local machine.
105105
self.client = (daskclient if daskclient is not None else
106106
Client(LocalCluster(n_workers=os.cpu_count(), threads_per_worker=1, processes=True)))
107+
108+
workers = self.client.scheduler_info()["workers"]
109+
110+
for worker in workers.values():
111+
threads = worker.get("nthreads", 1)
112+
113+
if threads > 1:
114+
raise RuntimeError(
115+
"DistRDF with Dask does not support threaded workers. "
116+
"Please use processes=True and threads_per_worker=1."
117+
)
107118

108119
def optimize_npartitions(self) -> int:
109120
"""

0 commit comments

Comments
 (0)