Skip to content

Commit 1c82e6b

Browse files
Merge pull request #38 from LCSB-BioCore/mk-retry-remote-macro-again
retry remote macro PR again
2 parents 5bd8cfd + ba0eb30 commit 1c82e6b

File tree

5 files changed

+64
-11
lines changed

5 files changed

+64
-11
lines changed

docs/make.jl

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
using Documenter, DistributedData
22

3-
makedocs(modules = [DistributedData],
3+
makedocs(
4+
modules = [DistributedData],
45
clean = false,
5-
format = Documenter.HTML(prettyurls = !("local" in ARGS),
6-
canonical = "https://lcsb-biocore.github.io/DistributedData.jl/stable/",
7-
assets = ["assets/logo.ico"]),
6+
format = Documenter.HTML(
7+
prettyurls = !("local" in ARGS),
8+
canonical = "https://lcsb-biocore.github.io/DistributedData.jl/stable/",
9+
assets = ["assets/logo.ico"],
10+
),
811
sitename = "DistributedData.jl",
912
authors = "The developers of DistributedData.jl",
1013
linkcheck = !("skiplinks" in ARGS),
@@ -20,5 +23,5 @@ deploydocs(
2023
target = "build",
2124
branch = "gh-pages",
2225
devbranch = "develop",
23-
push_preview = true
26+
push_preview = true,
2427
)

docs/slurm-example/run-analysis.jl

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
using Distributed, ClusterManagers, DistributedData
1+
using Distributed, ClusterManagers, DistributedData
22

33
# read the number of available workers from environment and start the worker processes
4-
n_workers = parse(Int , ENV["SLURM_NTASKS"])
5-
addprocs_slurm(n_workers , topology =:master_worker)
4+
n_workers = parse(Int, ENV["SLURM_NTASKS"])
5+
addprocs_slurm(n_workers, topology = :master_worker)
66

77
# load the required packages on all workers
88
@everywhere using DistributedData
99

1010
# generate a random dataset on all workers
11-
dataset = dtransform((), _ -> randn(10000,10000), workers(), :myData)
11+
dataset = dtransform((), _ -> randn(10000, 10000), workers(), :myData)
1212

1313
# for demonstration, sum the whole dataset
1414
totalResult = dmapreduce(dataset, sum, +)

src/DistributedData.jl

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ export save_at,
1919
dmap,
2020
dpmap,
2121
gather_array,
22-
tmp_symbol
22+
tmp_symbol,
23+
@remote
2324

2425
include("io.jl")
2526
export dstore, dload, dunlink

src/base.jl

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
12
"""
23
save_at(worker, sym, val)
34
@@ -46,7 +47,7 @@ Get a value `val` from a remote `worker`; quoting of `val` works just as with
4647
`save_at`. Returns a future with the requested value.
4748
"""
4849
function get_from(worker, val; mod = Main)
49-
remotecall(() -> Base.eval(mod, :($val)), worker)
50+
remotecall(() -> Base.eval(mod, val), worker)
5051
end
5152

5253
"""
@@ -358,3 +359,41 @@ Decorate the symbol from `dInfo` with prefix and suffix.
358359
function tmp_symbol(dInfo::Dinfo; prefix = "", suffix = "_tmp")
359360
return tmp_symbol(dInfo.val, prefix = prefix, suffix = suffix)
360361
end
362+
363+
"""
364+
@remote module expr
365+
366+
A version of [`@remote`](@ref) that adds additional choice of the module for
367+
scope.
368+
"""
369+
macro remote(mod, x)
370+
:(Base.eval($mod, $(QuoteNode(x))))
371+
end
372+
373+
"""
374+
@remote expr
375+
376+
In a function that will get evaluated on a remote worker, this ensures the
377+
evaluation scope of the expression `expr` (usually a variable) is taken on
378+
the remote side, preventing namespace clash with the local session.
379+
380+
This is mainly useful for making the functions from `Distributed` package (such
381+
as `pmap` and `remotecall`) work with the data stored by `DistributedData`
382+
package.
383+
384+
Internally, this is handled by wrapping in `eval`.
385+
386+
# Example
387+
```
388+
julia> save_at(2, :x, 321)
389+
Future(2, 1, 162, nothing)
390+
391+
julia> let x=123
392+
remotecall_fetch(() -> x + (@remote x), 2)
393+
end
394+
444
395+
```
396+
"""
397+
macro remote(x)
398+
:(@remote Main $x)
399+
end

test/base.jl

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,16 @@
129129
@test all([!isfile(f) for f in files])
130130
end
131131

132+
@testset "@remote macro" begin
133+
di = dtransform(:(), _ -> myid(), W, :test)
134+
135+
test = 333
136+
137+
for pid in W
138+
@test remotecall_fetch(() -> test + (@remote test), pid) == test + pid
139+
end
140+
end
141+
132142
rmprocs(W)
133143
W = nothing
134144

0 commit comments

Comments
 (0)