Julia parallel processing in separate modules

Question

I'm trying to write a simple program to do parallel processing with code in separate modules. I have the following code in two separate files:

main.jl

push!(LOAD_PATH, ".")

#using other  # This doesn't work either

importall other

np = 4
b = [Bounds(k, k+8) for k in 1:np]
fut = Array{Future}(np)

for k = 1:np
    fut[k] = @spawn center(b[k])
end

for k = 1:np
    xc = fetch(fut[k])
    println("Center for k ", k, " is ",  xc)
end

other.jl

@everywhere module other

export Bounds
export center

@everywhere type Bounds
    x1::Int
    x2::Int
end

@everywhere function center(bound::Bounds)
    return (bound.x1 + bound.x2) / 2
end 

end

When i run with a single process with "julia main.jl" it runs with no errors, but if I try to add processes with "julia -p4 main.jl" I get the error below. It looks like maybe the additional processes can't see the code in other.jl, but I have the @everywhere macro at all the right places it seems. What is the problem?

ERROR: LoadError: LoadError: On worker 2:
UndefVarError: ##5#7 not defined
 in deserialize_datatype at ./serialize.jl:823
 in handle_deserialize at ./serialize.jl:571
 in deserialize_msg at ./multi.jl:120
 in message_handler_loop at ./multi.jl:1317
 in process_tcp_streams at ./multi.jl:1276
 in #618 at ./event.jl:68
 in #remotecall_fetch#606(::Array{Any,1}, ::Function, ::Function, ::Base.Worker) at ./multi.jl:1070
 in remotecall_fetch(::Function, ::Base.Worker) at ./multi.jl:1062
 in #remotecall_fetch#609(::Array{Any,1}, ::Function, ::Function, ::Int64) at ./multi.jl:1080
 in remotecall_fetch(::Function, ::Int64) at ./multi.jl:1080
 in (::other.##6#8)() at ./multi.jl:1959

...and 3 other exceptions.

 in sync_end() at ./task.jl:311
 in macro expansion; at ./multi.jl:1968 [inlined]
 in anonymous at ./<missing>:?
 in eval(::Module, ::Any) at ./boot.jl:234
 in (::##1#3)() at ./multi.jl:1957
 in sync_end() at ./task.jl:311
 in macro expansion; at ./multi.jl:1968 [inlined]
 in anonymous at ./<missing>:?
 in include_from_node1(::String) at ./loading.jl:488
 in eval(::Module, ::Any) at ./boot.jl:234
 in require(::Symbol) at ./loading.jl:409
 in include_from_node1(::String) at ./loading.jl:488
 in process_options(::Base.JLOptions) at ./client.jl:262
 in _start() at ./client.jl:318
while loading /mnt/mint320/home/bmaier/BillHome/Programs/Julia/parallel/modules/other.jl, in expression starting on line 1
while loading /mnt/mint320/home/bmaier/BillHome/Programs/Julia/parallel/modules/main.jl, in expression starting on line 5

You need to '@everywhere importall other' in main.jl and you don't need the @everywhere's in other.jl — Alexander Morley
Thanks, that worked! I wouldn't have guessed I needed it on the importall statement. — ciric50

Alexander Morley Alexander Morley · Accepted Answer · 2016-11-03T15:54:42

Try @everywhere importall other in main.jl

This will load other on all of the current workers. There is no requirement for other to be anything other than a normal script or module, and so you don't need the @everywhere's in other.jl.

Julia parallel processing in separate modules

1 Answers