JuliaParallel / JuliaParallel/DistributedArrays.jl

Write DArray to HDF5/JLD without first converting to Array

Abierto
#108 3 comentarios 0 reacciones 0 asignados Ver en GitHub
Lenguaje dominante
Julia
Estrellas
205
Forks
34
Métricas de merge de PR
Sin PR fusionados en 30 d

Descripción

It would be useful to have a way to write `DArray`s to file without first collecting all of the data on the local process. I found a pretty hacky way of doing this that looks like this:

``` julia
function write_darray{T<:AbstractFloat}(filepath::AbstractString, darr::DArray{T})
function write_localpart(pid::Int)
jldopen(filepath, "r+") do file
write(file, "inds$pid", collect(localindexes(darr)))
write(file, "arr$pid", localpart(darr))
end
end

# Write DArray metadata
jldopen(filepath, "w") do file
write(file, "dims", darr.dims)
write(file, "pids", collect(darr.pids))
end

# For each process, write the local indices and local part
for pid in darr.pids
remotecall_wait(pid, write_localpart, pid)
sleep(0.001) # This is for some reason necessary, or else HDF5 complains that the new object already exists
end
end

function read_darray(filepath::AbstractString)
file = jldopen(filepath, "r")
dims = read(file, "dims")
pids = read(file, "pids")
out = zeros(dims...)

# Reassemble local parts
for pid in pids
inds = read(file, "inds$pid")
out[inds...] = read(file, "arr$pid")
end

close(file)
return out
end
```

However, there must be a better way than this, especially one that would create a single variable in the file `filepath` (rather than `2n+2` in my case, where `n` is the number of processes the `DArray` is stored on).

Guía de contribución

No hay ninguna guía de contribución indexada para este repositorio

Evaluación

Este issue todavía no se ha evaluado.

Recibe los nuevos issues en tu correo

Un resumen breve de issues de GitHub para principiantes.