Skip to content

Commit

Permalink
max_cache human writing (#334)
Browse files Browse the repository at this point in the history
* Update DAT.jl

```max_cache``` in bits is fine, but not very human friendly. This is a downstream implementation of the YAXArraysToolbox package to get ```max_cache``` as e.g. ```max_cache="100MB"``` that can be useful for users of the YAXArrays package.

* Update DAT.jl

Now this feature is non-breaking previous code.

* tests with different MB and GBs added

* test for human max_cache fixed
  • Loading branch information
dpabon authored Sep 26, 2023
1 parent 6abb756 commit e5447ed
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 3 deletions.
18 changes: 15 additions & 3 deletions src/DAT/DAT.jl
Original file line number Diff line number Diff line change
Expand Up @@ -401,9 +401,9 @@ Map a given function `fun` over slices of the data cube `cube`.
Use InDims to discribe the input dimensions and OutDims to describe the output dimensions of the function.
### Keyword arguments
* `max_cache=YAXDefaults.max_cache` maximum size of blocks that are read into memory, defaults to approx 10Mb
* `indims::InDims` List of input cube descriptors of type [`InDims`](@ref) for each input data cube
* `outdims::OutDims` List of output cube descriptors of type [`OutDims`](@ref) for each output cube
* `max_cache=YAXDefaults.max_cache` Float64 maximum size of blocks that are read into memory in bits e.g. ```max_cache=5.0e8```. Or String. e.g. ```max_cache="10MB" or ```max_cache=1GB``` defaults to approx 10Mb.
* `indims::InDims` List of input cube descriptors of type [`InDims`](@ref) for each input data cube.
* `outdims::OutDims` List of output cube descriptors of type [`OutDims`](@ref) for each output cube.
* `inplace` does the function write to an output array inplace or return a single value> defaults to `true`
* `ispar` boolean to determine if parallelisation should be applied, defaults to `true` if workers are available.
* `showprog` boolean indicating if a ProgressMeter shall be shown
Expand Down Expand Up @@ -434,6 +434,18 @@ function mapCube(
do_gc = true,
kwargs...,
)
if typeof(max_cache) == String
if last(max_cache, 2) == "MB"
max_cache = parse(Float64, max_cache[begin:end-2]) / (10^-6)

elseif last(max_cache, 2) == "GB"

max_cache = parse(Float64, max_cache[begin:end-2]) / (10^-9)

else
error("only MB or GB values are accepted for max_cache")
end
end

#Translate slices
if any(i -> isa(i, YAXSlice), cdata)
Expand Down
32 changes: 32 additions & 0 deletions test/DAT/mapcube.jl
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,36 @@
@test r.data[] == sum(yax.data)

end

@testset "max cache inputs" begin

x,y,z = X(1:4), Y(1:5), Z(1:6)
a1 = YAXArray((x,y,z), rand(4,5,6))
a2 = YAXArray((x,z,y), rand(4,6,5))
a3 = YAXArray((x,y), rand(4,5))
indims = InDims("x")
outdims = OutDims("x")

function simple_fun(xout, x1,x2)
xout .= x1 .+ x2
end

# Float64
r = mapCube(simple_fun, (a1, a2), indims=(indims, indims), outdims=outdims, max_cache = 6.0e8)
@test r.data == a1.data .+ permutedims(a2.data,(1,3,2))

# MB
r = mapCube(simple_fun, (a1, a2), indims=(indims, indims), outdims=outdims, max_cache = "0.5MB")
@test r.data == a1.data .+ permutedims(a2.data,(1,3,2))

r = mapCube(simple_fun, (a1, a2), indims=(indims, indims), outdims=outdims, max_cache = "3MB")
@test r.data == a1.data .+ permutedims(a2.data,(1,3,2))

r = mapCube(simple_fun, (a1, a2), indims=(indims, indims), outdims=outdims, max_cache = "10MB")
@test r.data == a1.data .+ permutedims(a2.data,(1,3,2))

# GB
r = mapCube(simple_fun, (a1, a2), indims=(indims, indims), outdims=outdims, max_cache = "0.1GB")
@test r.data == a1.data .+ permutedims(a2.data,(1,3,2))
end
end

0 comments on commit e5447ed

Please sign in to comment.