Merge pull request #9 from meggart/DiskArray-0.3

Switch to DiskArrays 0.3

.github/workflows/CI.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.3'
+          - '1.6'
           - '1'
         os:
           - ubuntu-latest

.github/workflows/CompatHelper.yml

@@ -1,19 +1,32 @@
 name: CompatHelper
-
 on:
   schedule:
-    - cron: '00 00 * * *'
-
+    - cron: 0 0 * * *
+  workflow_dispatch:
 jobs:
   CompatHelper:
     runs-on: ubuntu-latest
     steps:
-      - uses: julia-actions/setup-julia@latest
-        with:
-          version: 1.3
-      - name: Pkg.add("CompatHelper")
-        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
-      - name: CompatHelper.main()
+      - name: "Add the General registry via Git"
+        run: |
+          import Pkg
+          ENV["JULIA_PKG_SERVER"] = ""
+          Pkg.Registry.add("General")
+        shell: julia --color=yes {0}
+      - name: "Install CompatHelper"
+        run: |
+          import Pkg
+          name = "CompatHelper"
+          uuid = "aa819f21-2bde-4658-8897-bab36330d9b7"
+          version = "3"
+          Pkg.add(; name, uuid, version)
+        shell: julia --color=yes {0}
+      - name: "Run CompatHelper"
+        run: |
+          import CompatHelper
+          CompatHelper.main()
+        shell: julia --color=yes {0}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: julia -e 'using CompatHelper; CompatHelper.main()'
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+          # COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }}

Project.toml

@@ -1,7 +1,7 @@
 name = "DiskArrayTools"
 uuid = "fcd2136c-9f69-4db6-97e5-f31981721d63"
 authors = ["Fabian Gans <fgans@bgc-jena.mpg.de>"]
-version = "0.1.2"
+version = "0.1.3"
 
 [deps]
 DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3"
@@ -10,11 +10,11 @@ IterTools = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
 OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
 
 [compat]
-DiskArrays = "0.2.4"
+DiskArrays = "0.3"
 Interpolations = "0.12, 0.13"
 IterTools = "1"
 OffsetArrays = "1"
-julia = "1.3"
+julia = "1.6"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

src/DiskArrayTools.jl

@@ -1,33 +1,12 @@
 module DiskArrayTools
 import DiskArrays: AbstractDiskArray, eachchunk, haschunks, Chunked,
-estimate_chunksize, GridChunks, findints, readblock!, writeblock!
+estimate_chunksize, GridChunks, findints, readblock!, writeblock!, 
+RegularChunks, IrregularChunks, ChunkType, approx_chunksize
 using Interpolations
 using IterTools: imap
 using Base.Iterators: product
 using OffsetArrays: OffsetArray
 
-#Define a new chunk grid type for irregular, but still block-shaped grid
-struct IrregularGridChunks{N} <: AbstractArray{CartesianIndices{N,NTuple{N,UnitRange{Int}}},N}
-    cs::NTuple{N,Vector{UnitRange{Int}}}
-    s::NTuple{N,Int}
-end
-Base.size(cs::IrregularGridChunks) = cs.s
-Base.size(cs::IrregularGridChunks,i) = cs.s[i]
-function IrregularGridChunks(cs::Vector{UnitRange{Int}}...)
-    s = length.(cs)
-    IrregularGridChunks((cs...,),s)
-end
-Base.eltype(::IrregularGridChunks{N}) where N = CartesianIndices{N,NTuple{N,UnitRange{Int}}}
-function Base.show(io::IO,g::IrregularGridChunks)
-    stot = last.(g.cs)
-    griddes = join(string.(g.s), "x")
-    sizedes = join(string.(stot), "x")
-    print(io,"Irregular chunk grid of size $griddes over DiskArray of size $stot")
-end
-function Base.getindex(cs::IrregularGridChunks{N},i::Vararg{Int, N}) where N
-    rout = getindex.(cs.cs,i)
-    CartesianIndices(rout)
-end
 
 export DiskArrayStack, diskstack, ConcatDiskArray, CFDiskArray
 struct DiskArrayStack{T,N,M,NO}<:AbstractDiskArray{T,N}
@@ -40,11 +19,10 @@ end
 Base.size(r::DiskArrayStack) = (size(r.arrays[1])...,size(r.arrays)...)
 haschunks(a::DiskArrayStack) = haschunks(a.arrays[1])
 iscompressed(a::DiskArrayStack) = any(iscompressed,a.arrays)
-function eachchunk(a::DiskArrayStack{<:Any,<:Any,<:Any,NO}) where NO
-    iterold = eachchunk(a.arrays[1])
-    cs = (iterold.chunksize...,ntuple(one,NO)...)
-    co = (iterold.offset...,ntuple(zero,NO)...)
-    GridChunks(a,cs,offset=co)
+function eachchunk(a::DiskArrayStack)
+    oldchunks = eachchunk(a.arrays[1]).chunks
+    newchunks = map(s->RegularChunks(1,0,s),size(a.arrays))
+    GridChunks(oldchunks..., newchunks...)
 end
 
 function readblock!(a::DiskArrayStack{<:Any,N,<:Any,NO},aout,i::AbstractVector...) where {N,NO}
@@ -287,52 +265,31 @@ function writeblock!(a::ConcatDiskArray, aout, inds::AbstractUnitRange...)
   error("No method yet for writing into a ConcatDiskArray")
 end
 
-function eachchunk_fallback(aconc::ConcatDiskArray)
-  nested = imap(zip(aconc.parents, Iterators.product(aconc.startinds...))) do (par, si)
-      imap(eachchunk(par)) do cI
-          cI .+ CartesianIndex(si.-1)
-      end
+function mergechunks(a::RegularChunks, b::RegularChunks)
+  if a.s==0 || (a.cs == b.cs && length(last(a))==a.cs)
+    RegularChunks(a.cs, a.offset,a.s+b.s)
+  else
+    mergechunks_irregular(a,b)
   end
-  Iterators.flatten(nested)
+end
+
+mergechunks(a::ChunkType, b::ChunkType) = mergechunks_irregular(a,b)
+function mergechunks_irregular(a, b)
+  IrregularChunks(chunksizes = filter(!iszero,[length.(a); length.(b)]))
 end
 
 #This function will work for DiskArrayStack as well, we only need a fallback_chunksize function
 function eachchunk(aconc::ConcatDiskArray)
-  fbchunks = eachchunk_fallback(aconc)
-  nd = ndims(first(fbchunks))
-  dout = ntuple(_->Dict{NTuple{nd-1,UnitRange{Int}},Set{UnitRange{Int}}}(),nd)
-  foreach(fbchunks) do c
-    map(enumerate(c.indices)) do (ind,r)
-        other = (c.indices[1:ind-1]...,c.indices[ind+1:end]...)
-        s = get!(dout[ind],other) do 
-            Set{UnitRange{Int}}()
-        end
-        push!(s,c.indices[ind])
-    end
-  end
-  #First check if all 
-  uniqueranges = unique.(values.(dout)) # Here we give up
-  if any(length.(uniqueranges) .> 1)
-    return fbchunks
-  end
-  simplifyaxes = map(uniqueranges) do ur
-    allr = first(ur)
-    allr = sort!(collect(allr),by=first)
-    l = length.(allr)
-    if length(l)==1 
-        allr,(cs = l[1], offs = 0, l = l[1])
-    elseif all(isequal(l[2]),l[2:end-1]) && l[end]<=l[2]
-        allr,(cs = l[2], offs = l[2]-l[1], l = allr[end][end])
-    else
-        allr,nothing
-    end
-  end
-  if all(!isnothing,getindex.(simplifyaxes,2))
-    grid = getindex.(simplifyaxes,2)
-    cs,offs,l = getproperty.(grid,:cs),getproperty.(grid,:offs),getproperty.(grid,:l)
-    return GridChunks(l,cs, offset = offs)
-  else
-    return IrregularGridChunks(getindex.(simplifyaxes,1)...)
+  N = ndims(aconc)
+  s = size(aconc)
+  oldchunks = eachchunk.(aconc.parents)
+  newchunks = ntuple(N) do i
+    sliceinds = Base.setindex(map(_ -> 1,s),:,i)
+    v = map(c->c.chunks[i],oldchunks[sliceinds...])
+    init = RegularChunks(approx_chunksize(first(v)),0,0)
+    reduce(mergechunks, v, init=init)
   end
+  GridChunks(newchunks...)
 end
+
 end # module

test/concatdiskarray.jl

@@ -50,55 +50,35 @@ end
     ad = _DiskArray.(a)
     aconc = ConcatDiskArray(ad)
     @test isapprox(sum(aconc),sum(aar))
-    @test_broken copy(aconc .+ ones(1,15)) == aar .+ ones(1,15)
-    @test length(ad[3,5].getindex_list) == 1
+    @test copy(aconc .+ ones(1,15)) == aar .+ ones(1,15)
+    @test length(ad[3,5].getindex_list) == 2
 end
 
 @testset "Irregular chunks" begin
     a = [rand(i,j,4) for i = 1:10, j=1:5]
     a1 = [cat(x..., dims = 1) for x in eachcol(a)]
     aar = cat(a1...,dims = 2)
-    ad = _DiskArray.(a, chunksize = (size(a,1),1,2))
+    ad = [_DiskArray(aa, chunksize = (size(aa,1),1,2)) for aa in a]
     aconc = ConcatDiskArray(ad)
 
     cs1 = DiskArrays.eachchunk(aconc)
-    @test cs1 isa DiskArrayTools.IrregularGridChunks{3}
-    @test cs1.cs[1] == [1:1,2:3,4:6,7:10,11:15,16:21,22:28,29:36,37:45,46:55]
-    @test cs1.cs[2] == [i:i for i in 1:15]
-    @test cs1.cs[3] == [1:2,3:4]
+    @test cs1 isa DiskArrays.GridChunks
+    @test cs1.chunks[1] == [1:1,2:3,4:6,7:10,11:15,16:21,22:28,29:36,37:45,46:55]
+    @test cs1.chunks[2] == DiskArrays.RegularChunks(1,0,15)
+    @test cs1.chunks[3] == DiskArrays.RegularChunks(2,0,4)
 
     @test isapprox(sum(aconc),sum(aar))
-#    This will be fixed when broadcast can deal with irregular chunks
-    @test_broken copy(aconc .+ ones(1,15)) == aar .+ ones(1,15)
+
     @test length(ad[3,5].getindex_list) == 10
+#    This will be fixed when broadcast can deal with irregular chunks
+    @test copy(aconc .+ ones(1,15)) == aar .+ ones(1,15)
+    @test length(ad[3,5].getindex_list) == 20
 
     @test isapprox.(sum(aconc,dims=1), sum(aar,dims=1)) |> all
 
-    @test length(ad[3,5].getindex_list)==20
+    @test length(ad[3,5].getindex_list)==30
 end
 
-@testset "Random chunks" begin
-    a = [rand(i,j,4) for i = 1:10, j=1:5]
-    a1 = [cat(x..., dims = 1) for x in eachcol(a)]
-    aar = cat(a1...,dims = 2)
-    ad = [_DiskArray(a[i,j], chunksize = (iseven(j) ? 2 : 1,j,4)) for i in 1:10, j in 1:5]
-    aconc = ConcatDiskArray(ad)
-
-    cs1 = DiskArrays.eachchunk(aconc)
-    allchunks = collect(cs1)
-
-    @test length(allchunks) == 225
-    @test isapprox(sum(aconc),sum(aar))
-    # This will be fixed when broadcast can deal with irregular chunks
-    @test_broken copy(aconc .+ ones(1,15)) == aar .+ ones(1,15)
-    @test ad[3,5].getindex_list == [(1:1, 1:5, 1:4), (2:2, 1:5, 1:4),(3:3, 1:5, 1:4)]
-    @test ad[3,4].getindex_list == [(1:2, 1:4, 1:4), (3:3, 1:4, 1:4)]
-
-    @test isapprox.(sum(aconc,dims=1), sum(aar,dims=1)) |> all
-
-    @test length(ad[3,5].getindex_list)==6
-    @test length(ad[3,4].getindex_list)==4
-end
 
 @testset "Regular chunks" begin
     a = [rand(i,j,4) for i = 1:10, j=1:5]