Switch to extensions instead of requires for datasets (#24)

* Switch to extensions instead of requires for datasets

* Move all array interface implementations to extensions

This moves everything from using Requires to the new extensions setup.

* Switch the array tests to testitems

This ensures, that the tests for different backends are independent.

* Remove the Requires setup for lower julia versions

We decided to only support julia versions larger than 1.9

* Switch CI to test on julia 1.9 instead of 1.6

When we restrict the Julia version to 1.9 we should also stop testing on 1.6

* Remove Zarr and NetCDF as dependencies

* Remove AxisIndices

This seems to be not maintained anymore with the last commit three years ago

* Stop tracking Manifest.toml

* Remove AxisIndices also from test environment

* Add testing on nightly

* Bump version to 0.7.0

.github/workflows/CI.yml

@@ -13,8 +13,9 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.6'
+          - '1.9'
           - '1'
+          - 'nightly'
         os:
           - ubuntu-latest
           - macOS-latest

.gitignore

@@ -3,3 +3,4 @@
 *.jl.mem
 /deps/deps.jl
 /docs/build
+Manifest.toml

Project.toml

@@ -1,16 +1,38 @@
 name = "YAXArrayBase"
 uuid = "90b8fcef-0c2d-428d-9c56-5f86629e9d14"
 authors = ["Fabian Gans <fgans@bgc-jena.mpg.de>"]
-version = "0.6.1"
+version = "0.7.0"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
-Requires = "ae029012-a4dd-5104-9daa-d747884805df"
+
 
 [compat]
-DataStructures = "0.17, 0.18"
-Requires = "1"
-julia = "1.6"
+DataStructures = "0.17,0.18"
+julia = "1.9"
+ArchGDAL = "0.10"
+AxisArrays = "0.4"
+AxisKeys = "0.2"
+DimensionalData = "0.24"
+NetCDF = "0.11"
+Zarr = "0.8"
+
+[extensions]
+ArchGDALExt = "ArchGDAL"
+AxisArraysExt = "AxisArrays"
+AxisKeysExt = "AxisKeys"
+DimensionalDataExt = "DimensionalData"
+NamedDimsExt = "NamedDims"
+NetCDFExt = "NetCDF"
+ZarrExt = "Zarr"
 
+[weakdeps]
+ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
+AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9"
+AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5"
+DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0"
+NamedDims = "356022a1-0364-5f58-8944-0da4b18d706f"
+NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9"
+Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99"

src/axisarrays/archgdal.jl → ext/ArchGDALExt/ArchGDALExt.jl

@@ -1,6 +1,11 @@
-import .ArchGDAL: RasterDataset, AbstractRasterBand,
+module ArchGDALExt
+import ArchGDAL: RasterDataset, AbstractRasterBand,
   getgeotransform, width, height, getname, getcolorinterp,
   getband, nraster, getdataset
+using ArchGDAL: ArchGDAL as AG
+  import YAXArrayBase: dimname, dimnames, dimvals, iscontdim, getattributes, getdata, yaxcreate
+
+#include("archgdaldataset.jl")
 
 function dimname(a::RasterDataset, i)
     if i == 1
@@ -33,10 +38,10 @@ end
 iscontdim(a::RasterDataset, i) = i < 3 ? true : nraster(a)<8
 function getattributes(a::RasterDataset)
     globatts = Dict{String,Any}(
-        "projection_PROJ4"=>ArchGDAL.toPROJ4(ArchGDAL.newspatialref(ArchGDAL.getproj(a))),
-        "projection_WKT"=>ArchGDAL.toWKT(ArchGDAL.newspatialref(ArchGDAL.getproj(a))),
+        "projection_PROJ4"=>AG.toPROJ4(AG.newspatialref(AG.getproj(a))),
+        "projection_WKT"=>AG.toWKT(AG.newspatialref(AG.getproj(a))),
     )
-    bands = (getbandattributes(ArchGDAL.getband(a, i)) for i in 1:size(a, 3))
+    bands = (getbandattributes(AG.getband(a, i)) for i in 1:size(a, 3))
     allbands = mergewith(bands...) do a1,a2
         isequal(a1,a2) ? a1 : missing
     end
@@ -65,7 +70,7 @@ function dimvals(b::AbstractRasterBand, i)
 end
 iscontdim(a::AbstractRasterBand, i) = true
 function getattributes(a::AbstractRasterBand)
-  atts = getattributes(ArchGDAL.RasterDataset(ArchGDAL.getdataset(a)))
+  atts = getattributes(AG.RasterDataset(AG.getdataset(a)))
   bandatts = getbandattributes(a)
   merge(atts, bandatts)
 end
@@ -84,4 +89,5 @@ function getbandattributes(a::AbstractRasterBand)
   insertattifnot!(atts, AG.getoffset(a), "add_offset", iszero)
   insertattifnot!(atts, AG.getscale(a), "scale_factor", x->isequal(x, one(x)))
   atts
+end
 end

src/datasets/archgdal.jl → ext/ArchGDALExt/archgdaldataset.jl

@@ -1,9 +1,3 @@
-import .ArchGDAL: RasterDataset, AbstractRasterBand,
-  getgeotransform, width, height, getname, getcolorinterp,
-  getband, nraster, getdataset, ArchGDAL
-using .ArchGDAL.DiskArrays: GridChunks, DiskArrays, eachchunk
-const AG = ArchGDAL
-
 struct GDALBand{T} <: AG.DiskArrays.AbstractDiskArray{T,2}
     filename::String
     band::Int
@@ -221,8 +215,10 @@ allow_parallel_write(::GDALDataset) = false
 allow_missings(::Type{<:GDALDataset}) = false
 allow_missings(::GDALDataset) = false
 
-backendlist[:gdal] = GDALDataset
-push!(backendregex,r".tif$"=>GDALDataset)
-push!(backendregex,r".gtif$"=>GDALDataset)
-push!(backendregex,r".tiff$"=>GDALDataset)
-push!(backendregex,r".gtiff$"=>GDALDataset)
+function __init__()
+    backendlist[:gdal] = GDALDataset
+    push!(backendregex,r".tif$"=>GDALDataset)
+    push!(backendregex,r".gtif$"=>GDALDataset)
+    push!(backendregex,r".tiff$"=>GDALDataset)
+    push!(backendregex,r".gtiff$"=>GDALDataset)
+end

src/axisarrays/axisarrays.jl → ext/AxisArraysExt.jl

@@ -1,5 +1,6 @@
-using .AxisArrays: AxisArrays, AxisArray
-
+module AxisArraysExt
+using AxisArrays: AxisArrays, AxisArray
+import YAXArrayBase: dimname, dimnames, dimvals, iscontdim, getattributes, getdata, yaxcreate
 dimname(a::AxisArray, i) = AxisArrays.axisnames(a)[i]
 dimnames(a::AxisArray) = AxisArrays.axisnames(a)
 dimvals(a::AxisArray, i) = AxisArrays.axisvalues(a)[i]
@@ -11,3 +12,4 @@ function yaxcreate(::Type{<:AxisArray}, data, dnames, dvals, atts)
   end
   AxisArray(data; d...)
 end
+end

src/axisarrays/axiskeys.jl → ext/AxisKeysExt.jl

@@ -1,5 +1,6 @@
-import .AxisKeys
-
+module AxisKeysExt
+import AxisKeys
+import YAXArrayBase: dimname, dimnames, dimvals, iscontdim, getattributes, getdata, yaxcreate
 dimnames(a::AxisKeys.KeyedArray) = AxisKeys.dimnames(a)
 
 dimvals(a::AxisKeys.KeyedArray,i) = AxisKeys.getproperty(a,AxisKeys.dimnames(a,i))
@@ -8,3 +9,4 @@ getdata(a::AxisKeys.KeyedArray) = parent(parent(a))
 
 yaxcreate(::Type{<:AxisKeys.KeyedArray}, data, dnames, dvals, atts) =
   AxisKeys.KeyedArray(data; map(i->dnames[i]=>dvals[i],1:ndims(data))...)
+end

src/axisarrays/dimensionaldata.jl → ext/DimensionalDataExt.jl

@@ -1,5 +1,6 @@
-using .DimensionalData: DimArray, DimensionalData, data, Dim, metadata
-
+module DimensionalDataExt
+using DimensionalData: DimArray, DimensionalData, data, Dim, metadata
+import YAXArrayBase: dimname, dimnames, dimvals, iscontdim, getattributes, getdata, yaxcreate
 _dname(::DimensionalData.Dim{N}) where N = N
 _dname(d::DimensionalData.Dimension) = DimensionalData.name(d)
 dimname(x::DimArray, i) = _dname(DimensionalData.dims(x)[i])
@@ -17,3 +18,4 @@ function yaxcreate(::Type{<:DimArray},data,dnames,dvals,atts)
   end
   DimArray(data,d,metadata = atts)
 end
+end

src/axisarrays/nameddims.jl → ext/NamedDimsExt.jl

@@ -1,8 +1,11 @@
-using .NamedDims: NamedDimsArray
+module NamedDimsExt
+using NamedDims: NamedDimsArray
+import YAXArrayBase: dimname, dimnames, dimvals, iscontdim, getattributes, getdata, yaxcreate
 dimname(a::NamedDimsArray{N},i) where N = N[i]
 dimnames(a::NamedDimsArray{N}) where N = N
 getdata(a::NamedDimsArray) = parent(a)
 function yaxcreate(::Type{<:NamedDimsArray},data, dnames, dvals, atts)
   n = ntuple(i->dnames[i],ndims(data))
   NamedDimsArray(data,n)
 end
+end

src/datasets/netcdf.jl → ext/NetCDFExt.jl

@@ -1,4 +1,6 @@
-using .NetCDF
+module NetCDFExt
+import YAXArrayBase: YAXArrayBase as YAB
+using NetCDF
 
 """
     NetCDFDataset
@@ -43,37 +45,40 @@ function readblock!(v::NetCDFVariable, aout, r::AbstractUnitRange...)
     aout .= aouttemp
   end
 end
-iscompressed(v::NetCDFVariable) = NetCDF.open(v->v.compress > 0, v.filename, v.varname)
+YAB.iscompressed(v::NetCDFVariable) = NetCDF.open(v->v.compress > 0, v.filename, v.varname)
 
 Base.size(v::NetCDFVariable) = v.size
 
-get_var_dims(ds::NetCDFDataset,name) = NetCDF.open(v->map(i->i.name,v[name].dim),ds.filename)
-get_varnames(ds::NetCDFDataset) = NetCDF.open(v->collect(keys(v.vars)),ds.filename)
-get_var_attrs(ds::NetCDFDataset, name) = NetCDF.open(v->v[name].atts,ds.filename)
-get_global_attrs(ds::NetCDFDataset) = NetCDF.open(nc->nc.gatts, ds.filename)
+YAB.get_var_dims(ds::NetCDFDataset,name) = NetCDF.open(v->map(i->i.name,v[name].dim),ds.filename)
+YAB.get_varnames(ds::NetCDFDataset) = NetCDF.open(v->collect(keys(v.vars)),ds.filename)
+YAB.get_var_attrs(ds::NetCDFDataset, name) = NetCDF.open(v->v[name].atts,ds.filename)
+YAB.get_global_attrs(ds::NetCDFDataset) = NetCDF.open(nc->nc.gatts, ds.filename)
 function Base.getindex(ds::NetCDFDataset, i)
   s,et = NetCDF.open(j->(size(j),eltype(j)),ds.filename,i)
   NetCDFVariable{et,length(s)}(ds.filename, i, s)
 end
 Base.haskey(ds::NetCDFDataset,k) = NetCDF.open(nc->haskey(nc.vars,k),ds.filename)
 
-function add_var(p::NetCDFDataset, T::Type, varname, s, dimnames, attr;
+function YAB.add_var(p::NetCDFDataset, T::Type, varname, s, dimnames, attr;
   chunksize=s, compress = -1)
   dimsdescr = Iterators.flatten(zip(dimnames,s))
   nccreate(p.filename, varname, dimsdescr..., atts = attr, t=T, chunksize=chunksize, compress=compress)
   NetCDFVariable{T,length(s)}(p.filename,varname,(s...,))
 end
 
-function create_empty(::Type{NetCDFDataset}, path, gatts=Dict())
+function YAB.create_empty(::Type{NetCDFDataset}, path, gatts=Dict())
   NetCDF.create(_->nothing, path, NcVar[], gatts = gatts)
   NetCDFDataset(path)
 end
 
-allow_parallel_write(::Type{<:NetCDFDataset}) = false
-allow_parallel_write(::NetCDFDataset) = false
+YAB.allow_parallel_write(::Type{<:NetCDFDataset}) = false
+YAB.allow_parallel_write(::NetCDFDataset) = false
 
-allow_missings(::Type{<:NetCDFDataset}) = false
-allow_missings(::NetCDFDataset) = false
+YAB.allow_missings(::Type{<:NetCDFDataset}) = false
+YAB.allow_missings(::NetCDFDataset) = false
 
-backendlist[:netcdf] = NetCDFDataset
-push!(backendregex,r".nc$"=>NetCDFDataset)
+function __init__()
+YAB.backendlist[:netcdf] = NetCDFDataset
+push!(YAB.backendregex,r".nc$"=>NetCDFDataset)
+end
+end