Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make get_array work with all supported data types. #12

Merged
merged 3 commits into from
Jul 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion lib/metadata.ml
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,9 @@
| Bigarray.Float32, Extensions.Datatype.Float32
| Bigarray.Float64, Extensions.Datatype.Float64
| Bigarray.Complex32, Extensions.Datatype.Complex32
| Bigarray.Complex64, Extensions.Datatype.Complex64 -> true
| Bigarray.Complex64, Extensions.Datatype.Complex64
| Bigarray.Int, Extensions.Datatype.Int
| Bigarray.Nativeint, Extensions.Datatype.Nativeint -> true

Check warning on line 199 in lib/metadata.ml

View check run for this annotation

Codecov / codecov/patch

lib/metadata.ml#L198-L199

Added lines #L198 - L199 were not covered by tests
| _ -> false

let fillvalue_of_kind
Expand All @@ -208,6 +210,8 @@
| Bigarray.Int16_unsigned, FillValue.Int i -> Int64.to_int i
| Bigarray.Int32, FillValue.Int i -> Int64.to_int32 i
| Bigarray.Int64, FillValue.Int i -> i
| Bigarray.Int, FillValue.Int i -> Int64.to_int i
| Bigarray.Nativeint, FillValue.Int i -> Int64.to_nativeint i

Check warning on line 214 in lib/metadata.ml

View check run for this annotation

Codecov / codecov/patch

lib/metadata.ml#L213-L214

Added lines #L213 - L214 were not covered by tests
| Bigarray.Float32, FillValue.Float f -> f
| Bigarray.Float32, FillValue.FloatBits f -> f
| Bigarray.Float64, FillValue.Float f -> f
Expand Down
41 changes: 6 additions & 35 deletions lib/node.ml
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,7 @@
not (String.for_all (Char.equal '.') name) &&
not (String.starts_with ~prefix:"__" name)

module rec GroupNode : sig
type t
val root : t
val create : t -> string -> (t, [> error]) result
val ( / ) : t -> string -> (t, [> error]) result
val of_path : string -> (t, [> error]) result
val to_path : t -> string
val name : t -> string
val parent : t -> t option
val ( = ) : t -> t -> bool
val ancestors : t -> t list
val to_key : t -> string
val to_prefix : t -> string
val to_metakey : t -> string
val is_child_group : t -> t -> bool
val show : t -> string
val pp : Format.formatter -> t -> unit
end = struct
module GroupNode = struct
type t =
| Root
| Cons of t * string
Expand Down Expand Up @@ -107,23 +90,11 @@
let show n = to_path n
end

and ArrayNode : sig
type t
val create : GroupNode.t -> string -> (t, [> error]) result
val ( / ) : GroupNode.t -> string -> (t, [> error]) result
val of_path : string -> (t, [> error]) result
val to_path : t -> string
val name : t -> string
val parent : t -> GroupNode.t
val ( = ) : t -> t -> bool
val ancestors : t -> GroupNode.t list
val is_parent : t -> GroupNode.t -> bool
val to_key : t -> string
val to_metakey : t -> string
val show : t -> string
val pp : Format.formatter -> t -> unit
end = struct
type t = {parent : GroupNode.t; name : string} [@@deriving show]
module ArrayNode = struct
type t =

Check warning on line 94 in lib/node.ml

View check run for this annotation

Codecov / codecov/patch

lib/node.ml#L94

Added line #L94 was not covered by tests
{parent : GroupNode.t
;name : string}
[@@deriving show]

Check warning on line 97 in lib/node.ml

View check run for this annotation

Codecov / codecov/patch

lib/node.ml#L96-L97

Added lines #L96 - L97 were not covered by tests

let create parent name =
if rep_ok name then
Expand Down
21 changes: 8 additions & 13 deletions lib/storage/storage.ml
Original file line number Diff line number Diff line change
Expand Up @@ -175,19 +175,14 @@ module Make (M : STORE) : S with type t = M.t = struct
| Error _ ->
Ok (Ndarray.create repr.kind repr.shape repr.fill_value))
>>= fun arr ->
(* find_all returns bindings in reverse order. To restore the
* C-ordering of elements we must call List.rev. *)
let coords, vals =
List.split @@
List.rev @@
Arraytbl.find_all tbl idx in
let slice' = Indexing.slice_of_coords coords in
let shape' = Indexing.slice_shape slice' repr.shape in
let x' = Ndarray.of_array repr.kind (Array.of_list vals) shape' in
(* Ndarray.set_fancy* unfortunately doesn't work for array kinds
other than Float32, Float64, Complex32 and Complex64.
See: https://github.com/owlbarn/owl/issues/671 *)
Ndarray.set_fancy_ext slice' arr x'; (* possible to rewrite this function? *)
(* NOTE: Ndarray.set_fancy* functions unfortunately don't work for array
kinds other than Float32, Float64, Complex32 and Complex64.
See: https://github.com/owlbarn/owl/issues/671 . As a workaround
we manually set each coordinate one-at-time using the basic
set function which does not suffer from this bug. It is likely
much slower for large Zarr chunks but necessary for usability.*)
List.iter
(fun (c, v) -> Ndarray.set arr c v) @@ Arraytbl.find_all tbl idx;
Codecs.Chain.encode codecs arr >>| fun encoded ->
set t chunkkey encoded) cindices (Ok ())

Expand Down
Loading