From 00ade96953b1fca27a4e2e5fb0370c36eb2d3d64 Mon Sep 17 00:00:00 2001 From: Zolisa Bleki Date: Mon, 23 Dec 2024 17:34:09 +0200 Subject: [PATCH] Minor refactoring around code structuring. - Trim whitespace - compact multi-line code into one where possible. --- zarr-eio/src/storage.ml | 9 +---- zarr-lwt/src/storage.ml | 22 ++++------ zarr-sync/src/storage.ml | 12 ++---- zarr/src/extensions.ml | 16 +++----- zarr/src/metadata.ml | 27 ++++--------- zarr/src/ndarray.ml | 63 +++++++---------------------- zarr/src/node.ml | 50 ++++++++--------------- zarr/src/storage/storage.ml | 81 ++++++++++++++++--------------------- zarr/src/util.ml | 2 - zarr/src/util.mli | 2 +- 10 files changed, 92 insertions(+), 192 deletions(-) diff --git a/zarr-eio/src/storage.ml b/zarr-eio/src/storage.ml index cc73f2ca..ead57147 100644 --- a/zarr-eio/src/storage.ml +++ b/zarr-eio/src/storage.ml @@ -29,8 +29,7 @@ module FilesystemStore = struct type t = {root : Eio.Fs.dir_ty Eio.Path.t; perm : Eio.File.Unix_perm.t} let fspath_to_key t (path : Eio.Fs.dir_ty Eio.Path.t) = - let s = snd path in - let pos = String.length (snd t.root) + 1 in + let s = snd path and pos = String.length (snd t.root) + 1 in String.sub s pos (String.length s - pos) let key_to_fspath t key = Eio.Path.(t.root / key) @@ -98,12 +97,10 @@ module FilesystemStore = struct List.fold_left (add ~t ~dir) acc (Eio.Path.read_dir dir) let list t = walk t [] t.root - let list_prefix t prefix = walk t [] (key_to_fspath t prefix) - let is_member t key = Eio.Path.is_file (key_to_fspath t key) - let erase t key = Eio.Path.unlink (key_to_fspath t key) + let rename t k k' = Eio.Path.rename (key_to_fspath t k) (key_to_fspath t k') let erase_prefix t pre = (* if prefix points to the root of the store, only delete sub-dirs and files.*) @@ -119,8 +116,6 @@ module FilesystemStore = struct in let dir = key_to_fspath t prefix in List.partition_map (choose ~t ~dir) (Eio.Path.read_dir dir) - - let rename t k k' = Eio.Path.rename (key_to_fspath t k) (key_to_fspath t k') end let create ?(perm=0o700) ~env dirname = diff --git a/zarr-lwt/src/storage.ml b/zarr-lwt/src/storage.ml index 534ce002..7a568771 100644 --- a/zarr-lwt/src/storage.ml +++ b/zarr-lwt/src/storage.ml @@ -6,7 +6,7 @@ module Deferred = struct let return_unit = Lwt.return_unit let iter = Lwt_list.iter_s let fold_left = Lwt_list.fold_left_s - let concat_map f l = Lwt.map List.concat (Lwt_list.map_s f l) + let concat_map f l = Lwt.map List.concat (Lwt_list.map_p f l) module Infix = struct let (>>=) = Lwt.Infix.(>>=) @@ -47,8 +47,8 @@ module FilesystemStore = struct Lwt_unix.file_exists parent_dir >>= maybe_create ~perm parent_dir let size t key = - let file_length path () = Lwt.map Int64.to_int (Lwt_io.file_length path) in - let filepath = key_to_fspath t key in + let file_length path () = Lwt.map Int64.to_int (Lwt_io.file_length path) + and filepath = key_to_fspath t key in Lwt.catch (file_length filepath) (Fun.const @@ Deferred.return 0) let get t key = @@ -126,16 +126,6 @@ module FilesystemStore = struct in Lwt_stream.fold_s (accumulate ~t) (Lwt_unix.files_of_directory dir) acc - let list t = walk t [] (key_to_fspath t "") - - let list_prefix t prefix = walk t [] (key_to_fspath t prefix) - - let is_member t key = Lwt_unix.file_exists (key_to_fspath t key) - - let erase t key = Lwt_unix.unlink (key_to_fspath t key) - - let erase_prefix t pre = list_prefix t pre >>= Lwt_list.iter_s (erase t) - let list_dir t prefix = let choose ~t ~dir x = match Filename.concat dir x with | p when Sys.is_directory p -> Either.right @@ (fspath_to_key t p) ^ "/" @@ -147,6 +137,11 @@ module FilesystemStore = struct let+ dir_contents = Lwt_stream.to_list relevant in List.partition_map (choose ~t ~dir) dir_contents + let list t = walk t [] (key_to_fspath t "") + let list_prefix t prefix = walk t [] (key_to_fspath t prefix) + let is_member t key = Lwt_unix.file_exists (key_to_fspath t key) + let erase t key = Lwt_unix.unlink (key_to_fspath t key) + let erase_prefix t pre = list_prefix t pre >>= Lwt_list.iter_s (erase t) let rename t k k' = Lwt_unix.rename (key_to_fspath t k) (key_to_fspath t k') end @@ -192,7 +187,6 @@ module AmazonS3Store = struct Lwt.catch (return_or_raise res) (on_exception ~not_found) let raise_not_found k () = raise (Zarr.Storage.Key_not_found k) - let empty_Ls = Fun.const ([], S3.Ls.Done) let fold_continuation ~return ~more = function diff --git a/zarr-sync/src/storage.ml b/zarr-sync/src/storage.ml index 9871cc5c..60f8036a 100644 --- a/zarr-sync/src/storage.ml +++ b/zarr-sync/src/storage.ml @@ -26,7 +26,7 @@ module FilesystemStore = struct module IO = struct module Deferred = Deferred - type t = {dirname : string; perm : Unix.file_perm} + type t = {dirname : string; perm : int} let fspath_to_key t path = let pos = String.length t.dirname + 1 in @@ -72,7 +72,6 @@ module FilesystemStore = struct Out_channel.flush oc let is_member t key = Sys.file_exists (key_to_fspath t key) - let erase t key = Sys.remove (key_to_fspath t key) let size t key = @@ -88,12 +87,6 @@ module FilesystemStore = struct let dir_contents = Array.to_list (Sys.readdir dir) in List.fold_left (accumulate ~t) acc dir_contents - let list t = walk t [] (key_to_fspath t "") - - let list_prefix t prefix = walk t [] (key_to_fspath t prefix) - - let erase_prefix t pre = List.iter (erase t) (list_prefix t pre) - let list_dir t prefix = let choose ~t ~dir x = match Filename.concat dir x with | p when Sys.is_directory p -> Either.right @@ (fspath_to_key t p) ^ "/" @@ -103,6 +96,9 @@ module FilesystemStore = struct let dir_contents = Array.to_list (Sys.readdir dir) in List.partition_map (choose ~t ~dir) dir_contents + let list t = walk t [] (key_to_fspath t "") + let list_prefix t prefix = walk t [] (key_to_fspath t prefix) + let erase_prefix t pre = List.iter (erase t) (list_prefix t pre) let rename t k k' = Sys.rename (key_to_fspath t k) (key_to_fspath t k') end diff --git a/zarr/src/extensions.ml b/zarr/src/extensions.ml index c601b379..d379048c 100644 --- a/zarr/src/extensions.ml +++ b/zarr/src/extensions.ml @@ -4,21 +4,17 @@ module RegularGrid = struct type t = int array let chunk_shape : t -> int array = Fun.id + let ceildiv x y = Float.(to_int @@ ceil (of_int x /. of_int y)) + let floordiv x y = Float.(to_int @@ floor (of_int x /. of_int y)) + let grid_shape t array_shape = Array.map2 ceildiv array_shape t + let index_coord_pair t coord = (Array.map2 floordiv coord t, Array.map2 Int.rem coord t) + let ( = ) x y = x = y let create : array_shape:int array -> int array -> t = fun ~array_shape chunk_shape -> if Array.(length chunk_shape <> length array_shape) || Util.(max chunk_shape > max array_shape) then raise Grid_shape_mismatch else chunk_shape - let ceildiv x y = Float.(to_int @@ ceil (of_int x /. of_int y)) - - let floordiv x y = Float.(to_int @@ floor (of_int x /. of_int y)) - - let grid_shape t array_shape = Array.map2 ceildiv array_shape t - - let index_coord_pair t coord = - (Array.map2 floordiv coord t, Array.map2 Int.rem coord t) - (* returns all chunk indices in this regular grid *) let indices t array_shape = grid_shape t array_shape @@ -27,8 +23,6 @@ module RegularGrid = struct |> Ndarray.Indexing.cartesian_prod |> List.map Array.of_list - let ( = ) x y = x = y - let to_yojson : t -> Yojson.Safe.t = fun t -> let chunk_shape = `List (List.map (fun x -> `Int x) @@ Array.to_list t) in `Assoc diff --git a/zarr/src/metadata.ml b/zarr/src/metadata.ml index 8d84ad1f..495822e9 100644 --- a/zarr/src/metadata.ml +++ b/zarr/src/metadata.ml @@ -236,31 +236,21 @@ module Array = struct && x.storage_transformers = y.storage_transformers let shape t = t.shape - let codecs t = t.codecs - let dimension_names t = t.dimension_names - let attributes t = t.attributes - let chunk_shape t = RegularGrid.chunk_shape t.chunk_grid - let index_coord_pair t coord = RegularGrid.index_coord_pair t.chunk_grid coord - let chunk_key t index = ChunkKeyEncoding.encode t.chunk_key_encoding index - let chunk_indices t shape = RegularGrid.indices t.chunk_grid shape - let encode t = Yojson.Safe.to_string (to_yojson t) + let update_attributes t attrs = {t with attributes = attrs} + let update_shape t shape = {t with shape} let decode s = match of_yojson (Yojson.Safe.from_string s) with | Error e -> raise (Parse_error e) | Ok m -> m - let update_attributes t attrs = {t with attributes = attrs} - - let update_shape t shape = {t with shape} - let is_valid_kind : type a. t -> a Ndarray.dtype -> bool = fun t kind -> match kind, t.data_type with @@ -317,14 +307,17 @@ end module Group = struct type t = {zarr_format : int; node_type : string; attributes : Yojson.Safe.t} - let default = {zarr_format = 3; node_type = "group"; attributes = `Null} - let to_yojson : t -> Yojson.Safe.t = fun t -> let l = [("zarr_format", `Int t.zarr_format); ("node_type", `String t.node_type)] in match t.attributes with | `Null -> `Assoc l | x -> `Assoc (l @ [("attributes", x)]) + let default = {zarr_format = 3; node_type = "group"; attributes = `Null} + let encode t = Yojson.Safe.to_string (to_yojson t) + let update_attributes t attrs = {t with attributes = attrs} + let attributes t = t.attributes + let of_yojson x = let open Yojson.Safe.Util in let open Util.Result_syntax in @@ -348,12 +341,6 @@ module Group = struct | Error e -> raise (Parse_error e) | Ok m -> m - let encode t = Yojson.Safe.to_string (to_yojson t) - - let update_attributes t attrs = {t with attributes = attrs} - - let attributes t = t.attributes - let show t = Format.sprintf {|"{zarr_format=%d; node_type=%s; attributes=%s}"|} diff --git a/zarr/src/ndarray.ml b/zarr/src/ndarray.ml index 65ac835d..f17eaf94 100644 --- a/zarr/src/ndarray.ml +++ b/zarr/src/ndarray.ml @@ -15,12 +15,6 @@ type _ dtype = | Int : int dtype | Nativeint : nativeint dtype -type 'a t = - {shape : int array - ;strides : int array - ;dtype : 'a dtype - ;data : 'a array} - let dtype_size : type a. a dtype -> int = function | Char -> 1 | Bool -> 1 @@ -47,36 +41,25 @@ let make_strides shape = let n = Array.length shape - 1 in Array.init (n + 1) (fun i -> cumprod shape (i + 1) n) -let create dtype shape fv = - {shape - ;dtype - ;strides = make_strides shape - ;data = Array.make (Util.prod shape) fv} - -let init dtype shape f = - {shape - ;dtype - ;strides = make_strides shape - ;data = Array.init (Util.prod shape) f} - +type 'a t = {shape : int array; strides : int array; dtype : 'a dtype; data : 'a array} +let equal x y = x.data = y.data && x.shape = y.shape && x.dtype = y.dtype && x.strides = y.strides +(* 1d index of coord [i0; ...; in] is SUM(i0 * strides[0] + ... + in * strides[n-1] *) +let coord_to_index i s = Array.fold_left (fun a (x, y) -> Int.add a (x * y)) 0 @@ Array.combine i s +let create dtype shape fv = {shape; dtype; strides = make_strides shape; data = Array.make (Util.prod shape) fv} +let init dtype shape f = {shape; dtype; strides = make_strides shape; data = Array.init (Util.prod shape) f} +let of_array dtype shape xs = {shape; dtype; strides = make_strides shape; data = xs} let data_type t = t.dtype - let size t = Util.prod t.shape - let ndims t = Array.length t.shape - -let shape t = t.shape - +let get t i = t.data.(coord_to_index i t.strides) +let set t i x = t.data.(coord_to_index i t.strides) <- x +let fill t v = Array.iteri (fun i _ -> t.data.(i) <- v) t.data +let map f t = {t with data = Array.map f t.data} +let iteri f t = Array.iteri f t.data +let iter f t = Array.iter f t.data let byte_size t = size t * dtype_size t.dtype - let to_array t = t.data - -let of_array dtype shape xs = - {shape; dtype; strides = make_strides shape; data = xs} - -(* 1d index of coord [i0; ...; in] is SUM(i0 * strides[0] + ... + in * strides[n-1] *) -let coord_to_index i s = - Array.fold_left (fun a (x, y) -> Int.add a (x * y)) 0 @@ Array.combine i s +let shape t = t.shape (* This snippet is adapted from the Owl project. @@ -88,18 +71,6 @@ let index_to_coord ~strides i j = j.(k) <- i mod strides.(k - 1) / strides.(k) done -let get t i = t.data.(coord_to_index i t.strides) - -let set t i x = t.data.(coord_to_index i t.strides) <- x - -let iteri f t = Array.iteri f t.data - -let fill t v = Array.iteri (fun i _ -> t.data.(i) <- v) t.data - -let map f t = {t with data = Array.map f t.data} - -let iter f t = Array.iter f t.data - module B = Bigarray let to_bigarray : @@ -148,12 +119,6 @@ let of_bigarray : | B.Complex32 -> f Complex32 | B.Complex64 -> f Complex64 -let equal x y = - x.data = y.data - && x.shape = y.shape - && x.dtype = y.dtype - && x.strides = y.strides - (* validation for [axis] is done at the boundaries of the system and thus doing so inside this function would be redundant work. Also, the output array shares internal data with the input. Since this function is only ever diff --git a/zarr/src/node.ml b/zarr/src/node.ml index a90eb580..b0d72191 100644 --- a/zarr/src/node.ml +++ b/zarr/src/node.ml @@ -11,12 +11,7 @@ let rep_ok name = module Group = struct type t = Root | Cons of t * string - let create parent name = - if rep_ok name then Cons (parent, name) else raise Node_invariant - - let ( / ) = create - - let root = Root + let create parent name = if rep_ok name then Cons (parent, name) else raise Node_invariant let of_path = function | "/" -> Root @@ -54,8 +49,6 @@ module Group = struct | Root -> acc | Cons (p, _) -> p :: acc - let ancestors p = fold prepend_node [] p - let to_key p = let str = to_path p in String.sub str 1 (String.length str - 1) @@ -64,45 +57,31 @@ module Group = struct | Root -> "" | p -> to_key p ^ "/" - let to_metakey p = to_prefix p ^ "zarr.json" - let is_child_group x y = match x, y with | _, Root -> false | v, Cons (parent, _) -> parent = v - let show = to_path - - let pp fmt t = Format.fprintf fmt "%s" (show t) - let rename t str = match t with | Cons (parent, _) when rep_ok str -> Cons (parent, str) | Cons _ -> raise Node_invariant | Root -> raise Cannot_rename_root + + let root = Root + let ( / ) = create + let show = to_path + let ancestors p = fold prepend_node [] p + let pp fmt t = Format.fprintf fmt "%s" (show t) + let to_metakey p = to_prefix p ^ "zarr.json" end module Array = struct type t = {parent : Group.t option; name : string} - let create g name = - if rep_ok name then {parent = Some g; name} else raise Node_invariant - - let ( / ) = create - - let root = {parent = None; name = ""} - let of_path p = let g = Group.of_path p in match Group.parent g with | Some _ as parent -> {parent; name = Group.name g} | None -> raise Node_invariant - - let ( = ) - {parent = p; name = n} - {parent = q; name = m} = p = q && n = m - - let name {parent = _; name = n} = n - - let parent {parent = p; _} = p let to_path {parent = p; name} = match p with | None -> "/" @@ -124,13 +103,18 @@ module Array = struct let to_metakey = function | {parent = None; _} -> "zarr.json" | p -> to_key p ^ "/zarr.json" - - let show = to_path - - let pp fmt t = Format.fprintf fmt "%s" (show t) let rename t name = match t.parent with | Some _ when rep_ok name -> {t with name} | Some _ -> raise Node_invariant | None -> raise Cannot_rename_root + + let create g name = if rep_ok name then {parent = Some g; name} else raise Node_invariant + let ( / ) = create + let show = to_path + let root = {parent = None; name = ""} + let ( = ) {parent = p; name = n} {parent = q; name = m} = p = q && n = m + let parent {parent = p; _} = p + let name {parent = _; name = n} = n + let pp fmt t = Format.fprintf fmt "%s" (show t) end diff --git a/zarr/src/storage/storage.ml b/zarr/src/storage/storage.ml index b7b7057e..343fe596 100644 --- a/zarr/src/storage/storage.ml +++ b/zarr/src/storage/storage.ml @@ -1,13 +1,5 @@ include Storage_intf -module ArrayMap = Util.ArrayMap -module Indexing = Ndarray.Indexing - -module ArraySet = Set.Make (struct - type t = int array - let compare : t -> t -> int = Stdlib.compare -end) - module Make (Io : Types.IO) = struct module Io_chain = Codecs.Make(Io) module Deferred = Io.Deferred @@ -45,14 +37,16 @@ module Make (Io : Types.IO) = struct module Group = struct let exists t node = is_member t (Node.Group.to_metakey node) + let delete t node = erase_prefix t (Node.Group.to_prefix node) + let metadata t node = Deferred.map Metadata.Group.decode (get t @@ Node.Group.to_metakey node) (* This recursively creates parent group nodes if they don't exist.*) let rec create ?(attrs=`Null) t node = let maybe_create ~attrs t node = function | true -> Deferred.return_unit | false -> - let key = Node.Group.to_metakey node in - let meta = Metadata.Group.(update_attributes default attrs) in + let key = Node.Group.to_metakey node + and meta = Metadata.Group.(update_attributes default attrs) in let* () = set t key (Metadata.Group.encode meta) in match Node.Group.parent node with | None -> Deferred.return_unit @@ -60,8 +54,6 @@ module Make (Io : Types.IO) = struct in exists t node >>= maybe_create ~attrs t node - let metadata t node = Deferred.map Metadata.Group.decode (get t @@ Node.Group.to_metakey node) - let children t node = let add ~t (left, right) prefix = let path = "/" ^ String.sub prefix 0 (String.length prefix - 1) in @@ -75,8 +67,6 @@ module Make (Io : Types.IO) = struct in exists t node >>= maybe_enumerate t node - let delete t node = erase_prefix t (Node.Group.to_prefix node) - let rename t node str = let key = Node.Group.to_key node and key' = Node.Group.(rename node str |> to_key) in @@ -84,24 +74,21 @@ module Make (Io : Types.IO) = struct end module Array = struct + module ArrayMap = Util.ArrayMap + module Indexing = Ndarray.Indexing let exists t node = is_member t (Node.Array.to_metakey node) + let delete t node = erase_prefix t (Node.Array.to_key node ^ "/") + let metadata t node = Deferred.map Metadata.Array.decode (get t @@ Node.Array.to_metakey node) (* This recursively creates parent group nodes if they don't exist.*) - let create - ?(sep=`Slash) ?(dimension_names=[]) ?(attributes=`Null) - ~codecs ~shape ~chunks - kind fv node t = + let create ?(sep=`Slash) ?(dimension_names=[]) ?(attributes=`Null) ~codecs ~shape ~chunks kind fv node t = let c = Codecs.Chain.create chunks codecs in let m = Metadata.Array.create ~sep ~codecs:c ~dimension_names ~attributes ~shape kind fv chunks in - let key = Node.Array.to_metakey node in let value = Metadata.Array.encode m in + let key = Node.Array.to_metakey node in let* () = set t key value in Option.fold ~none:Deferred.return_unit ~some:(Group.create t) (Node.Array.parent node) - let metadata t node = Deferred.map Metadata.Array.decode (get t @@ Node.Array.to_metakey node) - - let delete t node = erase_prefix t (Node.Array.to_key node ^ "/") - let write t node slice x = let update_ndarray ~arr (c, v) = Ndarray.set arr c v in let add_coord_value ~meta acc (co, y) = @@ -111,9 +98,8 @@ module Make (Io : Types.IO) = struct let update_chunk ~t ~meta ~prefix ~chain ~fv ~repr (idx, pairs) = let ckey = prefix ^ Metadata.Array.chunk_key meta idx in if Io_chain.is_just_sharding chain then + let pget = get_partial_values t ckey and pset = set_partial_values t ckey in let* shardsize = size t ckey in - let pget = get_partial_values t ckey in - let pset = set_partial_values t ckey in Io_chain.partial_encode chain pget pset shardsize repr pairs fv else is_member t ckey >>= function | true -> @@ -137,16 +123,14 @@ module Make (Io : Types.IO) = struct let coords = Indexing.coords_of_slice slice shape in let coord_value_pair = Array.combine coords (Ndarray.to_array x) in let m = Array.fold_left (add_coord_value ~meta) ArrayMap.empty coord_value_pair in - let fv = Metadata.Array.fillvalue_of_kind meta kind in - let repr = Codecs.{kind; shape = Metadata.Array.chunk_shape meta} in - let prefix = Node.Array.to_key node ^ "/" in - let chain = Metadata.Array.codecs meta in - let bindings = ArrayMap.bindings m in + let fv = Metadata.Array.fillvalue_of_kind meta kind + and repr = Codecs.{kind; shape = Metadata.Array.chunk_shape meta} + and prefix = Node.Array.to_key node ^ "/" + and chain = Metadata.Array.codecs meta + and bindings = ArrayMap.bindings m in Deferred.iter (update_chunk ~t ~meta ~prefix ~chain ~fv ~repr) bindings let read (type a) t node slice (kind : a Ndarray.dtype) = - let indexed_fill_value ~fv (i, _) = (i, fv) in - let indexed_ndarray_value ~arr (i, c) = (i, Ndarray.get arr c) in let add_indexed_coord ~meta acc (i, y) = let chunk_idx, c = Metadata.Array.index_coord_pair meta y in ArrayMap.add_to_list chunk_idx (i, c) acc @@ -154,14 +138,14 @@ module Make (Io : Types.IO) = struct let read_chunk ~t ~meta ~prefix ~chain ~fv ~repr (idx, pairs) = let ckey = prefix ^ Metadata.Array.chunk_key meta idx in size t ckey >>= function - | 0 -> Deferred.return @@ List.map (indexed_fill_value ~fv) pairs + | 0 -> Deferred.return @@ List.map (fun (i, _) -> i, fv) pairs | shardsize when Io_chain.is_just_sharding chain -> let pget = get_partial_values t ckey in Io_chain.partial_decode chain pget shardsize repr pairs fv | _ -> let+ v = get t ckey in let arr = Codecs.Chain.decode chain repr v in - List.map (indexed_ndarray_value ~arr) pairs + List.map (fun (i, c) -> i, Ndarray.get arr c) pairs in let* meta = metadata t node in if not (Metadata.Array.is_valid_kind meta kind) then raise Invalid_data_type else @@ -170,19 +154,23 @@ module Make (Io : Types.IO) = struct | Assert_failure _ -> raise Invalid_array_slice in let icoords = Array.mapi (fun i v -> i, v) (Indexing.coords_of_slice slice shape) in - let m = Array.fold_left (add_indexed_coord ~meta) ArrayMap.empty icoords in - let chain = Metadata.Array.codecs meta in - let prefix = Node.Array.to_key node ^ "/" in - let fv = Metadata.Array.fillvalue_of_kind meta kind in - let repr = Codecs.{kind; shape = Metadata.Array.chunk_shape meta} in - let bs = ArrayMap.bindings m in - let+ ps = Deferred.concat_map (read_chunk ~t ~meta ~prefix ~chain ~fv ~repr) bs in + let m = Array.fold_left (add_indexed_coord ~meta) ArrayMap.empty icoords + and chain = Metadata.Array.codecs meta + and prefix = Node.Array.to_key node ^ "/" + and fv = Metadata.Array.fillvalue_of_kind meta kind + and repr = Codecs.{kind; shape = Metadata.Array.chunk_shape meta} in + let+ ps = Deferred.concat_map (read_chunk ~t ~meta ~prefix ~chain ~fv ~repr) (ArrayMap.bindings m) in (* sorting restores the C-order of the decoded array coordinates.*) let sorted_pairs = List.fast_sort (fun (x, _) (y, _) -> Int.compare x y) ps in let vs = List.map snd sorted_pairs in Ndarray.of_array kind slice_shape (Array.of_list vs) let reshape t node new_shape = + let module S = Set.Make (struct + type t = int array + let compare : t -> t -> int = Stdlib.compare + end) + in let maybe_erase t key = function | false -> Deferred.return_unit | true -> erase t key @@ -191,16 +179,15 @@ module Make (Io : Types.IO) = struct let key = prefix ^ Metadata.Array.chunk_key meta v in is_member t key >>= maybe_erase t key in - let mkey = Node.Array.to_metakey node in let* meta = metadata t node in let old_shape = Metadata.Array.shape meta in if Array.(length new_shape <> length old_shape) then raise Invalid_resize_shape else - let s = ArraySet.of_list (Metadata.Array.chunk_indices meta old_shape) in - let s' = ArraySet.of_list (Metadata.Array.chunk_indices meta new_shape) in - let prefix = Node.Array.to_key node ^ "/" in - let unreachable_chunks = ArraySet.elements (ArraySet.diff s s') in + let s = S.of_list (Metadata.Array.chunk_indices meta old_shape) + and s' = S.of_list (Metadata.Array.chunk_indices meta new_shape) in + let unreachable_chunks = S.elements (S.diff s s') + and prefix = Node.Array.to_key node ^ "/" in let* () = Deferred.iter (remove ~t ~meta ~prefix) unreachable_chunks in - set t mkey Metadata.Array.(encode @@ update_shape meta new_shape) + set t (Node.Array.to_metakey node) Metadata.Array.(encode @@ update_shape meta new_shape) let rename t node str = let key = Node.Array.to_key node diff --git a/zarr/src/util.ml b/zarr/src/util.ml index b5b45f88..6255e11b 100644 --- a/zarr/src/util.ml +++ b/zarr/src/util.ml @@ -18,9 +18,7 @@ module Result_syntax = struct end let get_name j = Yojson.Safe.Util.(member "name" j |> to_string) - let prod x = Array.fold_left Int.mul 1 x - let max = Array.fold_left Int.max Int.min_int (* Obtained from: https://discuss.ocaml.org/t/how-to-create-a-new-file-while-automatically-creating-any-intermediate-directories/14837/5?u=zoj613 *) diff --git a/zarr/src/util.mli b/zarr/src/util.mli index c20bdd9d..e836545d 100644 --- a/zarr/src/util.mli +++ b/zarr/src/util.mli @@ -23,7 +23,7 @@ val prod : int array -> int val max : int array -> int (** [max x] returns the maximum element of an integer array [x]. *) -val create_parent_dir : string -> Unix.file_perm -> unit +val create_parent_dir : string -> int -> unit (** [create_parent_dir f p] creates all the parent directories of file name [f] if they don't exist given file permissions [p]. *)