Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support optional chunk_key_encoding configuration. #29

Merged
merged 1 commit into from
Jul 11, 2024
Merged

Support optional chunk_key_encoding configuration. #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 25 additions & 19 deletions lib/extensions.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,14 @@ end

module ChunkKeyEncoding = struct
type kind = Default | V2
type t = {name : kind; sep : string}
type t = {name : kind; sep : string; is_default : bool}

let create = function
| `Dot -> {name = Default; sep = "."}
| `Slash -> {name = Default; sep = "/"}
| `Dot -> {name = Default; sep = "."; is_default = false}
| `Slash -> {name = Default; sep = "/"; is_default = false}

(* map a chunk coordinate index to a key. E.g, (2,3,1) maps to c/2/3/1 *)
let encode {name; sep} index =
let encode {name; sep; _} index =
let f i acc =
string_of_int i :: acc
in
Expand All @@ -95,31 +95,37 @@ module ChunkKeyEncoding = struct
Array.fold_right f index []

let ( = ) x y =
x.name = y.name && x.sep = y.sep
x.name = y.name && x.sep = y.sep && x.is_default = y.is_default

let to_yojson {name; sep} =
let to_yojson {name; sep; is_default} =
let str =
match name with
| Default -> "default"
| V2 -> "v2"
in
`Assoc
[("name", `String str)
;("configuration", `Assoc [("separator", `String sep)])]
if is_default then
`Assoc [("name", `String str)]
else
`Assoc
[("name", `String str)
;("configuration", `Assoc [("separator", `String sep)])]

let of_yojson x =
match
Util.get_name x,
Yojson.Safe.Util.(member "configuration" x |> to_assoc)
Util.get_name x, Yojson.Safe.Util.member "configuration" x
with
| "default", [("separator", `String "/")] ->
Ok {name = Default; sep = "/"}
| "default", [("separator", `String ".")] ->
Ok {name = Default; sep = "."}
| "v2", [("separator", `String "/")] ->
Ok {name = V2; sep = "/"}
| "v2", [("separator", `String ".")] ->
Ok {name = V2; sep = "."}
| "default", `Null ->
Ok {name = Default; sep = "/"; is_default = true}
| "default", `Assoc [("separator", `String "/")] ->
Ok {name = Default; sep = "/"; is_default = false}
| "default", `Assoc [("separator", `String ".")] ->
Ok {name = Default; sep = "."; is_default = false}
| "v2", `Null ->
Ok {name = V2; sep = "."; is_default = true}
| "v2", `Assoc [("separator", `String "/")] ->
Ok {name = V2; sep = "/"; is_default = false}
| "v2", `Assoc [("separator", `String ".")] ->
Ok {name = V2; sep = "."; is_default = false}
| _ -> Error "Invalid chunk key encoding configuration."
end

Expand Down
37 changes: 31 additions & 6 deletions test/test_metadata.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,7 @@ let test_encode_decode_fill_value fv =
"fill_value": %s,
"chunk_grid":
{"name": "regular", "configuration": {"chunk_shape": [100, 10]}},
"chunk_key_encoding":
{"name": "default", "configuration": {"separator": "."}},
"chunk_key_encoding": {"name": "default"},
"attributes": {"question": 7}}|} fv
in
let expected = Yojson.Safe.from_string str in
Expand Down Expand Up @@ -267,8 +266,7 @@ let array = [
"zarr_format": 3,
"shape": [10000, 1000],
"data_type": "float64",
"chunk_key_encoding":
{"name": "v2", "configuration": {"separator": "."}},
"chunk_key_encoding": {"name": "v2"},
"codecs": [
{"name": "bytes", "configuration": {"endian": "big"}}],
"fill_value": "0x7fc00000",
Expand Down Expand Up @@ -340,8 +338,7 @@ let array = [
"node_type": "array",
"shape": [10000, 1000],
"data_type": "float64",
"chunk_key_encoding":
{"name": "v2", "configuration": {"separator": "."}},
"chunk_key_encoding": {"name": "v2"},
"codecs":
{"name": "bytes", "configuration": {"endian": "big"}},
"fill_value": "0x7fc00000",
Expand Down Expand Up @@ -469,6 +466,34 @@ let array = [
~str:(template {|"UNKNOWN"|} {|[2, 4]|})
~msg:"Invalid Chunk grid name or configuration.";

(* test if decoding a chunk key encoding field without a configuration
leads to a default value being used. *)
let str = {|{
"zarr_format": 3,
"shape": [10000, 1000],
"node_type": "array",
"data_type": "float64",
"codecs": [
{"name": "bytes", "configuration": {"endian": "big"}}],
"fill_value": "0x7fc00000",
"chunk_grid":
{"name": "regular", "configuration": {"chunk_shape": [10, 10]}},
"chunk_key_encoding": {"name": "v2"}}|} in
(match ArrayMetadata.of_yojson @@ Yojson.Safe.from_string str with
| Ok meta ->
(* we except it to use the default "." separator. *)
assert_equal
~printer:Fun.id "2.0.1" @@ ArrayMetadata.chunk_key meta [|2; 0; 1|];
(* we expect the default (unspecified) config seperator to be
dropped when serializing the metadata to JSON format. *)
assert_equal
~printer:Fun.id
Yojson.Safe.(from_string str |> to_string) @@
ArrayMetadata.encode meta;
| Error _ ->
assert_failure
"Decoding a well formed array JSON metadata should not fail.");

(* test if the decoding fails if chunk key encoding contains unknown
* separator or name. *)
let str = {|{
Expand Down
Loading