From 6bda9a761359d9cb90045194c753753ca859f7e8 Mon Sep 17 00:00:00 2001 From: James McMullan Date: Thu, 7 Mar 2024 20:32:37 -0500 Subject: [PATCH] Code review changes --- .../org/hpccsystems/dfs/client/DFSReadWriteTest.java | 2 +- dfsclient/src/test/resources/generate-datasets.ecl | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java index 563b751e5..9ea4ea0a5 100644 --- a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java +++ b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java @@ -63,7 +63,7 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class DFSReadWriteTest extends BaseRemoteTest { - private static final String[] datasets = { "~benchmark::integer::20kb", "~unit_test::all_types::thor", "~unit_test::all_types::xml", "~unit_test::all_types::csv" }; + private static final String[] datasets = { "~benchmark::integer::20kb", "~unit_test::all_types::thor", "~unit_test::all_types::xml", "~unit_test::all_types::json", "~unit_test::all_types::csv" }; private static final int[] expectedCounts = { 1250, 10000, 10000, 10000, 10000, 10000}; private static final Version newProtocolVersion = new Version(8,12,10); diff --git a/dfsclient/src/test/resources/generate-datasets.ecl b/dfsclient/src/test/resources/generate-datasets.ecl index aa956e9b3..4e333ab9b 100644 --- a/dfsclient/src/test/resources/generate-datasets.ecl +++ b/dfsclient/src/test/resources/generate-datasets.ecl @@ -54,7 +54,9 @@ ds := DATASET(totalrecs1, transform(rec, dataset_name := '~unit_test::all_types::thor'; IF(~Std.File.FileExists(dataset_name), OUTPUT(ds,,dataset_name,overwrite)); -text_rec := { INTEGER8 int8, UNSIGNED8 uint8, INTEGER4 int4, UNSIGNED4 uint4, +// For the text files there appears to be an issue with reading sets from the datasets +// So, for those file formats create datasets wwith all types except SETs +recWithoutSet := { INTEGER8 int8, UNSIGNED8 uint8, INTEGER4 int4, UNSIGNED4 uint4, INTEGER2 int2, UNSIGNED2 uint2, REAL8 r8, REAL4 r4, DECIMAL16_8 dec16, @@ -72,7 +74,7 @@ text_rec := { INTEGER8 int8, UNSIGNED8 uint8, INTEGER4 int4, UNSIGNED4 uint4, VARUNICODE varUni, DATASET(childRec) childDataset, }; -text_ds := DATASET(totalrecs1, transform(text_rec, +dsWithoutSet := DATASET(totalrecs1, transform(recWithoutSet, self.int8 := (INTEGER)(random() % unique_keys); self.uint8 := (INTEGER)(random() % unique_values); self.int4 := (INTEGER)(random() % unique_values); @@ -98,13 +100,13 @@ text_ds := DATASET(totalrecs1, transform(text_rec, ), DISTRIBUTED); xml_dataset_name := '~unit_test::all_types::xml'; -IF(~Std.File.FileExists(xml_dataset_name), OUTPUT(text_ds,,xml_dataset_name,XML,overwrite)); +IF(~Std.File.FileExists(xml_dataset_name), OUTPUT(dsWithoutSet,,xml_dataset_name,XML,overwrite)); json_dataset_name := '~unit_test::all_types::json'; -IF(~Std.File.FileExists(json_dataset_name), OUTPUT(text_ds,,json_dataset_name,JSON,overwrite)); +IF(~Std.File.FileExists(json_dataset_name), OUTPUT(dsWithoutSet,,json_dataset_name,JSON,overwrite)); csv_dataset_name := '~unit_test::all_types::csv'; -IF(~Std.File.FileExists(csv_dataset_name), OUTPUT(text_ds,,csv_dataset_name,CSV,overwrite)); +IF(~Std.File.FileExists(csv_dataset_name), OUTPUT(dsWithoutSet,,csv_dataset_name,CSV,overwrite)); key_name := '~unit_test::all_types::key'; Ptbl := DATASET('~unit_test::all_types::thor', {rec,UNSIGNED8 RecPtr {virtual(fileposition)}}, FLAT);