Code review changes

hpcc-systems · Mar 8, 2024 · 6bda9a7 · 6bda9a7
1 parent c20c99d
commit 6bda9a7
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 6 deletions.
diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java
@@ -63,7 +63,7 @@
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public class DFSReadWriteTest extends BaseRemoteTest
 {
-    private static final String[] datasets       = { "~benchmark::integer::20kb", "~unit_test::all_types::thor", "~unit_test::all_types::xml", "~unit_test::all_types::csv" };
+    private static final String[] datasets       = { "~benchmark::integer::20kb", "~unit_test::all_types::thor", "~unit_test::all_types::xml", "~unit_test::all_types::json", "~unit_test::all_types::csv" };
     private static final int[]    expectedCounts = { 1250, 10000, 10000, 10000, 10000, 10000};
     private static final Version newProtocolVersion = new Version(8,12,10);
 

diff --git a/dfsclient/src/test/resources/generate-datasets.ecl b/dfsclient/src/test/resources/generate-datasets.ecl
@@ -54,7 +54,9 @@ ds := DATASET(totalrecs1, transform(rec,
 dataset_name := '~unit_test::all_types::thor';
 IF(~Std.File.FileExists(dataset_name), OUTPUT(ds,,dataset_name,overwrite));
 
-text_rec := {  INTEGER8 int8, UNSIGNED8 uint8, INTEGER4 int4, UNSIGNED4 uint4,
+// For the text files there appears to be an issue with reading sets from the datasets
+// So, for those file formats create datasets wwith all types except SETs
+recWithoutSet := {  INTEGER8 int8, UNSIGNED8 uint8, INTEGER4 int4, UNSIGNED4 uint4,
           INTEGER2 int2, UNSIGNED2 uint2,
           REAL8 r8, REAL4 r4,
           DECIMAL16_8 dec16,
@@ -72,7 +74,7 @@ text_rec := {  INTEGER8 int8, UNSIGNED8 uint8, INTEGER4 int4, UNSIGNED4 uint4,
           VARUNICODE varUni,
           DATASET(childRec) childDataset,
         };
-text_ds := DATASET(totalrecs1, transform(text_rec,
+dsWithoutSet := DATASET(totalrecs1, transform(recWithoutSet,
                                    self.int8 := (INTEGER)(random() % unique_keys);
                                    self.uint8 := (INTEGER)(random() % unique_values);
                                    self.int4 := (INTEGER)(random() % unique_values);
@@ -98,13 +100,13 @@ text_ds := DATASET(totalrecs1, transform(text_rec,
                                   ), DISTRIBUTED);
 
 xml_dataset_name := '~unit_test::all_types::xml';
-IF(~Std.File.FileExists(xml_dataset_name), OUTPUT(text_ds,,xml_dataset_name,XML,overwrite));
+IF(~Std.File.FileExists(xml_dataset_name), OUTPUT(dsWithoutSet,,xml_dataset_name,XML,overwrite));
 
 json_dataset_name := '~unit_test::all_types::json';
-IF(~Std.File.FileExists(json_dataset_name), OUTPUT(text_ds,,json_dataset_name,JSON,overwrite));
+IF(~Std.File.FileExists(json_dataset_name), OUTPUT(dsWithoutSet,,json_dataset_name,JSON,overwrite));
 
 csv_dataset_name := '~unit_test::all_types::csv';
-IF(~Std.File.FileExists(csv_dataset_name), OUTPUT(text_ds,,csv_dataset_name,CSV,overwrite));
+IF(~Std.File.FileExists(csv_dataset_name), OUTPUT(dsWithoutSet,,csv_dataset_name,CSV,overwrite));
 
 key_name := '~unit_test::all_types::key';
 Ptbl := DATASET('~unit_test::all_types::thor', {rec,UNSIGNED8 RecPtr {virtual(fileposition)}},  FLAT);