From 4dee534ecce4c22d4e738f87dae48b026a45bf08 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 27 Jul 2021 16:20:04 -0700 Subject: [PATCH 1/4] Remove the operation from configuration file --- .github/workflows/linux.yml | 18 +++++++-------- .gitlab-ci.yml | 4 ++-- h5bench_patterns/h5bench_read.c | 8 +------ h5bench_patterns/h5bench_write.c | 6 ----- .../sample_config/sample_2d_compressed.cfg | 1 - ...d_cc1d_small.cfg => sample_cc1d_small.cfg} | 1 - ...mall_EXP.cfg => sample_cc1d_small_EXP.cfg} | 1 - ...mall_IMP.cfg => sample_cc1d_small_IMP.cfg} | 1 - .../sample_config/sample_read_cc1d.cfg | 1 - .../sample_config/sample_read_cc1d_es1.cfg | 1 - .../sample_config/sample_write_cc1d.cfg | 2 +- .../sample_config/sample_write_cc1d_small.cfg | 23 ------------------- h5bench_patterns/sample_config/template.cfg | 1 - 13 files changed, 13 insertions(+), 55 deletions(-) rename h5bench_patterns/sample_config/{sample_read_cc1d_small.cfg => sample_cc1d_small.cfg} (97%) rename h5bench_patterns/sample_config/{sample_read_cc1d_small_EXP.cfg => sample_cc1d_small_EXP.cfg} (97%) rename h5bench_patterns/sample_config/{sample_read_cc1d_small_IMP.cfg => sample_cc1d_small_IMP.cfg} (97%) delete mode 100644 h5bench_patterns/sample_config/sample_write_cc1d_small.cfg diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3841a6fb..6022fd20 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -93,7 +93,7 @@ jobs: make - name: Build h5bench ASYNC - run: | + run: | mydir="$PWD" mkdir build_async cd build_async @@ -103,23 +103,23 @@ jobs: - name: Test h5bench SYNC Read & Write (NON) run: | cd build_sync - ./h5bench_write /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_write_cc1d_small.cfg testwrite.h5 - ./h5bench_read /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_read_cc1d_small.cfg testwrite.h5 + ./h5bench_write /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_cc1d_small.cfg file.h5 + ./h5bench_read /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_cc1d_small.cfg file.h5 - name: Test h5bench ASYNC Read & Write (NON) run: | cd build_async - ./h5bench_write /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_write_cc1d_small.cfg testwrite.h5 - ./h5bench_read /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_read_cc1d_small.cfg testwrite.h5 + ./h5bench_write /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_cc1d_small.cfg file.h5 + ./h5bench_read /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_cc1d_small.cfg file.h5 - name: Test h5bench ASYNC Read & Write (IMP) run: | cd build_async - ./h5bench_write /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_write_cc1d_small_IMP.cfg testwrite_IMP.h5 - ./h5bench_read /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_read_cc1d_small_IMP.cfg testwrite_IMP.h5 + ./h5bench_write /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_cc1d_small_IMP.cfg file_IMP.h5 + ./h5bench_read /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_cc1d_small_IMP.cfg file_IMP.h5 - name: Test h5bench ASYNC Read & Write (EXP) run: | cd build_async - ./h5bench_write /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_write_cc1d_small_EXP.cfg testwrite_EXP.h5 - ./h5bench_read /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_read_cc1d_small_EXP.cfg testwrite_EXP.h5 + ./h5bench_write /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_cc1d_small_EXP.cfg file_EXP.h5 + ./h5bench_read /home/runner/work/h5bench/h5bench/h5bench_patterns/sample_config/sample_cc1d_small_EXP.cfg file_EXP.h5 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0732c0c..7567b94a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,7 @@ test: - export LD_LIBRARY_PATH=$HDF5_DIR/lib:$LD_LIBRARY_PATH - export MPICH_MAX_THREAD_SAFETY=multiple - echo "Test h5bench SYNC write" - - srun -n 2 ./h5bench_write ../h5bench_patterns/sample_config/sample_write_cc1d_small.cfg file.h5 + - srun -n 2 ./h5bench_write ../h5bench_patterns/sample_config/sample_cc1d_small.cfg file.h5 - echo "Test h5bench SYNC read" - - srun -n 2 ./h5bench_read ../h5bench_patterns/sample_config/sample_read_cc1d_small.cfg file.h5 + - srun -n 2 ./h5bench_read ../h5bench_patterns/sample_config/sample_cc1d_small.cfg file.h5 - ls -larth file.h5 diff --git a/h5bench_patterns/h5bench_read.c b/h5bench_patterns/h5bench_read.c index 3e4e647d..8cf49458 100644 --- a/h5bench_patterns/h5bench_read.c +++ b/h5bench_patterns/h5bench_read.c @@ -385,13 +385,7 @@ main(int argc, char *argv[]) printf("Usage: ./%s /path/to/file #timestep [# mega particles]\n", argv[0]); return 0; } - - if (params.io_op != IO_READ) { - if (MY_RANK == 0) - printf("Make sure the configuration file has IO_OPERATION=READ defined\n"); - return 0; - } - + hid_t fapl, gapl; set_pl(&fapl, &gapl); diff --git a/h5bench_patterns/h5bench_write.c b/h5bench_patterns/h5bench_write.c index a077ba18..b7036549 100644 --- a/h5bench_patterns/h5bench_write.c +++ b/h5bench_patterns/h5bench_write.c @@ -964,12 +964,6 @@ main(int argc, char *argv[]) return 0; } - if (params.io_op != IO_WRITE) { - if (MY_RANK == 0) - printf("Make sure the configuration file has IO_OPERATION=WRITE defined\n"); - return 0; - } - if (params.useCompress) params.data_coll = 1; diff --git a/h5bench_patterns/sample_config/sample_2d_compressed.cfg b/h5bench_patterns/sample_config/sample_2d_compressed.cfg index 7dbebca5..eecd53f7 100644 --- a/h5bench_patterns/sample_config/sample_2d_compressed.cfg +++ b/h5bench_patterns/sample_config/sample_2d_compressed.cfg @@ -1,6 +1,5 @@ # this is a comment # Benchmark mode can only be one of these: CC/CI/IC/II/CC2D/CI2D/IC2D/II2D/CC2D/CC3D -IO_OPERATION=READ MEM_PATTERN=CONTIG FILE_PATTERN=CONTIG READ_OPTION=FULL diff --git a/h5bench_patterns/sample_config/sample_read_cc1d_small.cfg b/h5bench_patterns/sample_config/sample_cc1d_small.cfg similarity index 97% rename from h5bench_patterns/sample_config/sample_read_cc1d_small.cfg rename to h5bench_patterns/sample_config/sample_cc1d_small.cfg index 3ab6098b..08799edb 100644 --- a/h5bench_patterns/sample_config/sample_read_cc1d_small.cfg +++ b/h5bench_patterns/sample_config/sample_cc1d_small.cfg @@ -1,7 +1,6 @@ # this is a comment # Benchmark mode can only be one of these: CC/CI/IC/II/CC2D/CI2D/IC2D/II2D/CC2D/CC3D # Template cof include all options -IO_OPERATION=READ MEM_PATTERN=CONTIG FILE_PATTERN=CONTIG NUM_PARTICLES=16 M #16 K/G diff --git a/h5bench_patterns/sample_config/sample_read_cc1d_small_EXP.cfg b/h5bench_patterns/sample_config/sample_cc1d_small_EXP.cfg similarity index 97% rename from h5bench_patterns/sample_config/sample_read_cc1d_small_EXP.cfg rename to h5bench_patterns/sample_config/sample_cc1d_small_EXP.cfg index 4fb26e0e..48ca2f5c 100644 --- a/h5bench_patterns/sample_config/sample_read_cc1d_small_EXP.cfg +++ b/h5bench_patterns/sample_config/sample_cc1d_small_EXP.cfg @@ -1,7 +1,6 @@ # this is a comment # Benchmark mode can only be one of these: CC/CI/IC/II/CC2D/CI2D/IC2D/II2D/CC2D/CC3D # Template cof include all options -IO_OPERATION=READ MEM_PATTERN=CONTIG FILE_PATTERN=CONTIG NUM_PARTICLES=16 M #16 K/G diff --git a/h5bench_patterns/sample_config/sample_read_cc1d_small_IMP.cfg b/h5bench_patterns/sample_config/sample_cc1d_small_IMP.cfg similarity index 97% rename from h5bench_patterns/sample_config/sample_read_cc1d_small_IMP.cfg rename to h5bench_patterns/sample_config/sample_cc1d_small_IMP.cfg index 11bfa1cb..cc2ccb05 100644 --- a/h5bench_patterns/sample_config/sample_read_cc1d_small_IMP.cfg +++ b/h5bench_patterns/sample_config/sample_cc1d_small_IMP.cfg @@ -1,7 +1,6 @@ # this is a comment # Benchmark mode can only be one of these: CC/CI/IC/II/CC2D/CI2D/IC2D/II2D/CC2D/CC3D # Template cof include all options -IO_OPERATION=READ MEM_PATTERN=CONTIG FILE_PATTERN=CONTIG NUM_PARTICLES=16 M #16 K/G diff --git a/h5bench_patterns/sample_config/sample_read_cc1d.cfg b/h5bench_patterns/sample_config/sample_read_cc1d.cfg index 33f8ed05..89d1a74c 100644 --- a/h5bench_patterns/sample_config/sample_read_cc1d.cfg +++ b/h5bench_patterns/sample_config/sample_read_cc1d.cfg @@ -1,6 +1,5 @@ # this is a comment # Benchmark mode can only be one of these: CC/CI/IC/II/CC2D/CI2D/IC2D/II2D/CC2D/CC3D -IO_OPERATION=READ READ_OPTION=FULL NUM_DIMS=1 NUM_PARTICLES=8 M diff --git a/h5bench_patterns/sample_config/sample_read_cc1d_es1.cfg b/h5bench_patterns/sample_config/sample_read_cc1d_es1.cfg index 9cf4e3a1..16715f22 100644 --- a/h5bench_patterns/sample_config/sample_read_cc1d_es1.cfg +++ b/h5bench_patterns/sample_config/sample_read_cc1d_es1.cfg @@ -1,5 +1,4 @@ # this is a comment -IO_OPERATION=READ TO_READ_NUM_PARTICLES=16777216 READ_OPTION=READ_FULL MEM_PATTERN=CONTIG diff --git a/h5bench_patterns/sample_config/sample_write_cc1d.cfg b/h5bench_patterns/sample_config/sample_write_cc1d.cfg index a75c27ad..f63d41b9 100644 --- a/h5bench_patterns/sample_config/sample_write_cc1d.cfg +++ b/h5bench_patterns/sample_config/sample_write_cc1d.cfg @@ -17,7 +17,7 @@ NUM_DIMS=1 DIM_1=16777216 #16777216 # 16777216, 8388608 DIM_2=1 DIM_3=1 -ASYNC_MODE=NON #EXP #ASYNC_IMP ASYNC_NON ASYNC_EXP +ASYNC_MODE=IMP #EXP #ASYNC_IMP ASYNC_NON ASYNC_EXP CSV_FILE=perf_write_1d.csv #=========================== #WRITE_PATTERN=CC diff --git a/h5bench_patterns/sample_config/sample_write_cc1d_small.cfg b/h5bench_patterns/sample_config/sample_write_cc1d_small.cfg deleted file mode 100644 index a75c27ad..00000000 --- a/h5bench_patterns/sample_config/sample_write_cc1d_small.cfg +++ /dev/null @@ -1,23 +0,0 @@ -# this is a comment -# Benchmark mode can only be one of these: CC/CI/IC/II/CC2D/CI2D/IC2D/II2D/CC2D/CC3D -# Template cof include all options -IO_OPERATION=WRITE -MEM_PATTERN=CONTIG -FILE_PATTERN=CONTIG -NUM_PARTICLES=16 M #16 K/G -TIMESTEPS=5 -#IO_OPERATION=READ #WRITE -#MEM_PATTERN=CONTIG #INTERLEAVED STRIDED -#FILE_PATTERN=CONTIG #STRIDED -DELAYED_CLOSE_TIMESTEPS=2 -COLLECTIVE_DATA=NO #Optional, default for NO. -COLLECTIVE_METADATA=NO #Optional, default for NO. -EMULATED_COMPUTE_TIME_PER_TIMESTEP=1 s #1 ms, 1 min -NUM_DIMS=1 -DIM_1=16777216 #16777216 # 16777216, 8388608 -DIM_2=1 -DIM_3=1 -ASYNC_MODE=NON #EXP #ASYNC_IMP ASYNC_NON ASYNC_EXP -CSV_FILE=perf_write_1d.csv -#=========================== -#WRITE_PATTERN=CC diff --git a/h5bench_patterns/sample_config/template.cfg b/h5bench_patterns/sample_config/template.cfg index f908e822..52c189db 100644 --- a/h5bench_patterns/sample_config/template.cfg +++ b/h5bench_patterns/sample_config/template.cfg @@ -11,7 +11,6 @@ DIM_2=1 DIM_3=1 #======================================================== # IO pattern settings -IO_OPERATION=READ # WRITE MEM_PATTERN=CONTIG # INTERLEAVED STRIDED FILE_PATTERN=CONTIG # STRIDED #======================================================== From 693aebd8caeccdaea406ebfc216fc036477188c5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 27 Jul 2021 23:20:38 +0000 Subject: [PATCH 2/4] Committing clang-format changes --- h5bench_patterns/h5bench_read.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h5bench_patterns/h5bench_read.c b/h5bench_patterns/h5bench_read.c index 8cf49458..a455d4ce 100644 --- a/h5bench_patterns/h5bench_read.c +++ b/h5bench_patterns/h5bench_read.c @@ -385,7 +385,7 @@ main(int argc, char *argv[]) printf("Usage: ./%s /path/to/file #timestep [# mega particles]\n", argv[0]); return 0; } - + hid_t fapl, gapl; set_pl(&fapl, &gapl); From 56b2b1905de5642668a6cf43c0670464602c1152 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 27 Jul 2021 16:47:23 -0700 Subject: [PATCH 3/4] Fix for HDF5 1.8 where collective metadata operations is not implemented --- h5bench_patterns/h5bench_read.c | 6 ++++++ h5bench_patterns/h5bench_write.c | 2 ++ 2 files changed, 8 insertions(+) diff --git a/h5bench_patterns/h5bench_read.c b/h5bench_patterns/h5bench_read.c index 8cf49458..76179fe4 100644 --- a/h5bench_patterns/h5bench_read.c +++ b/h5bench_patterns/h5bench_read.c @@ -77,7 +77,9 @@ read_h5_data(time_step *ts, hid_t loc, hid_t *dset_ids, hid_t filespace, hid_t m hid_t dapl; unsigned long t1, t2, t3; dapl = H5Pcreate(H5P_DATASET_ACCESS); + #if H5_VERSION_GE(1,10,0) H5Pset_all_coll_metadata_ops(dapl, true); + #endif t1 = get_time_usec(); dset_ids[0] = H5Dopen_async(loc, "x", dapl, ts->es_meta_create); @@ -341,10 +343,14 @@ set_pl(hid_t *fapl, hid_t *gapl) { *fapl = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(*fapl, MPI_COMM_WORLD, MPI_INFO_NULL); + #if H5_VERSION_GE(1,10,0) H5Pset_all_coll_metadata_ops(*fapl, true); H5Pset_coll_metadata_write(*fapl, true); + #endif *gapl = H5Pcreate(H5P_GROUP_ACCESS); + #if H5_VERSION_GE(1,10,0) H5Pset_all_coll_metadata_ops(*gapl, true); + #endif } void diff --git a/h5bench_patterns/h5bench_write.c b/h5bench_patterns/h5bench_write.c index b7036549..81ad2956 100644 --- a/h5bench_patterns/h5bench_write.c +++ b/h5bench_patterns/h5bench_write.c @@ -885,8 +885,10 @@ set_metadata(hid_t fapl, int align, unsigned long threshold, unsigned long align if (meta_collective == 1) { if (MY_RANK == 0) printf("Collective write: enabled.\n"); + #if H5_VERSION_GE(1,10,0) H5Pset_all_coll_metadata_ops(fapl, 1); H5Pset_coll_metadata_write(fapl, 1); + #endif } else { if (MY_RANK == 0) From 1e52c929bf23028681fe3f7d09a6fae89d1681a6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 27 Jul 2021 23:48:13 +0000 Subject: [PATCH 4/4] Committing clang-format changes --- h5bench_patterns/h5bench_read.c | 12 ++++++------ h5bench_patterns/h5bench_write.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/h5bench_patterns/h5bench_read.c b/h5bench_patterns/h5bench_read.c index 3d93b63f..b56aad2b 100644 --- a/h5bench_patterns/h5bench_read.c +++ b/h5bench_patterns/h5bench_read.c @@ -77,9 +77,9 @@ read_h5_data(time_step *ts, hid_t loc, hid_t *dset_ids, hid_t filespace, hid_t m hid_t dapl; unsigned long t1, t2, t3; dapl = H5Pcreate(H5P_DATASET_ACCESS); - #if H5_VERSION_GE(1,10,0) +#if H5_VERSION_GE(1, 10, 0) H5Pset_all_coll_metadata_ops(dapl, true); - #endif +#endif t1 = get_time_usec(); dset_ids[0] = H5Dopen_async(loc, "x", dapl, ts->es_meta_create); @@ -343,14 +343,14 @@ set_pl(hid_t *fapl, hid_t *gapl) { *fapl = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(*fapl, MPI_COMM_WORLD, MPI_INFO_NULL); - #if H5_VERSION_GE(1,10,0) +#if H5_VERSION_GE(1, 10, 0) H5Pset_all_coll_metadata_ops(*fapl, true); H5Pset_coll_metadata_write(*fapl, true); - #endif +#endif *gapl = H5Pcreate(H5P_GROUP_ACCESS); - #if H5_VERSION_GE(1,10,0) +#if H5_VERSION_GE(1, 10, 0) H5Pset_all_coll_metadata_ops(*gapl, true); - #endif +#endif } void diff --git a/h5bench_patterns/h5bench_write.c b/h5bench_patterns/h5bench_write.c index 81ad2956..3027aa98 100644 --- a/h5bench_patterns/h5bench_write.c +++ b/h5bench_patterns/h5bench_write.c @@ -885,10 +885,10 @@ set_metadata(hid_t fapl, int align, unsigned long threshold, unsigned long align if (meta_collective == 1) { if (MY_RANK == 0) printf("Collective write: enabled.\n"); - #if H5_VERSION_GE(1,10,0) +#if H5_VERSION_GE(1, 10, 0) H5Pset_all_coll_metadata_ops(fapl, 1); H5Pset_coll_metadata_write(fapl, 1); - #endif +#endif } else { if (MY_RANK == 0)