From 2a4ce7d61f349faaf754cf0909ae065883d0a2ae Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Wed, 29 Nov 2017 14:42:22 -0500 Subject: [PATCH 01/33] Pull in diffs from kgen_extra.py into base/kgen_extra.py and kgen/parser/kgextra.py --- base/kgen_extra.py | 64 +++++++++++++++++++++++++++++++++++++++++- kgen/parser/kgextra.py | 64 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/base/kgen_extra.py b/base/kgen_extra.py index c58f5ca..6cf91ef 100644 --- a/base/kgen_extra.py +++ b/base/kgen_extra.py @@ -177,7 +177,8 @@ subroutine kgen_array_sumcheck(varname, sum1, sum2, finish) character(*), intent(in) :: varname real(kind=8), intent(in) :: sum1, sum2 - real(kind=8), parameter :: max_rel_diff = 1.E-10 + !real(kind=8), parameter :: max_rel_diff = 1.E-10 + real(kind=8), parameter :: max_rel_diff = 1.E-9 real(kind=8) :: diff, rel_diff logical, intent(in), optional :: finish logical checkresult @@ -550,6 +551,67 @@ END SUBROUTINE """ +kgen_mpifile = \ +""" +SUBROUTINE kgen_mpifile( str, rank, thread, cycle ) + CHARACTER(*), INTENT(INOUT) :: str + INTEGER, INTENT(IN) :: rank, thread, cycle + + CHARACTER(LEN=1024) :: format, string_format, string_format_format + CHARACTER(LEN=1024) :: rank_format + CHARACTER(LEN=1024) :: thread_format + CHARACTER(LEN=1024) :: cycle_format + INTEGER :: str_len + + str_len = INDEX(str,".") + + IF (str_len < 10) THEN + string_format_format = "(A1,I1)" + ELSE IF (str_len < 100) THEN + string_format_format = "(A1,I2)" + ELSE IF (str_len < 1000) THEN + string_format_format = "(A1,I3)" + ELSE IF (str_len < 10000) THEN + string_format_format = "(A1,I4)" + ENDIF + write(string_format,trim(string_format_format)) 'A',str_len + + IF (rank < 10) THEN + rank_format = "I1" + ELSE IF (rank < 100) THEN + rank_format = "I2" + ELSE IF (rank < 1000) THEN + rank_format = "I3" + ELSE IF (rank < 10000) THEN + rank_format = "I4" + ENDIF + + IF (thread < 10) THEN + thread_format = "I1" + ELSE IF (thread < 100) THEN + thread_format = "I2" + ELSE IF (thread < 1000) THEN + thread_format = "I3" + ELSE IF (thread < 10000) THEN + thread_format = "I4" + ENDIF + + IF (cycle < 10) THEN + cycle_format = "I1" + ELSE IF (cycle < 100) THEN + cycle_format = "I2" + ELSE IF (cycle < 1000) THEN + cycle_format = "I3" + ELSE IF (cycle < 10000) THEN + cycle_format = "I4" + ENDIF + + format = "("//trim(string_format)//","//trim(rank_format)//",A1,"//trim(thread_format)//",A1,"//trim(cycle_format)//")" + write(str,trim(format)) str(1:str_len), rank, ".", thread, ".", cycle + +END SUBROUTINE +""" + rdtsc = \ """ .file "rdtsc.s" .text diff --git a/kgen/parser/kgextra.py b/kgen/parser/kgextra.py index 57577fd..7bdc14c 100644 --- a/kgen/parser/kgextra.py +++ b/kgen/parser/kgextra.py @@ -177,7 +177,8 @@ subroutine kgen_array_sumcheck(varname, sum1, sum2, finish) character(*), intent(in) :: varname real(kind=8), intent(in) :: sum1, sum2 - real(kind=8), parameter :: max_rel_diff = 1.E-10 + !real(kind=8), parameter :: max_rel_diff = 1.E-10 + real(kind=8), parameter :: max_rel_diff = 1.E-9 real(kind=8) :: diff, rel_diff logical, intent(in), optional :: finish logical checkresult @@ -550,6 +551,67 @@ END SUBROUTINE """ +kgen_mpifile = \ +""" +SUBROUTINE kgen_mpifile( str, rank, thread, cycle ) + CHARACTER(*), INTENT(INOUT) :: str + INTEGER, INTENT(IN) :: rank, thread, cycle + + CHARACTER(LEN=1024) :: format, string_format, string_format_format + CHARACTER(LEN=1024) :: rank_format + CHARACTER(LEN=1024) :: thread_format + CHARACTER(LEN=1024) :: cycle_format + INTEGER :: str_len + + str_len = INDEX(str,".") + + IF (str_len < 10) THEN + string_format_format = "(A1,I1)" + ELSE IF (str_len < 100) THEN + string_format_format = "(A1,I2)" + ELSE IF (str_len < 1000) THEN + string_format_format = "(A1,I3)" + ELSE IF (str_len < 10000) THEN + string_format_format = "(A1,I4)" + ENDIF + write(string_format,trim(string_format_format)) 'A',str_len + + IF (rank < 10) THEN + rank_format = "I1" + ELSE IF (rank < 100) THEN + rank_format = "I2" + ELSE IF (rank < 1000) THEN + rank_format = "I3" + ELSE IF (rank < 10000) THEN + rank_format = "I4" + ENDIF + + IF (thread < 10) THEN + thread_format = "I1" + ELSE IF (thread < 100) THEN + thread_format = "I2" + ELSE IF (thread < 1000) THEN + thread_format = "I3" + ELSE IF (thread < 10000) THEN + thread_format = "I4" + ENDIF + + IF (cycle < 10) THEN + cycle_format = "I1" + ELSE IF (cycle < 100) THEN + cycle_format = "I2" + ELSE IF (cycle < 1000) THEN + cycle_format = "I3" + ELSE IF (cycle < 10000) THEN + cycle_format = "I4" + ENDIF + + format = "("//trim(string_format)//","//trim(rank_format)//",A1,"//trim(thread_format)//",A1,"//trim(cycle_format)//")" + write(str,trim(format)) str(1:str_len), rank, ".", thread, ".", cycle + +END SUBROUTINE +""" + rdtsc = \ """ .file "rdtsc.s" .text From ea20ea8b380c757731c12dfe2455a8bfd63028fb Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Wed, 29 Nov 2017 14:54:03 -0500 Subject: [PATCH 02/33] pull in changes to base/kgen_genfile.py and kgen/kggenfile.py --- base/kgen_genfile.py | 3 +++ kgen/kggenfile.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/base/kgen_genfile.py b/base/kgen_genfile.py index 1b175eb..b733ed1 100644 --- a/base/kgen_genfile.py +++ b/base/kgen_genfile.py @@ -902,6 +902,9 @@ def process_exclude(node, bag, depth): if isinstance(node, Name): for namepath, actions in bag['excludes'].iteritems(): if match_namepath(namepath, pack_innamepath(bag['stmt'], node.string)): + #jgw# for a remove_io exclude leave the type in + if actions[0] == 'remove_io': + return False bag['matched'] = True return True diff --git a/kgen/kggenfile.py b/kgen/kggenfile.py index 6e80e91..df4ec48 100644 --- a/kgen/kggenfile.py +++ b/kgen/kggenfile.py @@ -930,6 +930,9 @@ def process_exclude(node, bag, depth): if isinstance(node, Fortran2003.Name): for namepath, actions in bag['excludes'].iteritems(): if match_namepath(namepath, pack_innamepath(bag['stmt'], node.string)): + #jgw# for a remove_io exclude leave the type in + if actions[0] == 'remove_io': + return False bag['matched'] = True return True From 1cced999889ff36701457825988d25778fd735eb Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Wed, 29 Nov 2017 15:15:01 -0500 Subject: [PATCH 03/33] bring in changes from genmake.py into kgen/extractor/main.py and kgenapps/kernel_extractor/kext_make.py --- kgen/extractor/main.py | 12 +++++++----- kgenapps/kernel_extractor/kext_make.py | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/kgen/extractor/main.py b/kgen/extractor/main.py index 406d7f6..8a79484 100644 --- a/kgen/extractor/main.py +++ b/kgen/extractor/main.py @@ -281,6 +281,8 @@ def generate_kernel_makefile(self): #self.write(f, 'FC := ') for i, compiler in enumerate(compilers): self.write(f, 'FC_%d := %s'%(i, compiler)) + # jgw + self.write(f, 'FC_0 := $(shell which mpiifort)') if Config.kernel_option['FC_FLAGS']: #self.write(f, 'FC_FLAGS := %s'%Config.kernel_option['FC_FLAGS']) @@ -334,11 +336,11 @@ def generate_kernel_makefile(self): self.write(f, '') self.write(f, 'run: build') - if Config.add_mpi_frame['enabled']: - self.write(f, '%s%s -np %s ./kernel.exe'%(prerun_run_str, Config.add_mpi_frame['mpiexec'], Config.add_mpi_frame['np']), t=True) - else: - self.write(f, '%s./kernel.exe'%prerun_run_str, t=True) - self.write(f, '') + #jgw#if Config.add_mpi_frame['enabled']: + self.write(f, '%s%s -np %s ./kernel.exe'%(prerun_run_str, Config.add_mpi_frame['mpiexec'], Config.add_mpi_frame['np']), t=True) + #jgw#else: + #jgw# self.write(f, '%s./kernel.exe'%prerun_run_str, t=True) + #jgw#self.write(f, '') if Config.model['types']['papi']['enabled']: self.write(f, 'papi: build-papi') diff --git a/kgenapps/kernel_extractor/kext_make.py b/kgenapps/kernel_extractor/kext_make.py index 493c674..bc1646e 100644 --- a/kgenapps/kernel_extractor/kext_make.py +++ b/kgenapps/kernel_extractor/kext_make.py @@ -130,6 +130,8 @@ def generate_kernel_makefile(): write(f, 'FC := ') for i, compiler in enumerate(compilers): write(f, 'FC_%d := %s'%(i, compiler)) + # jgw + write(f, 'FC_0 := $(shell which mpiifort)') if Config.kernel_option['FC_FLAGS']: write(f, 'FC_FLAGS := %s'%Config.kernel_option['FC_FLAGS']) @@ -178,11 +180,11 @@ def generate_kernel_makefile(): write(f, '') write(f, 'run: build') - if Config.add_mpi_frame['enabled']: - write(f, '%s%s -np %s ./kernel.exe'%(prerun_run_str, Config.add_mpi_frame['mpiexec'], Config.add_mpi_frame['np']), t=True) - else: - write(f, '%s./kernel.exe'%prerun_run_str, t=True) - write(f, '') + #jgw#if Config.add_mpi_frame['enabled']: + write(f, '%s%s -np %s ./kernel.exe'%(prerun_run_str, Config.add_mpi_frame['mpiexec'], Config.add_mpi_frame['np']), t=True) + #jgw#else: + #jgw# write(f, '%s./kernel.exe'%prerun_run_str, t=True) + #jgw#write(f, '') write(f, 'build: ${ALL_OBJS}') From f2b24679a34ff874dc2361443cf3de718b5adc2f Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Wed, 29 Nov 2017 15:25:01 -0500 Subject: [PATCH 04/33] pull in changes from kgenapps/kernel_extractor/kext_tool.py into kgen/extractor/main.py and kgenapps/kernel_extractor/kext_tool.py --- kgen/extractor/main.py | 3 ++- kgenapps/kernel_extractor/kext_tool.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kgen/extractor/main.py b/kgen/extractor/main.py index 8a79484..f81f28b 100644 --- a/kgen/extractor/main.py +++ b/kgen/extractor/main.py @@ -12,7 +12,7 @@ append_program_in_root, set_indent from parser.kgparse import KGGenType from parser.kgextra import kgen_utils_file_head, kgen_utils_file_checksubr, kgen_get_newunit, kgen_error_stop, \ - kgen_utils_file_tostr, kgen_utils_array_sumcheck, kgen_rankthread + kgen_utils_file_tostr, kgen_utils_array_sumcheck, kgen_rankthread, kgen_mpifile KGUTIL = 'kgen_utils.f90' TPROF = 'tprof_mod.f90' @@ -520,4 +520,5 @@ def generate_kgen_utils(self): f.write(kgen_get_newunit) f.write(kgen_error_stop) f.write(kgen_rankthread) + f.write(kgen_mpifile) f.write('END MODULE kgen_utils_mod\n') diff --git a/kgenapps/kernel_extractor/kext_tool.py b/kgenapps/kernel_extractor/kext_tool.py index 4c344d3..ebae915 100755 --- a/kgenapps/kernel_extractor/kext_tool.py +++ b/kgenapps/kernel_extractor/kext_tool.py @@ -174,7 +174,7 @@ def fini(self): def generate_kgen_utils(self): from kgen_extra import kgen_utils_file_head, kgen_utils_file_checksubr, \ kgen_get_newunit, kgen_error_stop, kgen_utils_file_tostr, kgen_utils_array_sumcheck, \ - kgen_rankthread + kgen_rankthread, kgen_mpifile with open('%s/kgen_utils.f90'%Config.path['kernel'], 'wb') as f: f.write('MODULE kgen_utils_mod') @@ -188,6 +188,7 @@ def generate_kgen_utils(self): f.write(kgen_get_newunit) f.write(kgen_error_stop) f.write(kgen_rankthread) + f.write(kgen_mpifile) f.write('END MODULE kgen_utils_mod\n') def create_tree(self): From 0eeadc599acac98eb15d0286d040aa784bb8ba84 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 11:14:52 -0500 Subject: [PATCH 05/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/gen_driver.py to kgen/extractor/plugins/gencore/gen_driver.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/gen_driver.py. --- kgen/extractor/plugins/gencore/gen_driver.py | 51 ++++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/kgen/extractor/plugins/gencore/gen_driver.py b/kgen/extractor/plugins/gencore/gen_driver.py index cb12a46..14b3421 100644 --- a/kgen/extractor/plugins/gencore/gen_driver.py +++ b/kgen/extractor/plugins/gencore/gen_driver.py @@ -60,7 +60,8 @@ def create_kernel_driver_parts(self, node): namedpart_link_part(node, DRIVER_CONTAINS_PART, CONTAINS_PART) namedpart_link_part(node, DRIVER_SUBP_PART, SUBP_PART) - attrs = {'name':'kgen_utils_mod', 'isonly': True, 'items':['kgen_get_newunit', 'kgen_error_stop', 'kgen_dp', 'kgen_array_sumcheck', 'kgen_rankthreadinvoke']} + #jgw# add kgen_mpifile + attrs = {'name':'kgen_utils_mod', 'isonly': True, 'items':['kgen_get_newunit', 'kgen_error_stop', 'kgen_dp', 'kgen_array_sumcheck', 'kgen_rankthreadinvoke','kgen_mpifile']} part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) attrs = {'name':'tprof_mod', 'isonly': True, 'items':['tstart', 'tstop', 'tnull', 'tprnt']} @@ -71,6 +72,9 @@ def create_kernel_driver_parts(self, node): attrs = {'name':getinfo('topblock_stmt').name, 'isonly': True, 'items':[getinfo('parentblock_stmt').name]} part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) + #jgw# add use mpi + attrs = {'name':'mpi', 'isonly': False, 'items':[]} + part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) part_append_comment(node, USE_PART, '') part_append_genknode(node, IMPLICIT_PART, typedecl_statements.Implicit) @@ -115,6 +119,10 @@ def create_kernel_driver_parts(self, node): attrs = {'type_spec': 'INTEGER', 'entity_decls': ['kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke']} part_append_gensnode(node, DECL_PART, typedecl_statements.Integer, attrs=attrs) + #jgw# add NProcs and ProcID + attrs = {'type_spec': 'INTEGER', 'entity_decls': ['NProcs', 'ProcID']} + part_append_gensnode(node, DECL_PART, typedecl_statements.Integer, attrs=attrs) + attrs = {'type_spec': 'LOGICAL', 'entity_decls': ['kgen_evalstage', 'kgen_warmupstage', 'kgen_mainstage']} part_append_gensnode(node, DECL_PART, typedecl_statements.Logical, attrs=attrs) @@ -132,6 +140,17 @@ def create_kernel_driver_parts(self, node): part_append_comment(node, EXEC_PART, 'END IF') part_append_comment(node, EXEC_PART, '') + #jgw# Add calls to MPI_Init, MPI_Comm_Size, MPI_Comm_Rank + attrs = {'designator': 'MPI_Init', 'items': ( 'kgen_ierr', ) } + part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) + attrs = {'designator': 'MPI_Comm_Size', 'items': ( 'MPI_COMM_WORLD', '\ +NProcs', 'kgen_ierr', ) } + part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) + attrs = {'designator': 'MPI_Comm_Rank', 'items': ( 'MPI_COMM_WORLD', '\ +ProcID', 'kgen_ierr', ) } + part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) + part_append_comment(node, EXEC_PART, '') + #jgw# if getinfo('is_papi_enabled'): part_append_comment(node, EXEC_PART, '#ifdef KGEN_PAPI', style='rawtext') @@ -201,7 +220,9 @@ def create_kernel_driver_parts(self, node): # eval attrs = {'loopcontrol': 'WHILE ( kgen_ierr_list .EQ. 0 )'} - doobj1 = part_append_genknode(node, EXEC_PART, block_statements.Do, attrs=attrs) + #jgw# Get rid of while loop + #doobj1 = part_append_genknode(node, EXEC_PART, block_statements.Do, attrs=attrs) + doobj1 = node attrs = {'items': ['kgen_filepath'], 'specs': ['UNIT = kgen_unit_list', 'FMT="(A)"', 'IOSTAT=kgen_ierr_list']} part_append_genknode(doobj1, EXEC_PART, statements.Read, attrs=attrs) @@ -212,7 +233,9 @@ def create_kernel_driver_parts(self, node): attrs = {'variable': 'kgen_unit', 'sign': '=', 'expr': 'kgen_get_newunit()'} part_append_genknode(ifread, EXEC_PART, statements.Assignment, attrs=attrs) - attrs = {'designator': 'kgen_rankthreadinvoke', 'items': ( 'TRIM(ADJUSTL(kgen_filepath))', 'kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke' ) } + #jgw# call kgen_mpifile + #attrs = {'designator': 'kgen_rankthreadinvoke', 'items': ( 'TRIM(ADJUSTL(kgen_filepath))', 'kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke' ) } + attrs = {'designator': 'kgen_mpifile', 'items': ( 'kgen_filepath', 'ProcID', '0', '0' ) } part_append_genknode(ifread, EXEC_PART, statements.Call, attrs=attrs) attrs = {'specs': ['UNIT=kgen_unit', 'FILE=TRIM(ADJUSTL(kgen_filepath))', 'STATUS="OLD"', 'ACCESS="STREAM"', \ @@ -352,20 +375,25 @@ def create_kernel_driver_parts(self, node): part_append_comment(node, EXEC_PART, '') + #jgw# add ifzero node and replace several nodes below with ifzero + attrs = {'expr': 'ProcID == 0'} + ifzero = part_append_genknode(node, EXEC_PART, block_statements.IfThen\ +, attrs=attrs) + attrs = {'items': ['""']} - part_append_genknode(node, EXEC_PART, statements.Write, attrs=attrs) + part_append_genknode(ifzero, EXEC_PART, statements.Write, attrs=attrs) attrs = {'items': ['"****************************************************"'], 'specs': [ '*', '"(A)"' ]} - part_append_genknode(node, EXEC_PART, statements.Write, attrs=attrs) + part_append_genknode(ifzero, EXEC_PART, statements.Write, attrs=attrs) attrs = {'items': ['"kernel execution summary: %s"'%getinfo('kernel_name')], 'specs': [ '*', '"(4X,A)"' ]} - part_append_genknode(node, EXEC_PART, statements.Write, attrs=attrs) + part_append_genknode(ifzero, EXEC_PART, statements.Write, attrs=attrs) attrs = {'items': ['"****************************************************"'], 'specs': [ '*', '"(A)"' ]} - part_append_genknode(node, EXEC_PART, statements.Write, attrs=attrs) + part_append_genknode(ifzero, EXEC_PART, statements.Write, attrs=attrs) attrs = {'expr': 'kgen_case_count == 0'} - ifcount = part_append_genknode(node, EXEC_PART, block_statements.IfThen, attrs=attrs) + ifcount = part_append_genknode(ifzero, EXEC_PART, block_statements.IfThen, attrs=attrs) attrs = {'items': ['"No data file is verified."'] } part_append_genknode(ifcount, EXEC_PART, statements.Write, attrs=attrs) @@ -411,7 +439,7 @@ def create_kernel_driver_parts(self, node): part_append_comment(ifcount, EXEC_PART, '#endif', style='rawtext') attrs = {'items': ['"****************************************************"'], 'specs': [ '*', '"(A)"' ]} - part_append_genknode(node, EXEC_PART, statements.Write, attrs=attrs) + part_append_genknode(ifzero, EXEC_PART, statements.Write, attrs=attrs) if getinfo('add_mpi_frame'): part_append_comment(node, EXEC_PART, '') @@ -419,6 +447,11 @@ def create_kernel_driver_parts(self, node): part_append_comment(node, EXEC_PART, 'CALL mpi_finalize(kgen_ierr)') part_append_comment(node, EXEC_PART, '') + part_append_comment(node, EXEC_PART, '') + attrs = {'designator': 'MPI_Finalize', 'items': ( 'kgen_ierr', ) } + part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) + part_append_comment(node, EXEC_PART, '') + # Block Data attrs = {'name': 'KGEN'} cblock = part_append_gensnode(node.kgen_parent, UNIT_PART, block_statements.BlockData, attrs=attrs) From 945ae621d80e95fca8535436fca92cff4e2f41e8 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 11:24:38 -0500 Subject: [PATCH 06/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/gen_kernel_callsite_file.py to kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/gen_kernel_callsite_file.py. --- kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py b/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py index cd8149d..4e77be5 100644 --- a/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py @@ -99,6 +99,12 @@ def create_parentblock_parts(self, node): attrs = {'items': [ ( 'state', ('kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke', 'kgen_evalstage', 'kgen_warmupstage', 'kgen_mainstage') ) ]} part_append_genknode(node, DECL_PART, statements.Common, attrs=attrs) + #jgw# for mpi_allreduce + attrs = {'type_spec': 'INTEGER', 'entity_decls': ['send(1)=-1', 'recv(\ +1)=-1', 'kgen_ierr']} + part_append_gensnode(node, DECL_PART, typedecl_statements.Integer, att\ +rs=attrs) + part_append_comment(node, DECL_PART, '') namedpart_create_subpart(node, KERNEL_PBLOCK_READ_IN_LOCALS, EXEC_PART, index=0) From 60e970c36a3f08b8cf51f774ea0e1aeb21998fb0 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 11:55:33 -0500 Subject: [PATCH 07/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/gen_state_callsite_file.py to kgen/extractor/plugins/gencore/gen_state_callsite_file.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/gen_state_callsite_file.py. --- .../gencore/gen_state_callsite_file.py | 68 ++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py index f719ccd..46b842e 100644 --- a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py @@ -175,13 +175,15 @@ def create_parentblock_parts(self, node): part_append_gensnode(ifopen, EXEC_PART, statements.Exit, attrs=attrs) #SUBROUTINE kgen_init_vars - attrs = {'name': 'kgen_init_vars', 'args': ['mpi_s', 'mpi_e', 'omp_s', 'omp_e', 'invoke_e', 'msize', \ + #jgw# add kgen_mymid to argument list + attrs = {'name': 'kgen_init_vars', 'args': ['kgen_mymid', 'mpi_s', 'mpi_e', 'omp_s', 'omp_e', 'invoke_e', 'msize', \ 'osize', 'lockpath', 'last_invoke', 'isstop']} initvarsubr = part_append_gensnode(node, SUBP_PART, block_statements.Subroutine, attrs=attrs) part_append_comment(node, SUBP_PART, '') + #jgw# add kgen_mymid to argument list attrs = {'type_spec': 'INTEGER', 'attrspec': ['INTENT(IN)'], 'entity_decls': \ - ['mpi_s', 'mpi_e', 'omp_s', 'omp_e', 'invoke_e', 'msize', 'osize'], } + ['kgen_mymid', 'mpi_s', 'mpi_e', 'omp_s', 'omp_e', 'invoke_e', 'msize', 'osize'], } part_append_gensnode(initvarsubr, DECL_PART, typedecl_statements.Integer, attrs=attrs) attrs = {'type_spec': 'INTEGER', 'attrspec': ['INTENT(INOUT)', 'DIMENSION(0:osize-1)'], 'entity_decls': ['last_invoke']} @@ -216,11 +218,18 @@ def create_parentblock_parts(self, node): #attrs = {'variable': 'temp_unit', 'sign': '=', 'expr': 'kgen_get_newunit()'} #part_append_gensnode(doopenmp, EXEC_PART, statements.Assignment, attrs=attrs) + #jgw# add ifmymid0 node and attach several things to it. + attrs = {'expr': 'kgen_mymid .EQ. 0'} + ifmymid0 = part_append_gensnode(doopenmp, EXEC_PART, block_statements.\ +IfThen, attrs=attrs) + attrs = {'specs': ['NEWUNIT=temp_unit', 'FILE=lockpath(mpi_idx, openmp_idx)', 'STATUS="OLD"', 'IOSTAT=ierr']} - part_append_gensnode(doopenmp, EXEC_PART, statements.Open, attrs=attrs) + #jgw# + part_append_gensnode(ifmymid0, EXEC_PART, statements.Open, attrs=attrs) attrs = {'expr': 'ierr .EQ. 0'} - ifopenmp = part_append_gensnode(doopenmp, EXEC_PART, block_statements.IfThen, attrs=attrs) + #jgw# + ifopenmp = part_append_gensnode(ifmymid0, EXEC_PART, block_statements.IfThen, attrs=attrs) attrs = {'specs': [ 'UNIT=temp_unit', 'STATUS="DELETE"' ]} part_append_gensnode(ifopenmp, EXEC_PART, statements.Close, attrs=attrs) @@ -605,12 +614,20 @@ def create_callsite_parts1(self, node): part_append_gensnode(ifinit, EXEC_PART, statements.Assignment, attrs=attrs) for (mpi_s, mpi_e), (openmp_s, openmp_e), (invoke_s, invoke_e) in getinfo('invocations'): - attrs = {'designator': 'kgen_init_vars', 'items': ['INT(%s)'%mpi_s.replace('e', '(kgen_msize-1)'), \ + #jgw# add kgen_mymid argument to call + attrs = {'designator': 'kgen_init_vars', 'items': ['kgen_mymid', 'INT(%s)'%mpi_s.replace('e', '(kgen_msize-1)'), \ 'INT(%s)'%mpi_e.replace('e', '(kgen_msize-1)'), 'INT(%s)'%openmp_s.replace('e', '(kgen_osize-1)'), \ 'INT(%s)'%openmp_e.replace('e', '(kgen_osize-1)'), 'INT(%s)'%invoke_e, 'kgen_msize', 'kgen_osize', 'kgen_lockpath', \ 'kgen_last_invoke', 'kgen_isstop']} part_append_gensnode(ifinit, EXEC_PART, statements.Call, attrs=attrs) + #jgw# add mpi_barrier to avoid file race condition. + if getinfo('is_mpi_app'): + attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'\ +), 'kgen_ierr']} + namedpart_append_gensnode(node.kgen_kernel_id, BEFORE_CALLSITE, st\ +atements.Call, attrs=attrs) + # check save if getinfo('is_openmp_app'): @@ -939,15 +956,25 @@ def create_callsite_parts1(self, node): attrs = {'specs': [ 'UNIT=kgen_stopunit', 'STATUS="KEEP"' ]} part_append_gensnode(ifopen, EXEC_PART, statements.Close, attrs=attrs) + #jgw# only check stop on rank 0 + attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'\ +), 'kgen_ierr']} + part_append_gensnode(ifstop, EXEC_PART, statements.Call, attrs=att\ +rs) + attrs = {'expr': 'kgen_mymid == 0'} + ifzero = part_append_gensnode(ifstop, EXEC_PART, block_statements.\ +IfThen, attrs=attrs) for (mpi_s, mpi_e), (openmp_s, openmp_e), (invoke_s, invoke_e) in getinfo('invocations'): attrs = {'designator': 'kgen_check_stop', 'items': ['INT(%s)'%mpi_s.replace('e', '(kgen_msize-1)'), \ 'INT(%s)'%mpi_e.replace('e', '(kgen_msize-1)'), 'INT(%s)'%openmp_s.replace('e', '(kgen_osize-1)'), \ 'INT(%s)'%openmp_e.replace('e', '(kgen_osize-1)'), 'kgen_msize', 'kgen_osize', \ '0', 'kgen_lockpath', 'kgen_isstop']} - part_append_gensnode(ifstop, EXEC_PART, statements.Call, attrs=attrs) + #jgw# + part_append_gensnode(ifzero, EXEC_PART, statements.Call, attrs=attrs) - attrs = {'expr': 'ALL(kgen_isstop)'} + #jgw# add kgen_mymid == 0 to if test + attrs = {'expr': 'ALL(kgen_isstop) .and. kgen_mymid == 0'} ifallstop = part_append_gensnode(ifstop, EXEC_PART, block_statements.IfThen, attrs=attrs) #attrs = {'variable': 'kgen_stopunit', 'sign': '=', 'expr': 'kgen_get_newunit()'} @@ -981,10 +1008,10 @@ def create_callsite_parts1(self, node): if getinfo('is_mpi_app'): attrs = {'designator': 'SLEEP', 'items': ['5']} - part_append_gensnode(iflst, EXEC_PART, statements.Call, attrs=attrs) + #jgw#part_append_gensnode(iflst, EXEC_PART, statements.Call, attrs=attrs) - attrs = {'designator': 'mpi_abort', 'items': [getinfo('mpi_comm'), '0', 'kgen_ierr']} - part_append_gensnode(iflst, EXEC_PART, statements.Call, attrs=attrs) + #jgw#attrs = {'designator': 'mpi_abort', 'items': [getinfo('mpi_comm'), '0', 'kgen_ierr']} + #jgw#part_append_gensnode(iflst, EXEC_PART, statements.Call, attrs=attrs) else: attrs = {'designator': 'SLEEP', 'items': ['1']} part_append_gensnode(iflst, EXEC_PART, statements.Call, attrs=attrs) @@ -992,6 +1019,27 @@ def create_callsite_parts1(self, node): attrs = {'code': '0'} part_append_gensnode(iflst, EXEC_PART, statements.Stop, attrs=attrs) + #jgw# only check for finalization on rank 0. + if getinfo('is_mpi_app'): + attrs = {'expr': 'kgen_mymid == 0 .and. .not. ALL(kgen_isstop)\ +'} + ifallstop = part_append_gensnode(ifstop, EXEC_PART, block_stat\ +ements.IfThen, attrs=attrs) + attrs = {'designator': 'mpi_abort', 'items': [getinfo('mpi_com\ +m'), '0', 'kgen_ierr']} + part_append_gensnode(ifallstop, EXEC_PART, statements.Call, at\ +trs=attrs) + + elseifallstop = part_append_gensnode(ifallstop, EXEC_PART, blo\ +ck_statements.Else, attrs=attrs) + attrs = {'designator': 'mpi_finalize', 'items': ['kgen_ierr']} + part_append_gensnode(ifallstop, EXEC_PART, statements.Call, at\ +trs=attrs) + attrs = {} + part_append_gensnode(ifallstop, EXEC_PART, statements.Stop, at\ +trs=attrs) + # jgw + attrs = {'variable': 'kgen_invoke(0)', 'sign': '=', 'expr': 'kgen_invoke(0) + 1'} namedpart_append_gensnode(node.kgen_kernel_id, BEFORE_CALLSITE, statements.Assignment, attrs=attrs) From 8b25bd71d3313a98be8359f65b22d2ba514aadf0 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 12:03:18 -0500 Subject: [PATCH 08/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/gen_type.py to kgen/extractor/plugins/gencore/gen_type.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/gen_type.py. --- kgen/extractor/plugins/gencore/gen_type.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kgen/extractor/plugins/gencore/gen_type.py b/kgen/extractor/plugins/gencore/gen_type.py index 711180d..44b9b1c 100644 --- a/kgen/extractor/plugins/gencore/gen_type.py +++ b/kgen/extractor/plugins/gencore/gen_type.py @@ -5,7 +5,7 @@ from gencore_utils import get_dtype_writename, get_typedecl_writename, state_gencore_contains, \ get_dtype_readname, get_typedecl_readname, kernel_gencore_contains, gen_write_istrue, \ - gen_read_istrue, is_excluded, is_remove_state + gen_read_istrue, is_excluded, is_remove_state, is_remove_io class Gen_Type(Kgen_Plugin): def __init__(self): @@ -258,6 +258,8 @@ def create_dtype_read_subr(self, node): if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue + var = stmt.get_variable(entity_name) callname = get_typedecl_readname(stmt, entity_name) @@ -380,6 +382,8 @@ def create_dtype_write_subr(self, node): if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue + node.kgen_stmt.top.used4genstate = True var = stmt.get_variable(entity_name) From 68d14aff965d380856309bdc193a6574d3087e55 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 12:07:00 -0500 Subject: [PATCH 09/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/gen_typedecl_in_module.py to kgen/extractor/plugins/gencore/gen_typedecl_in_module.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/gen_typedecl_in_module.py. --- kgen/extractor/plugins/gencore/gen_typedecl_in_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kgen/extractor/plugins/gencore/gen_typedecl_in_module.py b/kgen/extractor/plugins/gencore/gen_typedecl_in_module.py index ff7002f..2faa8cc 100644 --- a/kgen/extractor/plugins/gencore/gen_typedecl_in_module.py +++ b/kgen/extractor/plugins/gencore/gen_typedecl_in_module.py @@ -419,7 +419,7 @@ def get_decls(names, decls, prefix=''): for entity_name, entity_decl in zip(entity_names, stmt.entity_decls): if node.kgen_parent.name+entity_name in self.kernel_extern_reads: continue - if is_remove_state(entity_name, stmt): continue + #jgw#if is_remove_state(entity_name, stmt): continue self.kernel_extern_reads.append(node.kgen_parent.name+entity_name) @@ -496,7 +496,7 @@ def create_subr_write_typedecl_in_module(self, node): for entity_name, entity_decl in zip(entity_names, stmt.entity_decls): if node.kgen_parent.name+entity_name in self.state_extern_writes: continue - if is_remove_state(entity_name, stmt): continue + #jgw#if is_remove_state(entity_name, stmt): continue self.state_extern_writes.append(node.kgen_parent.name+entity_name) From 56e5d87c74731e920bf434362f27dc5946492532 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 12:10:31 -0500 Subject: [PATCH 10/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/gen_typedecl_in_parentblock.py to kgen/extractor/plugins/gencore/gen_typedecl_in_parentblock.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/gen_typedecl_in_parentblock.py. --- .../plugins/gencore/gen_typedecl_in_parentblock.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kgen/extractor/plugins/gencore/gen_typedecl_in_parentblock.py b/kgen/extractor/plugins/gencore/gen_typedecl_in_parentblock.py index 8af2aab..1c7d901 100644 --- a/kgen/extractor/plugins/gencore/gen_typedecl_in_parentblock.py +++ b/kgen/extractor/plugins/gencore/gen_typedecl_in_parentblock.py @@ -6,7 +6,7 @@ DRIVER_READ_IN_ARGS, KERNEL_PBLOCK_READ_IN_LOCALS, KERNEL_PBLOCK_READ_OUT_LOCALS, \ DRIVER_DECL_PART, DRIVER_USE_PART, get_typedecl_writename, get_dtype_writename, state_gencore_contains, \ get_topname, get_typedecl_readname, get_dtype_readname, shared_objects, process_spec_stmts, is_zero_array, \ - is_excluded, is_remove_state, namedgen_read_istrue, namedgen_write_istrue, check_class_derived + is_excluded, is_remove_state, is_remove_io, namedgen_read_istrue, namedgen_write_istrue, check_class_derived from gencore_subr import create_write_subr, create_read_subr class Gen_Typedecl_In_Parentblock(Kgen_Plugin): @@ -118,6 +118,7 @@ def create_subr_read_typedecl_in_parentblock(self, node): if var.is_parameter(): continue if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue if self.check_intent(entity_name, stmt): if (entity_name,DRIVER_READ_IN_ARGS) not in argintype: @@ -191,6 +192,7 @@ def create_subr_read_typedecl_in_parentblock(self, node): if var.is_parameter(): continue if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue if (entity_name,KERNEL_PBLOCK_READ_OUT_LOCALS) not in localouttype: localouttype.append((uname.firstpartname(), KERNEL_PBLOCK_READ_OUT_LOCALS)) @@ -386,6 +388,7 @@ def create_subr_write_typedecl_in_parentblock(self, node): if var.is_parameter(): continue if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue if self.check_intent(entity_name, stmt): if (entity_name,STATE_PBLOCK_WRITE_IN_ARGS) not in argintype: @@ -399,6 +402,7 @@ def create_subr_write_typedecl_in_parentblock(self, node): if var.is_parameter(): continue if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue if (entity_name,STATE_PBLOCK_WRITE_OUT_LOCALS) not in localouttype: localouttype.append((uname.firstpartname(), STATE_PBLOCK_WRITE_OUT_LOCALS)) From 09994001e10fb43cde50c503346f17eb4c00323a Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 12:12:51 -0500 Subject: [PATCH 11/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/gen_typedecl_in_type.py to kgen/extractor/plugins/gencore/gen_typedecl_in_type.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/gen_typedecl_in_type.py. --- kgen/extractor/plugins/gencore/gen_typedecl_in_type.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kgen/extractor/plugins/gencore/gen_typedecl_in_type.py b/kgen/extractor/plugins/gencore/gen_typedecl_in_type.py index 97abd12..185d3b3 100644 --- a/kgen/extractor/plugins/gencore/gen_typedecl_in_type.py +++ b/kgen/extractor/plugins/gencore/gen_typedecl_in_type.py @@ -35,7 +35,7 @@ def create_read_subr(self, subrname, entity_name, parent, var, stmt, allocate=Fa checks = lambda n: isinstance(n.kgen_stmt, block_statements.Subroutine) and n.name==subrname if subrname not in self.kernel_created_subrs and not part_has_node(parent, SUBP_PART, checks): - if is_remove_state(entity_name, stmt): return + #jgw#if is_remove_state(entity_name, stmt): return self.kernel_created_subrs.append(subrname) @@ -225,7 +225,7 @@ def create_write_subr(self, subrname, entity_name, parent, var, stmt): checks = lambda n: isinstance(n.kgen_stmt, block_statements.Subroutine) and n.name==subrname if subrname not in self.state_created_subrs and not part_has_node(parent, SUBP_PART, checks): - if is_remove_state(entity_name, stmt): return + #jgw#if is_remove_state(entity_name, stmt): return self.state_created_subrs.append(subrname) From 6ba392c0ebe2e917b0e1591d08174d9f2ff0f8e7 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 12:15:12 -0500 Subject: [PATCH 12/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/gencore_utils.py to kgen/extractor/plugins/gencore/gencore_utils.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/gencore_utils.py. --- kgen/extractor/plugins/gencore/gencore_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kgen/extractor/plugins/gencore/gencore_utils.py b/kgen/extractor/plugins/gencore/gencore_utils.py index c302bba..751cc77 100644 --- a/kgen/extractor/plugins/gencore/gencore_utils.py +++ b/kgen/extractor/plugins/gencore/gencore_utils.py @@ -486,6 +486,14 @@ def is_remove_state(ename, stmt): return True return False +#jgw# +def is_remove_io(ename, stmt): + if hasattr(stmt, 'exclude_names'): + for name, actions in stmt.exclude_names.iteritems(): + if ename==name and 'remove_io' in actions: + return True + return False + def check_class_derived(stmt): if not stmt.is_class(): return False From d3034dacf8433d0d5684c15d3014223f15016f66 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 12:20:11 -0500 Subject: [PATCH 13/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/simple_timing.py to kgen/extractor/plugins/simple_timing/simple_timing.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/simple_timing.py. --- kgen/extractor/plugins/simple_timing/simple_timing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kgen/extractor/plugins/simple_timing/simple_timing.py b/kgen/extractor/plugins/simple_timing/simple_timing.py index 7401b31..db46488 100644 --- a/kgen/extractor/plugins/simple_timing/simple_timing.py +++ b/kgen/extractor/plugins/simple_timing/simple_timing.py @@ -121,7 +121,8 @@ def add_execblock(self, node): attrs = {'variable': 'kgen_measure', 'sign': '=', 'expr': '1.0D6*(kgen_stop_clock - kgen_start_clock)/DBLE(kgen_rate_clock)'} part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=attrs) - attrs = {'items': ['"%s : Time per call (usec): "'%getinfo('kernel_name'), 'kgen_measure']} + #jgw# add mpi rank to timing result. assumes kgen_mpirank defined. + attrs = {'items': ['"%s : Time per call (usec): "'%getinfo('kernel_name'), 'kgen_mpirank', 'kgen_measure']} part_append_gensnode(node, EXEC_PART, statements.Write, attrs=attrs) if getinfo('is_papi_enabled'): From c8b0bb6354dd83f3383ed19b85c197f3ea090134 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 19:47:22 -0500 Subject: [PATCH 14/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/verify_callsite_file.py to kgen/extractor/plugins/verification/verify_callsite_file.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/verify_callsite_file.py. --- .../verification/verify_callsite_file.py | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/kgen/extractor/plugins/verification/verify_callsite_file.py b/kgen/extractor/plugins/verification/verify_callsite_file.py index 36ac4a5..d8cd423 100644 --- a/kgen/extractor/plugins/verification/verify_callsite_file.py +++ b/kgen/extractor/plugins/verification/verify_callsite_file.py @@ -75,12 +75,39 @@ def create_verification_parts(self, node): attrs = {'items': ['""']} part_append_genknode(node, EXEC_PART, statements.Write, attrs=attrs) + #jgw# do mpi_allreduce on numOutTol + part_append_comment(node, EXEC_PART, '') + attrs = {'variable': 'send(1)', 'sign': '=', 'expr': 'check_status%num\ +OutTol'} + part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=att\ +rs) + attrs = {'designator': 'mpi_allreduce', 'items': ['send','recv','1','M\ +PI_INT','MPI_MAX', getinfo('mpi_comm'), 'kgen_ierr']} + part_append_gensnode(node, EXEC_PART, statements.Call, attrs=attrs) + + attrs = {'variable': 'check_status%numOutTol', 'sign': '=', 'expr': 'r\ +ecv(1)'} + part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=att\ +rs) + + attrs = {'designator': 'mpi_comm_rank', 'items': [getinfo('mpi_comm'),\ + 'kgen_mpirank', 'kgen_ierr']} + part_append_gensnode(node, EXEC_PART, statements.Call, attrs=attrs) + part_append_comment(node, EXEC_PART, '') + # jgw + # verification result attrs = {'expr': 'check_status%numOutTol > 0'} ifobj = part_append_genknode(node, EXEC_PART, block_statements.IfThen, attrs=attrs) + #jgw# only print verified on rank 0 + attrs = {'expr': 'kgen_mpirank == 0'} + ifzero = part_append_genknode(ifobj, EXEC_PART, block_statements.IfThe\ +n, attrs=attrs) + attrs = {'items': ['"Verification FAILED"']} - part_append_genknode(ifobj, EXEC_PART, statements.Write, attrs=attrs) + #jgw# + part_append_genknode(ifzero, EXEC_PART, statements.Write, attrs=attrs) attrs = {'variable': 'check_status%Passed', 'sign': '=', 'expr': '.FALSE.'} part_append_genknode(ifobj, EXEC_PART, statements.Assignment, attrs=attrs) @@ -90,8 +117,14 @@ def create_verification_parts(self, node): part_append_genknode(ifobj, EXEC_PART, statements.Else, attrs=attrs) + #jgw# only print verified on rank 0 + attrs = {'expr': 'kgen_mpirank == 0'} + ifzero = part_append_genknode(ifobj, EXEC_PART, block_statements.IfThe\ +n, attrs=attrs) + attrs = {'items': ['"Verification PASSED"']} - part_append_genknode(ifobj, EXEC_PART, statements.Write, attrs=attrs) + #jgw# + part_append_genknode(ifzero, EXEC_PART, statements.Write, attrs=attrs) attrs = {'variable': 'check_status%Passed', 'sign': '=', 'expr': '.TRUE.'} part_append_genknode(ifobj, EXEC_PART, statements.Assignment, attrs=attrs) @@ -110,5 +143,10 @@ def create_parentblock_parts(self, node): namedpart_link_part(node, VERIFY_PBLOCK_CONTAINS_PART, CONTAINS_PART) namedpart_link_part(node, VERIFY_PBLOCK_SUBP_PART, SUBP_PART) + #jgw# add use mpi. This was the best place I could find to put it, + # but it doesn't seem quite right. + attrs = {'name':'mpi', 'isonly': False, 'items':[]} + part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) + attrs = {'type_spec': 'TYPE', 'selector':(None, 'check_t'), 'entity_decls': ['check_status']} part_append_genknode(node, DECL_PART, typedecl_statements.Type, attrs=attrs) From e8e19be711bb117aa6081bbc3a358d6be93c9bd3 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 19:51:28 -0500 Subject: [PATCH 15/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/verify_subr.py to kgen/extractor/plugins/verification/verify_subr.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/verify_subr.py. --- kgen/extractor/plugins/verification/verify_subr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kgen/extractor/plugins/verification/verify_subr.py b/kgen/extractor/plugins/verification/verify_subr.py index 522ac7c..12f0508 100644 --- a/kgen/extractor/plugins/verification/verify_subr.py +++ b/kgen/extractor/plugins/verification/verify_subr.py @@ -168,7 +168,8 @@ def get_attrs(attrspec, allowed_attrs): part_append_genknode(subrobj, DECL_PART, var_class, attrs=attrs) # check result - attrs = {'type_spec': 'INTEGER', 'entity_decls': ['check_result']} + #jgw# + attrs = {'type_spec': 'INTEGER', 'entity_decls': ['check_result = -1']} part_append_genknode(subrobj, DECL_PART, typedecl_statements.Integer, attrs=attrs) # print check result From e6aa9e24d752330e998e49ce64e4a59163ab249b Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 19:54:26 -0500 Subject: [PATCH 16/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/verify_type.py to kgen/extractor/plugins/verification/verify_type.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/verify_type.py. --- kgen/extractor/plugins/verification/verify_type.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kgen/extractor/plugins/verification/verify_type.py b/kgen/extractor/plugins/verification/verify_type.py index 8d31c7b..508c3d2 100644 --- a/kgen/extractor/plugins/verification/verify_type.py +++ b/kgen/extractor/plugins/verification/verify_type.py @@ -2,7 +2,7 @@ from parser import statements, block_statements, typedecl_statements from kgplugin import Kgen_Plugin -from verify_utils import get_dtype_verifyname, get_typedecl_verifyname, kernel_verify_contains, kernel_verify_kgenutils, is_remove_state, \ +from verify_utils import get_dtype_verifyname, get_typedecl_verifyname, kernel_verify_contains, kernel_verify_kgenutils, is_remove_state, is_remove_io, \ is_zero_array class Verify_Type(Kgen_Plugin): @@ -203,6 +203,7 @@ def print_dummy_detail(parent, entity_name): var = stmt.get_variable(entity_name) if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue if var.is_array() and is_zero_array(var, stmt): continue callname = get_typedecl_verifyname(stmt, entity_name) From 9a72b4aea04ea6f0dcdb9de4c3ac73381fa48df7 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 20:00:53 -0500 Subject: [PATCH 17/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/verify_typedecl_in_module.py to kgen/extractor/plugins/verification/verify_typedecl_in_module.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/verify_typedecl_in_module.py. --- .../plugins/verification/verify_typedecl_in_module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kgen/extractor/plugins/verification/verify_typedecl_in_module.py b/kgen/extractor/plugins/verification/verify_typedecl_in_module.py index 92abfac..5978a1a 100644 --- a/kgen/extractor/plugins/verification/verify_typedecl_in_module.py +++ b/kgen/extractor/plugins/verification/verify_typedecl_in_module.py @@ -5,7 +5,7 @@ from kgplugin import Kgen_Plugin from verify_utils import get_module_verifyname, kernel_verify_contains, VERIFY_PBLOCK_USE_PART, VERIFY_PBLOCK_EXTERNS, \ - get_typedecl_verifyname, get_dtype_verifyname, is_remove_state, is_zero_array, check_class_derived + get_typedecl_verifyname, get_dtype_verifyname, is_remove_state, is_remove_io, is_zero_array, check_class_derived from verify_subr import create_verify_subr class Verify_Typedecl_In_Module(Kgen_Plugin): @@ -106,6 +106,7 @@ def create_subr_verify_typedecl_in_module(self, node): var = stmt.get_variable(entity_name) if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue if var.is_array() and is_zero_array(var, stmt): continue self.verify_extern.append(entity_name) From 577af99fc577baca69233184521d6ff6fa2765cb Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 20:05:51 -0500 Subject: [PATCH 18/33] pull in changes from kgenapps/kernel_extractor/plugins/gencore/verify_typedecl_in_parentblock.py to kgen/extractor/plugins/verification/verify_typedecl_in_parentblock.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/plugins/gencore/verify_typedecl_in_parentblock.py. --- .../plugins/verification/verify_typedecl_in_parentblock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kgen/extractor/plugins/verification/verify_typedecl_in_parentblock.py b/kgen/extractor/plugins/verification/verify_typedecl_in_parentblock.py index e545a9c..79fe3e0 100644 --- a/kgen/extractor/plugins/verification/verify_typedecl_in_parentblock.py +++ b/kgen/extractor/plugins/verification/verify_typedecl_in_parentblock.py @@ -2,7 +2,7 @@ from parser import statements, block_statements, typedecl_statements from kgplugin import Kgen_Plugin -from verify_utils import VERIFY_PBLOCK_LOCALS, get_typedecl_verifyname, get_dtype_verifyname, is_remove_state, is_zero_array, check_class_derived +from verify_utils import VERIFY_PBLOCK_LOCALS, get_typedecl_verifyname, get_dtype_verifyname, is_remove_state, is_remove_io, is_zero_array, check_class_derived from verify_subr import create_verify_subr class Verify_Typedecl_In_Parentblock(Kgen_Plugin): @@ -36,6 +36,7 @@ def create_subr_verify_typedecl_in_parentblock(self, node): var = stmt.get_variable(entity_name) if is_remove_state(entity_name, stmt): continue + if is_remove_io(entity_name, stmt): continue if var.is_array() and is_zero_array(var, stmt): continue self.verify_locals.append(entity_name) From 5f76a194d58edb11175c43b67f392c022c2e91d8 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 20:09:10 -0500 Subject: [PATCH 19/33] pull in changes from kgenapps/kernel_extractor/plugins/verification/verify_utils.py to kgen/extractor/plugins/verification/verify_utils.py. Did not also apply the changes to the obsolete version of kgenapps/kernel_extractor/verification/verify_utils.py. --- kgen/extractor/plugins/verification/verify_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kgen/extractor/plugins/verification/verify_utils.py b/kgen/extractor/plugins/verification/verify_utils.py index 5a60cd2..4ede2c4 100644 --- a/kgen/extractor/plugins/verification/verify_utils.py +++ b/kgen/extractor/plugins/verification/verify_utils.py @@ -128,6 +128,13 @@ def is_remove_state(ename, stmt): return True return False +def is_remove_io(ename, stmt): + if hasattr(stmt, 'exclude_names'): + for name, actions in stmt.exclude_names.iteritems(): + if ename==name and 'remove_io' in actions: + return True + return False + def is_param_zero(length, stmt): if hasattr(stmt, 'unknowns'): From 74c769a2574592f7aaa15dd724b2696573b42de1 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 20:32:16 -0500 Subject: [PATCH 20/33] fix bug in verbose mode. --- kgen/kgconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kgen/kgconfig.py b/kgen/kgconfig.py index 1199e58..31c7418 100644 --- a/kgen/kgconfig.py +++ b/kgen/kgconfig.py @@ -941,7 +941,7 @@ def _process_extract_flags(self, opts): # parsing logging options if opts.verbose_level: - self._attrs['verify']['verboselevel'] = str(opt) + self._attrs['verify']['verboselevel'] = str(opts) # mpi frame code in kernel driver if opts.add_mpi_frame: From 6508cbf812a9ff3c2aa105334c6c5ee697e97aa0 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 21:08:15 -0500 Subject: [PATCH 21/33] remove some spaces that were introduced by copy/paste. --- .../gencore/gen_kernel_callsite_file.py | 3 +-- .../gencore/gen_state_callsite_file.py | 26 +++++++------------ .../verification/verify_callsite_file.py | 12 +++------ 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py b/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py index 4e77be5..c3846ce 100644 --- a/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py @@ -102,8 +102,7 @@ def create_parentblock_parts(self, node): #jgw# for mpi_allreduce attrs = {'type_spec': 'INTEGER', 'entity_decls': ['send(1)=-1', 'recv(\ 1)=-1', 'kgen_ierr']} - part_append_gensnode(node, DECL_PART, typedecl_statements.Integer, att\ -rs=attrs) + part_append_gensnode(node, DECL_PART, typedecl_statements.Integer, attrs=attrs) part_append_comment(node, DECL_PART, '') diff --git a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py index 46b842e..001ddbf 100644 --- a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py @@ -625,8 +625,7 @@ def create_callsite_parts1(self, node): if getinfo('is_mpi_app'): attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'\ ), 'kgen_ierr']} - namedpart_append_gensnode(node.kgen_kernel_id, BEFORE_CALLSITE, st\ -atements.Call, attrs=attrs) + namedpart_append_gensnode(node.kgen_kernel_id, BEFORE_CALLSITE, statements.Call, attrs=attrs) # check save if getinfo('is_openmp_app'): @@ -959,8 +958,7 @@ def create_callsite_parts1(self, node): #jgw# only check stop on rank 0 attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'\ ), 'kgen_ierr']} - part_append_gensnode(ifstop, EXEC_PART, statements.Call, attrs=att\ -rs) + part_append_gensnode(ifstop, EXEC_PART, statements.Call, attrs=attrs) attrs = {'expr': 'kgen_mymid == 0'} ifzero = part_append_gensnode(ifstop, EXEC_PART, block_statements.\ IfThen, attrs=attrs) @@ -1023,21 +1021,15 @@ def create_callsite_parts1(self, node): if getinfo('is_mpi_app'): attrs = {'expr': 'kgen_mymid == 0 .and. .not. ALL(kgen_isstop)\ '} - ifallstop = part_append_gensnode(ifstop, EXEC_PART, block_stat\ -ements.IfThen, attrs=attrs) - attrs = {'designator': 'mpi_abort', 'items': [getinfo('mpi_com\ -m'), '0', 'kgen_ierr']} - part_append_gensnode(ifallstop, EXEC_PART, statements.Call, at\ -trs=attrs) - - elseifallstop = part_append_gensnode(ifallstop, EXEC_PART, blo\ -ck_statements.Else, attrs=attrs) + ifallstop = part_append_gensnode(ifstop, EXEC_PART, block_statements.IfThen, attrs=attrs) + attrs = {'designator': 'mpi_abort', 'items': [getinfo('mpi_comm'), '0', 'kgen_ierr']} + part_append_gensnode(ifallstop, EXEC_PART, statements.Call, attrs=attrs) + + elseifallstop = part_append_gensnode(ifallstop, EXEC_PART, block_statements.Else, attrs=attrs) attrs = {'designator': 'mpi_finalize', 'items': ['kgen_ierr']} - part_append_gensnode(ifallstop, EXEC_PART, statements.Call, at\ -trs=attrs) + part_append_gensnode(ifallstop, EXEC_PART, statements.Call, attrs=attrs) attrs = {} - part_append_gensnode(ifallstop, EXEC_PART, statements.Stop, at\ -trs=attrs) + part_append_gensnode(ifallstop, EXEC_PART, statements.Stop, attrs=attrs) # jgw attrs = {'variable': 'kgen_invoke(0)', 'sign': '=', 'expr': 'kgen_invoke(0) + 1'} diff --git a/kgen/extractor/plugins/verification/verify_callsite_file.py b/kgen/extractor/plugins/verification/verify_callsite_file.py index d8cd423..78f7da7 100644 --- a/kgen/extractor/plugins/verification/verify_callsite_file.py +++ b/kgen/extractor/plugins/verification/verify_callsite_file.py @@ -79,16 +79,14 @@ def create_verification_parts(self, node): part_append_comment(node, EXEC_PART, '') attrs = {'variable': 'send(1)', 'sign': '=', 'expr': 'check_status%num\ OutTol'} - part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=att\ -rs) + part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=attrs) attrs = {'designator': 'mpi_allreduce', 'items': ['send','recv','1','M\ PI_INT','MPI_MAX', getinfo('mpi_comm'), 'kgen_ierr']} part_append_gensnode(node, EXEC_PART, statements.Call, attrs=attrs) attrs = {'variable': 'check_status%numOutTol', 'sign': '=', 'expr': 'r\ ecv(1)'} - part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=att\ -rs) + part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=attrs) attrs = {'designator': 'mpi_comm_rank', 'items': [getinfo('mpi_comm'),\ 'kgen_mpirank', 'kgen_ierr']} @@ -102,8 +100,7 @@ def create_verification_parts(self, node): #jgw# only print verified on rank 0 attrs = {'expr': 'kgen_mpirank == 0'} - ifzero = part_append_genknode(ifobj, EXEC_PART, block_statements.IfThe\ -n, attrs=attrs) + ifzero = part_append_genknode(ifobj, EXEC_PART, block_statements.IfThen, attrs=attrs) attrs = {'items': ['"Verification FAILED"']} #jgw# @@ -119,8 +116,7 @@ def create_verification_parts(self, node): #jgw# only print verified on rank 0 attrs = {'expr': 'kgen_mpirank == 0'} - ifzero = part_append_genknode(ifobj, EXEC_PART, block_statements.IfThe\ -n, attrs=attrs) + ifzero = part_append_genknode(ifobj, EXEC_PART, block_statements.IfThen, attrs=attrs) attrs = {'items': ['"Verification PASSED"']} #jgw# From 0a210c3e34a52970f49f0870f90c6fc654c61b4b Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 21:25:32 -0500 Subject: [PATCH 22/33] change inital values of kgen_invoke and kgen_last_invoke --- kgen/extractor/plugins/gencore/gen_state_callsite_file.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py index 001ddbf..829ccd6 100644 --- a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py @@ -569,10 +569,12 @@ def create_callsite_parts1(self, node): attrs = {'variable': 'kgen_issave(:)', 'sign': '=', 'expr': '.FALSE.'} part_append_gensnode(ifinit, EXEC_PART, statements.Assignment, attrs=attrs) - attrs = {'variable': 'kgen_invoke(:)', 'sign': '=', 'expr': '1'} + #jgw# change initialization to 0 + attrs = {'variable': 'kgen_invoke(:)', 'sign': '=', 'expr': '0'} part_append_gensnode(ifinit, EXEC_PART, statements.Assignment, attrs=attrs) - attrs = {'variable': 'kgen_last_invoke(:)', 'sign': '=', 'expr': '1'} + #jgw# change initialization to 0 + attrs = {'variable': 'kgen_last_invoke(:)', 'sign': '=', 'expr': '0'} part_append_gensnode(ifinit, EXEC_PART, statements.Assignment, attrs=attrs) if getinfo('is_mpi_app'): From 182cf63fd75b5c375816896a67e86f6b0a5c0b0a Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 21:53:11 -0500 Subject: [PATCH 23/33] remove some carriage returns introduced by copy paste. --- kgen/extractor/plugins/gencore/gen_driver.py | 9 +++------ .../plugins/gencore/gen_kernel_callsite_file.py | 3 +-- .../plugins/gencore/gen_state_callsite_file.py | 15 +++++---------- .../plugins/verification/verify_callsite_file.py | 9 +++------ 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/kgen/extractor/plugins/gencore/gen_driver.py b/kgen/extractor/plugins/gencore/gen_driver.py index 14b3421..b1e8cf7 100644 --- a/kgen/extractor/plugins/gencore/gen_driver.py +++ b/kgen/extractor/plugins/gencore/gen_driver.py @@ -143,11 +143,9 @@ def create_kernel_driver_parts(self, node): #jgw# Add calls to MPI_Init, MPI_Comm_Size, MPI_Comm_Rank attrs = {'designator': 'MPI_Init', 'items': ( 'kgen_ierr', ) } part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) - attrs = {'designator': 'MPI_Comm_Size', 'items': ( 'MPI_COMM_WORLD', '\ -NProcs', 'kgen_ierr', ) } + attrs = {'designator': 'MPI_Comm_Size', 'items': ( 'MPI_COMM_WORLD', 'NProcs', 'kgen_ierr', ) } part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) - attrs = {'designator': 'MPI_Comm_Rank', 'items': ( 'MPI_COMM_WORLD', '\ -ProcID', 'kgen_ierr', ) } + attrs = {'designator': 'MPI_Comm_Rank', 'items': ( 'MPI_COMM_WORLD', 'ProcID', 'kgen_ierr', ) } part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) part_append_comment(node, EXEC_PART, '') #jgw# @@ -377,8 +375,7 @@ def create_kernel_driver_parts(self, node): #jgw# add ifzero node and replace several nodes below with ifzero attrs = {'expr': 'ProcID == 0'} - ifzero = part_append_genknode(node, EXEC_PART, block_statements.IfThen\ -, attrs=attrs) + ifzero = part_append_genknode(node, EXEC_PART, block_statements.IfThen, attrs=attrs) attrs = {'items': ['""']} part_append_genknode(ifzero, EXEC_PART, statements.Write, attrs=attrs) diff --git a/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py b/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py index c3846ce..1b42c4d 100644 --- a/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py @@ -100,8 +100,7 @@ def create_parentblock_parts(self, node): part_append_genknode(node, DECL_PART, statements.Common, attrs=attrs) #jgw# for mpi_allreduce - attrs = {'type_spec': 'INTEGER', 'entity_decls': ['send(1)=-1', 'recv(\ -1)=-1', 'kgen_ierr']} + attrs = {'type_spec': 'INTEGER', 'entity_decls': ['send(1)=-1', 'recv(1)=-1', 'kgen_ierr']} part_append_gensnode(node, DECL_PART, typedecl_statements.Integer, attrs=attrs) part_append_comment(node, DECL_PART, '') diff --git a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py index 829ccd6..ad0f782 100644 --- a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py @@ -220,8 +220,7 @@ def create_parentblock_parts(self, node): #jgw# add ifmymid0 node and attach several things to it. attrs = {'expr': 'kgen_mymid .EQ. 0'} - ifmymid0 = part_append_gensnode(doopenmp, EXEC_PART, block_statements.\ -IfThen, attrs=attrs) + ifmymid0 = part_append_gensnode(doopenmp, EXEC_PART, block_statements.IfThen, attrs=attrs) attrs = {'specs': ['NEWUNIT=temp_unit', 'FILE=lockpath(mpi_idx, openmp_idx)', 'STATUS="OLD"', 'IOSTAT=ierr']} #jgw# @@ -625,8 +624,7 @@ def create_callsite_parts1(self, node): #jgw# add mpi_barrier to avoid file race condition. if getinfo('is_mpi_app'): - attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'\ -), 'kgen_ierr']} + attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'), 'kgen_ierr']} namedpart_append_gensnode(node.kgen_kernel_id, BEFORE_CALLSITE, statements.Call, attrs=attrs) # check save @@ -958,12 +956,10 @@ def create_callsite_parts1(self, node): part_append_gensnode(ifopen, EXEC_PART, statements.Close, attrs=attrs) #jgw# only check stop on rank 0 - attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'\ -), 'kgen_ierr']} + attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'), 'kgen_ierr']} part_append_gensnode(ifstop, EXEC_PART, statements.Call, attrs=attrs) attrs = {'expr': 'kgen_mymid == 0'} - ifzero = part_append_gensnode(ifstop, EXEC_PART, block_statements.\ -IfThen, attrs=attrs) + ifzero = part_append_gensnode(ifstop, EXEC_PART, block_statements.IfThen, attrs=attrs) for (mpi_s, mpi_e), (openmp_s, openmp_e), (invoke_s, invoke_e) in getinfo('invocations'): attrs = {'designator': 'kgen_check_stop', 'items': ['INT(%s)'%mpi_s.replace('e', '(kgen_msize-1)'), \ @@ -1021,8 +1017,7 @@ def create_callsite_parts1(self, node): #jgw# only check for finalization on rank 0. if getinfo('is_mpi_app'): - attrs = {'expr': 'kgen_mymid == 0 .and. .not. ALL(kgen_isstop)\ -'} + attrs = {'expr': 'kgen_mymid == 0 .and. .not. ALL(kgen_isstop)'} ifallstop = part_append_gensnode(ifstop, EXEC_PART, block_statements.IfThen, attrs=attrs) attrs = {'designator': 'mpi_abort', 'items': [getinfo('mpi_comm'), '0', 'kgen_ierr']} part_append_gensnode(ifallstop, EXEC_PART, statements.Call, attrs=attrs) diff --git a/kgen/extractor/plugins/verification/verify_callsite_file.py b/kgen/extractor/plugins/verification/verify_callsite_file.py index 78f7da7..a0f8f68 100644 --- a/kgen/extractor/plugins/verification/verify_callsite_file.py +++ b/kgen/extractor/plugins/verification/verify_callsite_file.py @@ -77,15 +77,12 @@ def create_verification_parts(self, node): #jgw# do mpi_allreduce on numOutTol part_append_comment(node, EXEC_PART, '') - attrs = {'variable': 'send(1)', 'sign': '=', 'expr': 'check_status%num\ -OutTol'} + attrs = {'variable': 'send(1)', 'sign': '=', 'expr': 'check_status%numOutTol'} part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=attrs) - attrs = {'designator': 'mpi_allreduce', 'items': ['send','recv','1','M\ -PI_INT','MPI_MAX', getinfo('mpi_comm'), 'kgen_ierr']} + attrs = {'designator': 'mpi_allreduce', 'items': ['send','recv','1','MPI_INT','MPI_MAX', getinfo('mpi_comm'), 'kgen_ierr']} part_append_gensnode(node, EXEC_PART, statements.Call, attrs=attrs) - attrs = {'variable': 'check_status%numOutTol', 'sign': '=', 'expr': 'r\ -ecv(1)'} + attrs = {'variable': 'check_status%numOutTol', 'sign': '=', 'expr': 'recv(1)'} part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=attrs) attrs = {'designator': 'mpi_comm_rank', 'items': [getinfo('mpi_comm'),\ From a67b6d37e0c34b497c15565cfecf02475613da06 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 22:12:11 -0500 Subject: [PATCH 24/33] fix verbose level parsing error --- kgen/kgconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kgen/kgconfig.py b/kgen/kgconfig.py index 31c7418..d8039a5 100644 --- a/kgen/kgconfig.py +++ b/kgen/kgconfig.py @@ -941,7 +941,7 @@ def _process_extract_flags(self, opts): # parsing logging options if opts.verbose_level: - self._attrs['verify']['verboselevel'] = str(opts) + self._attrs['verify']['verboselevel'] = str(opts.verbose_level) # mpi frame code in kernel driver if opts.add_mpi_frame: From 7d6498a3e2d537e20411f352c49c7990bcc1471a Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 30 Nov 2017 23:01:17 -0500 Subject: [PATCH 25/33] Add some example results under examples/MPI_kernel --- examples/MPI_kernel/Makefile | 31 ++ examples/MPI_kernel/README | 14 + examples/MPI_kernel/state_callsite.f90 | 592 ++++++++++++++++++++++++ examples/MPI_kernel/verify_callsite.f90 | 378 +++++++++++++++ 4 files changed, 1015 insertions(+) create mode 100644 examples/MPI_kernel/Makefile create mode 100644 examples/MPI_kernel/README create mode 100644 examples/MPI_kernel/state_callsite.f90 create mode 100644 examples/MPI_kernel/verify_callsite.f90 diff --git a/examples/MPI_kernel/Makefile b/examples/MPI_kernel/Makefile new file mode 100644 index 0000000..03b1b70 --- /dev/null +++ b/examples/MPI_kernel/Makefile @@ -0,0 +1,31 @@ +#KGEN_HOME := $(HOME)/devel/packages/KGen +KGEN_HOME := $(HOME)/tmp/KGen +KGEN := ${KGEN_HOME}/bin/kgen + +SRC_DIR := ${PWD}/../src +SRC := ${SRC_DIR}/radtrans/rad_diffusion.f90 + +# don't have invocations beyond first cycle if the function is only called 1x! +# there is still a race condition somewhere in the src/Makefile. Change to -j8 +# until we can find it. + +test: + ${KGEN} \ + --check tolerance=1.0D-10 \ + --cmd-build="cd ${SRC_DIR}; make -j8" \ + --cmd-clean="cd ${SRC_DIR}; make clean; cd ${SRC_DIR}/run; ./clean.sh" \ + --cmd-run="cd ${SRC_DIR}; make run" \ + --exclude-ini=exclude.ini \ + --invocation=0-15:0:0 \ + --mpi=enable \ + --repr-etime=disable \ + --verbose=3 \ + ${SRC} + + +# --invocation 0-383:0:0 \ + +clean: + ${MAKE} clean -C ../src + $(RM) -rf _kgen_compflag_cmdwrapper.sh elapsedtime kernel kgen.log model state + #$(RM) include.ini strace.log diff --git a/examples/MPI_kernel/README b/examples/MPI_kernel/README new file mode 100644 index 0000000..b2af367 --- /dev/null +++ b/examples/MPI_kernel/README @@ -0,0 +1,14 @@ +These are some example files generated by my modified KGen. They are meant +to show the kind of things that are needed to have an MPI extracted kernel. +None of this has been tested with an OpenMP code. + +Makefile +- Showing the options that I'm invoking KGen with + +state_callsite.f90 +- need to set kgen_invoke and kgen_last_invoke to 0 to get the thing to run +- add mpi_comm_rank, mpi_comm_size, mpi_barrier +- write file per rank + +verify_callsite.f90 +- mpi_allreduce max on numOutTol diff --git a/examples/MPI_kernel/state_callsite.f90 b/examples/MPI_kernel/state_callsite.f90 new file mode 100644 index 0000000..1f0a2a9 --- /dev/null +++ b/examples/MPI_kernel/state_callsite.f90 @@ -0,0 +1,592 @@ + + +module raddiff_mod !jgw! + implicit none +contains + subroutine rad_diffusion + ! Calculates radiation transport using multi-group flux-limited + ! diffusion model + ! + ! + USE def_kind, ONLY : ik, rk + USE def_mpi, ONLY : NProcs, ProcID, MASTER, & + Idomain1, Idomain2, Idomain3, & + Ndomain1, Ndomain2, Ndomain3 + USE def_vars, ONLY : var, dtime, clight, TINY_NUMBER, PI1 + USE def_rad, ONLY : opac_table_groups, grid, fr + USE def_ursos, ONLY : n_e + USE multigrid_solver_mod, ONLY : multigrid_solver !jgw! + USE s_b_z_1, ONLY : set_boundary_zone_1 !jgw! + USE s_b_z_2, ONLY : set_boundary_zone_2 !jgw! + USE s_b_z_3, ONLY : set_boundary_zone_3 !jgw! + USE mpi + USE def_rad, ONLY: kw_def_rad_grid + USE def_rad, ONLY: kw_externs_in_def_rad + USE def_vars, ONLY: kw_externs_in_def_vars + USE def_vars, ONLY: kw_externs_out_def_vars + USE def_mpi, ONLY: kw_externs_in_def_mpi + IMPLICIT NONE + !----------------------------------------------------------------------- + ! Local variables + !----------------------------------------------------------------------- + !kgen variables + INTEGER :: kgen_openmp_issave + INTEGER :: kgen_mpirank + COMMON / state / kgen_mpirank, kgen_openmp_issave + LOGICAL :: kgen_istrue + INTEGER :: kgen_count + INTEGER, SAVE :: kgen_unit, kgen_stopunit + INTEGER, SAVE :: kgen_mymid, kgen_msize, kgen_osize, kgen_ierr + REAL(KIND=8), SAVE :: kgen_array_sum, kgen_realnum + LOGICAL, SAVE, ALLOCATABLE, DIMENSION(:,:) :: kgen_isstop + CHARACTER(LEN=1024), SAVE, ALLOCATABLE, DIMENSION(:,:) :: kgen_filepath, kgen_lockpath + LOGICAL, SAVE, ALLOCATABLE, DIMENSION(:) :: kgen_islast, kgen_issave, kgen_ischecked + INTEGER, SAVE, ALLOCATABLE, DIMENSION(:) :: kgen_invoke, kgen_last_invoke + integer (ik) :: i1, i2, i3, ierr, ifg, n2, n3 + integer (ik) :: icycle, ilevel, bottom_level + integer (ik), pointer :: i1s, i1e, i2s, i2e, i3s, i3e, nfg + + real (rk) :: tmp, tmp1, tmp2, tmp3, tmp4, cdt, err + + real (rk), parameter :: errmax = 1.0e-5_rk + real (rk), parameter :: zone_optical_depth_floor = 1.0e-7_rk + + type (grid), pointer :: sg, top_grid + + logical, save :: first_call = .true. + !----------------------------------------------------------------------- + + i1s => var%i1s + i1e => var%i1e + i2s => var%i2s + i2e => var%i2e + i3s => var%i3s + i3e => var%i3e + nfg => opac_table_groups + !----------------------------------------------------------------------- + ! Initialize the top subgrid + ! + ! + ! + allocate(sg) + top_grid => sg ! remember the top grid + sg%level=0 ! the top-subgrid level is zero + ! Allocate and define global coordinates sg%x2c and sg%x3c, + ! and global indeces sg%i2s, sg%i2e, sg%i3s, sg%i3e + + ! + call get_global_coordinates_2 (sg) + call get_global_coordinates_3 (sg) + + call allocate_subgrid(sg) + ! Calculate flux-limited radiation diffusion coefficient + + ! + cdt=clight*dtime + do i1=i1s,i1e + tmp1=var%x1c(i1+1)-var%x1c(i1-1) + do i2=i2s,i2e + tmp2=var%x1c(i1)*(var%x2c(i2+1)-var%x2c(i2-1)) + tmp4=var%x1c(i1)*var%x2c_sin(i2) + do i3=i3s,i3e + tmp3=tmp4*(var%x3c(i3+1)-var%x3c(i3-1)) + do ifg=1,nfg + if(fr(i1,i2,i3,ifg) < 0.0_rk) then + write(*,"('fr < 0 entering rad_diffusion, fr=',1pd12.5)") var%fr(i1,i2,i3,ifg) + write(*,"('ProcID=',i4,' i1=',i4,' i1s=',i4,' i1e=',i4)") ProcID, i1, i1s, i1e + write(*,"('ProcID=',i4,' i2=',i4,' i2s=',i4,' i2e=',i4)") ProcID, i2, i2s, i2e + write(*,"('ProcID=',i4,' i3=',i4,' i3s=',i4,' i3e=',i4)") ProcID, i3, i3s, i3e + stop + end if + tmp=((fr(i1+1,i2,i3,ifg)-fr(i1-1,i2,i3,ifg))/tmp1)**2+ & + ((fr(i1,i2+1,i3,ifg)-fr(i1,i2-1,i3,ifg))/tmp2)**2+ & + ((fr(i1,i2,i3+1,ifg)-fr(i1,i2,i3-1,ifg))/tmp3)**2 + tmp=sqrt(tmp)/(fr(i1,i2,i3,ifg)+TINY_NUMBER) ! inverse spatial scale of rad. en. density + ! assume a floor for the zone's optical depth + ! + ! + tmp=tmp+zone_optical_depth_floor/(0.5_rk*sqrt(tmp1**2+tmp2**2+tmp3**2)) + ! Photon mean-free path with flux limitation + ! + ! + tmp=1.0_rk/(3.0_rk*var%opar(i1,i2,i3,ifg)+tmp) + ! + sg%dif(i1,i2,i3,ifg)=cdt*tmp ! diff. coef. in [cm^2] + end do + end do + end do + end do + ! + do ifg=1,nfg + call set_boundary_zone_1(i2s,i2e,i3s,i3e,sg%dif(i1s-1:i1e+1,i2s-1:i2e+1,i3s-1:i3e+1,ifg)) + call set_boundary_zone_2(i2s,i2e,i3s,i3e,sg%dif(i1s-1:i1e+1,i2s-1:i2e+1,i3s-1:i3e+1,ifg)) + call set_boundary_zone_3(i2s,i2e,i3s,i3e,sg%dif(i1s-1:i1e+1,i2s-1:i2e+1,i3s-1:i3e+1,ifg)) + end do + !jgw!forall(i1=i1s:i1e,i2=i2s:i2e,i3=i3s:i3e,ifg=1:nfg) sg%opp(i1,i2,i3,ifg)=var%opap(i1,i2,i3,ifg) + ! + sg%opp(i1s:i1e,i2s:i2e,i3s:i3e,1:nfg)=var%opap(i1s:i1e,i2s:i2e,i3s:i3e,1:nfg) + do ifg=1,nfg + call set_boundary_zone_1(i2s,i2e,i3s,i3e,sg%opp(i1s-1:i1e+1,i2s-1:i2e+1,i3s-1:i3e+1,ifg)) + call set_boundary_zone_2(i2s,i2e,i3s,i3e,sg%opp(i1s-1:i1e+1,i2s-1:i2e+1,i3s-1:i3e+1,ifg)) + call set_boundary_zone_3(i2s,i2e,i3s,i3e,sg%opp(i1s-1:i1e+1,i2s-1:i2e+1,i3s-1:i3e+1,ifg)) + end do + + call get_diff_coefs (sg) + !----------------------------------------------------------------------- + ! Build the whole subgrid hierarchy + !----------------------------------------------------------------------- + ! The top level subgrid is of the zero level. + ! The lowest subgrid can be up to sg%level=N+1, + ! where 2**N is the number of theta/phi zones in subdomain. + ! If Ndomain2 is the odd number then the lowest sg%level=N. + ! The lowest subgrid level = bottom_level + + ! + ! + select case (i2e-i2s+1) + case ( 2) + bottom_level=2 + case ( 4) + bottom_level=3 + case ( 8) + bottom_level=4 + case ( 16) + bottom_level=5 + case ( 32) + bottom_level=6 + case ( 64) + bottom_level=7 + case (128) + bottom_level=8 + case (256) + bottom_level=9 + case default + write(*,"('*** Wrong bottom_level in rad_diffusion')") + call terminate + end select + if(Ndomain2 /= 2*(Ndomain2/2)) bottom_level=bottom_level-1 + ! + do + if(.not.associated(sg%crse)) allocate(sg%crse) + sg%crse%fine => sg + sg => sg%crse + sg%level=sg%fine%level+1 + call build_coarse_subgrid(sg) + call allocate_subgrid(sg) + call restrict_dif(sg) + call get_diff_coefs (sg) + if(sg%level == bottom_level) exit + ! The top level subgrid is of the zero level. + ! The lowest subgrid can be up to sg%level=N+1, + ! where 2**N is the number of phi zones in subdomain. + ! If Ndomain2 is the odd number then the lowest sg%level=N. + !if(sg%level==4) exit ! for 8 zones + !if(sg%level==5) exit ! for 16 zones + !if(sg%level==6) exit ! for 32 zones + !if(sg%level==7) exit ! for 64 zones + !if(maxval(sg%i2e(1:Ndomain2)-sg%i2s(1:Ndomain2)) == 0) exit + ! + ! + end do + !bottom_level = sg%level ! remember the bottom grid + !----------------------------------------------------------------------- + ! This is to print out the subgrid coordinates + ! + ! + if(.false.) then + if(ProcID==MASTER) then + write(*,*)'bottom_level =',bottom_level + sg => top_grid + do + + write(*,"('*** grid_level=',g12.5)") sg%level + i2=sg%i2s(1)-1 + write(*,"('i2=',i3,' x2c=',1pd12.5)") i2, sg%x2c(i2)/PI1 + if(sg%level==0) then + i2=sg%i2s(1)-1 + write(*,"('i2=',i3,' x2c=',1pd12.5)") i2, sg%x2c(i2)/PI1 + do n2=1,Ndomain2 + do i2=sg%i2s(n2),sg%i2e(n2) + write(*,"('i2=',i3,' x2c=',1pd12.5,' i2=',i3,' ',i3,' ii2=',i3,' ',i3,' Idomain2=',i3)") & + i2, sg%x2c(i2)/PI1, 0, 0, sg%ii2m(i2), sg%ii2p(i2), n2 + end do + end do + else if(sg%level==bottom_level) then + do n2=1,Ndomain2 + do i2=sg%i2s(n2),sg%i2e(n2) + write(*,"('i2=',i3,' x2c=',1pd12.5,' i2=',i3,' ',i3,' ii2=',i3,' ',i3,' Idomain2=',i3)") & + i2, sg%x2c(i2)/PI1, sg%i2m(i2), sg%i2p(i2), 0, 0, n2 + end do + end do + else + do n2=1,Ndomain2 + do i2=sg%i2s(n2),sg%i2e(n2) + write(*,"('i2=',i3,' x2c=',1pd12.5,' i2=',i3,' ',i3,' ii2=',i3,' ',i3,' Idomain2=',i3)") & + i2, sg%x2c(i2)/PI1, sg%i2m(i2), sg%i2p(i2), sg%ii2m(i2), sg%ii2p(i2), n2 + end do + end do + end if + i2=sg%i2e(Ndomain2)+1 + write(*,"('i2=',i3,' x2c=',1pd12.5)") i2, sg%x2c(i2)/PI1 + + write(*,"('*** ')") + i3=sg%i3s(1)-1 + write(*,"('i3=',i3,' x3c=',1pd12.5)") i3, sg%x3c(i3)/PI1 + if(sg%level==0) then + do n3=1,Ndomain3 + do i3=sg%i3s(n3),sg%i3e(n3) + write(*,"('i3=',i3,' x3c=',1pd12.5,' i3=',i3,' ',i3,' ii3=',i3,' ',i3,' Idomain3=',i3)") & + i3, sg%x3c(i3)/PI1, 0, 0, sg%ii3m(i3), sg%ii3p(i3), n3 + end do + end do + else if(sg%level==bottom_level) then + do n3=1,Ndomain3 + do i3=sg%i3s(n3),sg%i3e(n3) + write(*,"('i3=',i3,' x3c=',1pd12.5,' i3=',i3,' ',i3,' ii3=',i3,' ',i3,' Idomain3=',i3)") & + i3, sg%x3c(i3)/PI1, sg%i3m(i3), sg%i3p(i3), 0, 0, n3 + end do + end do + else + do n3=1,Ndomain3 + do i3=sg%i3s(n3),sg%i3e(n3) + write(*,"('i3=',i3,' x3c=',1pd12.5,' i3=',i3,' ',i3,' ii3=',i3,' ',i3,' Idomain3=',i3)") & + i3, sg%x3c(i3)/PI1, sg%i3m(i3), sg%i3p(i3), sg%ii3m(i3), sg%ii3p(i3), n3 + end do + end do + end if + i3=sg%i3e(Ndomain3)+1 + write(*,"('i3=',i3,' x3c=',1pd12.5)") i3, sg%x3c(i3)/PI1 + + if(sg%level==bottom_level) exit + sg => sg%crse + end do + end if + call MPI_BARRIER(MPI_COMM_WORLD,ierr) + stop + end if + !----------------------------------------------------------------------- + !$kgen begin_callsite multigrid + !START OF KGEN REGION + IF (.NOT. ALLOCATED(kgen_isstop)) THEN + kgen_osize = 1 + kgen_unit = -1 + kgen_stopunit = -1 + ALLOCATE (kgen_ischecked(0:kgen_osize-1)) + ALLOCATE (kgen_islast(0:kgen_osize-1)) + ALLOCATE (kgen_issave(0:kgen_osize-1)) + ALLOCATE (kgen_invoke(0:kgen_osize-1)) + ALLOCATE (kgen_last_invoke(0:kgen_osize-1)) + kgen_ischecked(:) = .FALSE. + kgen_islast(:) = .FALSE. + kgen_issave(:) = .FALSE. + kgen_invoke(:) = 0 + kgen_last_invoke(:) = 0 + CALL mpi_comm_rank(1140850688, kgen_mymid, kgen_ierr) + CALL kgen_error_stop(kgen_ierr, "mpi_comm_rank is failed") + CALL mpi_comm_size(1140850688, kgen_msize, kgen_ierr) + CALL kgen_error_stop(kgen_ierr, "mpi_comm_size is failed") + kgen_mpirank = kgen_mymid + ALLOCATE (kgen_filepath(0:kgen_msize-1, 0:kgen_osize-1)) + ALLOCATE (kgen_lockpath(0:kgen_msize-1, 0:kgen_osize-1)) + ALLOCATE (kgen_isstop(0:kgen_msize-1, 0:kgen_osize-1)) + kgen_isstop(:,:) = .TRUE. + CALL kgen_init_vars(kgen_mymid, INT(0), INT(15), INT(0), INT(0), INT(0), kgen_msize, kgen_osize, kgen_lockpath, & + &kgen_last_invoke, kgen_isstop) + END IF + CALL mpi_barrier(1140850688, kgen_ierr) + kgen_issave(0) = .FALSE. + kgen_islast(0) = .FALSE. + CALL kgen_check_save(INT(0), INT(15), INT(0), INT(0), INT(0), INT(0), kgen_mymid, 0, kgen_osize, kgen_invoke, & + &kgen_last_invoke, kgen_issave, kgen_islast) + IF (kgen_issave(0)) THEN + WRITE (kgen_filepath(kgen_mymid, 0), FMT="(A,I0,A,I0,A,I0)") "/p/work1/wohlbier/devel/aster/kgen/kernel/multigrid.", & + &kgen_mymid, ".", 0, ".", kgen_invoke(0) + OPEN (NEWUNIT=kgen_unit, FILE=TRIM(ADJUSTL(kgen_filepath(kgen_mymid, 0))), STATUS="REPLACE", ACCESS="STREAM", & + &FORM="UNFORMATTED", ACTION="WRITE", CONVERT="BIG_ENDIAN", IOSTAT=kgen_ierr) + CALL kgen_error_stop(kgen_ierr, "File open error: " // TRIM(ADJUSTL(kgen_filepath(kgen_mymid, 0)))) + + !argument input variables + + !extern input variables + CALL kw_externs_in_def_rad(kgen_unit) + CALL kw_externs_in_def_vars(kgen_unit) + CALL kw_externs_in_def_mpi(kgen_unit) + + !local input variables + WRITE (UNIT = kgen_unit) ifg + CALL kw_rad_diffusion_integer__ik_ptr(nfg, kgen_unit, "nfg", .FALSE.) + CALL kw_rad_diffusion_grid__grid_ptr(sg, kgen_unit, "sg", .FALSE.) + CALL kw_rad_diffusion_grid__grid_ptr(top_grid, kgen_unit, "top_grid", .FALSE.) + END IF + IF (kgen_issave(0)) THEN + kgen_openmp_issave = kgen_invoke(0) + ELSE + kgen_openmp_issave = -1 + END IF + !jgw! %fine is not written to kgen files so this association is addeded + sg => top_grid + do while(associated(sg%crse)) + sg%crse%fine => sg + sg => sg%crse + end do + + ! the real kernel + ! remove "freq_groups_loop" label as kgen can't handle it. + !freq_groups_loop: do ifg=1,nfg + do ifg=1,nfg + sg => top_grid + call multigrid_solver(ifg,errmax,sg) + end do !freq_groups_loop + kgen_openmp_issave = -1 + IF (kgen_issave(0)) THEN + + !extern output variables + CALL kw_externs_out_def_vars(kgen_unit) + + !local output variables + WRITE (UNIT = kgen_unit) ifg + CALL kw_rad_diffusion_grid__grid_ptr(sg, kgen_unit, "sg", .FALSE.) + CLOSE (UNIT=kgen_unit) + WRITE (*, *) "Collected Kernel Input/Ouput state from: ", kgen_mymid, 0, kgen_invoke(0) + END IF + IF (.NOT. kgen_ischecked(0) .AND. kgen_islast(0)) THEN + kgen_ischecked(0) = .TRUE. + OPEN (NEWUNIT=kgen_stopunit, FILE=TRIM(ADJUSTL(kgen_lockpath(kgen_mymid, 0))), STATUS="NEW", IOSTAT=kgen_ierr) + IF (kgen_ierr == 0) THEN + CLOSE (UNIT=kgen_stopunit, STATUS="KEEP") + END IF + CALL mpi_barrier(1140850688, kgen_ierr) + IF (kgen_mymid == 0) THEN + CALL kgen_check_stop(INT(0), INT(15), INT(0), INT(0), kgen_msize, kgen_osize, 0, kgen_lockpath, kgen_isstop) + END IF + IF (ALL(kgen_isstop) .and. kgen_mymid == 0) THEN + OPEN (NEWUNIT=kgen_stopunit, FILE="/p/work1/wohlbier/devel/aster/kgen/kernel/kgen_statefile.lst", STATUS="REPLACE", & + &FORM="FORMATTED", ACCESS="SEQUENTIAL", ACTION="WRITE", IOSTAT=kgen_ierr) + IF (kgen_ierr .EQ. 0) THEN + FLUSH (kgen_stopunit) + CALL kgen_write_list(kgen_stopunit, INT(0), INT(15), INT(0), INT(0), INT(0), INT(0)) + FLUSH (kgen_stopunit) + CLOSE (UNIT=kgen_stopunit, STATUS="KEEP") + WRITE (*, *) "Stopping application..." + END IF + END IF + IF (kgen_mymid == 0 .and. .not. ALL(kgen_isstop)) THEN + CALL mpi_abort(1140850688, 0, kgen_ierr) + ELSE + CALL mpi_finalize(kgen_ierr) + STOP + END IF + END IF + kgen_invoke(0) = kgen_invoke(0) + 1 + !END OF KGEN REGION + ! + + + !$kgen end_callsite multigrid + !----------------------------------------------------------------------- + ! Deallocate subgrids + !----------------------------------------------------------------------- + ! Cycle down to the coarsest (bottom) subgrid + + ! + ! + sg => top_grid + do while(associated(sg%crse)) + sg => sg%crse + end do + ! Cosequently deallocate subgrids from bottom to top + ! + ! + do while(associated(sg%fine)) + sg => sg%fine + deallocate(sg%crse) + end do + nullify(top_grid) + deallocate(sg) + !----------------------------------------------------------------------- + + call MPI_BARRIER(MPI_COMM_WORLD,ierr) + ! do i1=i1s,i1e + ! do i2=i2s,i2e + ! do i3=i3s,i3e + ! do ifg=1,nfg + ! if(var%fr(i1,i2,i3,ifg) < 0.0_rk) then + ! write(*,"('fr < 0 exiting rad_diffusion, fr=',1pd12.5)") var%fr(i1,i2,i3,ifg) + ! write(*,"('ProcID=',i4,' i1=',i4,' i1s=',i4,' i1e=',i4)") ProcID, i1, i1s, i1e + ! write(*,"('ProcID=',i4,' i2=',i4,' i2s=',i4,' i2e=',i4)") ProcID, i2, i2s, i2e + ! write(*,"('ProcID=',i4,' i3=',i4,' i3s=',i4,' i3e=',i4)") ProcID, i3, i3s, i3e + ! stop + ! end if + ! end do + ! end do + ! end do + ! end do + + CONTAINS + + + + FUNCTION kgen_get_newunit() RESULT ( new_unit ) + INTEGER, PARAMETER :: UNIT_MIN=100, UNIT_MAX=1000000 + LOGICAL :: is_opened + INTEGER :: nunit, new_unit, counter + + new_unit = -1 + DO counter=UNIT_MIN, UNIT_MAX + INQUIRE (UNIT=counter, OPENED=is_opened) + IF (.NOT. is_opened) THEN + new_unit = counter + EXIT + END IF + END DO + END FUNCTION kgen_get_newunit + + SUBROUTINE kgen_init_vars(kgen_mymid, mpi_s, mpi_e, omp_s, omp_e, invoke_e, msize, osize, lockpath, last_invoke, isstop) + INTEGER, INTENT(IN) :: kgen_mymid, mpi_s, mpi_e, omp_s, omp_e, invoke_e, msize, osize + INTEGER, INTENT(INOUT), DIMENSION(0:osize-1) :: last_invoke + LOGICAL, INTENT(INOUT), DIMENSION(0:msize-1,0:osize-1) :: isstop + CHARACTER(LEN=1024), INTENT(INOUT), DIMENSION(0:msize-1,0:osize-1) :: lockpath + INTEGER :: mpi_idx, openmp_idx, temp_unit, ierr + DO mpi_idx=mpi_s,mpi_e + DO openmp_idx=omp_s,omp_e + WRITE (lockpath(mpi_idx, openmp_idx), FMT="(A,I0,A,I0)") "/p/work1/wohlbier/devel/aster/kgen/kernel/done.", & + &mpi_idx, ".", openmp_idx + isstop(mpi_idx, openmp_idx) = .FALSE. + last_invoke(openmp_idx) = MAX( last_invoke(openmp_idx), invoke_e) + IF (kgen_mymid .EQ. 0) THEN + OPEN (NEWUNIT=temp_unit, FILE=lockpath(mpi_idx, openmp_idx), STATUS="OLD", IOSTAT=ierr) + IF (ierr .EQ. 0) THEN + CLOSE (UNIT=temp_unit, STATUS="DELETE") + END IF + END IF + END DO + END DO + END SUBROUTINE kgen_init_vars + + SUBROUTINE kgen_check_save(mpi_s, mpi_e, omp_s, omp_e, invoke_s, invoke_e, mymid, myoid, osize, invoke, last_invoke, issave, & + &islast) + INTEGER, INTENT(IN) :: mpi_s, mpi_e, omp_s, omp_e, invoke_s, invoke_e, osize, mymid, myoid + INTEGER, INTENT(IN), DIMENSION(0:osize-1) :: invoke, last_invoke + LOGICAL, INTENT(OUT), DIMENSION(0:osize-1) :: issave, islast + IF ((mymid .GE. mpi_s) .AND. (mymid .LE. mpi_e)) THEN + IF ((myoid .GE. omp_s) .AND. (myoid .LE. omp_e)) THEN + IF ((invoke(myoid) .GE. invoke_s) .AND. (invoke(myoid) .LE. invoke_e)) THEN + issave(myoid) = .TRUE. + END IF + IF (invoke(myoid) .GE. last_invoke(myoid)) THEN + islast(myoid) = .TRUE. + END IF + END IF + END IF + END SUBROUTINE kgen_check_save + + SUBROUTINE kgen_check_stop(mpi_s, mpi_e, omp_s, omp_e, msize, osize, myoid, lockpath, isstop) + INTEGER, INTENT(IN) :: mpi_s, mpi_e, omp_s, omp_e, msize, osize, myoid + CHARACTER(LEN=1024), INTENT(IN), DIMENSION(0:msize-1,0:osize-1) :: lockpath + LOGICAL, INTENT(OUT), DIMENSION(0:msize-1,0:osize-1) :: isstop + INTEGER :: mpi_idx, openmp_idx, ierr, myunit + DO mpi_idx=mpi_s,mpi_e + DO openmp_idx=omp_s,omp_e + IF (.NOT. isstop(mpi_idx, openmp_idx)) THEN + OPEN (NEWUNIT=myunit, FILE=TRIM(ADJUSTL(lockpath(mpi_idx, openmp_idx))), STATUS="OLD", ACTION="READ", & + &IOSTAT=ierr) + IF (ierr .EQ. 0) THEN + isstop(mpi_idx, openmp_idx) = .TRUE. + CLOSE (UNIT=myunit) + END IF + END IF + END DO + END DO + END SUBROUTINE kgen_check_stop + + SUBROUTINE kgen_write_list(myunit, mpi_s, mpi_e, omp_s, omp_e, invoke_s, invoke_e) + INTEGER, INTENT(IN) :: myunit, mpi_s, mpi_e, omp_s, omp_e, invoke_s, invoke_e + INTEGER :: mpi_idx, openmp_idx, invoke_idx, temp_unit, ierr + CHARACTER(LEN=16) :: mpi_str, openmp_str, invoke_str + DO mpi_idx=mpi_s,mpi_e + WRITE (mpi_str, "(I16)") mpi_idx + DO openmp_idx=omp_s,omp_e + WRITE (openmp_str, "(I16)") openmp_idx + DO invoke_idx=invoke_s,invoke_e + WRITE (invoke_str, "(I16)") invoke_idx + WRITE (UNIT = myunit, FMT="(A)") "multigrid." // TRIM(ADJUSTL(mpi_str)) // "." // TRIM(ADJUSTL(openmp_str)) & + &// "." // TRIM(ADJUSTL(invoke_str)) + END DO + OPEN (NEWUNIT=temp_unit, FILE="/p/work1/wohlbier/devel/aster/kgen/kernel/done." // TRIM(ADJUSTL(mpi_str)) // "." & + &// TRIM(ADJUSTL(openmp_str)), STATUS="OLD", IOSTAT=ierr) + IF (ierr .EQ. 0) THEN + CLOSE (UNIT=temp_unit, STATUS="DELETE") + END IF + END DO + END DO + END SUBROUTINE kgen_write_list + + SUBROUTINE kgen_error_stop(ierr, errmsg) + INTEGER, INTENT(IN) :: ierr + CHARACTER(LEN=*), INTENT(IN) :: errmsg + INTEGER :: kgen_ierr + IF (ierr /= 0) THEN + WRITE (*, *) errmsg + CALL mpi_abort(1140850688, 0, kgen_ierr) + END IF + END SUBROUTINE kgen_error_stop + + SUBROUTINE kgen_print_counter(counter) + INTEGER, INTENT(IN) :: counter + WRITE (*, *) "KGEN writes input state variables at count = ", counter + END SUBROUTINE kgen_print_counter + + !write state subroutine for kw_rad_diffusion_integer__ik_ptr + SUBROUTINE kw_rad_diffusion_integer__ik_ptr(var, kgen_unit, printname, printvar) + INTEGER(KIND=ik), INTENT(IN), POINTER :: var + INTEGER, INTENT(IN) :: kgen_unit + CHARACTER(LEN=*), INTENT(IN) :: printname + LOGICAL, INTENT(IN), OPTIONAL :: printvar + LOGICAL :: kgen_istrue + REAL(KIND=8) :: kgen_array_sum + + kgen_istrue = .TRUE. + IF (.NOT. ASSOCIATED(var)) THEN + kgen_istrue = .FALSE. + END IF + WRITE (UNIT = kgen_unit) kgen_istrue + IF (kgen_istrue) THEN + WRITE (UNIT = kgen_unit) var + IF (PRESENT( printvar ) .AND. printvar) THEN + WRITE (*, *) "KGEN DEBUG: " // printname // " = ", var + END IF + END IF + + END SUBROUTINE kw_rad_diffusion_integer__ik_ptr + + !write state subroutine for kw_rad_diffusion_grid__grid_ptr + SUBROUTINE kw_rad_diffusion_grid__grid_ptr(var, kgen_unit, printname, printvar) + TYPE(grid), INTENT(IN), POINTER :: var + INTEGER, INTENT(IN) :: kgen_unit + CHARACTER(LEN=*), INTENT(IN) :: printname + LOGICAL, INTENT(IN), OPTIONAL :: printvar + LOGICAL :: kgen_istrue + REAL(KIND=8) :: kgen_array_sum + + kgen_istrue = .TRUE. + IF (.NOT. ASSOCIATED(var)) THEN + kgen_istrue = .FALSE. + END IF + WRITE (UNIT = kgen_unit) kgen_istrue + IF (kgen_istrue) THEN + IF (PRESENT( printvar ) .AND. printvar) THEN + CALL kw_def_rad_grid(var, kgen_unit, printname, .TRUE.) + ELSE + CALL kw_def_rad_grid(var, kgen_unit, printname, .FALSE.) + END IF + END IF + + END SUBROUTINE kw_rad_diffusion_grid__grid_ptr + + end subroutine rad_diffusion +end module raddiff_mod !jgw! +BLOCK DATA KGEN + INTEGER :: kgen_mpirank = 0 + INTEGER :: kgen_openmp_issave = -1 + COMMON / state / kgen_mpirank, kgen_openmp_issave +END BLOCK DATA KGEN \ No newline at end of file diff --git a/examples/MPI_kernel/verify_callsite.f90 b/examples/MPI_kernel/verify_callsite.f90 new file mode 100644 index 0000000..c8eecba --- /dev/null +++ b/examples/MPI_kernel/verify_callsite.f90 @@ -0,0 +1,378 @@ +!KGEN-generated Fortran source file + +!Generated at : 2017-11-30 22:38:01 +!KGEN version : 0.7.3 + + + +module raddiff_mod !jgw! + USE kgen_utils_mod, ONLY: kgen_dp, kgen_array_sumcheck + USE tprof_mod, ONLY: tstart, tstop, tnull, tprnt + USE kgen_utils_mod, ONLY: check_t, kgen_init_check, kgen_tolerance, kgen_minvalue, CHECK_IDENTICAL, CHECK_IN_TOL, & + &CHECK_OUT_TOL + IMPLICIT NONE + PUBLIC rad_diffusion +contains +SUBROUTINE rad_diffusion(kgen_unit, kgen_measure, kgen_isverified) + ! Calculates radiation transport using multi-group flux-limited + ! diffusion model + ! + ! + USE def_kind, ONLY: ik, rk + USE def_rad, ONLY: grid + USE multigrid_solver_mod, ONLY: multigrid_solver + USE kgen_utils_mod, ONLY: kgen_dp, kgen_array_sumcheck + USE mpi + USE kgen_utils_mod, ONLY: kgen_perturb_real + USE def_rad, ONLY: kr_def_rad_grid + USE def_vars, ONLY: kr_externs_out_def_vars + USE def_rad, ONLY: kv_def_rad_grid + USE kgen_utils_mod, ONLY: check_t, kgen_init_check, kgen_tolerance, kgen_minvalue, CHECK_IDENTICAL, CHECK_IN_TOL, & + &CHECK_OUT_TOL + USE def_vars, ONLY: kv_externs_def_vars + IMPLICIT NONE + !----------------------------------------------------------------------- + ! Local variables + !----------------------------------------------------------------------- + INTEGER(KIND=ik) :: ifg + INTEGER(KIND=ik), pointer :: nfg + + + real (rk), parameter :: errmax = 1.0e-5_rk + + TYPE(grid), pointer :: sg, top_grid + + !----------------------------------------------------------------------- + INTEGER, INTENT(IN) :: kgen_unit + REAL(KIND=kgen_dp), INTENT(OUT) :: kgen_measure + LOGICAL, INTENT(OUT) :: kgen_isverified + LOGICAL :: kgen_istrue + REAL(KIND=8) :: kgen_array_sum + INTEGER :: kgen_mpirank, kgen_openmptid, kgen_kernelinvoke + LOGICAL :: kgen_evalstage, kgen_warmupstage, kgen_mainstage + COMMON / state / kgen_mpirank, kgen_openmptid, kgen_kernelinvoke, kgen_evalstage, kgen_warmupstage, kgen_mainstage + INTEGER :: send(1)=-1, recv(1)=-1, kgen_ierr + + TYPE(check_t) :: check_status + INTEGER*8 :: kgen_start_clock, kgen_stop_clock, kgen_rate_clock + INTEGER(KIND=ik) :: kgenref_ifg + TYPE(grid), pointer :: kgenref_sg + + !local input variables + READ (UNIT = kgen_unit) ifg + CALL kr_rad_diffusion_integer__ik_ptr(nfg, kgen_unit, "nfg", .FALSE.) + CALL kr_rad_diffusion_grid__grid_ptr(sg, kgen_unit, "sg", .FALSE.) + CALL kr_rad_diffusion_grid__grid_ptr(top_grid, kgen_unit, "top_grid", .FALSE.) + + !extern output variables + CALL kr_externs_out_def_vars(kgen_unit) + + !local output variables + READ (UNIT = kgen_unit) kgenref_ifg + CALL kr_rad_diffusion_grid__grid_ptr(kgenref_sg, kgen_unit, "kgenref_sg", .FALSE.) + + !----------------------------------------------------------------------- + ! Initialize the top subgrid + ! + ! + ! + ! Allocate and define global coordinates sg%x2c and sg%x3c, + ! and global indeces sg%i2s, sg%i2e, sg%i3s, sg%i3e + + ! + + ! Calculate flux-limited radiation diffusion coefficient + + ! + + ! + + !jgw!forall(i1=i1s:i1e,i2=i2s:i2e,i3=i3s:i3e,ifg=1:nfg) sg%opp(i1,i2,i3,ifg)=var%opap(i1,i2,i3,ifg) + ! + + + !----------------------------------------------------------------------- + ! Build the whole subgrid hierarchy + !----------------------------------------------------------------------- + ! The top level subgrid is of the zero level. + ! The lowest subgrid can be up to sg%level=N+1, + ! where 2**N is the number of theta/phi zones in subdomain. + ! If Ndomain2 is the odd number then the lowest sg%level=N. + ! The lowest subgrid level = bottom_level + + ! + ! + + + ! + + + !bottom_level = sg%level ! remember the bottom grid + !----------------------------------------------------------------------- + ! This is to print out the subgrid coordinates + ! + ! + + + !----------------------------------------------------------------------- + !$kgen begin_callsite multigrid + IF (kgen_evalstage) THEN + END IF + IF (kgen_warmupstage) THEN + END IF + IF (kgen_mainstage) THEN + END IF + + !Uncomment following call statement to turn on perturbation experiment. + !Adjust perturbation value and/or kind parameter if required. + !CALL kgen_perturb_real( your_variable, 1.0E-15_8 ) + + + !call to kgen kernel + !jgw! %fine is not written to kgen files so this association is addeded + sg => top_grid + do while(associated(sg%crse)) + sg%crse%fine => sg + sg => sg%crse + end do + + ! the real kernel + ! remove "freq_groups_loop" label as kgen can't handle it. + !freq_groups_loop: do ifg=1,nfg + do ifg=1,nfg + sg => top_grid + call multigrid_solver(ifg,errmax,sg) + end do !freq_groups_loop + IF (kgen_mainstage) THEN + + !verify init + CALL kgen_init_check(check_status, tolerance=1.0d-10, verboseLevel=3) + + !extern verify variables + CALL kv_externs_def_vars(check_status) + + !local verify variables + CALL kv_rad_diffusion_integer__ik("ifg", check_status, ifg, kgenref_ifg) + CALL kv_rad_diffusion_grid__grid_ptr("sg", check_status, sg, kgenref_sg) + WRITE (*, *) "" + IF (check_status%verboseLevel > 0) THEN + WRITE (*, *) "Number of output variables: ", check_status%numTotal + WRITE (*, *) "Number of identical variables: ", check_status%numIdentical + WRITE (*, *) "Number of non-identical variables within tolerance: ", check_status%numInTol + WRITE (*, *) "Number of non-identical variables out of tolerance: ", check_status%numOutTol + WRITE (*, *) "Tolerance: ", kgen_tolerance + END IF + WRITE (*, *) "" + + send(1) = check_status%numOutTol + CALL mpi_allreduce(send, recv, 1, MPI_INT, MPI_MAX, 1140850688, kgen_ierr) + check_status%numOutTol = recv(1) + CALL mpi_comm_rank(1140850688, kgen_mpirank, kgen_ierr) + + IF (check_status%numOutTol > 0) THEN + IF (kgen_mpirank == 0) THEN + WRITE (*, *) "Verification FAILED" + END IF + check_status%Passed = .FALSE. + kgen_isverified = .FALSE. + ELSE + IF (kgen_mpirank == 0) THEN + WRITE (*, *) "Verification PASSED" + END IF + check_status%Passed = .TRUE. + kgen_isverified = .TRUE. + END IF + WRITE (*, *) "" + CALL SYSTEM_CLOCK(kgen_start_clock, kgen_rate_clock) + !jgw! %fine is not written to kgen files so this association is addeded + sg => top_grid + do while(associated(sg%crse)) + sg%crse%fine => sg + sg => sg%crse + end do + + ! the real kernel + ! remove "freq_groups_loop" label as kgen can't handle it. + !freq_groups_loop: do ifg=1,nfg + do ifg=1,nfg + sg => top_grid + call multigrid_solver(ifg,errmax,sg) + end do !freq_groups_loop + CALL SYSTEM_CLOCK(kgen_stop_clock, kgen_rate_clock) + kgen_measure = 1.0D6*(kgen_stop_clock - kgen_start_clock)/DBLE(kgen_rate_clock) + WRITE (*, *) "multigrid : Time per call (usec): ", kgen_mpirank, kgen_measure + END IF + IF (kgen_warmupstage) THEN + END IF + IF (kgen_evalstage) THEN + END IF + ! + + + !$kgen end_callsite multigrid + !----------------------------------------------------------------------- + ! Deallocate subgrids + !----------------------------------------------------------------------- + ! Cycle down to the coarsest (bottom) subgrid + + ! + ! + + ! Cosequently deallocate subgrids from bottom to top + ! + ! + + !----------------------------------------------------------------------- + + ! do i1=i1s,i1e + ! do i2=i2s,i2e + ! do i3=i3s,i3e + ! do ifg=1,nfg + ! if(var%fr(i1,i2,i3,ifg) < 0.0_rk) then + ! write(*,"('fr < 0 exiting rad_diffusion, fr=',1pd12.5)") var%fr(i1,i2,i3,ifg) + ! write(*,"('ProcID=',i4,' i1=',i4,' i1s=',i4,' i1e=',i4)") ProcID, i1, i1s, i1e + ! write(*,"('ProcID=',i4,' i2=',i4,' i2s=',i4,' i2e=',i4)") ProcID, i2, i2s, i2e + ! write(*,"('ProcID=',i4,' i3=',i4,' i3s=',i4,' i3e=',i4)") ProcID, i3, i3s, i3e + ! stop + ! end if + ! end do + ! end do + ! end do + ! end do + + CONTAINS + + + + !read state subroutine for kr_rad_diffusion_integer__ik_ptr + SUBROUTINE kr_rad_diffusion_integer__ik_ptr(var, kgen_unit, printname, printvar) + INTEGER(KIND=ik), INTENT(INOUT), POINTER :: var + INTEGER, INTENT(IN) :: kgen_unit + CHARACTER(LEN=*), INTENT(IN) :: printname + LOGICAL, INTENT(IN), OPTIONAL :: printvar + LOGICAL :: kgen_istrue + REAL(KIND=8) :: kgen_array_sum + + READ (UNIT = kgen_unit) kgen_istrue + IF (kgen_istrue) THEN + IF (ASSOCIATED( var )) THEN + NULLIFY (var) + END IF + ALLOCATE (var) + READ (UNIT = kgen_unit) var + IF (PRESENT( printvar ) .AND. printvar) THEN + WRITE (*, *) "KGEN DEBUG: " // printname // " = ", var + END IF + END IF + END SUBROUTINE kr_rad_diffusion_integer__ik_ptr + + !read state subroutine for kr_rad_diffusion_grid__grid_ptr + SUBROUTINE kr_rad_diffusion_grid__grid_ptr(var, kgen_unit, printname, printvar) + TYPE(grid), INTENT(INOUT), POINTER :: var + INTEGER, INTENT(IN) :: kgen_unit + CHARACTER(LEN=*), INTENT(IN) :: printname + LOGICAL, INTENT(IN), OPTIONAL :: printvar + LOGICAL :: kgen_istrue + REAL(KIND=8) :: kgen_array_sum + + READ (UNIT = kgen_unit) kgen_istrue + IF (kgen_istrue) THEN + IF (ASSOCIATED( var )) THEN + NULLIFY (var) + END IF + ALLOCATE (var) + IF (PRESENT( printvar ) .AND. printvar) THEN + CALL kr_def_rad_grid(var, kgen_unit, printname, .TRUE.) + ELSE + CALL kr_def_rad_grid(var, kgen_unit, printname, .FALSE.) + END IF + END IF + END SUBROUTINE kr_rad_diffusion_grid__grid_ptr + + !verify state subroutine for kv_rad_diffusion_integer__ik + RECURSIVE SUBROUTINE kv_rad_diffusion_integer__ik(varname, check_status, var, kgenref_var) + CHARACTER(LEN=*), INTENT(IN) :: varname + TYPE(check_t), INTENT(INOUT) :: check_status + INTEGER(KIND=ik), INTENT(IN) :: var, kgenref_var + INTEGER :: check_result = -1 + LOGICAL :: is_print = .FALSE. + + integer(KIND=ik) :: diff + + check_status%numTotal = check_status%numTotal + 1 + + IF (var == kgenref_var) THEN + check_status%numIdentical = check_status%numIdentical + 1 + IF (check_status%verboseLevel > 1) THEN + WRITE (*, *) trim(adjustl(varname)), " is IDENTICAL." + END IF + check_result = CHECK_IDENTICAL + ELSE + diff = ABS(var - kgenref_var) + IF (diff <= kgen_tolerance) THEN + check_status%numInTol = check_status%numInTol + 1 + IF (check_status%verboseLevel > 0) THEN + WRITE (*, *) trim(adjustl(varname)), " is NOT IDENTICAL(within tolerance)." + END IF + check_result = CHECK_IN_TOL + ELSE + check_status%numOutTol = check_status%numOutTol + 1 + IF (check_status%verboseLevel > 0) THEN + WRITE (*, *) trim(adjustl(varname)), " is NOT IDENTICAL(out of tolerance)." + END IF + check_result = CHECK_OUT_TOL + END IF + END IF + IF (check_result == CHECK_IDENTICAL) THEN + IF (check_status%verboseLevel > 2) THEN + WRITE (*, *) "Difference is ", 0 + WRITE (*, *) "" + END IF + ELSE IF (check_result == CHECK_OUT_TOL) THEN + IF (check_status%verboseLevel > 0) THEN + WRITE (*, *) "Difference is ", diff + WRITE (*, *) "" + END IF + ELSE IF (check_result == CHECK_IN_TOL) THEN + IF (check_status%verboseLevel > 1) THEN + WRITE (*, *) "Difference is ", diff + WRITE (*, *) "" + END IF + END IF + + END SUBROUTINE kv_rad_diffusion_integer__ik + + !verify state subroutine for kv_rad_diffusion_grid__grid_ptr + RECURSIVE SUBROUTINE kv_rad_diffusion_grid__grid_ptr(varname, check_status, var, kgenref_var) + CHARACTER(LEN=*), INTENT(IN) :: varname + TYPE(check_t), INTENT(INOUT) :: check_status + TYPE(grid), pointer, INTENT(IN) :: var, kgenref_var + INTEGER :: check_result = -1 + LOGICAL :: is_print = .FALSE. + + + IF (ASSOCIATED(var)) THEN + check_status%numTotal = check_status%numTotal + 1 + + IF (check_result == CHECK_IDENTICAL) THEN + IF (check_status%verboseLevel > 2) THEN + WRITE (*, *) "NOT IMPLEMENTED" + WRITE (*, *) "" + END IF + ELSE IF (check_result == CHECK_OUT_TOL) THEN + IF (check_status%verboseLevel > 0) THEN + WRITE (*, *) "NOT IMPLEMENTED" + WRITE (*, *) "" + END IF + ELSE IF (check_result == CHECK_IN_TOL) THEN + IF (check_status%verboseLevel > 1) THEN + WRITE (*, *) "NOT IMPLEMENTED" + WRITE (*, *) "" + END IF + END IF + + END IF + END SUBROUTINE kv_rad_diffusion_grid__grid_ptr + +END SUBROUTINE rad_diffusion +end module raddiff_mod !jgw! \ No newline at end of file From af54e0df032e0b2fd04383d9d706e1a90005d2b7 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Wed, 14 Mar 2018 12:23:35 -0400 Subject: [PATCH 26/33] comment out warning --- base/statements.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/statements.py b/base/statements.py index 519ad3e..5f85496 100644 --- a/base/statements.py +++ b/base/statements.py @@ -1729,7 +1729,8 @@ def analyze(self): # start of KGEN addition def resolve_uname(self, uname, request): - Logger.warn('resolve_uname is not implemented: %s'%self.__class__) + # comment out based on input from Youngsung + #Logger.warn('resolve_uname is not implemented: %s'%self.__class__) pass def tokgen(self): From f91299025add7889942d59ed7869049f113823a1 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Wed, 4 Apr 2018 11:38:40 -0400 Subject: [PATCH 27/33] Add another forced "use mpi" in the generated kernel files. --- kgen/extractor/plugins/gencore/gen_driver.py | 2 ++ kgen/extractor/plugins/gencore/gen_typedecl_in_module.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/kgen/extractor/plugins/gencore/gen_driver.py b/kgen/extractor/plugins/gencore/gen_driver.py index b1e8cf7..b734578 100644 --- a/kgen/extractor/plugins/gencore/gen_driver.py +++ b/kgen/extractor/plugins/gencore/gen_driver.py @@ -72,7 +72,9 @@ def create_kernel_driver_parts(self, node): attrs = {'name':getinfo('topblock_stmt').name, 'isonly': True, 'items':[getinfo('parentblock_stmt').name]} part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) + #jgw# add use mpi + part_append_comment(node, USE_PART, 'forced mpi from gen_driver.py') attrs = {'name':'mpi', 'isonly': False, 'items':[]} part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) part_append_comment(node, USE_PART, '') diff --git a/kgen/extractor/plugins/gencore/gen_typedecl_in_module.py b/kgen/extractor/plugins/gencore/gen_typedecl_in_module.py index 2faa8cc..9a6b196 100644 --- a/kgen/extractor/plugins/gencore/gen_typedecl_in_module.py +++ b/kgen/extractor/plugins/gencore/gen_typedecl_in_module.py @@ -124,6 +124,12 @@ def add_default_stmts(self, node): #attrs = {'name':'IEEE_ARITHMETIC', 'nature': 'INTRINSIC', 'isonly': True, 'items':['ieee_is_normal']} #part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) + #jgw# add use mpi + part_append_comment(node, USE_PART, 'forced mpi from gen_typedecl_in_module.py') + attrs = {'name':'mpi', 'isonly': False, 'items':[]} + part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) + part_append_comment(node, USE_PART, '') + def create_kernel_module_parts(self, node): in_subrobj = None in_subrname = get_module_in_readname(node.kgen_stmt) From 2a2906118e41a4097ad9750698789083ef5d48ff Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Tue, 10 Apr 2018 08:35:43 -0400 Subject: [PATCH 28/33] add trim to kgen_util.f90 kgen_mpifile subroutine to make it work with llvm. --- kgen/parser/kgextra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kgen/parser/kgextra.py b/kgen/parser/kgextra.py index 7bdc14c..ec17c0e 100644 --- a/kgen/parser/kgextra.py +++ b/kgen/parser/kgextra.py @@ -607,7 +607,7 @@ ENDIF format = "("//trim(string_format)//","//trim(rank_format)//",A1,"//trim(thread_format)//",A1,"//trim(cycle_format)//")" - write(str,trim(format)) str(1:str_len), rank, ".", thread, ".", cycle + write(str,trim(format)) trim(str(1:str_len)), rank, ".", thread, ".", cycle END SUBROUTINE """ From 3c337015e887740cdb976c684fc22821354eaf83 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Tue, 10 Apr 2018 08:37:13 -0400 Subject: [PATCH 29/33] make the change also to the base/kgen_extra.py file even though it is not used. --- base/kgen_extra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/kgen_extra.py b/base/kgen_extra.py index 6cf91ef..025968b 100644 --- a/base/kgen_extra.py +++ b/base/kgen_extra.py @@ -607,7 +607,7 @@ ENDIF format = "("//trim(string_format)//","//trim(rank_format)//",A1,"//trim(thread_format)//",A1,"//trim(cycle_format)//")" - write(str,trim(format)) str(1:str_len), rank, ".", thread, ".", cycle + write(str,trim(format)) trim(str(1:str_len)), rank, ".", thread, ".", cycle END SUBROUTINE """ From 9260f87426eda947842f2f1d941fef96052662e7 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Thu, 19 Apr 2018 12:17:55 -0400 Subject: [PATCH 30/33] add some code to generated files and Makefile to enable use of intels itt_notify. --- kgen/extractor/main.py | 12 ++++++++++-- kgen/extractor/plugins/gencore/gen_driver.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/kgen/extractor/main.py b/kgen/extractor/main.py index f81f28b..92bc967 100644 --- a/kgen/extractor/main.py +++ b/kgen/extractor/main.py @@ -282,7 +282,7 @@ def generate_kernel_makefile(self): for i, compiler in enumerate(compilers): self.write(f, 'FC_%d := %s'%(i, compiler)) # jgw - self.write(f, 'FC_0 := $(shell which mpiifort)') + self.write(f, 'FC_0 := ${shell which mpiifort}') if Config.kernel_option['FC_FLAGS']: #self.write(f, 'FC_FLAGS := %s'%Config.kernel_option['FC_FLAGS']) @@ -311,6 +311,12 @@ def generate_kernel_makefile(self): new_options.append(add_opt) self.write(f, 'FC_FLAGS_SET_%d := %s'%(i, ' '.join(new_options))) + #jgw + self.write(f,'\n#INTEL_PATH=/usr/local/install/intel-2018-update2/vtune_amplifier') + self.write(f,'#FC_FLAGS_SET_0 += -D__ITT_NOTIFY__ -I${INTEL_PATH}/include/intel64') + self.write(f,'#LD_FLAGS = -L${INTEL_PATH}/lib64 -littnotify') + #jgw + prerun_build_str = '' if Config.prerun['kernel_build']: self.write(f, 'PRERUN_BUILD := %s'%Config.prerun['kernel_build']) @@ -358,7 +364,9 @@ def generate_kernel_makefile(self): #if len(compilers)>0 and not Config.kernel_option['FC']: fc_str += '_0' #if len(compiler_options)>0 and not Config.kernel_option['FC_FLAGS']: fc_flags_str += '_SET_0' - self.write(f, '%s${%s} ${%s} -o kernel.exe $^ %s %s'%(prerun_build_str, fc_str, fc_flags_str, link_flags, objects), t=True) + #jgw# + #self.write(f, '%s${%s} ${%s} -o kernel.exe $^ %s %s'%(prerun_build_str, fc_str, fc_flags_str, link_flags, objects), t=True) + self.write(f, '%s${%s} ${%s} -o kernel.exe $^ %s %s ${LD_FLAGS}'%(prerun_build_str, fc_str, fc_flags_str, link_flags, objects), t=True) self.write(f, '') if Config.model['types']['papi']['enabled']: diff --git a/kgen/extractor/plugins/gencore/gen_driver.py b/kgen/extractor/plugins/gencore/gen_driver.py index b734578..96b5739 100644 --- a/kgen/extractor/plugins/gencore/gen_driver.py +++ b/kgen/extractor/plugins/gencore/gen_driver.py @@ -64,6 +64,12 @@ def create_kernel_driver_parts(self, node): attrs = {'name':'kgen_utils_mod', 'isonly': True, 'items':['kgen_get_newunit', 'kgen_error_stop', 'kgen_dp', 'kgen_array_sumcheck', 'kgen_rankthreadinvoke','kgen_mpifile']} part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) + #jgw# add itt_notify + part_append_comment(node, USE_PART, '#ifdef __ITT_NOTIFY__', style='rawtext') + attrs = {'name':'ittnotify', 'isonly': False, 'items':[]} + part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) + part_append_comment(node, USE_PART, '#endif', style='rawtext') + attrs = {'name':'tprof_mod', 'isonly': True, 'items':['tstart', 'tstop', 'tnull', 'tprnt']} part_append_genknode(node, USE_PART, statements.Use, attrs=attrs) @@ -142,6 +148,12 @@ def create_kernel_driver_parts(self, node): part_append_comment(node, EXEC_PART, 'END IF') part_append_comment(node, EXEC_PART, '') + #jgw# Add calls to itt_pause + part_append_comment(node, EXEC_PART, '#ifdef __ITT_NOTIFY__', style='rawtext') + attrs = {'designator': 'ITT_PAUSE', 'items':('',)} + part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) + part_append_comment(node, EXEC_PART, '#endif', style='rawtext') + #jgw# Add calls to MPI_Init, MPI_Comm_Size, MPI_Comm_Rank attrs = {'designator': 'MPI_Init', 'items': ( 'kgen_ierr', ) } part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) From 04b2a5ae5eeeba18a00cc323325114354b62a1ef Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Tue, 24 Apr 2018 13:22:25 -0400 Subject: [PATCH 31/33] fix bug with cray ftn --- kgen/compflag/kgcompiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kgen/compflag/kgcompiler.py b/kgen/compflag/kgcompiler.py index abe7262..506469a 100644 --- a/kgen/compflag/kgcompiler.py +++ b/kgen/compflag/kgcompiler.py @@ -185,10 +185,10 @@ class CrayFortranCompiler(GenericFortranCompiler): discard_opts_arg = [ '-J' ] def get_discard_opts_noarg(self): - return super(IbmxlFortranCompiler, self).get_discard_opts_noarg() + self.discard_opts_noarg + return super(CrayFortranCompiler, self).get_discard_opts_noarg() + self.discard_opts_noarg def get_discard_opts_arg(self): - return super(IbmxlFortranCompiler, self).get_discard_opts_arg() + self.discard_opts_arg + return super(CrayFortranCompiler, self).get_discard_opts_arg() + self.discard_opts_arg class CompilerFactory(object): From b8a141679b52c4cc38d56c97b49450dee99c96a8 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Wed, 29 Aug 2018 15:45:49 -0400 Subject: [PATCH 32/33] Remove evalstage and warmupstage from kernel driver and callsite file. Other minor edits to generated files. --- kgen/extractor/main.py | 11 +++ kgen/extractor/plugins/gencore/gen_driver.py | 80 ++----------------- .../gencore/gen_kernel_callsite_file.py | 30 +------ .../gencore/gen_state_callsite_file.py | 6 +- .../plugins/gencore/gen_write_in_module.py | 9 ++- .../verification/verify_callsite_file.py | 6 +- 6 files changed, 34 insertions(+), 108 deletions(-) diff --git a/kgen/extractor/main.py b/kgen/extractor/main.py index 92bc967..c601f15 100644 --- a/kgen/extractor/main.py +++ b/kgen/extractor/main.py @@ -283,12 +283,19 @@ def generate_kernel_makefile(self): self.write(f, 'FC_%d := %s'%(i, compiler)) # jgw self.write(f, 'FC_0 := ${shell which mpiifort}') + self.write(f, 'FC_0 := ${shell which mpif90}') + #self.write(f, 'FC_0 := ${shell which ftn}') if Config.kernel_option['FC_FLAGS']: #self.write(f, 'FC_FLAGS := %s'%Config.kernel_option['FC_FLAGS']) self.write(f, 'FC_FLAGS_SET_0 := %s'%Config.kernel_option['FC_FLAGS']) else: #self.write(f, 'FC_FLAGS := ') + self.write(f, 'FC_FLAGS_SET_0 := -g -O2 -cpp') #jgw# + #self.write(f, 'FC_FLAGS_SET_0 := -g -O0 -cpp') #jgw# + #self.write(f, 'FC_FLAGS_SET_0 := -g -O0 -eF') #jgw# + #self.write(f, 'FC_FLAGS_SET_0 := -g -O2 -eF') #jgw# + #self.write(f, 'FC_FLAGS_SET_0 := -g -O3 -eF') #jgw# for i, options in enumerate(compiler_options): opt_list = options.split() L = len(opt_list) @@ -348,6 +355,10 @@ def generate_kernel_makefile(self): #jgw# self.write(f, '%s./kernel.exe'%prerun_run_str, t=True) #jgw#self.write(f, '') + #jgw# + self.write(f, 'test: build') + self.write(f, '\tmpirun -n 32 ./kernel.exe | grep PASSED') + if Config.model['types']['papi']['enabled']: self.write(f, 'papi: build-papi') if Config.add_mpi_frame['enabled']: diff --git a/kgen/extractor/plugins/gencore/gen_driver.py b/kgen/extractor/plugins/gencore/gen_driver.py index 96b5739..76ea779 100644 --- a/kgen/extractor/plugins/gencore/gen_driver.py +++ b/kgen/extractor/plugins/gencore/gen_driver.py @@ -4,11 +4,7 @@ from parser import statements, block_statements, typedecl_statements from gencore_utils import DRIVER_USE_PART, DRIVER_READ_IN_ARGS, DRIVER_CALLSITE_PART, DRIVER_DECL_PART, \ DRIVER_EXEC_PART, DRIVER_CONTAINS_PART, DRIVER_SUBP_PART, DRIVER_ALLOC_PART, DRIVER_DEALLOC_PART, \ - shared_objects, DRIVER_READ_IN_EXTERNS, DRIVER_WARMUP_ALLOC_PART, DRIVER_WARMUP_READ_IN_ARGS, \ - DRIVER_WARMUP_READ_IN_EXTERNS, DRIVER_WARMUP_CALLSITE_PART, DRIVER_WARMUP_DEALLOC_PART, \ - DRIVER_EVAL_ALLOC_PART, DRIVER_EVAL_READ_IN_ARGS, \ - DRIVER_EVAL_READ_IN_EXTERNS, DRIVER_EVAL_CALLSITE_PART, DRIVER_EVAL_DEALLOC_PART - + shared_objects, DRIVER_READ_IN_EXTERNS class Gen_K_Driver(Kgen_Plugin): def __init__(self): @@ -35,20 +31,6 @@ def finalize_kernel_driver_parts(self, node): attrs = {'designator': getinfo('parentblock_stmt').name, 'items': getinfo('kernel_driver_callsite_args')} namedpart_append_genknode(node.kgen_kernel_id, DRIVER_CALLSITE_PART, statements.Call, attrs=attrs) - for partname, wpartname in [ (DRIVER_ALLOC_PART, DRIVER_WARMUP_ALLOC_PART), (DRIVER_READ_IN_ARGS, DRIVER_WARMUP_READ_IN_ARGS), \ - (DRIVER_READ_IN_EXTERNS, DRIVER_WARMUP_READ_IN_EXTERNS), (DRIVER_CALLSITE_PART, DRIVER_WARMUP_CALLSITE_PART), \ - (DRIVER_DEALLOC_PART, DRIVER_WARMUP_DEALLOC_PART) ]: - pnode, rawname, named_part = get_namedpart(node.kgen_kernel_id, partname) - for item in named_part: - namedpart_append_node(node.kgen_kernel_id, wpartname, item) - - for partname, epartname in [ (DRIVER_ALLOC_PART, DRIVER_EVAL_ALLOC_PART), (DRIVER_READ_IN_ARGS, DRIVER_EVAL_READ_IN_ARGS), \ - (DRIVER_READ_IN_EXTERNS, DRIVER_EVAL_READ_IN_EXTERNS), (DRIVER_CALLSITE_PART, DRIVER_EVAL_CALLSITE_PART), \ - (DRIVER_DEALLOC_PART, DRIVER_EVAL_DEALLOC_PART) ]: - pnode, rawname, named_part = get_namedpart(node.kgen_kernel_id, partname) - for item in named_part: - namedpart_append_node(node.kgen_kernel_id, epartname, item) - # process after node creation def create_kernel_driver_parts(self, node): @@ -131,10 +113,10 @@ def create_kernel_driver_parts(self, node): attrs = {'type_spec': 'INTEGER', 'entity_decls': ['NProcs', 'ProcID']} part_append_gensnode(node, DECL_PART, typedecl_statements.Integer, attrs=attrs) - attrs = {'type_spec': 'LOGICAL', 'entity_decls': ['kgen_evalstage', 'kgen_warmupstage', 'kgen_mainstage']} + attrs = {'type_spec': 'LOGICAL', 'entity_decls': ['kgen_mainstage']} part_append_gensnode(node, DECL_PART, typedecl_statements.Logical, attrs=attrs) - attrs = {'items': [ ( 'state', ('kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke', 'kgen_evalstage', 'kgen_warmupstage', 'kgen_mainstage') ) ]} + attrs = {'items': [ ( 'state', ('kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke', 'kgen_mainstage') ) ]} part_append_gensnode(node, DECL_PART, statements.Common, attrs=attrs) part_append_comment(node, DECL_PART, '') @@ -251,7 +233,8 @@ def create_kernel_driver_parts(self, node): part_append_genknode(ifread, EXEC_PART, statements.Call, attrs=attrs) attrs = {'specs': ['UNIT=kgen_unit', 'FILE=TRIM(ADJUSTL(kgen_filepath))', 'STATUS="OLD"', 'ACCESS="STREAM"', \ - 'FORM="UNFORMATTED"', 'ACTION="READ"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} +#jgw# 'FORM="UNFORMATTED"', 'ACTION="READ"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} + 'FORM="UNFORMATTED"', 'ACTION="READ"', 'IOSTAT=kgen_ierr']} part_append_genknode(ifread, EXEC_PART, statements.Open, attrs=attrs) attrs = {'expr': 'kgen_ierr == 0'} @@ -263,56 +246,7 @@ def create_kernel_driver_parts(self, node): attrs = {'items': ['"***************** Verification against \'" // trim(adjustl(kgen_filepath)) // "\' *****************"']} part_append_genknode(ifopen, EXEC_PART, statements.Write, attrs=attrs) - attrs = {'variable': 'kgen_evalstage', 'sign': '=', 'expr': '.TRUE.'} - part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) - - attrs = {'variable': 'kgen_warmupstage', 'sign': '=', 'expr': '.FALSE.'} - part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) - - attrs = {'variable': 'kgen_mainstage', 'sign': '=', 'expr': '.FALSE.'} - part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) - - part_append_comment(ifopen, EXEC_PART, '') - - # register gencore parts - namedpart_create_subpart(ifopen, DRIVER_EVAL_ALLOC_PART, EXEC_PART) - namedpart_create_subpart(ifopen, DRIVER_EVAL_READ_IN_ARGS, EXEC_PART) - namedpart_create_subpart(ifopen, DRIVER_EVAL_READ_IN_EXTERNS, EXEC_PART) - namedpart_create_subpart(ifopen, DRIVER_EVAL_CALLSITE_PART, EXEC_PART) - namedpart_create_subpart(ifopen, DRIVER_EVAL_DEALLOC_PART, EXEC_PART) - - attrs = {'specs': ['UNIT=kgen_unit']} - part_append_genknode(ifopen, EXEC_PART, statements.Rewind, attrs=attrs) - - # warm up - attrs = {'variable': 'kgen_evalstage', 'sign': '=', 'expr': '.FALSE.'} - part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) - - attrs = {'variable': 'kgen_warmupstage', 'sign': '=', 'expr': '.TRUE.'} - part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) - - attrs = {'variable': 'kgen_mainstage', 'sign': '=', 'expr': '.FALSE.'} - part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) - - part_append_comment(ifopen, EXEC_PART, '') - - # register gencore parts - namedpart_create_subpart(ifopen, DRIVER_WARMUP_ALLOC_PART, EXEC_PART) - namedpart_create_subpart(ifopen, DRIVER_WARMUP_READ_IN_ARGS, EXEC_PART) - namedpart_create_subpart(ifopen, DRIVER_WARMUP_READ_IN_EXTERNS, EXEC_PART) - namedpart_create_subpart(ifopen, DRIVER_WARMUP_CALLSITE_PART, EXEC_PART) - namedpart_create_subpart(ifopen, DRIVER_WARMUP_DEALLOC_PART, EXEC_PART) - - attrs = {'specs': ['UNIT=kgen_unit']} - part_append_genknode(ifopen, EXEC_PART, statements.Rewind, attrs=attrs) - # main - attrs = {'variable': 'kgen_evalstage', 'sign': '=', 'expr': '.FALSE.'} - part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) - - attrs = {'variable': 'kgen_warmupstage', 'sign': '=', 'expr': '.FALSE.'} - part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) - attrs = {'variable': 'kgen_mainstage', 'sign': '=', 'expr': '.TRUE.'} part_append_genknode(ifopen, EXEC_PART, statements.Assignment, attrs=attrs) @@ -470,9 +404,9 @@ def create_kernel_driver_parts(self, node): attrs = {'type_spec': 'INTEGER', 'entity_decls': ['kgen_mpirank = 0', 'kgen_openmptid = 0', 'kgen_kernelinvoke = 0']} part_append_gensnode(cblock, DECL_PART, typedecl_statements.Integer, attrs=attrs) - attrs = {'type_spec': 'LOGICAL', 'entity_decls': ['kgen_evalstage = .TRUE.', 'kgen_warmupstage = .FALSE.', 'kgen_mainstage = .FALSE.']} + attrs = {'type_spec': 'LOGICAL', 'entity_decls': ['kgen_mainstage = .FALSE.']} part_append_gensnode(cblock, DECL_PART, typedecl_statements.Logical, attrs=attrs) - attrs = {'items': [ ( 'state', ('kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke', 'kgen_evalstage', 'kgen_warmupstage', 'kgen_mainstage') ) ]} + attrs = {'items': [ ( 'state', ('kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke', 'kgen_mainstage') ) ]} part_append_gensnode(cblock, DECL_PART, statements.Common, attrs=attrs) diff --git a/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py b/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py index 1b42c4d..fe5c624 100644 --- a/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_kernel_callsite_file.py @@ -93,14 +93,14 @@ def create_parentblock_parts(self, node): attrs = {'type_spec': 'INTEGER', 'entity_decls': ['kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke']} part_append_genknode(node, DECL_PART, typedecl_statements.Integer, attrs=attrs) - attrs = {'type_spec': 'LOGICAL', 'entity_decls': ['kgen_evalstage', 'kgen_warmupstage', 'kgen_mainstage']} + attrs = {'type_spec': 'LOGICAL', 'entity_decls': ['kgen_mainstage']} part_append_genknode(node, DECL_PART, typedecl_statements.Logical, attrs=attrs) - attrs = {'items': [ ( 'state', ('kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke', 'kgen_evalstage', 'kgen_warmupstage', 'kgen_mainstage') ) ]} + attrs = {'items': [ ( 'state', ('kgen_mpirank', 'kgen_openmptid', 'kgen_kernelinvoke', 'kgen_mainstage') ) ]} part_append_genknode(node, DECL_PART, statements.Common, attrs=attrs) #jgw# for mpi_allreduce - attrs = {'type_spec': 'INTEGER', 'entity_decls': ['send(1)=-1', 'recv(1)=-1', 'kgen_ierr']} + attrs = {'type_spec': 'INTEGER', 'entity_decls': ['ksend(1)=-1', 'krecv(1)=-1', 'kgen_ierr']} part_append_gensnode(node, DECL_PART, typedecl_statements.Integer, attrs=attrs) part_append_comment(node, DECL_PART, '') @@ -178,21 +178,6 @@ def create_callsite_parts(self, node): idx = index - attrs = {'expr': 'kgen_evalstage'} - ifeval = part_insert_genknode(node.kgen_parent, EXEC_PART, block_statements.IfThen, attrs=attrs, index=idx) - setinfo('blocknode_beforecallsite_eval', ifeval) - idx += 1 - - attrs = {'expr': 'kgen_warmupstage'} - ifwarmup = part_insert_genknode(node.kgen_parent, EXEC_PART, block_statements.IfThen, attrs=attrs, index=idx) - setinfo('blocknode_beforecallsite_warmup', ifwarmup) - idx += 1 - - attrs = {'expr': 'kgen_mainstage'} - ifmain = part_insert_genknode(node.kgen_parent, EXEC_PART, block_statements.IfThen, attrs=attrs, index=idx) - setinfo('blocknode_beforecallsite_main', ifmain) - idx += 1 - namedpart_create_subpart(node.kgen_parent, KERNEL_PBLOCK_BEFORE_KERNEL, EXEC_PART, index=idx) self.plugin_common[node.kgen_kernel_id]['ext.gencore']['blocks']['before_kernel'] = KERNEL_PBLOCK_BEFORE_KERNEL idx += 1 @@ -238,15 +223,6 @@ def create_callsite_parts(self, node): setinfo('blocknode_aftercallsite_main', ifmain) idx += 1 - attrs = {'expr': 'kgen_warmupstage'} - ifwarmup = part_insert_genknode(node.kgen_parent, EXEC_PART, block_statements.IfThen, attrs=attrs, index=idx) - setinfo('blocknode_aftercallsite_warmup', ifwarmup) - idx += 1 - - attrs = {'expr': 'kgen_evalstage'} - ifeval = part_insert_genknode(node.kgen_parent, EXEC_PART, block_statements.IfThen, attrs=attrs, index=idx) - setinfo('blocknode_aftercallsite_eval', ifeval) - def invalid_kernel_stmts(self, node): kernel_stmts = getinfo('callsite_stmts') diff --git a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py index ad0f782..714bc93 100644 --- a/kgen/extractor/plugins/gencore/gen_state_callsite_file.py +++ b/kgen/extractor/plugins/gencore/gen_state_callsite_file.py @@ -668,7 +668,8 @@ def create_callsite_parts1(self, node): #part_append_gensnode(ifsave, EXEC_PART, statements.Assignment, attrs=attrs) attrs = {'specs': ['NEWUNIT=kgen_save_unit(OMP_GET_THREAD_NUM())', 'FILE=TRIM(ADJUSTL(kgen_filepath(kgen_mymid, OMP_GET_THREAD_NUM())))', \ - 'STATUS="REPLACE"', 'ACCESS="STREAM"', 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} +#jgw# 'STATUS="REPLACE"', 'ACCESS="STREAM"', 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} + 'STATUS="REPLACE"', 'ACCESS="STREAM"', 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'IOSTAT=kgen_ierr']} part_append_gensnode(ifsave, EXEC_PART, statements.Open, attrs=attrs) attrs = {'variable': 'kgen_unit', 'sign': '=', 'expr': 'kgen_save_unit(OMP_GET_THREAD_NUM())'} @@ -718,7 +719,8 @@ def create_callsite_parts1(self, node): #part_append_gensnode(ifsave, EXEC_PART, statements.Assignment, attrs=attrs) attrs = {'specs': ['NEWUNIT=kgen_unit', 'FILE=TRIM(ADJUSTL(kgen_filepath(kgen_mymid, 0)))', \ - 'STATUS="REPLACE"', 'ACCESS="STREAM"', 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} +#jgw# 'STATUS="REPLACE"', 'ACCESS="STREAM"', 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} + 'STATUS="REPLACE"', 'ACCESS="STREAM"', 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'IOSTAT=kgen_ierr']} part_append_gensnode(ifsave, EXEC_PART, statements.Open, attrs=attrs) attrs = {'designator': 'kgen_error_stop', 'items': ['kgen_ierr', '"File open error: " // TRIM(ADJUSTL(kgen_filepath(kgen_mymid, 0)))']} diff --git a/kgen/extractor/plugins/gencore/gen_write_in_module.py b/kgen/extractor/plugins/gencore/gen_write_in_module.py index 9c4d959..6ff28f7 100644 --- a/kgen/extractor/plugins/gencore/gen_write_in_module.py +++ b/kgen/extractor/plugins/gencore/gen_write_in_module.py @@ -166,7 +166,8 @@ def write_state(self, node): part_append_gensnode(ifsave, EXEC_PART, statements.Write, attrs=attrs) attrs = {'specs': ['NEWUNIT=kgen_write_unit', 'FILE=kgen_write_filepath_L%d'%lineno, 'STATUS="NEW"', 'ACCESS="STREAM"', \ - 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} +#jgw# 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} + 'FORM="UNFORMATTED"', 'ACTION="WRITE"', 'IOSTAT=kgen_ierr']} part_append_gensnode(ifsave, EXEC_PART, statements.Open, attrs=attrs) @@ -396,7 +397,8 @@ def read_state(self, node): # file open attrs = {'specs': ['NEWUNIT=kgen_read_unit', 'FILE=kgen_read_filepath_L%d'%lineno, 'STATUS="OLD"', 'ACCESS="STREAM"', \ - 'FORM="UNFORMATTED"', 'ACTION="READ"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} +#jgw# 'FORM="UNFORMATTED"', 'ACTION="READ"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} + 'FORM="UNFORMATTED"', 'ACTION="READ"', 'IOSTAT=kgen_ierr']} part_append_genknode(ifeval, EXEC_PART, statements.Open, attrs=attrs) for varstr, var in vars: @@ -498,7 +500,8 @@ def read_state(self, node): # file open attrs = {'specs': ['NEWUNIT=kgen_read_unit', 'FILE=kgen_read_filepath_L%d'%lineno, 'STATUS="OLD"', 'ACCESS="STREAM"', \ - 'FORM="UNFORMATTED"', 'ACTION="READ"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} +#jgw# 'FORM="UNFORMATTED"', 'ACTION="READ"', 'CONVERT="BIG_ENDIAN"', 'IOSTAT=kgen_ierr']} + 'FORM="UNFORMATTED"', 'ACTION="READ"', 'IOSTAT=kgen_ierr']} part_append_genknode(ifwarmup, EXEC_PART, statements.Open, attrs=attrs) for varstr, var in vars: diff --git a/kgen/extractor/plugins/verification/verify_callsite_file.py b/kgen/extractor/plugins/verification/verify_callsite_file.py index a0f8f68..0ceca05 100644 --- a/kgen/extractor/plugins/verification/verify_callsite_file.py +++ b/kgen/extractor/plugins/verification/verify_callsite_file.py @@ -77,12 +77,12 @@ def create_verification_parts(self, node): #jgw# do mpi_allreduce on numOutTol part_append_comment(node, EXEC_PART, '') - attrs = {'variable': 'send(1)', 'sign': '=', 'expr': 'check_status%numOutTol'} + attrs = {'variable': 'ksend(1)', 'sign': '=', 'expr': 'check_status%numOutTol'} part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=attrs) - attrs = {'designator': 'mpi_allreduce', 'items': ['send','recv','1','MPI_INT','MPI_MAX', getinfo('mpi_comm'), 'kgen_ierr']} + attrs = {'designator': 'mpi_allreduce', 'items': ['ksend','krecv','1','MPI_INT','MPI_MAX', getinfo('mpi_comm'), 'kgen_ierr']} part_append_gensnode(node, EXEC_PART, statements.Call, attrs=attrs) - attrs = {'variable': 'check_status%numOutTol', 'sign': '=', 'expr': 'recv(1)'} + attrs = {'variable': 'check_status%numOutTol', 'sign': '=', 'expr': 'krecv(1)'} part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=attrs) attrs = {'designator': 'mpi_comm_rank', 'items': [getinfo('mpi_comm'),\ From 6fd2d0ef3ee398abaed4ec07b6ab681e794a7254 Mon Sep 17 00:00:00 2001 From: John Wohlbier Date: Tue, 2 Oct 2018 09:40:32 -0400 Subject: [PATCH 33/33] update intel path to update 3. add mpi barriers before and after timing --- kgen/extractor/main.py | 2 +- kgen/extractor/plugins/simple_timing/simple_timing.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/kgen/extractor/main.py b/kgen/extractor/main.py index c601f15..a98fd97 100644 --- a/kgen/extractor/main.py +++ b/kgen/extractor/main.py @@ -319,7 +319,7 @@ def generate_kernel_makefile(self): self.write(f, 'FC_FLAGS_SET_%d := %s'%(i, ' '.join(new_options))) #jgw - self.write(f,'\n#INTEL_PATH=/usr/local/install/intel-2018-update2/vtune_amplifier') + self.write(f,'\n#INTEL_PATH=/usr/local/install/intel-2018-update3/vtune_amplifier') self.write(f,'#FC_FLAGS_SET_0 += -D__ITT_NOTIFY__ -I${INTEL_PATH}/include/intel64') self.write(f,'#LD_FLAGS = -L${INTEL_PATH}/lib64 -littnotify') #jgw diff --git a/kgen/extractor/plugins/simple_timing/simple_timing.py b/kgen/extractor/plugins/simple_timing/simple_timing.py index db46488..67a3774 100644 --- a/kgen/extractor/plugins/simple_timing/simple_timing.py +++ b/kgen/extractor/plugins/simple_timing/simple_timing.py @@ -70,6 +70,10 @@ def add_execblock(self, node): part_append_comment(node, EXEC_PART, '#else', style='rawtext') + + attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'), 'kgen_ierr']} + part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) + attrs = {'designator': 'SYSTEM_CLOCK', 'items': ['kgen_start_clock', 'kgen_rate_clock']} part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) @@ -117,6 +121,9 @@ def add_execblock(self, node): attrs = {'designator': 'SYSTEM_CLOCK', 'items': ['kgen_stop_clock', 'kgen_rate_clock']} part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) + attrs = {'designator': 'mpi_barrier', 'items': [getinfo('mpi_comm'), 'kgen_ierr']} + part_append_genknode(node, EXEC_PART, statements.Call, attrs=attrs) + attrs = {'variable': 'kgen_measure', 'sign': '=', 'expr': '1.0D6*(kgen_stop_clock - kgen_start_clock)/DBLE(kgen_rate_clock)'} part_append_genknode(node, EXEC_PART, statements.Assignment, attrs=attrs)