MPI kernel

base/kgen_extra.py

@@ -177,7 +177,8 @@
 subroutine kgen_array_sumcheck(varname, sum1, sum2, finish)
     character(*), intent(in) :: varname
     real(kind=8), intent(in) :: sum1, sum2
-    real(kind=8), parameter  :: max_rel_diff = 1.E-10
+    !real(kind=8), parameter  :: max_rel_diff = 1.E-10
+    real(kind=8), parameter  :: max_rel_diff = 1.E-9
     real(kind=8)  :: diff, rel_diff
     logical, intent(in), optional :: finish
     logical checkresult
@@ -550,6 +551,67 @@
 END SUBROUTINE
 """
 
+kgen_mpifile = \
+"""
+SUBROUTINE kgen_mpifile( str, rank, thread, cycle )
+    CHARACTER(*), INTENT(INOUT) :: str
+    INTEGER, INTENT(IN) :: rank, thread, cycle
+
+    CHARACTER(LEN=1024) :: format, string_format, string_format_format
+    CHARACTER(LEN=1024) :: rank_format
+    CHARACTER(LEN=1024) :: thread_format
+    CHARACTER(LEN=1024) :: cycle_format
+    INTEGER :: str_len
+
+    str_len = INDEX(str,".")
+
+    IF (str_len < 10) THEN
+      string_format_format = "(A1,I1)"
+    ELSE IF (str_len < 100) THEN
+      string_format_format = "(A1,I2)"
+    ELSE IF (str_len < 1000) THEN
+      string_format_format = "(A1,I3)"
+    ELSE IF (str_len < 10000) THEN
+      string_format_format = "(A1,I4)"
+    ENDIF
+    write(string_format,trim(string_format_format)) 'A',str_len
+
+    IF (rank < 10) THEN
+      rank_format = "I1"
+    ELSE IF (rank < 100) THEN
+      rank_format = "I2"
+    ELSE IF (rank < 1000) THEN
+      rank_format = "I3"
+    ELSE IF (rank < 10000) THEN
+      rank_format = "I4"
+    ENDIF
+
+    IF (thread < 10) THEN
+      thread_format = "I1"
+    ELSE IF (thread < 100) THEN
+      thread_format = "I2"
+    ELSE IF (thread < 1000) THEN
+      thread_format = "I3"
+    ELSE IF (thread < 10000) THEN
+      thread_format = "I4"
+    ENDIF
+
+    IF (cycle < 10) THEN
+      cycle_format = "I1"
+    ELSE IF (cycle < 100) THEN
+      cycle_format = "I2"
+    ELSE IF (cycle < 1000) THEN
+      cycle_format = "I3"
+    ELSE IF (cycle < 10000) THEN
+      cycle_format = "I4"
+    ENDIF
+
+    format = "("//trim(string_format)//","//trim(rank_format)//",A1,"//trim(thread_format)//",A1,"//trim(cycle_format)//")"
+    write(str,trim(format)) trim(str(1:str_len)), rank, ".", thread, ".", cycle
+
+END SUBROUTINE
+"""
+
 rdtsc = \
 """         .file   "rdtsc.s"
          .text

base/kgen_genfile.py

@@ -902,6 +902,9 @@ def process_exclude(node, bag, depth):
             if isinstance(node, Name):
                 for namepath, actions in bag['excludes'].iteritems():
                     if match_namepath(namepath, pack_innamepath(bag['stmt'], node.string)):
+                        #jgw# for a remove_io exclude leave the type in
+                        if actions[0] == 'remove_io':
+                            return False
                         bag['matched'] = True
                         return True
 

base/statements.py

@@ -1729,7 +1729,8 @@ def analyze(self):
 
     # start of KGEN addition
     def resolve_uname(self, uname, request):
-        Logger.warn('resolve_uname is not implemented: %s'%self.__class__)
+        # comment out based on input from Youngsung
+        #Logger.warn('resolve_uname is not implemented: %s'%self.__class__)
         pass
 
     def tokgen(self):

examples/MPI_kernel/Makefile

@@ -0,0 +1,31 @@
+#KGEN_HOME := $(HOME)/devel/packages/KGen
+KGEN_HOME := $(HOME)/tmp/KGen
+KGEN := ${KGEN_HOME}/bin/kgen
+
+SRC_DIR := ${PWD}/../src
+SRC := ${SRC_DIR}/radtrans/rad_diffusion.f90
+
+# don't have invocations beyond first cycle if the function is only called 1x!
+# there is still a race condition somewhere in the src/Makefile. Change to -j8
+# until we can find it.
+
+test:
+	${KGEN} \
+	--check tolerance=1.0D-10 \
+	--cmd-build="cd ${SRC_DIR}; make -j8" \
+	--cmd-clean="cd ${SRC_DIR}; make clean; cd ${SRC_DIR}/run; ./clean.sh" \
+	--cmd-run="cd ${SRC_DIR}; make run" \
+	--exclude-ini=exclude.ini \
+	--invocation=0-15:0:0 \
+	--mpi=enable \
+	--repr-etime=disable \
+	--verbose=3 \
+	${SRC}
+
+
+#	--invocation 0-383:0:0 \
+
+clean:
+	${MAKE} clean -C ../src
+	$(RM) -rf  _kgen_compflag_cmdwrapper.sh elapsedtime kernel kgen.log model state 
+	#$(RM) include.ini strace.log

examples/MPI_kernel/README

@@ -0,0 +1,14 @@
+These are some example files generated by my modified KGen. They are meant
+to show the kind of things that are needed to have an MPI extracted kernel.
+None of this has been tested with an OpenMP code.
+
+Makefile
+- Showing the options that I'm invoking KGen with
+
+state_callsite.f90
+- need to set kgen_invoke and kgen_last_invoke to 0 to get the thing to run
+- add mpi_comm_rank, mpi_comm_size, mpi_barrier
+- write file per rank
+
+verify_callsite.f90
+- mpi_allreduce max on numOutTol