(Closes #450) Ensure set_dirty/clean calls are outside ACC regions.

examples/lfric/code/dg_matrix_vector_kernel_mod.F90

@@ -45,7 +45,7 @@ module dg_matrix_vector_kernel_mod
                                 GH_FIELD, GH_OPERATOR,      &
                                 GH_REAL, GH_READ, GH_WRITE, &
                                 ANY_DISCONTINUOUS_SPACE_1,  &
-                                ANY_SPACE_1,                &
+                                ANY_SPACE_1, GH_READWRITE,  &
                                 CELL_COLUMN
 
   use constants_mod,     only : r_def, i_def

examples/lfric/eg14/Makefile

@@ -1,7 +1,7 @@
 # -----------------------------------------------------------------------------
 # BSD 3-Clause License
 #
-# Copyright (c) 2020, Science and Technology Facilities Council.
+# Copyright (c) 2020-2022, Science and Technology Facilities Council.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -45,6 +45,11 @@ kernels:
   ../code/gw_mixed_schur_preconditioner_alg_mod.x90 \
   -oalg /dev/null
 
+kernels_redundant:
+	${PSYCLONE} -s ./acc_kernels_redundant.py \
+  ../code/helmholtz_solver_alg_mod.x90 \
+  -oalg /dev/null
+
 parallel_no_dm:
 	${PSYCLONE} -s ./acc_parallel.py -nodm \
   ../code/gw_mixed_schur_preconditioner_alg_mod.x90 \

examples/lfric/eg14/acc_kernels.py

@@ -40,6 +40,7 @@
 
 '''
 from __future__ import print_function
+from psyclone.psyGen import HaloExchange
 from psyclone.transformations import ACCKernelsTrans
 
 
@@ -53,7 +54,20 @@ def trans(psy):
 
         print("Transforming invoke '"+invoke.name+"'...")
         schedule = invoke.schedule
-        kernels_trans.apply(schedule)
+        schedule.view()
+
+        node_list = []
+        for node in schedule.children[:]:
+            if not isinstance(node, HaloExchange):
+                node_list.append(node)
+            else:
+                if node_list:
+                    kernels_trans.apply(node_list)
+                node_list = []
+                continue
+        if node_list:
+            kernels_trans.apply(node_list)
+
         schedule.view()
 
     return psy

examples/lfric/eg14/acc_kernels_redundant.py

@@ -0,0 +1,83 @@
+# -----------------------------------------------------------------------------
+# BSD 3-Clause License
+#
+# Copyright (c) 2019, Science and Technology Facilities Council
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+# Authors: R. W. Ford and A. R. Porter, STFC Daresbury Laboratory
+
+
+'''File containing a PSyclone transformation script for the LFRic (Dynamo0p3)
+API to apply redundant computation and OpenACC Kernels directives generically.
+This can be applied via the -s option to the psyclone script.
+
+'''
+from __future__ import print_function
+from psyclone.psyGen import HaloExchange
+from psyclone.psyir.nodes import Loop
+from psyclone.transformations import (ACCKernelsTrans, MoveTrans,
+                                      Dynamo0p3RedundantComputationTrans)
+
+
+def trans(psy):
+    ''' PSyclone transformation script for the dynamo0p3 api to apply
+    OpenACC Kernels directives generically in the presence of redundant
+    computation..'''
+    kernels_trans = ACCKernelsTrans()
+    rc_trans = Dynamo0p3RedundantComputationTrans()
+    m_trans = MoveTrans()
+
+    # Loop over all of the Invokes in the PSy object
+    for invoke in psy.invokes.invoke_list:
+
+        print("Transforming invoke '"+invoke.name+"'...")
+        schedule = invoke.schedule
+        schedule.view()
+
+        loops = schedule.walk(Loop)
+        rc_trans.apply(loops[1], {"depth": 2})
+        rc_trans.apply(loops[2], {"depth": 2})
+
+        node_list = []
+        for node in schedule.children[:]:
+            # We can't include halo exchange calls within a KERNELS region.
+            if not isinstance(node, HaloExchange):
+                node_list.append(node)
+            else:
+                if node_list:
+                    kernels_trans.apply(node_list)
+                node_list = []
+                continue
+        if node_list:
+            kernels_trans.apply(node_list)
+
+        schedule.view()
+
+    return psy

examples/lfric/eg8/Makefile

@@ -1,7 +1,7 @@
 # -----------------------------------------------------------------------------
 # BSD 3-Clause License
 #
-# Copyright (c) 2020, Science and Technology Facilities Council.
+# Copyright (c) 2020-2022, Science and Technology Facilities Council.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -41,10 +41,10 @@ include ../../common.mk
 transform: vanilla redundant
 
 vanilla:
-	${PSYCLONE} helmholtz_solver_alg_mod.x90
+	${PSYCLONE} ../code/helmholtz_solver_alg_mod.x90
 
 redundant:
-	${PSYCLONE} -s ./redundant_script.py helmholtz_solver_alg_mod.x90
+	${PSYCLONE} -s ./redundant_script.py ../code/helmholtz_solver_alg_mod.x90
 
 compile: transform
 	@echo "No compilation supported for lfric/eg8"

examples/psyad/eg2/tl_hydrostatic_kernel_mod.F90

@@ -215,16 +215,10 @@ subroutine tl_hydrostatic_code(nlayers,          &
       exner_e(df) = exner( map_w3(df) + k )
     end do
     do df = 1, ndf_wt
-      !*** Issue #1491 Unsupported form of rhs expression (multiple
-      !*** active terms multiplied by an inactive term, as PSyclone sees
-      !*** this as a single term).
-      !***    theta_v_e(df) = ls_theta_v_e(df) * &
-      !***       ( theta( map_wt(df) + k ) /  ls_theta( map_wt(df) + k )  +               &
-      !***         moist_dyn_gas( map_wt(df) + k ) / ls_moist_dyn_gas( map_wt(df) + k ) - &
-      !***         moist_dyn_tot( map_wt(df) + k ) / ls_moist_dyn_tot( map_wt(df) + k ) )
-      theta_v_e(df) = ls_theta_v_e(df) * theta( map_wt(df) + k ) /  ls_theta( map_wt(df) + k )  + &
-           ls_theta_v_e(df) * moist_dyn_gas( map_wt(df) + k ) / ls_moist_dyn_gas( map_wt(df) + k ) - &
-           ls_theta_v_e(df) * moist_dyn_tot( map_wt(df) + k ) / ls_moist_dyn_tot( map_wt(df) + k )
+      theta_v_e(df) = ls_theta_v_e(df) * &
+         ( theta( map_wt(df) + k ) /  ls_theta( map_wt(df) + k )  +               &
+           moist_dyn_gas( map_wt(df) + k ) / ls_moist_dyn_gas( map_wt(df) + k ) - &
+           moist_dyn_tot( map_wt(df) + k ) / ls_moist_dyn_tot( map_wt(df) + k ) )
     end do
     ! Compute the RHS integrated over one cell
     do qp2 = 1, nqp_v
@@ -260,20 +254,12 @@ subroutine tl_hydrostatic_code(nlayers,          &
           v  = w2_basis(:,df,qp1,qp2)
           dv = w2_diff_basis(1,df,qp1,qp2)
           ! Pressure gradient term
-
-          !*** Issue #1491 Unsupported form of rhs expression (multiple
-          !*** active terms multiplied by an inactive term, as PSyclone sees
-          !*** this as a single term).
-          !*** grad_term = cp * ls_exner_at_quad * (                        &
-          !***             theta_v_at_quad * dv                             &
-          !***           + dot_product( grad_theta_v_at_quad(:),v) ) +      &
-          !***             cp * exner_at_quad * (                           &
-          !***             ls_theta_v_at_quad * dv                          &
-          !***           + dot_product( ls_grad_theta_v_at_quad(:),v) )
-          grad_term = cp * ls_exner_at_quad * theta_v_at_quad * dv                           &
-                    + cp * ls_exner_at_quad * dot_product( grad_theta_v_at_quad(:),v)        &
-                    + cp * exner_at_quad * ls_theta_v_at_quad * dv                           &
-                    + cp * exner_at_quad * dot_product( ls_grad_theta_v_at_quad(:),v)
+          grad_term = cp * ls_exner_at_quad * (                        &
+                      theta_v_at_quad * dv                             &
+                    + dot_product( grad_theta_v_at_quad(:),v) ) +      &
+                      cp * exner_at_quad * (                           &
+                      ls_theta_v_at_quad * dv                          &
+                    + dot_product( ls_grad_theta_v_at_quad(:),v) )
           r_u( map_w2(df) + k ) = r_u( map_w2(df) + k ) &
                                 + wqp_h(qp1)*wqp_v(qp2)*grad_term
         end do

src/psyclone/core/symbolic_maths.py

@@ -32,13 +32,13 @@
 # POSSIBILITY OF SUCH DAMAGE.
 # -----------------------------------------------------------------------------
 # Author J. Henrichs, Bureau of Meteorology
+# Modified: R. W. Ford, STFC Daresbury Lab
 # -----------------------------------------------------------------------------
 
 ''' This module provides access to sympy-based symbolic maths
 functions.'''
 
-
-from sympy import simplify, true
+from sympy import simplify, expand, true
 
 
 class SymbolicMaths:
@@ -115,3 +115,44 @@ def equal(exp1, exp2):
 
         # Convert SymPy boolean to python boolean.
         return result is true
+
+    # -------------------------------------------------------------------------
+    @staticmethod
+    def expand(expr):
+        '''Expand a PSyIR expression. This is done by converting the PSyIR
+        expression to a sympy expression, applying the expansion
+        operation and then converting the resultant output back into
+        PSyIR.
+
+        Currently does not work if the PSyIR expression contains Range
+        nodes, see issue #1655.
+
+        :param expr: the expression to be expanded.
+        :type expr: py:class:`psyclone.psyir.nodes.Node`
+
+        '''
+        # Avoid circular import
+        # pylint: disable=import-outside-toplevel
+        from psyclone.psyir.backend.sympy_writer import SymPyWriter
+        from psyclone.psyir.frontend.fortran import FortranReader
+        from psyclone.psyir.nodes import Reference, Literal, Routine
+
+        # variables and literals do not require expansion
+        if isinstance(expr, (Reference, Literal)):
+            return
+        # Convert the PSyIR expression to a sympy expression
+        sympy_expression = SymPyWriter.convert_to_sympy_expressions([expr])
+        # Expand the expression
+        result = expand(sympy_expression[0])
+        # If the expanded result is the same as the original then
+        # nothing needs to be done.
+        if result == sympy_expression[0]:
+            return
+        # Find the required symbol table in the original PSyIR
+        symbol_table = expr.ancestor(Routine).symbol_table
+        # Convert the new sympy expression to PSyIR
+        reader = FortranReader()
+        new_expr = reader.psyir_from_expression(str(result), symbol_table)
+        # Replace the old PSyIR expression with the new expanded PSyIR
+        # expression
+        expr.replace_with(new_expr)

src/psyclone/dynamo0p3.py

@@ -75,7 +75,7 @@
 from psyclone.psyir.frontend.fortran import FortranReader
 from psyclone.psyir.nodes import (Loop, Literal, Schedule, Reference,
                                   ArrayReference, ACCEnterDataDirective,
-                                  OMPParallelDoDirective)
+                                  ACCRegionDirective, OMPParallelDoDirective)
 from psyclone.psyir.symbols import (
     INTEGER_TYPE, INTEGER_SINGLE_TYPE, DataSymbol, SymbolTable, ScalarType,
     DeferredType, DataTypeSymbol, ContainerSymbol, ImportInterface, ArrayType)
@@ -7639,9 +7639,13 @@ def gen_code(self, parent):
             return
 
         # Set halo clean/dirty for all fields that are modified
-        fields = self.unique_modified_args("gh_field")
+        if not self.unique_modified_args("gh_field"):
+            return
 
-        if not fields:
+        if self.ancestor(ACCRegionDirective):
+            # We cannot include calls to set halos dirty/clean within OpenACC
+            # regions. This is handled by the appropriate Directive class
+            # instead.
             return
 
         parent.add(CommentGen(parent, ""))
@@ -7652,6 +7656,7 @@ def gen_code(self, parent):
         parent.add(CommentGen(parent, f" Set halos dirty/clean for fields "
                               f"modified in the above {prev_node_name}"))
         parent.add(CommentGen(parent, ""))
+
         use_omp_master = False
         if self.is_openmp_parallel():
             if not self.ancestor(OMPParallelDoDirective):
@@ -7660,9 +7665,35 @@ def gen_code(self, parent):
                 # set_dirty() and set_clean() with OpenMP Master
                 parent.add(DirectiveGen(parent, "omp", "begin", "master", ""))
 
+        self.gen_mark_halos_clean_dirty(parent)
+
+        if use_omp_master:
+            # I am within an OpenMP Do directive so protect
+            # set_dirty() and set_clean() with OpenMP Master
+            parent.add(DirectiveGen(parent, "omp", "end", "master", ""))
+        parent.add(CommentGen(parent, ""))
+
+    def gen_mark_halos_clean_dirty(self, parent):
+        '''
+        Generates the necessary code to mark halo regions as clean or dirty
+        following execution of this loop.
+
+        :param parent: the node in the f2pygen AST to which to add content.
+        :type parent: :py:class:`psyclone.f2pygen.BaseGen`
+
+        '''
+        if not Config.get().distributed_memory:
+            return
+
+        # Set halo clean/dirty for all fields that are modified
+        fields = self.unique_modified_args("gh_field")
+
+        if not fields:
+            return
+
         sym_table = self.ancestor(InvokeSchedule).symbol_table
 
-        # first set all of the halo dirty unless we are
+        # First set all of the halo dirty unless we are
         # subsequently going to set all of the halo clean
         for field in fields:
             # The HaloWriteAccess class provides information about how the
@@ -7679,9 +7710,8 @@ def gen_code(self, parent):
                 else:
                     parent.add(CallGen(parent, name=field.proxy_name +
                                        "%set_dirty()"))
-        # now set appropriate parts of the halo clean where
-        # redundant computation has been performed
-        for field in fields:
+            # now set appropriate parts of the halo clean where
+            # redundant computation has been performed
             # The HaloWriteAccess class provides information about how the
             # supplied field is accessed within its parent loop
             hwa = HaloWriteAccess(field, sym_table)
@@ -7725,12 +7755,6 @@ def gen_code(self, parent):
                                    f"set_clean({halo_depth})")
                     parent.add(call)
 
-        if use_omp_master:
-            # I am within an OpenMP Do directive so protect
-            # set_dirty() and set_clean() with OpenMP Master
-            parent.add(DirectiveGen(parent, "omp", "end", "master", ""))
-        parent.add(CommentGen(parent, ""))
-
 
 class DynKern(CodedKern):
     ''' Stores information about Dynamo Kernels as specified by the

src/psyclone/psyad/tl2ad.py

@@ -96,7 +96,7 @@ def generate_adjoint_str(tl_fortran_str, active_variables, create_test=False):
     # logger.debug(tl_psyir.view())
 
     # Apply any required transformations to the TL PSyIR
-    preprocess_trans(tl_psyir)
+    preprocess_trans(tl_psyir, active_variables)
 
     # Addressing issue #1238 will allow the view() method to be output
     # to the logger.