compiler: refine nested parallel support with decl check

Author: mloubout

conftest.py

@@ -38,7 +38,7 @@ def skipif(items, whole_module=False):
     accepted.update({'device', 'device-C', 'device-openmp', 'device-openacc',
                      'device-aomp', 'cpu64-icc', 'cpu64-icx', 'cpu64-nvc',
                      'noadvisor', 'cpu64-arm', 'cpu64-icpx', 'chkpnt'})
-    accepted.update({'nodevice', 'noomp', 'nointel'})
+    accepted.update({'nodevice', 'noomp'})
     unknown = sorted(set(items) - accepted)
     if unknown:
         raise ValueError(f"Illegal skipif argument(s) `{unknown}`")
@@ -93,11 +93,6 @@ def skipif(items, whole_module=False):
         if i == 'noomp' and 'openmp' not in configuration['language']:
             skipit = "Must use openmp"
             break
-        # Skip if not using an Intel compiler
-        if i == 'nointel' and \
-           not isinstance(configuration['compiler'], (IntelCompiler, OneapiCompiler)):
-            skipit = "Must use an Intel compiler"
-            break
         # Skip if it won't run on Arm
         if i == 'cpu64-arm' and isinstance(configuration['platform'], Arm):
             skipit = "Arm doesn't support x86-specific instructions"

devito/arch/compiler.py

@@ -894,7 +894,7 @@ def __lookup_cmds__(self):
             check_output(["mpiicc", f"-cc={self.CC}", "--version"]).decode("utf-8")
             self.MPICC = 'mpiicc'
             self.MPICXX = 'mpicxx'
-        except FileNotFoundError:
+        except (FileNotFoundError, CalledProcessError):
             self.MPICC = 'mpicc'
             self.MPICXX = 'mpicxx'
 

devito/passes/iet/languages/openmp.py

@@ -283,10 +283,10 @@ def _support_nested_parallelism(cls, compiler):
         # In case we have a CustomCompiler
         if isinstance(compiler, CustomCompiler):
             compiler = compiler._base()
-        if isinstance(compiler, (IntelCompiler, OneapiCompiler)):  # noqa: SIM103
-            return True
-        else:
-            return False
+        # Only supported by icc (IntelCompiler) but not by
+        # OneAPI's DPC++ compiler (OneapiCompiler) that inherits from IntelCompiler
+        return isinstance(compiler, IntelCompiler) and not \
+            isinstance(compiler, OneapiCompiler)
 
 
 class Ompizer(AbstractOmpizer):

devito/passes/iet/parpragma.py

@@ -346,10 +346,18 @@ def _make_parregion(self, partree, parrays):
     def _make_guard(self, parregion):
         return parregion
 
+    def _support_uindices(self, uindices):
+        if not uindices:
+            # No secondary indices, so we can apply nested parallelism
+            return True
+        else:
+            # Compiler supports nested parallelism with multiple indices
+            # such as for(int i = 0, j=1; ...)
+            return self._support_nested_parallelism(self.compiler)
+
     def _make_nested_partree(self, partree):
         # Apply heuristic
-        if self.nhyperthreads <= self.nested or \
-           not self._support_nested_parallelism(self.compiler):
+        if self.nhyperthreads <= self.nested:
             return partree
 
         # Note: there might be multiple sub-trees amenable to nested parallelism,
@@ -371,7 +379,8 @@ def _make_nested_partree(self, partree):
             # within a block)
             candidates = []
             for i in inner:
-                if self.key(i) and any((j.dim.root is i.dim.root) for j in outer):
+                if self.key(i) and any((j.dim.root is i.dim.root) for j in outer) and \
+                   self._support_uindices(i.uindices):
                     candidates.append(i)
                 elif candidates:
                     # If there's at least one candidate but `i` doesn't honor the

tests/test_caching.py

@@ -5,12 +5,11 @@
 import pytest
 from sympy import Expr
 
-from conftest import skipif
 from devito import (
     ConditionalDimension, Constant, DefaultDimension, Dimension, Eq, Function, Grid,
     Operator, SparseFunction, SparseTimeFunction, SubDimension, TensorFunction,
     TensorTimeFunction, TimeFunction, VectorFunction, VectorTimeFunction, _SymbolCache,
-    clear_cache, solve
+    clear_cache, solve, switchconfig
 )
 from devito.types import (
     DeviceID, LocalObject, NPThreads, NThreadsBase, Object, Scalar, Symbol, ThreadID
@@ -468,7 +467,7 @@ def test_grid_dtypes(self):
 
         assert hash(grid0) != hash(grid1)
 
-    @skipif('nointel')
+    @switchconfig(compiler='icc')
     def test_special_symbols(self):
         """
         This test checks the singletonization, through the caching infrastructure,

tests/test_dle.py

@@ -9,7 +9,8 @@
 from devito import (
     CustomDimension, DefaultDimension, Dimension, Eq, Function, Grid, Inc, Operator,
     PrecomputedSparseTimeFunction, ReduceMax, ReduceMin, ReduceMinMax, SpaceDimension,
-    SparseTimeFunction, SubDimension, TimeFunction, configuration, cos, dimensions, info
+    SparseTimeFunction, SubDimension, TimeFunction, configuration, cos, dimensions, info,
+    switchconfig
 )
 from devito.arch.compiler import IntelCompiler, OneapiCompiler
 from devito.exceptions import InvalidArgument
@@ -1238,7 +1239,6 @@ def test_parallel_prec_inject(self):
 
 class TestNestedParallelism:
 
-    @skipif('nointel')
     def test_basic(self):
         grid = Grid(shape=(3, 3, 3))
 
@@ -1251,7 +1251,6 @@ def test_basic(self):
                                                   'par-dynamic-work': 0}))
 
         # Does it compile? Honoring the OpenMP specification isn't trivial
-        print(op)
         assert op.cfunction
 
         # Does it produce the right result
@@ -1271,7 +1270,6 @@ def test_basic(self):
         assert iterations[2].pragmas[0].ccode.value ==\
             'omp parallel for schedule(dynamic,1) num_threads(nthreads_nested)'
 
-    @skipif('nointel')
     def test_collapsing(self):
         grid = Grid(shape=(3, 3, 3))
 
@@ -1280,7 +1278,6 @@ def test_collapsing(self):
 
         op = Operator(Eq(u.forward, u + f + 1),
                       opt=('blocking', 'openmp', {'par-nested': 0,
-                                                  'cire-rotate': True,
                                                   'par-collapse-ncores': 1,
                                                   'par-collapse-work': 0,
                                                   'par-dynamic-work': 0}))
@@ -1302,7 +1299,7 @@ def test_collapsing(self):
             ('omp parallel for collapse(2) schedule(dynamic,1) '
              'num_threads(nthreads_nested)')
 
-    @skipif('nointel')
+    @switchconfig(compiler='icc')
     def test_multiple_subnests_v0(self):
         grid = Grid(shape=(3, 3, 3))
         x, y, z = grid.dimensions
@@ -1335,7 +1332,7 @@ def test_multiple_subnests_v0(self):
             ('omp parallel for collapse(2) schedule(dynamic,1) '
              'num_threads(nthreads_nested)')
 
-    @skipif('nointel')
+    @switchconfig(compiler='icc')
     def test_multiple_subnests_v1(self):
         """
         Unlike ``test_multiple_subnestes_v0``, now we use the ``cire-rotate=True``
@@ -1374,7 +1371,6 @@ def test_multiple_subnests_v1(self):
         assert trees[-1][3].pragmas[0].ccode.value ==\
             'omp parallel for schedule(dynamic,1) num_threads(nthreads_nested)'
 
-    @skipif('nointel')
     @pytest.mark.parametrize('blocklevels', [1, 2])
     def test_nested_cache_blocking_structure_subdims(self, blocklevels):
         """
@@ -1438,7 +1434,6 @@ def test_nested_cache_blocking_structure_subdims(self, blocklevels):
             ('omp parallel for collapse(2) schedule(dynamic,1) '
              'num_threads(nthreads_nested)')
 
-    @skipif('nointel')
     @pytest.mark.parametrize('exprs,collapsed,scheduling', [
         (['Eq(u.forward, u.dx)'], '2', 'static'),
         (['Eq(u.forward, u.dy)'], '2', 'static'),
@@ -1471,7 +1466,6 @@ def test_collapsing_w_wo_halo(self, exprs, collapsed, scheduling):
         assert iterations[1].pragmas[0].ccode.value ==\
             "".join([ompfor_string, scheduling_string])
 
-    @skipif('device')
     def test_nested_parallelism_support(self):
         grid = Grid(shape=(10, 10, 10))
 
@@ -1484,7 +1478,9 @@ def test_nested_parallelism_support(self):
         v1.data_with_halo[:] = 1.
 
         eqn = Eq(v.forward, (v.dx * (1 + 2*f) * f).dx)
-        op = Operator(eqn, opt=('advanced', {'openmp': True, 'par-nested': 0}))
+        op = Operator(eqn, opt=('advanced', {'openmp': True,
+                                             'par-collapse-ncores': 1,
+                                             'par-nested': 0}))
 
         bns, _ = assert_blocking(op, {'x0_blk0'})
         trees = retrieve_iteration_tree(bns['x0_blk0'])
@@ -1493,13 +1489,15 @@ def test_nested_parallelism_support(self):
         # Check omp pargams
         assert trees[0][0].pragmas[0].ccode.value == \
             'omp for collapse(2) schedule(dynamic,1)'
-        if isinstance(configuration['compiler'], (IntelCompiler, OneapiCompiler)):
+        if isinstance(configuration['compiler'], IntelCompiler) and \
+           not isinstance(configuration['compiler'], OneapiCompiler):
             # Supports nested parallelism
             assert trees[0][2].pragmas[0].ccode.value == \
-                '#pragma omp parallel for collapse(2) schedule(dynamic,1)'\
+                'omp parallel for collapse(2) schedule(dynamic,1)'\
                 ' num_threads(nthreads_nested)'
             assert trees[1][2].pragmas[0].ccode.value == \
-                trees[0][2].pragmas[0].ccode.value
+                'omp parallel for collapse(2) schedule(static,1)'\
+                ' num_threads(nthreads_nested)'
         else:
             # Most compiler don't support nested parallelism
             assert not trees[0][2].pragmas