Intel Compiler Bug

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ on:
   pull_request:
 
   workflow_dispatch:
-
+    
 jobs:
   github:
     name: Github
@@ -24,24 +24,25 @@ jobs:
         os:    ['ubuntu', 'macos']
         mpi:   ['mpi', 'no-mpi']
         debug: ['debug', 'no-debug']
-        intel: [false]
+        intel: [true, false]
       fail-fast: false
     continue-on-error: true
     runs-on: ${{ matrix.os }}-latest
     steps:
       - name: Clone
+        if: matrix.os == 'ubuntu' || (matrix.os == 'macos' && matrix.intel == false)
         uses: actions/checkout@v3
 
       - name: Setup MacOS
-        if:   matrix.os == 'macos'
+        if:   matrix.os == 'macos' && matrix.intel == false
         run: |
           echo "CC=gcc-13"      >> $GITHUB_ENV
           echo "CXX=g++-13"     >> $GITHUB_ENV
           echo "FC=gfortran-13" >> $GITHUB_ENV
           brew install wget make python make cmake coreutils gcc@13
 
       - name: (MacOS) Build OpenMPI
-        if:   matrix.os == 'macos' && matrix.mpi == 'mpi'
+        if:   matrix.os == 'macos' && matrix.mpi == 'mpi' && matrix.intel == false
         run: |
           echo "OMPI_FC=gfortran-13" >> $GITHUB_ENV
           echo "OMPI_CXX=g++-13"     >> $GITHUB_ENV
@@ -51,9 +52,9 @@ jobs:
       - name: Setup Ubuntu
         if:   matrix.os == 'ubuntu' && matrix.intel == false
         run: |
-          sudo apt update -y
-          sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
-
+           sudo apt update -y
+           sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
+     
       - name: Setup Ubuntu (Intel)
         if:   matrix.os == 'ubuntu' && matrix.intel == true
         run: |
@@ -72,12 +73,29 @@ jobs:
           echo "OMPI_CXX=$(which icpc)"     >> $GITHUB_ENV
           echo "OMPI_MPICC=$(which icc)"    >> $GITHUB_ENV
           echo "MPI_HOME=/opt/intel/oneapi/mpi/2021.7.1/"    >> $GITHUB_ENV
+          echo "I_MPI_ROOT=/opt/intel/oneapi/mpi/2021.7.1/"    >> $GITHUB_ENV
+
+      - name: Build-intel
+        if:   matrix.intel == true && matrix.os == 'ubuntu'
+        run:  |
+          source /opt/intel/oneapi/setvars.sh
+          /bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} 
 
       - name: Build
-        run:  /bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
+        if:   matrix.intel == false
+        run:  |
+          /bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} 
+
+      - name: Test-intel
+        if:   matrix.intel == true && matrix.os == 'ubuntu'
+        run:  |
+          source /opt/intel/oneapi/setvars.sh
+          /bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
 
       - name: Test
-        run:  /bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
+        if:   matrix.intel == false
+        run:  |
+          /bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
 
   docker:
     name:    Github | Docker

src/common/m_variables_conversion.fpp

@@ -690,7 +690,6 @@ contains
             s_convert_to_mixture_variables => &
                 s_convert_species_to_mixture_variables
         end if
-
     end subroutine s_initialize_variables_conversion_module ! --------------
 
         !Initialize mv at the quadrature nodes based on the initialized moments and sigma  

src/post_process/m_data_input.f90

@@ -405,15 +405,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
 
             call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'x')
 
-            do i = 1, offset_x%beg
-                x_cb(-1 - i) = x_cb(-i) - dx(-i)
-            end do
+        end if
 
-            do i = 1, buff_size
-                x_cc(-i) = x_cc(1 - i) - (dx(1 - i) + dx(-i))/2d0
-            end do
+        do i = 1, offset_x%beg
+            x_cb(-1 - i) = x_cb(-i) - dx(-i)
+        end do
 
-        end if
+        do i = 1, buff_size
+            x_cc(-i) = x_cc(1 - i) - (dx(1 - i) + dx(-i))/2d0
+        end do
 
         ! Ghost-cell extrapolation BC at the end
         if (bc_x%end <= -3) then
@@ -441,15 +441,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
 
             call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'x')
 
-            do i = 1, offset_x%end
-                x_cb(m + i) = x_cb(m + (i - 1)) + dx(m + i)
-            end do
+        end if
 
-            do i = 1, buff_size
-                x_cc(m + i) = x_cc(m + (i - 1)) + (dx(m + (i - 1)) + dx(m + i))/2d0
-            end do
+        do i = 1, offset_x%end
+            x_cb(m + i) = x_cb(m + (i - 1)) + dx(m + i)
+        end do
 
-        end if
+        do i = 1, buff_size
+            x_cc(m + i) = x_cc(m + (i - 1)) + (dx(m + (i - 1)) + dx(m + i))/2d0
+        end do
 
         ! END: Populating Buffer Regions in the x-direction ================
 
@@ -483,15 +483,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
 
                 call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'y')
 
-                do i = 1, offset_y%beg
-                    y_cb(-1 - i) = y_cb(-i) - dy(-i)
-                end do
+            end if
 
-                do i = 1, buff_size
-                    y_cc(-i) = y_cc(1 - i) - (dy(1 - i) + dy(-i))/2d0
-                end do
+            do i = 1, offset_y%beg
+                y_cb(-1 - i) = y_cb(-i) - dy(-i)
+            end do
 
-            end if
+            do i = 1, buff_size
+                y_cc(-i) = y_cc(1 - i) - (dy(1 - i) + dy(-i))/2d0
+            end do
 
             ! Ghost-cell extrapolation BC at the end
             if (bc_y%end <= -3) then
@@ -519,15 +519,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
 
                 call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'y')
 
-                do i = 1, offset_y%end
-                    y_cb(n + i) = y_cb(n + (i - 1)) + dy(n + i)
-                end do
+            end if
 
-                do i = 1, buff_size
-                    y_cc(n + i) = y_cc(n + (i - 1)) + (dy(n + (i - 1)) + dy(n + i))/2d0
-                end do
+            do i = 1, offset_y%end
+                y_cb(n + i) = y_cb(n + (i - 1)) + dy(n + i)
+            end do
 
-            end if
+            do i = 1, buff_size
+                y_cc(n + i) = y_cc(n + (i - 1)) + (dy(n + (i - 1)) + dy(n + i))/2d0
+            end do
 
             ! END: Populating Buffer Regions in the y-direction ================
 
@@ -561,15 +561,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
 
                     call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'z')
 
-                    do i = 1, offset_z%beg
-                        z_cb(-1 - i) = z_cb(-i) - dz(-i)
-                    end do
+                end if
 
-                    do i = 1, buff_size
-                        z_cc(-i) = z_cc(1 - i) - (dz(1 - i) + dz(-i))/2d0
-                    end do
+                do i = 1, offset_z%beg
+                    z_cb(-1 - i) = z_cb(-i) - dz(-i)
+                end do
 
-                end if
+                do i = 1, buff_size
+                    z_cc(-i) = z_cc(1 - i) - (dz(1 - i) + dz(-i))/2d0
+                end do
 
                 ! Ghost-cell extrapolation BC at the end
                 if (bc_z%end <= -3) then
@@ -597,15 +597,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
 
                     call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'z')
 
-                    do i = 1, offset_z%end
-                        z_cb(p + i) = z_cb(p + (i - 1)) + dz(p + i)
-                    end do
+                end if
 
-                    do i = 1, buff_size
-                        z_cc(p + i) = z_cc(p + (i - 1)) + (dz(p + (i - 1)) + dz(p + i))/2d0
-                    end do
+                do i = 1, offset_z%end
+                    z_cb(p + i) = z_cb(p + (i - 1)) + dz(p + i)
+                end do
 
-                end if
+                do i = 1, buff_size
+                    z_cc(p + i) = z_cc(p + (i - 1)) + (dz(p + (i - 1)) + dz(p + i))/2d0
+                end do
 
             end if
 

src/post_process/m_mpi_proxy.fpp

@@ -19,6 +19,8 @@ module m_mpi_proxy
     use m_global_parameters     !< Global parameters for the code
 
     use m_mpi_common
+
+    use ieee_arithmetic
     ! ==========================================================================
 
     implicit none
@@ -516,8 +518,8 @@ contains
             ! Boundary condition at the beginning
             if (proc_coords(2) > 0 .or. bc_y%beg == -1) then
                 proc_coords(2) = proc_coords(2) - 1
-                call MPI_CART_RANK(MPI_COMM_CART, proc_coords, bc_y%beg, &
-                                   ierr)
+                call MPI_CART_RANK(MPI_COMM_CART, proc_coords, &
+                                   bc_y%beg, ierr)
                 proc_coords(2) = proc_coords(2) + 1
             end if
 
@@ -531,8 +533,8 @@ contains
             ! Boundary condition at the end
             if (proc_coords(2) < num_procs_y - 1 .or. bc_y%end == -1) then
                 proc_coords(2) = proc_coords(2) + 1
-                call MPI_CART_RANK(MPI_COMM_CART, proc_coords, bc_y%end, &
-                                   ierr)
+                call MPI_CART_RANK(MPI_COMM_CART, proc_coords, &
+                                   bc_y%end, ierr)
                 proc_coords(2) = proc_coords(2) - 1
             end if
 
@@ -607,7 +609,7 @@ contains
         else
             offset_x%beg = 0
         end if
-
+        
         ! Boundary condition at the end
         if (proc_coords(1) < num_procs_x - 1 .or. bc_x%end == -1) then
             proc_coords(1) = proc_coords(1) + 1
@@ -928,7 +930,13 @@ contains
                                 r = sys_size*(j + buff_size) &
                                     + sys_size*buff_size*k + (i - 1) &
                                     + sys_size*buff_size*(n + 1)*l
-                                q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
+                                q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)                             
+#if defined(__INTEL_COMPILER)
+                                if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
+                                    print *, "Error", j, k, l, i
+                                    error stop "NaN(s) in recv"
+                                end if
+#endif                                
                             end do
                         end do
                     end do
@@ -1002,7 +1010,13 @@ contains
                                 r = (i - 1) + sys_size*(j - m - 1) &
                                     + sys_size*buff_size*k &
                                     + sys_size*buff_size*(n + 1)*l
-                                q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
+                                q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)                                
+#if defined(__INTEL_COMPILER)
+                                if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
+                                    print *, "Error", j, k, l, i
+                                    error stop "NaN(s) in recv"
+                                end if
+#endif                                 
                             end do
                         end do
                     end do
@@ -1091,6 +1105,12 @@ contains
                                     (k + buff_size) + sys_size* &
                                     (m + 2*buff_size + 1)*buff_size*l
                                 q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
+#if defined(__INTEL_COMPILER)
+                                if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
+                                    print *, "Error", j, k, l, i
+                                    error stop "NaN(s) in recv"
+                                end if
+#endif            
                             end do
                         end do
                     end do
@@ -1171,6 +1191,12 @@ contains
                                     (k - n - 1) + sys_size* &
                                     (m + 2*buff_size + 1)*buff_size*l
                                 q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
+#if defined(__INTEL_COMPILER)
+                                if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
+                                    print *, "Error", j, k, l, i
+                                    error stop "NaN(s) in recv"
+                                end if
+#endif 
                             end do
                         end do
                     end do
@@ -1264,6 +1290,12 @@ contains
                                     + sys_size*(m + 2*buff_size + 1)* &
                                     (n + 2*buff_size + 1)*(l + buff_size)
                                 q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
+#if defined(__INTEL_COMPILER)
+                                if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
+                                    print *, "Error", j, k, l, i
+                                    error stop "NaN(s) in recv"
+                                end if
+#endif                           
                             end do
                         end do
                     end do
@@ -1349,6 +1381,12 @@ contains
                                     + sys_size*(m + 2*buff_size + 1)* &
                                     (n + 2*buff_size + 1)*(l - p - 1)
                                 q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
+#if defined(__INTEL_COMPILER)
+                                if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
+                                    print *, "Error", j, k, l, i
+                                    error stop "NaN(s) in recv"
+                                end if
+#endif 
                             end do
                         end do
                     end do