From 4d5972f9e280ca08c5f2ae03a330977d76a6ebe9 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 3 Dec 2020 15:55:50 +0000
Subject: [PATCH 001/198] git ignore *.d and profile_*

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 58f2516546..e652edd6d9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 *.o
+*.d
 *.f90
 *.mod
 *.a
@@ -10,6 +11,7 @@ milc_interface/*
 *.pyc
 tunecache.tsv
 profile.tsv
+profile_*.tsv
 config.log
 CMakeCache.txt
 CMakeFiles

From b8d4b2e779814d3be60622268f8b21366fe6c729 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 3 Dec 2020 15:56:57 +0000
Subject: [PATCH 002/198] omptarget replicates cuda

---
 include/quda_api.h                            | 215 +++++
 include/targets/omptarget/FFT_Plans.h         | 137 +++
 .../omptarget/block_reduction_kernel.h        |  70 ++
 include/targets/omptarget/kernel.h            |  61 ++
 include/targets/omptarget/quda_fp16.cuh       |  21 +
 include/targets/omptarget/random_helper.h     |  81 ++
 include/targets/omptarget/reduction_kernel.h  |  54 ++
 include/targets/omptarget/target_device.h     | 139 +++
 lib/targets/omptarget/blas_lapack_cublas.cpp  | 191 ++++
 lib/targets/omptarget/device.cpp              | 376 ++++++++
 lib/targets/omptarget/malloc.cpp              | 707 ++++++++++++++
 lib/targets/omptarget/quda_api.cpp            | 549 +++++++++++
 lib/targets/omptarget/tune.cpp                | 876 ++++++++++++++++++
 13 files changed, 3477 insertions(+)
 create mode 100644 include/targets/omptarget/FFT_Plans.h
 create mode 100644 include/targets/omptarget/block_reduction_kernel.h
 create mode 100644 include/targets/omptarget/kernel.h
 create mode 100644 include/targets/omptarget/quda_fp16.cuh
 create mode 100644 include/targets/omptarget/random_helper.h
 create mode 100644 include/targets/omptarget/reduction_kernel.h
 create mode 100644 include/targets/omptarget/target_device.h
 create mode 100644 lib/targets/omptarget/blas_lapack_cublas.cpp
 create mode 100644 lib/targets/omptarget/device.cpp
 create mode 100644 lib/targets/omptarget/malloc.cpp
 create mode 100644 lib/targets/omptarget/quda_api.cpp
 create mode 100644 lib/targets/omptarget/tune.cpp

diff --git a/include/quda_api.h b/include/quda_api.h
index 3f8b7bc9e5..2bf284386a 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -1,10 +1,225 @@
 #pragma once
 
+#ifdef QUDA_BACKEND_OMPTARGET
+
+#include <omp.h>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <functional>
+#include <string_view>
+
+#include <util_quda.h>
+
+template <typename T>
+constexpr auto type_name() noexcept {
+  std::string_view name = "Error: unsupported compiler", prefix, suffix;
+#ifdef __clang__
+  name = __PRETTY_FUNCTION__;
+  prefix = "auto type_name() [T = ";
+  suffix = "]";
+#elif defined(__GNUC__)
+  name = __PRETTY_FUNCTION__;
+  prefix = "constexpr auto type_name() [with T = ";
+  suffix = "]";
+#endif
+  name.remove_prefix(prefix.size());
+  name.remove_suffix(suffix.size());
+  return name;
+}
+
+template <typename...Arg>
+struct seq_args_call{
+  static constexpr size_t n = sizeof...(Arg);
+  template <typename F, std::size_t...IX>
+  void operator()(F *func, void *args[n], std::index_sequence<IX...>)
+  {(*func)(*(Arg*)args[IX]...);}
+};
+
+template <typename T>
+T * to_device(const T * x, size_t s) {
+  if(0<omp_get_num_devices()){
+    const int d = omp_get_default_device();
+    const int h = omp_get_initial_device();
+    void *p = omp_target_alloc(s, d);
+    printfQuda("# to_device: host_ptr@%d = %p  device_ptr@%d = %p  size = %zu\n", h, x, d, p, s);
+    omp_target_memcpy(p, (void *)x, s, 0, 0, d, h);
+    return (T *)p;
+  }else{
+    return x;
+  }
+}
+
+template <typename T>
+T * to_device(const T& x) {
+  constexpr size_t s = sizeof(T);
+  return to_device(&x, s);
+}
+
+#define __host__
+#define __device__
+#define __shared__
+#define __global__
+#define __constant__ static
+#define __launch_bounds__(x)
+#define __syncthreads() _Pragma("omp barrier")
+
+using size_t = std::size_t;
+
+#define __forceinline__ inline __attribute__((always_inline))
+
+enum cudaMemcpyKind{cudaMemcpyHostToHost, cudaMemcpyHostToDevice, cudaMemcpyDeviceToHost, cudaMemcpyDeviceToDevice, cudaMemcpyDefault};
+enum cudaError_t{cudaSuccess,cudaErrorNotReady};
+enum {cudaEventDisableTiming,cudaEventInterprocess};
+enum {cudaHostRegisterDefault};
+enum {cudaIpcMemLazyEnablePeerAccess};
+enum {cudaStreamDefault};
+
+static inline cudaError_t
+ompwip_(const char * const file, const size_t line, const char * const func, const char * const msg, std::function<void(void)>f = [](){})
+{
+  if(0==omp_get_team_num()&&0==omp_get_thread_num()) std::cerr<<"OMP WIP:"<<msg<<": "<<file<<':'<<line<<' '<<func<<std::endl;
+  f();
+  return cudaSuccess;
+}
+static inline cudaError_t
+ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f = [](){})
+{return ompwip_(file,line,func,"",f);}
+#define ompwip(...) ompwip_(__FILE__,__LINE__,__PRETTY_FUNCTION__,##__VA_ARGS__)
+
+#define __shfl_down_sync(a,b,c) ompwip("__shfl_down_sync")
+
+#define cuMemcpy(a,b,c) ompwip()
+#define cuMemcpyAsync(a,b,c,d) ompwip()
+#define cuMemcpyDtoD(a,b,c) ompwip()
+#define cuMemcpyDtoDAsync(a,b,c,d) ompwip()
+#define cuMemcpyDtoH(a,b,c) ompwip()
+#define cuMemcpyDtoHAsync(a,b,c,d) ompwip()
+#define cuMemcpyHtoD(a,b,c) ompwip()
+#define cuMemcpyHtoDAsync(a,b,c,d) ompwip()
+#define cudaMemcpy(a,b,c,d) ompwip([&](){printfQuda("memcpy %p <- %p\n",a,b);ompwipMemcpy(a,(void*)b,c,d);})
+#define cudaMemcpy2D(a,b,c,d,e,f,g) ompwip()
+#define cudaMemcpy2DAsync(a,b,c,d,e,f,g,h) ompwip()
+#define cudaMemcpyAsync(a,b,c,d,e) ompwip()
+#define cudaMemcpyToSymbolAsync(a,b,c,d,e,f) ompwip()
+#define cudaMemset(a,b,c) ompwip([&](){printfQuda("memset %p\n",a);ompwipMemset(a,b,c);})
+#define cudaMemset2D(a,b,c,d,e) ompwip()
+#define cudaMemset2DAsync(a,b,c,d,e,f) ompwip()
+#define cudaMemsetAsync(a,b,c,d) ompwip()
+
+static inline void
+ompwipMemset(void *p, unsigned char b, std::size_t s)
+{
+#pragma omp target teams distribute parallel for simd is_device_ptr(p)
+  for(std::size_t i=0;i<s;++i) *(unsigned char *)p = b;
+}
+
+static inline void
+ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
+{
+  switch(k){
+  case cudaMemcpyHostToHost: memcpy(d,s,c); break;
+  case cudaMemcpyHostToDevice:
+    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+    else warningQuda("cudaMemcpyHostToDevice without a device");
+    break;
+  case cudaMemcpyDeviceToHost:
+    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
+    else warningQuda("cudaMemcpyDeviceToHost without a device");
+    break;
+  case cudaMemcpyDeviceToDevice:
+    warningQuda("unimplemented for cudaMemcpyDeviceToDevice");
+    break;
+  case cudaMemcpyDefault:
+    warningQuda("unimplemented for cudaMemcpyDefault");
+    break;
+  default: errorQuda("Unsupported cudaMemcpyType %d", k);
+  }
+}
+
+#define cudaDeviceSynchronize() ompwip()
+#define cudaEventCreate(a,...) ompwip([&](){(*(a))=0;})
+#define cudaEventCreateWithFlags(a,b) ompwip()
+#define cudaEventDestroy(a) ompwip()
+#define cudaEventElapsedTime(a,b,c) ompwip([&](){(*(a))=0;})
+#define cudaEventQuery(a) ompwip()
+#define cudaEventRecord(a,b) ompwip()
+#define cudaEventSynchronize(a) ompwip()
+#define cudaFuncGetAttributes(a,b) ompwip()
+#define cudaFuncSetAttribute(a,b,c) ompwip()
+#define cudaGetErrorString(a) "FIXME"
+#define cudaGetLastError() ompwip()
+#define cudaGetSymbolAddress(a,b) ompwip()
+#define cudaHostRegister(a,b,c) ompwip()
+#define cudaHostUnregister(a) ompwip()
+#define cudaIpcCloseMemHandle(a) ompwip()
+#define cudaIpcGetEventHandle(a,b) ompwip([&](){(*(a))=0;})
+#define cudaIpcGetMemHandle(a,b) ompwip([&](){(*(a))=0;})
+#define cudaIpcOpenEventHandle(a,b) ompwip([&](){(*(a))=0;})
+#define cudaIpcOpenMemHandle(a,b,c) ompwip()
+#define cudaStreamCreate(a) ompwip([&](){(*(a))=0;})
+#define cudaStreamDestroy(a) ompwip()
+#define cudaStreamSynchronize(a) ompwip()
+#define cudaStreamWaitEvent(a,b,c) ompwip()
+
+static char FIXME[]="FIXME";
+#define cuGetErrorString(a,b) ompwip([&](){(*(b))=FIXME;})
+#define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
+
+using qudaStream_t = int;
+using cudaStream_t = int;
+using CUresult = int;
+using cudaIpcMemHandle_t = int;
+using cudaIpcEventHandle_t = int;
+using cudaEvent_t = int;
+
+struct dim3 {unsigned int x,y,z;
+  dim3():x(1u),y(1u),z(1u){}
+  dim3(unsigned int x):x(x),y(1u),z(1u){}
+  dim3(unsigned int x,unsigned int y):x(x),y(y),z(1u){}
+  dim3(unsigned int x,unsigned int y,unsigned int z):x(x),y(y),z(z){}
+};
+struct int2 {int x,y;};
+struct int4 {int x,y,z,w;};
+static inline int4 make_int4(int x,int y,int z,int w){return int4{x,y,z,w};}
+struct uint2 {unsigned int x,y;};
+static inline uint2 make_uint2(unsigned int x,unsigned int y){return uint2{x,y};}
+struct uint3 {unsigned int x,y,z;};
+static inline uint3 make_uint3(unsigned int x,unsigned int y,unsigned int z){return uint3{x,y,z};}
+struct char2 {char x,y;};
+static inline char2 make_char2(char x,char y){return char2{x,y};}
+struct char3 {char x,y,z;};
+struct char4 {char x,y,z,w;};
+struct short2 {short x,y;};
+static inline short2 make_short2(short x,short y){return short2{x,y};}
+struct short3 {short x,y,z;};
+struct short4 {short x,y,z,w;};
+struct float2 {float x,y;};
+static inline float2 make_float2(float x,float y){return float2{x,y};}
+struct float3 {float x,y,z;};
+static inline float3 make_float3(float x,float y,float z){return float3{x,y,z};}
+struct float4 {float x,y,z,w;};
+static inline float4 make_float4(float x,float y,float z,float w){return float4{x,y,z,w};}
+struct double2 {double x,y;};
+static inline double2 make_double2(double x,double y){return double2{x,y};}
+struct double3 {double x,y,z;};
+static inline double3 make_double3(double x,double y,double z){return double3{x,y,z};}
+struct double4 {double x,y,z,w;};
+static inline double4 make_double4(double x,double y,double z,double w){return double4{x,y,z,w};}
+
+#define QUDA_RT_CONSTS ompwip();const dim3 threadIdx(omp_get_thread_num(),0,0),blockDim(omp_get_num_threads()),blockIdx(omp_get_team_num(),0,0),gridDim(omp_get_num_teams())
+
+#else  // QUDA_BACKEND_OMPTARGET
+
 #ifndef __CUDACC_RTC__
 #include <cuda.h>
 #include <cuda_runtime.h>
 #endif
 
+using qudaStream_t = cudaStream_t;
+
+#endif  // QUDA_BACKEND_OMPTARGET
+
 #include <enum_quda.h>
 
 /**
diff --git a/include/targets/omptarget/FFT_Plans.h b/include/targets/omptarget/FFT_Plans.h
new file mode 100644
index 0000000000..e53bfe1a5f
--- /dev/null
+++ b/include/targets/omptarget/FFT_Plans.h
@@ -0,0 +1,137 @@
+#pragma once
+
+#include <quda_internal.h>
+#include <quda_matrix.h>
+
+using FFTPlanHandle = int;
+/*
+#include <cufft.h>
+
+using FFTPlanHandle = cufftHandle;
+#define FFT_FORWARD     CUFFT_FORWARD
+#define FFT_INVERSE     CUFFT_INVERSE
+
+#ifndef GPU_GAUGE_ALG
+*/
+#ifdef QUDA_BACKEND_OMPTARGET
+#define CUFFT_SAFE_CALL(call)
+
+inline void ApplyFFT(FFTPlanHandle &, float2 *, float2 *, int)
+{
+  errorQuda("unimplemented");
+}
+
+inline void ApplyFFT(FFTPlanHandle &, double2 *, double2 *, int)
+{
+  errorQuda("unimplemented");
+}
+
+inline void SetPlanFFTMany(FFTPlanHandle &, int4, int, QudaPrecision)
+{
+  errorQuda("unimplemented");
+}
+
+inline void SetPlanFFT2DMany(FFTPlanHandle &, int4, int, QudaPrecision)
+{
+  errorQuda("unimplemented");
+}
+
+inline void FFTDestroyPlan(FFTPlanHandle &)
+{
+  errorQuda("unimplemented");
+}
+#else
+
+/*-------------------------------------------------------------------------------*/
+#define CUFFT_SAFE_CALL( call) {                                      \
+    cufftResult err = call;                                         \
+    if ( CUFFT_SUCCESS != err ) {                                     \
+      fprintf(stderr, "CUFFT error in file '%s' in line %i.\n",   \
+              __FILE__, __LINE__);                                \
+      exit(EXIT_FAILURE);                                         \
+    } }
+/*-------------------------------------------------------------------------------*/
+
+/**
+ * @brief Call CUFFT to perform a single-precision complex-to-complex
+ * transform plan in the transform direction as specified by direction
+ * parameter
+ * @param[in] CUFFT plan
+ * @param[in] data_in, pointer to the complex input data (in GPU memory) to transform
+ * @param[out] data_out, pointer to the complex output data (in GPU memory)
+ * @param[in] direction, the transform direction: CUFFT_FORWARD or CUFFT_INVERSE
+ */
+inline void ApplyFFT(FFTPlanHandle &plan, float2 *data_in, float2 *data_out, int direction){
+  CUFFT_SAFE_CALL(cufftExecC2C(plan, (cufftComplex *)data_in, (cufftComplex *)data_out, direction));
+}
+
+/**
+ * @brief Call CUFFT to perform a double-precision complex-to-complex transform plan in the transform direction
+as specified by direction parameter
+ * @param[in] CUFFT plan
+ * @param[in] data_in, pointer to the complex input data (in GPU memory) to transform
+ * @param[out] data_out, pointer to the complex output data (in GPU memory)
+ * @param[in] direction, the transform direction: CUFFT_FORWARD or CUFFT_INVERSE
+ */
+inline void ApplyFFT(FFTPlanHandle &plan, double2 *data_in, double2 *data_out, int direction){
+  CUFFT_SAFE_CALL(cufftExecZ2Z(plan, (cufftDoubleComplex *)data_in, (cufftDoubleComplex *)data_out, direction));
+}
+
+/**
+ * @brief Creates a CUFFT plan supporting 4D (1D+3D) data layouts for complex-to-complex
+ * @param[out] plan, CUFFT plan
+ * @param[in] size, int4 with lattice size dimensions, (.x,.y,.z,.w) -> (Nx, Ny, Nz, Nt)
+ * @param[in] dim, 1 for 1D plan along the temporal direction with batch size Nx*Ny*Nz, 3 for 3D plan along Nx, Ny and Nz with batch size Nt
+ * @param[in] precision The precision of the computation
+ */
+
+inline void SetPlanFFTMany(FFTPlanHandle &plan, int4 size, int dim, QudaPrecision precision)
+{
+  auto type = precision == QUDA_DOUBLE_PRECISION ? CUFFT_Z2Z : CUFFT_C2C;
+  switch (dim) {
+  case 1:
+  {
+    int n[1] = { size.w };
+    CUFFT_SAFE_CALL(cufftPlanMany(&plan, 1, n, NULL, 1, 0, NULL, 1, 0, type, size.x * size.y * size.z));
+  }
+  break;
+  case 3:
+  {
+    int n[3] = { size.x, size.y, size.z };
+    CUFFT_SAFE_CALL(cufftPlanMany(&plan, 3, n, NULL, 1, 0, NULL, 1, 0, type, size.w));
+  }
+  break;
+  }
+}
+
+/**
+ * @brief Creates a CUFFT plan supporting 4D (2D+2D) data layouts for complex-to-complex
+ * @param[out] plan, CUFFT plan
+ * @param[in] size, int4 with lattice size dimensions, (.x,.y,.z,.w) -> (Nx, Ny, Nz, Nt)
+ * @param[in] dim, 0 for 2D plan in Z-T planes with batch size Nx*Ny, 1 for 2D plan in X-Y planes with batch size Nz*Nt
+ * @param[in] precision The precision of the computation
+ */
+inline void SetPlanFFT2DMany(cufftHandle &plan, int4 size, int dim, QudaPrecision precision)
+{
+  auto type = precision == QUDA_DOUBLE_PRECISION ? CUFFT_Z2Z : CUFFT_C2C;
+  switch (dim) {
+  case 0:
+  {
+    int n[2] = { size.w, size.z };
+    CUFFT_SAFE_CALL(cufftPlanMany(&plan, 2, n, NULL, 1, 0, NULL, 1, 0, type, size.x * size.y));
+  }
+  break;
+  case 1:
+  {
+    int n[2] = { size.x, size.y };
+    CUFFT_SAFE_CALL(cufftPlanMany(&plan, 2, n, NULL, 1, 0, NULL, 1, 0, type, size.z * size.w));
+  }
+  break;
+  }
+}
+
+inline void FFTDestroyPlan( FFTPlanHandle &plan) {
+   CUFFT_SAFE_CALL(cufftDestroy(plan));
+}
+
+#endif
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
new file mode 100644
index 0000000000..2a825cae8b
--- /dev/null
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -0,0 +1,70 @@
+#pragma once
+
+#include <reduce_helper.h>
+
+namespace quda {
+
+  /**
+     @brief This helper function swizzles the block index through
+     mapping the block index onto a matrix and tranposing it.  This is
+     done to potentially increase the cache utilization.  Requires
+     that the argument class has a member parameter "swizzle" which
+     determines if we are swizzling and a parameter "swizzle_factor"
+     which is the effective matrix dimension that we are tranposing in
+     this mapping.
+   */
+  template <typename Arg> __device__ int virtual_block_idx(const Arg &arg)
+  {
+    QUDA_RT_CONSTS;
+    int block_idx = blockIdx.x;
+    if (arg.swizzle) {
+      // the portion of the grid that is exactly divisible by the number of SMs
+      const int gridp = gridDim.x - gridDim.x % arg.swizzle_factor;
+
+      block_idx = blockIdx.x;
+      if (blockIdx.x < gridp) {
+        // this is the portion of the block that we are going to transpose
+        const int i = blockIdx.x % arg.swizzle_factor;
+        const int j = blockIdx.x / arg.swizzle_factor;
+
+        // transpose the coordinates
+        block_idx = i * (gridp / arg.swizzle_factor) + j;
+      }
+    }
+    return block_idx;
+  }
+
+  /**
+     @brief Generic block reduction kernel.  Here, we ensure that each
+     thread block maps exactly to a logical block to be reduced, with
+     number of threads equal to the number of sites per block.  The y
+     thread dimension is a trivial vectorizable dimension.
+
+     TODO: add a Reducer class for non summation reductions
+  */
+  template <int block_size, template <typename> class Transformer, typename Arg>
+  __global__ void BlockReductionKernel2D(Arg arg)
+  {
+    QUDA_RT_CONSTS;
+    using reduce_t = typename Transformer<Arg>::reduce_t;
+    Transformer<Arg> t(arg);
+
+    const int block = virtual_block_idx(arg);
+    const int i = threadIdx.x;
+    const int j = blockDim.y*blockIdx.y + threadIdx.y;
+    const int j_local = threadIdx.y;
+    if (j >= arg.threads.y) return;
+
+    reduce_t value; // implicitly we assume here that default constructor zeros reduce_t
+    // only active threads call the transformer
+    if (i < arg.threads.x) value = t(block, i, j);
+/*
+    // but all threads take part in the reduction
+    using BlockReduce = cub::BlockReduce<reduce_t, block_size, cub::BLOCK_REDUCE_WARP_REDUCTIONS>;
+    __shared__ typename BlockReduce::TempStorage temp_storage[Arg::n_vector_y];
+    value = BlockReduce(temp_storage[j_local]).Sum(value);
+*/
+    if (i == 0) t.store(value, block, j);
+  }
+
+}
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
new file mode 100644
index 0000000000..4c7ff3e240
--- /dev/null
+++ b/include/targets/omptarget/kernel.h
@@ -0,0 +1,61 @@
+#pragma once
+
+namespace quda {
+
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  __global__ void Kernel1D(Arg arg)
+  {
+    QUDA_RT_CONSTS;
+    Functor<Arg> f(arg);
+
+    int i = threadIdx.x + blockIdx.x * blockDim.x;
+
+    while (i < arg.threads.x) {
+      f(i);
+      if (grid_stride) i += gridDim.x * blockDim.x; else break;
+    }
+  }
+
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  __global__ void Kernel2D(Arg arg)
+  {
+    QUDA_RT_CONSTS;
+    Functor<Arg> f(arg);
+
+    int i = threadIdx.x + blockIdx.x * blockDim.x;
+    int j = threadIdx.y + blockIdx.y * blockDim.y;
+    if (j >= arg.threads.y) return;
+
+    while (i < arg.threads.x) {
+      f(i, j);
+      if (grid_stride) i += gridDim.x * blockDim.x; else break;
+    }
+  }
+
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  __global__ void Kernel3D(Arg arg)
+  {
+    QUDA_RT_CONSTS;
+    Functor<Arg> f(arg);
+
+    int i = threadIdx.x + blockIdx.x * blockDim.x;
+    int j = threadIdx.y + blockIdx.y * blockDim.y;
+    int k = threadIdx.z + blockIdx.z * blockDim.z;
+    if (j >= arg.threads.y) return;
+    if (k >= arg.threads.z) return;
+
+    while (i < arg.threads.x) {
+      f(i, j, k);
+      if (grid_stride) i += gridDim.x * blockDim.x; else break;
+    }
+  }
+
+  template <template <typename> class Functor, typename Arg>
+  // __launch_bounds__(Arg::block_dim, Arg::min_blocks)
+  __global__ void raw_kernel(Arg arg)
+  {
+    Functor<Arg> f(arg);
+    f();
+  }
+
+}
diff --git a/include/targets/omptarget/quda_fp16.cuh b/include/targets/omptarget/quda_fp16.cuh
new file mode 100644
index 0000000000..c5ad585dda
--- /dev/null
+++ b/include/targets/omptarget/quda_fp16.cuh
@@ -0,0 +1,21 @@
+#pragma once
+
+// #include <cuda_fp16.h>
+
+namespace quda
+{
+
+  __device__ inline half2 habs2(half2 input) {
+/*
+#if CUDA_VERSION >= 10020
+    return __habs2(input);
+#else
+*/
+    static constexpr uint32_t maximum_mask = 0x7fff7fffu; // 0111 1111 1111 1111 0111 1111 1111 1111
+
+    uint32_t input_masked = *reinterpret_cast<const uint32_t *>(&input) & maximum_mask;
+    return *reinterpret_cast<half2 *>(&input_masked);
+//#endif
+  }
+
+} // namespace quda
diff --git a/include/targets/omptarget/random_helper.h b/include/targets/omptarget/random_helper.h
new file mode 100644
index 0000000000..d7449cf6a2
--- /dev/null
+++ b/include/targets/omptarget/random_helper.h
@@ -0,0 +1,81 @@
+#pragma once
+
+#include <random>
+
+#include <random_quda.h>
+// #include <curand_kernel.h>
+
+namespace quda {
+/*
+#if defined(XORWOW)
+  using rng_state_t = curandStateXORWOW;
+#elif defined(MRG32k3a)
+  using rng_state_t = curandStateMRG32k3a;
+#else
+  using rng_state_t = curandStateMRG32k3a;
+#endif
+*/
+  using rng_state_t = std::mt19937_64;
+
+  struct RNGState {
+    rng_state_t state;
+  };
+
+  /**
+   * \brief random init
+   * @param [in] seed -- The RNG seed
+   * @param [in] sequence -- The sequence
+   * @param [in] offset -- the offset
+   * @param [in,out] state - the RNG State
+   */
+  __device__ inline void random_init(unsigned long long seed, unsigned long long sequence, unsigned long long offset, RNGState &state)
+  {
+    ompwip();
+    std::seed_seq ss{seed, sequence, offset};
+    state.state.seed(ss);
+    // curand_init(seed, sequence, offset, &state.state);
+  }
+
+  template<class Real>
+    struct uniform {
+
+    /**
+     * \brief Return a uniform deviate between 0 and 1
+     * @param [in,out] the RNG State
+     */
+    __device__ static inline Real rand(RNGState &state)
+    {
+      ompwip();
+      std::uniform_real_distribution<Real> u;
+      return u(state.state);
+    }
+
+    /**
+     * \brief return a uniform deviate between a and b
+     * @param [in,out] the RNG state
+     * @param [in] a (the lower end of the range)
+     * @param [in] b (the upper end of the range)
+     */
+    __device__ static inline Real rand(RNGState &state, Real a, Real b)
+    {
+      ompwip();
+      std::uniform_real_distribution<Real> u(a,b);
+      return u(state.state);
+    }
+
+  };
+
+  template<class Real>
+    struct normal {    /**
+     * \brief return a gaussian normal deviate with mean of 0
+     * @param [in,out] state
+     */
+    __device__ static inline Real rand(RNGState &state)
+    {
+      ompwip();
+      std::normal_distribution<Real> n;
+      return n(state.state);
+    }
+  };
+
+}
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
new file mode 100644
index 0000000000..2fc92e3cbd
--- /dev/null
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include <reduce_helper.h>
+
+namespace quda {
+
+  template <int block_size_x, int block_size_y, template <typename> class Transformer,
+            template <typename> class Reducer, typename Arg>
+  __global__ void Reduction2D(Arg arg)
+  {/*
+    using reduce_t = typename Transformer<Arg>::reduce_t;
+    Transformer<Arg> t(arg);
+    Reducer<reduce_t> r;
+
+    int idx = threadIdx.x + blockIdx.x * blockDim.x;
+    int j = threadIdx.y;
+
+    reduce_t value = arg.init();
+
+    while (idx < arg.threads.x) {
+      value = t(value, r, idx, j);
+      idx += blockDim.x * gridDim.x;
+    }
+
+    // perform final inter-block reduction and write out result
+    reduce<block_size_x, block_size_y, decltype(r)>(arg, value);
+  */}
+
+  template <int block_size_x, int block_size_y, template <typename> class Transformer,
+            template <typename> class Reducer, typename Arg>
+  __global__ void MultiReduction(Arg arg)
+  {/*
+    using reduce_t = typename Transformer<Arg>::reduce_t;
+    Transformer<Arg> t(arg);
+    Reducer<reduce_t> r;
+
+    int idx = threadIdx.x + blockIdx.x * blockDim.x;
+    int j = threadIdx.y + blockIdx.y * blockDim.y;
+    int k = threadIdx.z;
+
+    if (j >= arg.threads.y) return;
+
+    reduce_t value = arg.init();
+
+    while (idx < arg.threads.x) {
+      value = t(value, r, idx, j, k);
+      idx += blockDim.x * gridDim.x;
+    }
+
+    // perform final inter-block reduction and write out result
+    reduce<block_size_x, block_size_y, decltype(r)>(arg, value, j);
+  */}
+
+}
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
new file mode 100644
index 0000000000..b514595aa7
--- /dev/null
+++ b/include/targets/omptarget/target_device.h
@@ -0,0 +1,139 @@
+#pragma once
+
+namespace quda {
+
+  namespace device {
+
+    /**
+       @brief Helper function that returns if the current execution
+       region is on the device
+    */
+#pragma omp begin declare variant match(device={kind(host)})
+    constexpr bool is_device() {return false;}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(nohost)})
+    constexpr bool is_device() {return true;}
+#pragma omp end declare variant
+    constexpr bool is_device() {return false;}
+
+    /**
+       @brief Helper function that returns if the current execution
+       region is on the host
+    */
+#pragma omp begin declare variant match(device={kind(host)})
+    constexpr bool is_host() {return true;}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(nohost)})
+    constexpr bool is_host() {return false;}
+#pragma omp end declare variant
+    constexpr bool is_host() {return true;}
+
+    /**
+       @brief Helper function that returns the thread block
+       dimensions.  On CUDA this returns the intrinsic blockDim.
+       @param[in] arg Kernel argument struct
+    */
+    __device__ __host__ inline dim3 block_dim()
+    {
+      return dim3(omp_get_num_threads());
+/*
+#ifdef __CUDA_ARCH__
+      return dim3(blockDim.x, blockDim.y, blockDim.z);
+#else
+      return dim3(0, 0, 0);
+#endif
+*/
+    }
+
+    /**
+       @brief Helper function that returns the thread block
+       dimensions.  On CUDA this returns the intrinsic blockDim,
+       whereas on the host this grabs the dimensions from the kernel
+       argument struct
+       @param[in] arg Kernel argument struct
+    */
+    __device__ __host__ inline dim3 thread_idx()
+    {
+      return dim3(omp_get_thread_num(),0,0);
+/*
+#ifdef __CUDA_ARCH__
+      return dim3(threadIdx.x, threadIdx.y, threadIdx.z);
+#else
+      return dim3(0, 0, 0);
+#endif
+*/
+    }
+
+    /**
+       @brief Helper function that returns the warp-size of the
+       architecture we are running on.
+    */
+    constexpr int warp_size() { return 32; }
+
+    /**
+       @brief Return the thread mask for a converged warp.
+    */
+    constexpr unsigned int warp_converged_mask() { return 0xffffffff; }
+
+    /**
+       @brief Helper function that returns the maximum number of threads
+       in a block in the x dimension.
+    */
+    template <int block_size_y = 1, int block_size_z = 1>
+      constexpr unsigned int max_block_size()
+      {
+        return std::max(warp_size(), 1024 / (block_size_y * block_size_z));
+      }
+
+    /**
+       @brief Helper function that returns the maximum number of threads
+       in a block in the x dimension for reduction kernels.
+    */
+    template <int block_size_y = 1, int block_size_z = 1>
+      constexpr unsigned int max_reduce_block_size()
+      {
+#ifdef QUDA_FAST_COMPILE_REDUCE
+        // This is the specialized variant used when we have fast-compilation mode enabled
+        return warp_size();
+#else
+        return max_block_size<block_size_y, block_size_z>();
+#endif
+      }
+
+    /**
+       @brief Helper function that returns the maximum number of threads
+       in a block in the x dimension for reduction kernels.
+    */
+    constexpr unsigned int max_multi_reduce_block_size()
+    {
+#ifdef QUDA_FAST_COMPILE_REDUCE
+      // This is the specialized variant used when we have fast-compilation mode enabled
+      return warp_size();
+#else
+      return 128;
+#endif
+    }
+
+    /**
+       @brief Helper function that returns the maximum size of a
+       constant_param_t buffer on the target architecture.  For CUDA,
+       this corresponds to the maximum __constant__ buffer size.
+    */
+    constexpr size_t max_constant_param_size() { return 8192; }
+
+    /**
+       @brief Helper function that returns the maximum static size of
+       the kernel arguments passed to a kernel on the target
+       architecture.
+    */
+    constexpr size_t max_kernel_arg_size() { return 4096; }
+
+    /**
+       @brief Helper function that returns the bank width of the
+       shared memory bank width on the target architecture.
+    */
+    constexpr int shared_memory_bank_width() { return 32; }
+
+  }
+
+}
diff --git a/lib/targets/omptarget/blas_lapack_cublas.cpp b/lib/targets/omptarget/blas_lapack_cublas.cpp
new file mode 100644
index 0000000000..94f1cf7f9f
--- /dev/null
+++ b/lib/targets/omptarget/blas_lapack_cublas.cpp
@@ -0,0 +1,191 @@
+#include <blas_lapack.h>
+#ifdef NATIVE_LAPACK_LIB
+#include <cublas_v2.h>
+#include <malloc_quda.h>
+#endif
+
+//#define _DEBUG
+
+#ifdef _DEBUG
+#include <eigen_helper.h>
+#endif
+
+namespace quda
+{
+
+  namespace blas_lapack
+  {
+
+    namespace native
+    {
+
+#ifdef NATIVE_LAPACK_LIB
+      static cublasHandle_t handle;
+#endif
+      static bool cublas_init = false;
+
+      void init()
+      {
+        if (!cublas_init) {
+#ifdef NATIVE_LAPACK_LIB
+          cublasStatus_t error = cublasCreate(&handle);
+          if (error != CUBLAS_STATUS_SUCCESS) errorQuda("cublasCreate failed with error %d", error);
+          cublas_init = true;
+#endif
+        }
+      }
+
+      void destroy()
+      {
+        if (cublas_init) {
+#ifdef NATIVE_LAPACK_LIB
+          cublasStatus_t error = cublasDestroy(handle);
+          if (error != CUBLAS_STATUS_SUCCESS)
+            errorQuda("\nError indestroying cublas context, error code = %d\n", error);
+          cublas_init = false;
+#endif
+        }
+      }
+
+#ifdef _DEBUG
+      template <typename EigenMatrix, typename Float>
+      __host__ void checkEigen(std::complex<Float> *A_h, std::complex<Float> *Ainv_h, int n, uint64_t batch)
+      {
+        EigenMatrix A = EigenMatrix::Zero(n, n);
+        EigenMatrix Ainv = EigenMatrix::Zero(n, n);
+        for (int j = 0; j < n; j++) {
+          for (int k = 0; k < n; k++) {
+            A(k, j) = A_h[batch * n * n + j * n + k];
+            Ainv(k, j) = Ainv_h[batch * n * n + j * n + k];
+          }
+        }
+
+        // Check result:
+        EigenMatrix unit = EigenMatrix::Identity(n, n);
+        EigenMatrix prod = A * Ainv;
+        Float L2norm = ((prod - unit).norm() / (n * n));
+        printfQuda("cuBLAS: Norm of (A * Ainv - I) batch %lu = %e\n", batch, L2norm);
+      }
+#endif
+
+      // FIXME do this in pipelined fashion to reduce memory overhead.
+      long long BatchInvertMatrix(void *Ainv, void *A, const int n, const uint64_t batch, QudaPrecision prec,
+                                  QudaFieldLocation location)
+      {
+#ifdef NATIVE_LAPACK_LIB
+        init();
+        if (getVerbosity() >= QUDA_VERBOSE)
+          printfQuda("BatchInvertMatrix (native - cuBLAS): Nc = %d, batch = %lu\n", n, batch);
+
+        long long flops = 0;
+        timeval start, stop;
+        gettimeofday(&start, NULL);
+
+        size_t size = 2 * n * n * prec * batch;
+        void *A_d = location == QUDA_CUDA_FIELD_LOCATION ? A : pool_device_malloc(size);
+        void *Ainv_d = location == QUDA_CUDA_FIELD_LOCATION ? Ainv : pool_device_malloc(size);
+        if (location == QUDA_CPU_FIELD_LOCATION) qudaMemcpy(A_d, A, size, cudaMemcpyHostToDevice);
+
+#ifdef _DEBUG
+        // Debug code: Copy original A matrix to host
+        std::complex<float> *A_h
+          = (location == QUDA_CUDA_FIELD_LOCATION ? static_cast<std::complex<float> *>(pool_pinned_malloc(size)) :
+                                                    static_cast<std::complex<float> *>(A_d));
+        if (location == QUDA_CUDA_FIELD_LOCATION) qudaMemcpy((void *)A_h, A_d, size, cudaMemcpyDeviceToHost);
+#endif
+
+        int *dipiv = static_cast<int *>(pool_device_malloc(batch * n * sizeof(int)));
+        int *dinfo_array = static_cast<int *>(pool_device_malloc(batch * sizeof(int)));
+        int *info_array = static_cast<int *>(pool_pinned_malloc(batch * sizeof(int)));
+        memset(info_array, '0', batch * sizeof(int)); // silence memcheck warnings
+
+        if (prec == QUDA_SINGLE_PRECISION) {
+          typedef cuFloatComplex C;
+          C **A_array = static_cast<C **>(pool_device_malloc(2 * batch * sizeof(C *)));
+          C **Ainv_array = A_array + batch;
+          C **A_array_h = static_cast<C **>(pool_pinned_malloc(2 * batch * sizeof(C *)));
+          C **Ainv_array_h = A_array_h + batch;
+          for (uint64_t i = 0; i < batch; i++) {
+            A_array_h[i] = static_cast<C *>(A_d) + i * n * n;
+            Ainv_array_h[i] = static_cast<C *>(Ainv_d) + i * n * n;
+          }
+          qudaMemcpy(A_array, A_array_h, 2 * batch * sizeof(C *), cudaMemcpyHostToDevice);
+
+          cublasStatus_t error = cublasCgetrfBatched(handle, n, A_array, n, dipiv, dinfo_array, batch);
+          flops += batch * FLOPS_CGETRF(n, n);
+
+          if (error != CUBLAS_STATUS_SUCCESS)
+            errorQuda("\nError in LU decomposition (cublasCgetrfBatched), error code = %d\n", error);
+
+          qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), cudaMemcpyDeviceToHost);
+          for (uint64_t i = 0; i < batch; i++) {
+            if (info_array[i] < 0) {
+              errorQuda("%lu argument had an illegal value or another error occured, such as memory allocation failed",
+                        i);
+            } else if (info_array[i] > 0) {
+              errorQuda("%lu factorization completed but the factor U is exactly singular", i);
+            }
+          }
+
+          error = cublasCgetriBatched(handle, n, (const C **)A_array, n, dipiv, Ainv_array, n, dinfo_array, batch);
+          flops += batch * FLOPS_CGETRI(n);
+
+          if (error != CUBLAS_STATUS_SUCCESS)
+            errorQuda("\nError in matrix inversion (cublasCgetriBatched), error code = %d\n", error);
+
+          qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), cudaMemcpyDeviceToHost);
+
+          for (uint64_t i = 0; i < batch; i++) {
+            if (info_array[i] < 0) {
+              errorQuda("%lu argument had an illegal value or another error occured, such as memory allocation failed",
+                        i);
+            } else if (info_array[i] > 0) {
+              errorQuda("%lu factorization completed but the factor U is exactly singular", i);
+            }
+          }
+
+          pool_device_free(A_array);
+          pool_pinned_free(A_array_h);
+
+#ifdef _DEBUG
+          // Debug code: Copy computed Ainv to host
+          std::complex<float> *Ainv_h = static_cast<std::complex<float> *>(pool_pinned_malloc(size));
+          qudaMemcpy((void *)Ainv_h, Ainv_d, size, cudaMemcpyDeviceToHost);
+
+          for (uint64_t i = 0; i < batch; i++) { checkEigen<MatrixXcf, float>(A_h, Ainv_h, n, i); }
+          pool_pinned_free(Ainv_h);
+          pool_pinned_free(A_h);
+#endif
+        } else {
+          errorQuda("%s not implemented for precision=%d", __func__, prec);
+        }
+
+        if (location == QUDA_CPU_FIELD_LOCATION) {
+          qudaMemcpy(Ainv, Ainv_d, size, cudaMemcpyDeviceToHost);
+          pool_device_free(Ainv_d);
+          pool_device_free(A_d);
+        }
+
+        pool_device_free(dipiv);
+        pool_device_free(dinfo_array);
+        pool_pinned_free(info_array);
+
+        qudaDeviceSynchronize();
+        gettimeofday(&stop, NULL);
+        long ds = stop.tv_sec - start.tv_sec;
+        long dus = stop.tv_usec - start.tv_usec;
+        double time = ds + 0.000001 * dus;
+
+        if (getVerbosity() >= QUDA_VERBOSE)
+          printfQuda("Batched matrix inversion completed in %f seconds with GFLOPS = %f\n", time, 1e-9 * flops / time);
+
+        return flops;
+#else
+        errorQuda("Native BLAS not built. Please build and use native BLAS or use generic BLAS");
+        return 0; // Stops a compiler warning
+#endif
+      }
+
+    } // namespace native
+  }   // namespace blas_lapack
+} // namespace quda
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
new file mode 100644
index 0000000000..a2cbf0d681
--- /dev/null
+++ b/lib/targets/omptarget/device.cpp
@@ -0,0 +1,376 @@
+#include <omp.h>
+#include <util_quda.h>
+#include <quda_internal.h>
+
+#ifdef QUDA_NVML
+#include <nvml.h>
+#endif
+
+#ifdef NUMA_NVML
+#include <numa_affinity.h>
+#endif
+
+static char ompdevname[] = "OpenMP Target Device";
+static char omphostname[] = "OpenMP Host Device";
+struct cudaDeviceProp{
+  char*name;
+  int major,minor;
+  size_t sharedMemPerBlock;
+  unsigned int maxThreadsPerBlock;
+  unsigned int maxThreadsPerMultiProcessor;
+  unsigned int maxThreadsDim[3];
+  unsigned int maxGridSize[3];
+  unsigned int multiProcessorCount;
+};
+
+static void cudaDriverGetVersion(int*v){*v=0;}
+static void cudaRuntimeGetVersion(int*v){*v=0;}
+static void cudaGetDeviceCount(int*c){*c=omp_get_num_devices();}
+static void cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
+{
+  if(0<omp_get_num_devices()){
+    p->name = ompdevname;
+    p->major = 7;
+    p->minor = 0;
+    p->sharedMemPerBlock = 0;
+    p->maxThreadsPerBlock = 65536;
+    p->maxThreadsPerMultiProcessor = 65536;
+    p->maxThreadsDim[0] = 65536;
+    p->maxThreadsDim[1] = 2;
+    p->maxThreadsDim[2] = 1;
+    p->maxGridSize[0] = 65536;
+    p->maxGridSize[1] = 8;
+    p->maxGridSize[2] = 8;
+    p->multiProcessorCount = 4096;
+  }else{
+    p->name = omphostname;
+    p->major = 3;
+    p->minor = 0;
+    p->sharedMemPerBlock = 0;
+    p->maxThreadsPerBlock = 4096;
+    p->maxThreadsPerMultiProcessor = 4096;
+    p->maxThreadsDim[0] = 4096;
+    p->maxThreadsDim[1] = 2;
+    p->maxThreadsDim[2] = 1;
+    p->maxGridSize[0] = 128;
+    p->maxGridSize[1] = 8;
+    p->maxGridSize[2] = 8;
+    p->multiProcessorCount = 64;
+  }
+}
+
+#define cudaSetDevice omp_set_default_device
+#define cudaDeviceSetCacheConfig(a)
+static void cudaDeviceGetStreamPriorityRange(int*lo,int*hi){lo=0;hi=0;}
+static void cudaStreamCreateWithPriority(qudaStream_t*s,int lo,int hi){*s=0;}
+#define cudaDeviceReset()
+enum {cudaDevAttrMaxSharedMemoryPerBlockOptin};
+static void cudaDeviceGetAttribute(int*b,int o,int i){*b=98304;}
+
+static cudaDeviceProp deviceProp;
+static cudaStream_t *streams;
+static const int Nstream = 9;
+
+namespace quda
+{
+
+  namespace device
+  {
+
+    static bool initialized = false;
+
+    void init(int dev)
+    {
+      if (initialized) return;
+      initialized = true;
+
+      int driver_version;
+      cudaDriverGetVersion(&driver_version);
+      printfQuda("CUDA Driver version = %d\n", driver_version);
+
+      int runtime_version;
+      cudaRuntimeGetVersion(&runtime_version);
+      printfQuda("CUDA Runtime version = %d\n", runtime_version);
+
+#ifdef QUDA_NVML
+      nvmlReturn_t result = nvmlInit();
+      if (NVML_SUCCESS != result) errorQuda("NVML Init failed with error %d", result);
+      const int length = 80;
+      char graphics_version[length];
+      result = nvmlSystemGetDriverVersion(graphics_version, length);
+      if (NVML_SUCCESS != result) errorQuda("nvmlSystemGetDriverVersion failed with error %d", result);
+      printfQuda("Graphic driver version = %s\n", graphics_version);
+      result = nvmlShutdown();
+      if (NVML_SUCCESS != result) errorQuda("NVML Shutdown failed with error %d", result);
+#endif
+
+      int deviceCount;
+      cudaGetDeviceCount(&deviceCount);
+      if (deviceCount == 0) {
+        warningQuda("No CUDA devices found");
+      }
+
+      for (int i = 0; i < deviceCount; i++) {
+        cudaGetDeviceProperties(&deviceProp, i);
+        checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
+        if (getVerbosity() >= QUDA_SUMMARIZE) {
+          printfQuda("Found device %d: %s\n", i, deviceProp.name);
+        }
+      }
+
+      cudaGetDeviceProperties(&deviceProp, dev);
+      checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
+      if (deviceProp.major < 1) {
+        errorQuda("Device %d does not support CUDA", dev);
+      }
+
+      // Check GPU and QUDA build compatibiliy
+      // 4 cases:
+      // a) QUDA and GPU match: great
+      // b) QUDA built for higher compute capability: error
+      // c) QUDA built for lower major compute capability: warn if QUDA_ALLOW_JIT, else error
+      // d) QUDA built for same major compute capability but lower minor: warn
+
+      const int my_major = __COMPUTE_CAPABILITY__ / 100;
+      const int my_minor = (__COMPUTE_CAPABILITY__  - my_major * 100) / 10;
+      // b) UDA was compiled for a higher compute capability
+      if (deviceProp.major * 100 + deviceProp.minor * 10 < __COMPUTE_CAPABILITY__)
+        warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. ** \n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
+
+      // c) QUDA was compiled for a lower compute capability
+      if (deviceProp.major < my_major) {
+        char *allow_jit_env = getenv("QUDA_ALLOW_JIT");
+        if (allow_jit_env && strcmp(allow_jit_env, "1") == 0) {
+          if (getVerbosity() > QUDA_SILENT) warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- Jitting the PTX since QUDA_ALLOW_JIT=1 was set. Note that this will take some time.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
+        } else {
+          warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n If you want the PTX to be jitted for your current GPU arch please set the enviroment variable QUDA_ALLOW_JIT=1.", deviceProp.major, deviceProp.minor, my_major, my_minor);
+        }
+      }
+      // d) QUDA built for same major compute capability but lower minor
+      if (deviceProp.major == my_major and deviceProp.minor > my_minor) {
+        warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- This might result in a lower performance. Please consider adjusting QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
+      }
+
+      if (getVerbosity() >= QUDA_SUMMARIZE) {
+        printfQuda("Using device %d: %s\n", dev, deviceProp.name);
+      }
+#ifndef USE_QDPJIT
+      cudaSetDevice(dev);
+      checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
+#endif
+
+#ifdef NUMA_NVML
+      char *enable_numa_env = getenv("QUDA_ENABLE_NUMA");
+      if (enable_numa_env && strcmp(enable_numa_env, "0") == 0) {
+        if (getVerbosity() > QUDA_SILENT) printfQuda("Disabling numa_affinity\n");
+      } else{
+        setNumaAffinityNVML(dev);
+      }
+#endif
+
+      cudaDeviceSetCacheConfig(cudaFuncCachePreferL1);
+      //cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte);
+      // cudaGetDeviceProperties(&deviceProp, dev);
+    }
+
+    void print_device_properties()
+    {
+
+      int dev_count;
+      cudaGetDeviceCount(&dev_count);
+      int device;
+      for (device = 0; device < dev_count; device++) {
+
+        // cudaDeviceProp deviceProp;
+        cudaGetDeviceProperties(&deviceProp, device);
+        printfQuda("%d - name:                    %s\n", device, deviceProp.name);
+//        printfQuda("%d - totalGlobalMem:          %lu bytes ( %.2f Gbytes)\n", device, deviceProp.totalGlobalMem,
+//                   deviceProp.totalGlobalMem / (float)(1024 * 1024 * 1024));
+        printfQuda("%d - sharedMemPerBlock:       %lu bytes ( %.2f Kbytes)\n", device, deviceProp.sharedMemPerBlock,
+                   deviceProp.sharedMemPerBlock / (float)1024);
+//        printfQuda("%d - regsPerBlock:            %d\n", device, deviceProp.regsPerBlock);
+//        printfQuda("%d - warpSize:                %d\n", device, deviceProp.warpSize);
+//        printfQuda("%d - memPitch:                %lu\n", device, deviceProp.memPitch);
+        printfQuda("%d - maxThreadsPerBlock:      %d\n", device, deviceProp.maxThreadsPerBlock);
+        printfQuda("%d - maxThreadsDim[0]:        %d\n", device, deviceProp.maxThreadsDim[0]);
+        printfQuda("%d - maxThreadsDim[1]:        %d\n", device, deviceProp.maxThreadsDim[1]);
+        printfQuda("%d - maxThreadsDim[2]:        %d\n", device, deviceProp.maxThreadsDim[2]);
+        printfQuda("%d - maxGridSize[0]:          %d\n", device, deviceProp.maxGridSize[0]);
+        printfQuda("%d - maxGridSize[1]:          %d\n", device, deviceProp.maxGridSize[1]);
+        printfQuda("%d - maxGridSize[2]:          %d\n", device, deviceProp.maxGridSize[2]);
+//        printfQuda("%d - totalConstMem:           %lu bytes ( %.2f Kbytes)\n", device, deviceProp.totalConstMem,
+//                   deviceProp.totalConstMem / (float)1024);
+        printfQuda("%d - compute capability:      %d.%d\n", device, deviceProp.major, deviceProp.minor);
+//        printfQuda("%d - deviceOverlap            %s\n", device, (deviceProp.deviceOverlap ? "true" : "false"));
+        printfQuda("%d - multiProcessorCount      %d\n", device, deviceProp.multiProcessorCount);
+/*
+        printfQuda("%d - kernelExecTimeoutEnabled %s\n", device,
+                   (deviceProp.kernelExecTimeoutEnabled ? "true" : "false"));
+        printfQuda("%d - integrated               %s\n", device, (deviceProp.integrated ? "true" : "false"));
+        printfQuda("%d - canMapHostMemory         %s\n", device, (deviceProp.canMapHostMemory ? "true" : "false"));
+        switch (deviceProp.computeMode) {
+        case 0: printfQuda("%d - computeMode              0: cudaComputeModeDefault\n", device); break;
+        case 1: printfQuda("%d - computeMode              1: cudaComputeModeExclusive\n", device); break;
+        case 2: printfQuda("%d - computeMode              2: cudaComputeModeProhibited\n", device); break;
+        case 3: printfQuda("%d - computeMode              3: cudaComputeModeExclusiveProcess\n", device); break;
+        default: errorQuda("Unknown deviceProp.computeMode.");
+        }
+        printfQuda("%d - surfaceAlignment         %lu\n", device, deviceProp.surfaceAlignment);
+        printfQuda("%d - concurrentKernels        %s\n", device, (deviceProp.concurrentKernels ? "true" : "false"));
+        printfQuda("%d - ECCEnabled               %s\n", device, (deviceProp.ECCEnabled ? "true" : "false"));
+        printfQuda("%d - pciBusID                 %d\n", device, deviceProp.pciBusID);
+        printfQuda("%d - pciDeviceID              %d\n", device, deviceProp.pciDeviceID);
+        printfQuda("%d - pciDomainID              %d\n", device, deviceProp.pciDomainID);
+        printfQuda("%d - tccDriver                %s\n", device, (deviceProp.tccDriver ? "true" : "false"));
+        switch (deviceProp.asyncEngineCount) {
+        case 0: printfQuda("%d - asyncEngineCount         1: host -> device only\n", device); break;
+        case 1: printfQuda("%d - asyncEngineCount         2: host <-> device\n", device); break;
+        case 2: printfQuda("%d - asyncEngineCount         0: not supported\n", device); break;
+        default: errorQuda("Unknown deviceProp.asyncEngineCount.");
+        }
+        printfQuda("%d - unifiedAddressing        %s\n", device, (deviceProp.unifiedAddressing ? "true" : "false"));
+        printfQuda("%d - memoryClockRate          %d kilohertz\n", device, deviceProp.memoryClockRate);
+        printfQuda("%d - memoryBusWidth           %d bits\n", device, deviceProp.memoryBusWidth);
+        printfQuda("%d - l2CacheSize              %d bytes\n", device, deviceProp.l2CacheSize);
+*/
+        printfQuda("%d - maxThreadsPerMultiProcessor          %d\n\n", device, deviceProp.maxThreadsPerMultiProcessor);
+      }
+    }
+
+    void create_context()
+    {
+      streams = new cudaStream_t[Nstream];
+
+      int greatestPriority;
+      int leastPriority;
+      cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority);
+      for (int i=0; i<Nstream-1; i++) {
+        cudaStreamCreateWithPriority(&streams[i], cudaStreamDefault, greatestPriority);
+      }
+      cudaStreamCreateWithPriority(&streams[Nstream-1], cudaStreamDefault, leastPriority);
+
+      checkCudaError();
+
+    }
+
+    void destroy()
+    {
+      if (streams) {
+        for (int i=0; i<Nstream; i++) cudaStreamDestroy(streams[i]);
+        delete []streams;
+        streams = nullptr;
+      }
+
+      char *device_reset_env = getenv("QUDA_DEVICE_RESET");
+      if (device_reset_env && strcmp(device_reset_env,"1") == 0) {
+        // end this CUDA context
+        cudaDeviceReset();
+      }
+    }
+
+    cudaStream_t get_cuda_stream(const qudaStream_t &stream)
+    {
+      return streams[stream.idx];
+    }
+
+    qudaStream_t get_stream(unsigned int i)
+    {
+      if (i > Nstream) errorQuda("Invalid stream index %u", i);
+      qudaStream_t stream;
+      stream.idx = i;
+      return stream;
+      //return qudaStream_t(i);
+      // return streams[i];
+    }
+
+    qudaStream_t get_default_stream()
+    {
+      qudaStream_t stream;
+      stream.idx = Nstream - 1;
+      return stream;
+      //return qudaStream_t(Nstream - 1);
+      //return streams[Nstream - 1];
+    }
+
+    unsigned int get_default_stream_idx()
+    {
+      return Nstream - 1;
+    }
+
+    bool managed_memory_supported()
+    {
+      // managed memory is supported on Pascal and up
+      return deviceProp.major >= 6;
+    }
+
+    bool shared_memory_atomic_supported()
+    {
+      // shared memory atomics are supported on Maxwell and up
+      return deviceProp.major >= 5;
+    }
+
+    size_t max_default_shared_memory() { return deviceProp.sharedMemPerBlock; }
+
+    size_t max_dynamic_shared_memory()
+    {
+      static int max_shared_bytes = 0;
+      if (!max_shared_bytes)
+        cudaDeviceGetAttribute(&max_shared_bytes, cudaDevAttrMaxSharedMemoryPerBlockOptin, comm_gpuid());
+      return max_shared_bytes;
+    }
+
+    unsigned int max_threads_per_block() { return deviceProp.maxThreadsPerBlock; }
+
+    unsigned int max_threads_per_processor() { return deviceProp.maxThreadsPerMultiProcessor; }
+
+    unsigned int max_threads_per_block_dim(int i) { return deviceProp.maxThreadsDim[i]; }
+
+    unsigned int max_grid_size(int i) { return deviceProp.maxGridSize[i]; }
+
+    unsigned int processor_count() { return deviceProp.multiProcessorCount; }
+
+    unsigned int max_blocks_per_processor()
+    {
+#if CUDA_VERSION >= 11000
+      static int max_blocks_per_sm = 0;
+      if (!max_blocks_per_sm) cudaDeviceGetAttribute(&max_blocks_per_sm, cudaDevAttrMaxBlocksPerMultiprocessor, comm_gpuid());
+      return max_blocks_per_sm;
+#else
+      // these variables are taken from Table 14 of the CUDA 10.2 prgramming guide
+      switch (deviceProp.major) {
+      case 2:
+	return 8;
+      case 3:
+	return 16;
+      case 5:
+      case 6: return 32;
+      case 7:
+        switch (deviceProp.minor) {
+        case 0: return 32;
+        case 2: return 32;
+        case 5: return 16;
+        }
+      default:
+        warningQuda("Unknown SM architecture %d.%d - assuming limit of 32 blocks per SM\n",
+                    deviceProp.major, deviceProp.minor);
+        return 32;
+      }
+#endif
+    }
+
+    namespace profile {
+
+      void start()
+      {
+        // cudaProfilerStart();
+      }
+
+      void stop()
+      {
+        // cudaProfilerStop();
+      }
+
+    }
+
+  }
+}
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
new file mode 100644
index 0000000000..ad357f8ed5
--- /dev/null
+++ b/lib/targets/omptarget/malloc.cpp
@@ -0,0 +1,707 @@
+#include <cstdlib>
+#include <cstdio>
+#include <string>
+#include <map>
+#include <unistd.h>   // for getpagesize()
+#include <execinfo.h> // for backtrace
+#include <quda_internal.h>
+#include <device.h>
+
+#ifdef USE_QDPJIT
+#include "qdp_quda.h"
+#include "qdp_config.h"
+#endif
+
+#ifdef QUDA_BACKWARDSCPP
+#include "backward.hpp"
+#endif
+namespace quda
+{
+
+  enum AllocType { DEVICE, DEVICE_PINNED, HOST, PINNED, MAPPED, MANAGED, N_ALLOC_TYPE };
+
+  class MemAlloc
+  {
+
+  public:
+    std::string func;
+    std::string file;
+    int line;
+    size_t size;
+    size_t base_size;
+#ifdef QUDA_BACKWARDSCPP
+    backward::StackTrace st;
+#endif
+
+    MemAlloc() : line(-1), size(0), base_size(0) {}
+
+    MemAlloc(std::string func, std::string file, int line) : func(func), file(file), line(line), size(0), base_size(0)
+    {
+#ifdef QUDA_BACKWARDSCPP
+      st.load_here(32);
+      st.skip_n_firsts(1);
+#endif
+    }
+
+    MemAlloc(const MemAlloc &a) = default;
+    MemAlloc(MemAlloc &&a) = default;
+    virtual ~MemAlloc() = default;
+    MemAlloc &operator=(const MemAlloc &a) = default;
+  };
+
+  static std::map<void *, MemAlloc> alloc[N_ALLOC_TYPE];
+  static long total_bytes[N_ALLOC_TYPE] = {0};
+  static long max_total_bytes[N_ALLOC_TYPE] = {0};
+  static long total_host_bytes, max_total_host_bytes;
+  static long total_pinned_bytes, max_total_pinned_bytes;
+
+  long device_allocated_peak() { return max_total_bytes[DEVICE]; }
+
+  long pinned_allocated_peak() { return max_total_bytes[PINNED]; }
+
+  long mapped_allocated_peak() { return max_total_bytes[MAPPED]; }
+
+  long managed_allocated_peak() { return max_total_bytes[MANAGED]; }
+
+  long host_allocated_peak() { return max_total_bytes[HOST]; }
+
+  static void print_trace(void)
+  {
+    void *array[10];
+    size_t size;
+    char **strings;
+    size = backtrace(array, 10);
+    strings = backtrace_symbols(array, size);
+    printfQuda("Obtained %zd stack frames.\n", size);
+    for (size_t i = 0; i < size; i++) printfQuda("%s\n", strings[i]);
+    free(strings);
+  }
+
+  static void print_alloc_header()
+  {
+    printfQuda("Type    Pointer          Size             Location\n");
+    printfQuda("----------------------------------------------------------\n");
+  }
+
+  static void print_alloc(AllocType type)
+  {
+    const char *type_str[] = {"Device", "Device Pinned", "Host  ", "Pinned", "Mapped", "Managed"};
+    std::map<void *, MemAlloc>::iterator entry;
+
+    for (entry = alloc[type].begin(); entry != alloc[type].end(); entry++) {
+      void *ptr = entry->first;
+      MemAlloc a = entry->second;
+      printfQuda("%s  %15p  %15lu  %s(), %s:%d\n", type_str[type], ptr, (unsigned long)a.base_size, a.func.c_str(),
+                 a.file.c_str(), a.line);
+#ifdef QUDA_BACKWARDSCPP
+      if (getRankVerbosity()) {
+        backward::Printer p;
+        p.print(a.st);
+      }
+#endif
+    }
+  }
+
+  static void track_malloc(const AllocType &type, const MemAlloc &a, void *ptr)
+  {
+    total_bytes[type] += a.base_size;
+    if (total_bytes[type] > max_total_bytes[type]) { max_total_bytes[type] = total_bytes[type]; }
+    if (type != DEVICE && type != DEVICE_PINNED) {
+      total_host_bytes += a.base_size;
+      if (total_host_bytes > max_total_host_bytes) { max_total_host_bytes = total_host_bytes; }
+    }
+    if (type == PINNED || type == MAPPED) {
+      total_pinned_bytes += a.base_size;
+      if (total_pinned_bytes > max_total_pinned_bytes) { max_total_pinned_bytes = total_pinned_bytes; }
+    }
+    alloc[type][ptr] = a;
+  }
+
+  static void track_free(const AllocType &type, void *ptr)
+  {
+    size_t size = alloc[type][ptr].base_size;
+    total_bytes[type] -= size;
+    if (type != DEVICE && type != DEVICE_PINNED) { total_host_bytes -= size; }
+    if (type == PINNED || type == MAPPED) { total_pinned_bytes -= size; }
+    alloc[type].erase(ptr);
+  }
+
+  /**
+   * Under CUDA 4.0, cudaHostRegister seems to require that both the
+   * beginning and end of the buffer be aligned on page boundaries.
+   * This local function takes care of the alignment and gets called
+   * by pinned_malloc_() and mapped_malloc_()
+   */
+  static void *aligned_malloc(MemAlloc &a, size_t size)
+  {
+    void *ptr = nullptr;
+
+    a.size = size;
+
+#if 0
+    a.base_size = size;
+    ptr = malloc(size);
+    if (!ptr) {
+#else
+    // we need to manually align to page boundaries to allow us to bind a texture to mapped memory
+    static int page_size = 2 * getpagesize();
+    a.base_size = ((size + page_size - 1) / page_size) * page_size; // round up to the nearest multiple of page_size
+    int align = posix_memalign(&ptr, page_size, a.base_size);
+    if (!ptr || align != 0) {
+#endif
+      errorQuda("Failed to allocate aligned host memory of size %zu (%s:%d in %s())\n", size, a.file.c_str(), a.line,
+                a.func.c_str());
+    }
+    return ptr;
+  }
+
+  bool use_managed_memory()
+  {
+    static bool managed = false;
+    static bool init = false;
+
+    if (!init) {
+      char *enable_managed_memory = getenv("QUDA_ENABLE_MANAGED_MEMORY");
+      if (enable_managed_memory && strcmp(enable_managed_memory, "1") == 0) {
+        warningQuda("Using managed memory for CUDA allocations");
+        managed = true;
+
+        if (!device::managed_memory_supported())
+          warningQuda("Target device does not report supporting managed memory");
+      }
+
+      init = true;
+    }
+
+    return managed;
+  }
+
+  bool is_prefetch_enabled()
+  {
+    static bool prefetch = false;
+    static bool init = false;
+
+    if (!init) {
+      if (use_managed_memory()) {
+        char *enable_managed_prefetch = getenv("QUDA_ENABLE_MANAGED_PREFETCH");
+        if (enable_managed_prefetch && strcmp(enable_managed_prefetch, "1") == 0) {
+          warningQuda("Enabling prefetch support for managed memory");
+          prefetch = true;
+        }
+      }
+
+      init = true;
+    }
+
+    return prefetch;
+  }
+
+  /**
+   * Perform a standard cudaMalloc() with error-checking.  This
+   * function should only be called via the device_malloc() macro,
+   * defined in malloc_quda.h
+   */
+  void *device_malloc_(const char *func, const char *file, int line, size_t size)
+  {
+    if (use_managed_memory()) return managed_malloc_(func, file, line, size);
+
+#ifndef QDP_USE_CUDA_MANAGED_MEMORY
+    MemAlloc a(func, file, line);
+    void *ptr;
+
+    a.size = a.base_size = size;
+
+    if(0<omp_get_num_devices())
+      ptr = omp_target_alloc(size, omp_get_default_device());
+    else{
+      warningQuda("%s:%d %s() allocate on host instead of device", file, line, func);
+      ptr = std::aligned_alloc(64, size);
+    }
+    if (!ptr) {
+      errorQuda("Failed to allocate device memory of size %zu (%s:%d in %s())\n", size, file, line, func);
+    }
+    track_malloc(DEVICE, a, ptr);
+#ifdef HOST_DEBUG
+    cudaMemset(ptr, 0xff, size);
+#endif
+    return ptr;
+#else
+    // when QDO uses managed memory we can bypass the QDP memory manager
+    return device_pinned_malloc_(func, file, line, size);
+#endif
+  }
+
+  /**
+   * Perform a cuMemAlloc with error-checking.  This function is to
+   * guarantee a unique memory allocation on the device, since
+   * cudaMalloc can be redirected (as is the case with QDPJIT).  This
+   * should only be called via the device_pinned_malloc() macro,
+   * defined in malloc_quda.h.
+   */
+  void *device_pinned_malloc_(const char *func, const char *file, int line, size_t size)
+  {ompwip("return nullptr");return nullptr;/*
+    if (!comm_peer2peer_present()) return device_malloc_(func, file, line, size);
+
+    MemAlloc a(func, file, line);
+    void *ptr;
+
+    a.size = a.base_size = size;
+
+    CUresult err = cuMemAlloc((CUdeviceptr *)&ptr, size);
+    if (err != CUDA_SUCCESS) {
+      errorQuda("Failed to allocate device memory of size %zu (%s:%d in %s())\n", size, file, line, func);
+    }
+    track_malloc(DEVICE_PINNED, a, ptr);
+#ifdef HOST_DEBUG
+    cudaMemset(ptr, 0xff, size);
+#endif
+    return ptr;
+  */}
+
+  /**
+   * Perform a standard malloc() with error-checking.  This function
+   * should only be called via the safe_malloc() macro, defined in
+   * malloc_quda.h
+   */
+  void *safe_malloc_(const char *func, const char *file, int line, size_t size)
+  {
+    MemAlloc a(func, file, line);
+    a.size = a.base_size = size;
+
+    void *ptr = malloc(size);
+    ompwip([=](){printfQuda("malloc: %p\n",ptr);});
+    if (!ptr) { errorQuda("Failed to allocate host memory of size %zu (%s:%d in %s())\n", size, file, line, func); }
+    track_malloc(HOST, a, ptr);
+#ifdef HOST_DEBUG
+    memset(ptr, 0xff, size);
+#endif
+    return ptr;
+  }
+
+  /**
+   * Allocate page-locked ("pinned") host memory.  This function
+   * should only be called via the pinned_malloc() macro, defined in
+   * malloc_quda.h
+   *
+   * Note that we do not rely on cudaHostAlloc(), since buffers
+   * allocated in this way have been observed to cause problems when
+   * shared with MPI via GPU Direct on some systems.
+   */
+  void *pinned_malloc_(const char *func, const char *file, int line, size_t size)
+  {ompwip("return nullptr");return nullptr;/*
+    MemAlloc a(func, file, line);
+    void *ptr = aligned_malloc(a, size);
+
+    cudaError_t err = cudaHostRegister(ptr, a.base_size, cudaHostRegisterDefault);
+    if (err != cudaSuccess) {
+      errorQuda("Failed to register pinned memory of size %zu (%s:%d in %s())\n", size, file, line, func);
+    }
+    track_malloc(PINNED, a, ptr);
+#ifdef HOST_DEBUG
+    memset(ptr, 0xff, a.base_size);
+#endif
+    return ptr;
+  */}
+
+  /**
+   * Allocate page-locked ("pinned") host memory, and map it into the
+   * GPU address space.  This function should only be called via the
+   * mapped_malloc() macro, defined in malloc_quda.h
+   */
+  void *mapped_malloc_(const char *func, const char *file, int line, size_t size)
+  {
+    MemAlloc a(func, file, line);
+
+#if 0
+    void *ptr;
+    static int page_size = 2*getpagesize();
+    a.base_size = ((size + page_size - 1) / page_size) * page_size; // round up to the nearest multiple of page_size
+    a.size = size;
+    cudaError_t err = cudaHostAlloc(&ptr, a.base_size, cudaHostAllocMapped | cudaHostAllocPortable);
+    if (err != cudaSuccess) {
+      errorQuda("cudaHostAlloc failed of size %zu (%s:%d in %s())\n", size, file, line, func); }
+    }
+#else
+    void *ptr = aligned_malloc(a, size);
+    ompwip([=](){printfQuda("mapped_malloc_ host: %p\n",ptr);});
+    if(0<omp_get_num_devices()){
+      int d = omp_get_default_device();
+      void *dp = omp_target_alloc(a.base_size, d);  // FIXME keep it somewhere so we can free it
+      ompwip("require special memcpy",[=](){printfQuda("mapped_malloc_ device: %p\n",dp);});
+      if(!dp)
+        errorQuda("%s:%d %s() Failed to allocate device memory of size %zu for mapped malloc\n", file, line, func, size);
+      if(!omp_target_associate_ptr(ptr, dp, a.base_size, 0, d))
+        errorQuda("%s:%d %s() Failed to assocaite device memory to host pointer\n", file, line, func);
+    }else
+      warningQuda("%s:%d %s() mapped malloc without a device", file, line, func);
+#endif
+    track_malloc(MAPPED, a, ptr);
+#ifdef HOST_DEBUG
+    memset(ptr, 0xff, a.base_size);
+#endif
+    return ptr;
+  }
+
+  /**
+   * Perform a standard cudaMallocManaged() with error-checking.  This
+   * function should only be called via the managed_malloc() macro,
+   * defined in malloc_quda.h
+   */
+  void *managed_malloc_(const char *func, const char *file, int line, size_t size)
+  {ompwip("return nullptr");return nullptr;/*
+    MemAlloc a(func, file, line);
+    void *ptr;
+
+    a.size = a.base_size = size;
+
+    cudaError_t err = cudaMallocManaged(&ptr, size);
+    if (err != cudaSuccess) {
+      errorQuda("Failed to allocate managed memory of size %zu (%s:%d in %s())\n", size, file, line, func);
+    }
+    track_malloc(MANAGED, a, ptr);
+#ifdef HOST_DEBUG
+    cudaMemset(ptr, 0xff, size);
+#endif
+    return ptr;
+  */}
+
+  /**
+   * Free device memory allocated with device_malloc().  This function
+   * should only be called via the device_free() macro, defined in
+   * malloc_quda.h
+   */
+  void device_free_(const char *func, const char *file, int line, void *ptr)
+  {
+    if (use_managed_memory()) {ompwip();
+      managed_free_(func, file, line, ptr);
+      return;
+    }
+
+#ifndef QDP_USE_CUDA_MANAGED_MEMORY
+    ompwip();
+    if (!ptr) { errorQuda("Attempt to free NULL device pointer (%s:%d in %s())\n", file, line, func); }
+    if (!alloc[DEVICE].count(ptr)) {
+      errorQuda("Attempt to free invalid device pointer (%s:%d in %s())\n", file, line, func);
+    }
+    if(0<omp_get_num_devices())
+      omp_target_free(ptr, omp_get_default_device());
+    else
+      free(ptr);
+/*
+    cudaError_t err = cudaFree(ptr);
+    if (err != cudaSuccess) { errorQuda("Failed to free device memory (%s:%d in %s())\n", file, line, func); }
+*/
+    track_free(DEVICE, ptr);
+#else
+    device_pinned_free_(func, file, line, ptr);
+#endif
+  }
+
+  /**
+   * Free device memory allocated with device_pinned malloc().  This
+   * function should only be called via the device_pinned_free()
+   * macro, defined in malloc_quda.h
+   */
+  void device_pinned_free_(const char *func, const char *file, int line, void *ptr)
+  {
+    if (!comm_peer2peer_present()) {
+      device_free_(func, file, line, ptr);
+      return;
+    }
+
+    if (!ptr) { errorQuda("Attempt to free NULL device pointer (%s:%d in %s())\n", file, line, func); }
+    if (!alloc[DEVICE_PINNED].count(ptr)) {
+      errorQuda("Attempt to free invalid device pointer (%s:%d in %s())\n", file, line, func);
+    }
+    if(0<omp_get_num_devices())
+      omp_target_free(ptr, omp_get_default_device());
+    else
+      free(ptr);
+/*
+    CUresult err = cuMemFree((CUdeviceptr)ptr);
+    if (err != CUDA_SUCCESS) { printfQuda("Failed to free device memory (%s:%d in %s())\n", file, line, func); }
+*/
+    track_free(DEVICE_PINNED, ptr);
+  }
+
+  /**
+   * Free device memory allocated with device_malloc().  This function
+   * should only be called via the device_free() macro, defined in
+   * malloc_quda.h
+   */
+  void managed_free_(const char *func, const char *file, int line, void *ptr)
+  {ompwip("doing nothing");/*
+    if (!ptr) { errorQuda("Attempt to free NULL managed pointer (%s:%d in %s())\n", file, line, func); }
+    if (!alloc[MANAGED].count(ptr)) {
+      errorQuda("Attempt to free invalid managed pointer (%s:%d in %s())\n", file, line, func);
+    }
+    cudaError_t err = cudaFree(ptr);
+    if (err != cudaSuccess) { errorQuda("Failed to free device memory (%s:%d in %s())\n", file, line, func); }
+    track_free(MANAGED, ptr);
+  */}
+
+  /**
+   * Free host memory allocated with safe_malloc(), pinned_malloc(),
+   * or mapped_malloc().  This function should only be called via the
+   * host_free() macro, defined in malloc_quda.h
+   */
+  void host_free_(const char *func, const char *file, int line, void *ptr)
+  {
+    if (!ptr) { errorQuda("Attempt to free NULL host pointer (%s:%d in %s())\n", file, line, func); }
+    if (alloc[HOST].count(ptr)) {
+      track_free(HOST, ptr);
+      free(ptr);
+    } else if (alloc[PINNED].count(ptr)) {ompwip("alloc[PINNED].count(ptr)");/*
+      cudaError_t err = cudaHostUnregister(ptr);
+      if (err != cudaSuccess) { errorQuda("Failed to unregister pinned memory (%s:%d in %s())\n", file, line, func); }
+      track_free(PINNED, ptr);
+      free(ptr);
+    */} else if (alloc[MAPPED].count(ptr)) {
+#ifdef HOST_ALLOC
+      ompwip("HOST_ALLOC alloc[MAPPED].count(ptr)");/*
+      cudaError_t err = cudaFreeHost(ptr);
+      if (err != cudaSuccess) { errorQuda("Failed to free host memory (%s:%d in %s())\n", file, line, func); }
+      */
+#else
+      ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr)");/*
+      cudaError_t err = cudaHostUnregister(ptr);
+      if (err != cudaSuccess) {
+        errorQuda("Failed to unregister host-mapped memory (%s:%d in %s())\n", file, line, func);
+      }
+      free(ptr);
+      */
+#endif
+      // track_free(MAPPED, ptr);
+    } else {
+      printfQuda("ERROR: Attempt to free invalid host pointer (%s:%d in %s())\n", file, line, func);
+      print_trace();
+      errorQuda("Aborting");
+    }
+  }
+
+  void printPeakMemUsage()
+  {
+    printfQuda("Device memory used = %.1f MB\n", max_total_bytes[DEVICE] / (double)(1 << 20));
+    printfQuda("Pinned device memory used = %.1f MB\n", max_total_bytes[DEVICE_PINNED] / (double)(1 << 20));
+    printfQuda("Managed memory used = %.1f MB\n", max_total_bytes[MANAGED] / (double)(1 << 20));
+    printfQuda("Page-locked host memory used = %.1f MB\n", max_total_pinned_bytes / (double)(1 << 20));
+    printfQuda("Total host memory used >= %.1f MB\n", max_total_host_bytes / (double)(1 << 20));
+  }
+
+  void assertAllMemFree()
+  {
+    if (!alloc[DEVICE].empty() || !alloc[DEVICE_PINNED].empty() || !alloc[HOST].empty() || !alloc[PINNED].empty()
+        || !alloc[MAPPED].empty()) {
+      warningQuda("The following internal memory allocations were not freed.");
+      printfQuda("\n");
+      print_alloc_header();
+      print_alloc(DEVICE);
+      print_alloc(DEVICE_PINNED);
+      print_alloc(HOST);
+      print_alloc(PINNED);
+      print_alloc(MAPPED);
+      printfQuda("\n");
+    }
+  }
+
+  QudaFieldLocation get_pointer_location(const void *ptr)
+  {ompwip("return QUDA_INVALID_FIELD_LOCATION;");return QUDA_INVALID_FIELD_LOCATION;/*
+
+    CUpointer_attribute attribute[] = {CU_POINTER_ATTRIBUTE_MEMORY_TYPE};
+    CUmemorytype mem_type;
+    void *data[] = {&mem_type};
+    CUresult error = cuPointerGetAttributes(1, attribute, data, reinterpret_cast<CUdeviceptr>(ptr));
+    if (error != CUDA_SUCCESS) {
+      const char *string;
+      cuGetErrorString(error, &string);
+      errorQuda("cuPointerGetAttributes failed with error %s", string);
+    }
+
+    // catch pointers that have not been created in CUDA
+    if (mem_type == 0) mem_type = CU_MEMORYTYPE_HOST;
+
+    switch (mem_type) {
+    case CU_MEMORYTYPE_DEVICE:
+    case CU_MEMORYTYPE_UNIFIED: return QUDA_CUDA_FIELD_LOCATION;
+    case CU_MEMORYTYPE_HOST: return QUDA_CPU_FIELD_LOCATION;
+    default: errorQuda("Unknown memory type %d", mem_type); return QUDA_INVALID_FIELD_LOCATION;
+    }
+  */}
+
+  void *get_mapped_device_pointer_(const char *func, const char *file, int line, const void *host)
+  {ompwip("return nullptr");return nullptr;/*
+    void *device;
+    auto error = cudaHostGetDevicePointer(&device, const_cast<void *>(host), 0);
+    if (error != cudaSuccess) {
+      errorQuda("cudaHostGetDevicePointer failed with error %s (%s:%d in %s()",
+                cudaGetErrorString(error), file, line, func);
+    }
+    return device;
+  */}
+
+
+  namespace pool
+  {
+
+    /** Cache of inactive pinned-memory allocations.  We cache pinned
+        memory allocations so that fields can reuse these with minimal
+        overhead.*/
+    static std::multimap<size_t, void *> pinnedCache;
+
+    /** Sizes of active pinned-memory allocations.  For convenience,
+        we keep track of the sizes of active allocations (i.e., those not
+        in the cache). */
+    static std::map<void *, size_t> pinnedSize;
+
+    /** Cache of inactive device-memory allocations.  We cache pinned
+        memory allocations so that fields can reuse these with minimal
+        overhead.*/
+    static std::multimap<size_t, void *> deviceCache;
+
+    /** Sizes of active device-memory allocations.  For convenience,
+        we keep track of the sizes of active allocations (i.e., those not
+        in the cache). */
+    static std::map<void *, size_t> deviceSize;
+
+    static bool pool_init = false;
+
+    /** whether to use a memory pool allocator for device memory */
+    static bool device_memory_pool = true;
+
+    /** whether to use a memory pool allocator for pinned memory */
+    static bool pinned_memory_pool = true;
+
+    void init()
+    {
+      if (!pool_init) {
+        // device memory pool
+        char *enable_device_pool = getenv("QUDA_ENABLE_DEVICE_MEMORY_POOL");
+        if (!enable_device_pool || strcmp(enable_device_pool, "0") != 0) {
+          warningQuda("Using device memory pool allocator");
+          device_memory_pool = true;
+        } else {
+          warningQuda("Not using device memory pool allocator");
+          device_memory_pool = false;
+        }
+
+        // pinned memory pool
+        char *enable_pinned_pool = getenv("QUDA_ENABLE_PINNED_MEMORY_POOL");
+        if (!enable_pinned_pool || strcmp(enable_pinned_pool, "0") != 0) {
+          warningQuda("Using pinned memory pool allocator");
+          pinned_memory_pool = true;
+        } else {
+          warningQuda("Not using pinned memory pool allocator");
+          pinned_memory_pool = false;
+        }
+        pool_init = true;
+      }
+    }
+
+    void *pinned_malloc_(const char *func, const char *file, int line, size_t nbytes)
+    {ompwip();
+      void *ptr = nullptr;
+      if (pinned_memory_pool) {
+        std::multimap<size_t, void *>::iterator it;
+
+        if (pinnedCache.empty()) {
+          ptr = quda::pinned_malloc_(func, file, line, nbytes);
+        } else {
+          it = pinnedCache.lower_bound(nbytes);
+          if (it != pinnedCache.end()) { // sufficiently large allocation found
+            nbytes = it->first;
+            ptr = it->second;
+            pinnedCache.erase(it);
+          } else { // sacrifice the smallest cached allocation
+            it = pinnedCache.begin();
+            ptr = it->second;
+            pinnedCache.erase(it);
+            host_free(ptr);
+            ptr = quda::pinned_malloc_(func, file, line, nbytes);
+          }
+        }
+        pinnedSize[ptr] = nbytes;
+      } else {
+        ptr = quda::pinned_malloc_(func, file, line, nbytes);
+      }
+      return ptr;
+    }
+
+    void pinned_free_(const char *func, const char *file, int line, void *ptr)
+    {ompwip();
+      if (pinned_memory_pool) {
+        if (!pinnedSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
+        pinnedCache.insert(std::make_pair(pinnedSize[ptr], ptr));
+        pinnedSize.erase(ptr);
+      } else {
+        quda::host_free_(func, file, line, ptr);
+      }
+    }
+
+    void *device_malloc_(const char *func, const char *file, int line, size_t nbytes)
+    {
+      void *ptr = nullptr;
+      if (device_memory_pool) {
+        std::multimap<size_t, void *>::iterator it;
+
+        if (deviceCache.empty()) {
+          ptr = quda::device_malloc_(func, file, line, nbytes);
+        } else {
+          it = deviceCache.lower_bound(nbytes);
+          if (it != deviceCache.end()) { // sufficiently large allocation found
+            nbytes = it->first;
+            ptr = it->second;
+            deviceCache.erase(it);
+          } else { // sacrifice the smallest cached allocation
+            it = deviceCache.begin();
+            ptr = it->second;
+            deviceCache.erase(it);
+            quda::device_free_(func, file, line, ptr);
+            ptr = quda::device_malloc_(func, file, line, nbytes);
+          }
+        }
+        deviceSize[ptr] = nbytes;
+      } else {
+        ptr = quda::device_malloc_(func, file, line, nbytes);
+      }
+      ompwip([=](){printfQuda("device malloc: %p\n",ptr);});
+      return ptr;
+    }
+
+    void device_free_(const char *func, const char *file, int line, void *ptr)
+    {ompwip();
+      if (device_memory_pool) {
+        if (!deviceSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
+        deviceCache.insert(std::make_pair(deviceSize[ptr], ptr));
+        deviceSize.erase(ptr);
+      } else {
+        quda::device_free_(func, file, line, ptr);
+      }
+    }
+
+    void flush_pinned()
+    {ompwip();
+      if (pinned_memory_pool) {
+        std::multimap<size_t, void *>::iterator it;
+        for (it = pinnedCache.begin(); it != pinnedCache.end(); it++) {
+          void *ptr = it->second;
+          host_free(ptr);
+        }
+        pinnedCache.clear();
+      }
+    }
+
+    void flush_device()
+    {ompwip();
+      if (device_memory_pool) {
+        std::multimap<size_t, void *>::iterator it;
+        for (it = deviceCache.begin(); it != deviceCache.end(); it++) {
+          void *ptr = it->second;
+          device_free(ptr);
+        }
+        deviceCache.clear();
+      }
+    }
+
+  } // namespace pool
+
+} // namespace quda
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
new file mode 100644
index 0000000000..c18088cbd6
--- /dev/null
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -0,0 +1,549 @@
+#include <unordered_set>
+#include <tune_quda.h>
+#include <uint_to_char.h>
+#include <quda_internal.h>
+#include <device.h>
+
+#include <quda_api.h>
+
+enum cudaFuncAttribute{cudaFuncAttributePreferredSharedMemoryCarveout};
+using cudaFuncAttributes = int;
+using CUdeviceptr = void*;
+
+// if this macro is defined then we use the driver API, else use the
+// runtime API.  Typically the driver API has 10-20% less overhead
+// #define USE_DRIVER_API
+
+// if this macro is defined then we profile the CUDA API calls
+//#define API_PROFILE
+
+#ifdef API_PROFILE
+#define PROFILE(f, idx)                                 \
+  apiTimer.TPSTART(idx);				\
+  f;                                                    \
+  apiTimer.TPSTOP(idx);
+#else
+#define PROFILE(f, idx) f;
+#endif
+
+namespace quda {
+
+  // No need to abstract these across the library so keep these definitions local to CUDA target
+
+  /**
+     @brief Wrapper around cudaFuncSetAttribute with built-in error checking
+     @param[in] kernel Kernel function for which we are setting the attribute
+     @param[in] attr Attribute to set
+     @param[in] value Value to set
+  */
+  void qudaFuncSetAttribute_(const void *kernel, cudaFuncAttribute attr, int value, const char *func, const char *file,
+                             const char *line);
+
+  /**
+     @brief Wrapper around cudaFuncGetAttributes with built-in error checking
+     @param[in] attr the cudaFuncGetAttributes object to store the output
+     @param[in] kernel Kernel function for which we are setting the attribute
+  */
+  void qudaFuncGetAttributes_(cudaFuncAttributes &attr, const void *kernel, const char *func, const char *file,
+                              const char *line);
+
+#define qudaFuncSetAttribute(kernel, attr, value)                                                                      \
+  ::quda::qudaFuncSetAttribute_(kernel, attr, value, __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
+
+#define qudaFuncGetAttributes(attr, kernel)                                                                            \
+  ::quda::qudaFuncGetAttributes_(attr, kernel, __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
+
+#ifdef USE_DRIVER_API
+  static TimeProfile apiTimer("CUDA API calls (driver)");
+#else
+  static TimeProfile apiTimer("CUDA API calls (runtime)");
+#endif
+
+  qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, void **args, qudaStream_t stream)
+  {
+    ompwip("PRETENDING",[&](){std::cerr<<"launch "<<func<<' '<<tp<<' '<<*args<<' '<<device::get_cuda_stream(stream)<<std::endl;});
+    // if launch requests the maximum shared memory and the device supports it then opt in
+    if (tp.set_max_shared_bytes && device::max_dynamic_shared_memory() > device::max_default_shared_memory()) {
+      ompwip([](){warningQuda("Unimplemented for maximum shared memory");});
+/*
+      static std::unordered_set<const void *> cache;
+      auto search = cache.find(func);
+      if (search == cache.end()) {
+        cache.insert(func);
+        qudaFuncSetAttribute(func, cudaFuncAttributePreferredSharedMemoryCarveout, (int)cudaSharedmemCarveoutMaxShared);
+        cudaFuncAttributes attributes;
+        qudaFuncGetAttributes(attributes, func);
+        qudaFuncSetAttribute(func, cudaFuncAttributeMaxDynamicSharedMemorySize,
+                             device::max_dynamic_shared_memory() - attributes.sharedSizeBytes);
+      }
+*/
+    }
+/*
+    // no driver API variant here since we have C++ functions
+    PROFILE(cudaError_t error = cudaLaunchKernel(func, tp.grid, tp.block, args, tp.shared_bytes, device::get_cuda_stream(stream)),
+            QUDA_PROFILE_LAUNCH_KERNEL);
+    if (error != cudaSuccess && !activeTuning()) errorQuda("(CUDA) %s", cudaGetErrorString(error));
+    return error == cudaSuccess ? QUDA_SUCCESS : QUDA_ERROR;
+*/
+    return QUDA_SUCCESS;
+  }
+
+  class QudaMem : public Tunable
+  {
+    void *dst;
+    const void *src;
+    const size_t count;
+    const int value;
+    const bool copy;
+    const cudaMemcpyKind kind;
+    const bool async;
+    const char *name;
+    const bool active_tuning;
+
+    unsigned int sharedBytesPerThread() const { return 0; }
+    unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
+
+  public:
+    inline QudaMem(void *dst, const void *src, size_t count, cudaMemcpyKind kind, const qudaStream_t &stream,
+                   bool async, const char *func, const char *file, const char *line) :
+      dst(dst), src(src), count(count), value(0), copy(true), kind(kind), async(async), active_tuning(activeTuning())
+    {
+      if (!async) {
+        switch (kind) {
+        case cudaMemcpyDeviceToHost:   name = "cudaMemcpyDeviceToHost";   break;
+        case cudaMemcpyHostToDevice:   name = "cudaMemcpyHostToDevice";   break;
+        case cudaMemcpyHostToHost:     name = "cudaMemcpyHostToHost";     break;
+        case cudaMemcpyDeviceToDevice: name = "cudaMemcpyDeviceToDevice"; break;
+        case cudaMemcpyDefault:        name = "cudaMemcpyDefault";        break;
+        default: errorQuda("Unsupported cudaMemcpyKind %d", kind);
+        }
+      } else {
+        switch(kind) {
+        case cudaMemcpyDeviceToHost:   name = "cudaMemcpyAsyncDeviceToHost";   break;
+        case cudaMemcpyHostToDevice:   name = "cudaMemcpyAsyncHostToDevice";   break;
+        case cudaMemcpyHostToHost:     name = "cudaMemcpyAsyncHostToHost";     break;
+        case cudaMemcpyDeviceToDevice: name = "cudaMemcpyAsyncDeviceToDevice"; break;
+        case cudaMemcpyDefault:        name = "cudaMemcpyAsyncDefault";        break;
+        default: errorQuda("Unsupported cudaMemcpyKind %d", kind);
+        }
+      }
+      strcpy(aux, func);
+      strcat(aux, ",");
+      strcat(aux, file);
+      strcat(aux, ",");
+      strcat(aux, line);
+
+      apply(stream);
+    }
+
+    inline QudaMem(void *dst, int value, size_t count, const qudaStream_t &stream, bool async, const char *func,
+                   const char *file, const char *line) :
+      dst(dst),
+      src(nullptr),
+      count(count),
+      value(value),
+      copy(false),
+      kind(cudaMemcpyDefault),
+      async(async),
+      active_tuning(activeTuning())
+    {
+      name = !async ? "cudaMemset" : "cudaMemsetAsync";
+      strcpy(aux, func);
+      strcat(aux, ",");
+      strcat(aux, file);
+      strcat(aux, ",");
+      strcat(aux, line);
+
+      apply(stream);
+    }
+
+    inline void apply(const qudaStream_t &stream)
+    {
+      if (!active_tuning) tuneLaunch(*this, getTuning(), getVerbosity());
+
+      if (copy) {
+        if (async) {
+#ifdef USE_DRIVER_API
+          switch (kind) {
+          case cudaMemcpyDeviceToHost:
+            PROFILE(cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
+            break;
+          case cudaMemcpyHostToDevice:
+            PROFILE(cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
+            break;
+          case cudaMemcpyDeviceToDevice:
+            PROFILE(cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
+            break;
+          case cudaMemcpyDefault:
+            PROFILE(cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
+            break;
+          default: errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
+          }
+#else
+#ifdef API_PROFILE
+          QudaProfileType type = QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC;
+          switch (kind) {
+          case cudaMemcpyDeviceToHost: type = QUDA_PROFILE_MEMCPY_D2H_ASYNC; break;
+          case cudaMemcpyHostToDevice: type = QUDA_PROFILE_MEMCPY_H2D_ASYNC; break;
+          case cudaMemcpyDeviceToDevice: type = QUDA_PROFILE_MEMCPY_D2D_ASYNC; break;
+          case cudaMemcpyDefault: type = QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC; break;
+          default: errorQuda("Unsupported cudaMemcpyTypeAsync %d", kind);
+          }
+#endif
+          PROFILE(cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)), type);
+#endif
+        } else {
+#ifdef USE_DRIVER_API
+          switch (kind) {
+          case cudaMemcpyDeviceToHost: cuMemcpyDtoH(dst, (CUdeviceptr)src, count); break;
+          case cudaMemcpyHostToDevice: cuMemcpyHtoD((CUdeviceptr)dst, src, count); break;
+          case cudaMemcpyHostToHost: memcpy(dst, src, count); break;
+          case cudaMemcpyDeviceToDevice: cuMemcpyDtoD((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
+          case cudaMemcpyDefault: cuMemcpy((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
+          default: errorQuda("Unsupported cudaMemcpyType %d", kind);
+          }
+#else
+          cudaMemcpy(dst, src, count, kind);
+#endif
+        }
+      } else {
+#ifdef USE_DRIVER_API
+        if (async)
+          cuMemsetD32Async((CUdeviceptr)dst, value, count / 4, device::get_cuda_stream(stream));
+        else
+          cuMemsetD32((CUdeviceptr)dst, value, count / 4);
+#else
+        if (async)
+          cudaMemsetAsync(dst, value, count, device::get_cuda_stream(stream));
+        else
+          cudaMemset(dst, value, count);
+#endif
+      }
+    }
+
+    bool advanceTuneParam(TuneParam &) const { return false; }
+
+    TuneKey tuneKey() const {
+      char vol[128];
+      strcpy(vol,"bytes=");
+      u64toa(vol+6, (uint64_t)count);
+      return TuneKey(vol, name, aux);
+    }
+
+    long long flops() const { return 0; }
+    long long bytes() const { return kind == cudaMemcpyDeviceToDevice ? 2*count : count; }
+  };
+
+  void qudaMemcpy_(void *dst, const void *src, size_t count, cudaMemcpyKind kind,
+                   const char *func, const char *file, const char *line) {
+    if (count == 0) return;
+    QudaMem copy(dst, src, count, kind, device::get_default_stream(), false, func, file, line);
+    cudaError_t error = cudaGetLastError();
+    if (error != cudaSuccess)
+      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void qudaMemcpyAsync_(void *dst, const void *src, size_t count, cudaMemcpyKind kind, const qudaStream_t &stream,
+                        const char *func, const char *file, const char *line)
+  {
+    if (count == 0) return;
+
+    if (kind == cudaMemcpyDeviceToDevice) {
+      QudaMem copy(dst, src, count, kind, stream, true, func, file, line);
+    } else {
+#ifdef USE_DRIVER_API
+      switch (kind) {
+      case cudaMemcpyDeviceToHost:
+        PROFILE(cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
+        break;
+      case cudaMemcpyHostToDevice:
+        PROFILE(cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
+        break;
+      case cudaMemcpyDeviceToDevice:
+        PROFILE(cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
+        break;
+      case cudaMemcpyDefault:
+        PROFILE(cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
+        break;
+      default:
+        errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
+      }
+#else
+      PROFILE(cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)),
+              kind == cudaMemcpyDeviceToHost ? QUDA_PROFILE_MEMCPY_D2H_ASYNC : QUDA_PROFILE_MEMCPY_H2D_ASYNC);
+#endif
+    }
+  }
+
+  void qudaMemcpyP2PAsync_(void *dst, const void *src, size_t count, const qudaStream_t &stream,
+                           const char *func, const char *file, const char *line)
+  {
+    if (count == 0) return;
+    auto error = cudaMemcpyAsync(dst, src, count, cudaMemcpyDeviceToDevice, device::get_cuda_stream(stream));
+    if (error != cudaSuccess)
+      errorQuda("cudaMemcpyAsync returned %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void qudaMemcpy2D_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
+                     cudaMemcpyKind kind, const char *func, const char *file, const char *line)
+  {
+#ifdef USE_DRIVER_API
+    CUDA_MEMCPY2D param;
+    param.srcPitch = spitch;
+    param.srcY = 0;
+    param.srcXInBytes = 0;
+    param.dstPitch = dpitch;
+    param.dstY = 0;
+    param.dstXInBytes = 0;
+    param.WidthInBytes = width;
+    param.Height = height;
+
+    switch (kind) {
+    case cudaMemcpyDeviceToHost:
+      param.srcDevice = (CUdeviceptr)src;
+      param.srcMemoryType = CU_MEMORYTYPE_DEVICE;
+      param.dstHost = dst;
+      param.dstMemoryType = CU_MEMORYTYPE_HOST;
+      break;
+    default: errorQuda("Unsupported cuMemcpyType2DAsync %d", kind);
+    }
+    PROFILE(auto error = cuMemcpy2D(&param), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
+    if (error != CUDA_SUCCESS) {
+      const char *str;
+      cuGetErrorName(error, &str);
+      errorQuda("cuMemcpy2D returned error %s\n (%s:%s in %s())", str, file, line, func);
+    }
+#else
+    PROFILE(auto error = cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
+    if (error != cudaSuccess)
+      errorQuda("cudaMemcpy2D returned error %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+#endif
+  }
+
+  void qudaMemcpy2DAsync_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
+                          cudaMemcpyKind kind, const qudaStream_t &stream, const char *func, const char *file,
+                          const char *line)
+  {
+#ifdef USE_DRIVER_API
+    CUDA_MEMCPY2D param;
+    param.srcPitch = spitch;
+    param.srcY = 0;
+    param.srcXInBytes = 0;
+    param.dstPitch = dpitch;
+    param.dstY = 0;
+    param.dstXInBytes = 0;
+    param.WidthInBytes = width;
+    param.Height = height;
+
+    switch (kind) {
+    case cudaMemcpyDeviceToHost:
+      param.srcDevice = (CUdeviceptr)src;
+      param.srcMemoryType = CU_MEMORYTYPE_DEVICE;
+      param.dstHost = dst;
+      param.dstMemoryType = CU_MEMORYTYPE_HOST;
+      break;
+    default:
+      errorQuda("Unsupported cuMemcpyType2DAsync %d", kind);
+    }
+    PROFILE(auto error = cuMemcpy2DAsync(&param, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
+    if (error != CUDA_SUCCESS) {
+      const char *str;
+      cuGetErrorName(error, &str);
+      errorQuda("cuMemcpy2DAsync returned error %s\n (%s:%s in %s())", str, file, line, func);
+    }
+#else
+    PROFILE(auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
+    if (error != cudaSuccess)
+      errorQuda("cudaMemcpy2DAsync returned error %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+#endif
+  }
+
+  void qudaMemcpy2DP2PAsync_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
+                             const qudaStream_t &stream, const char *func, const char *file, const char *line)
+  {
+    auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, device::get_cuda_stream(stream));
+    if (error != cudaSuccess)
+      errorQuda("cudaMemcpy2DAsync returned %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void qudaMemset_(void *ptr, int value, size_t count, const char *func, const char *file, const char *line)
+  {
+    if (count == 0) return;
+    QudaMem set(ptr, value, count, device::get_default_stream(), false, func, file, line);
+    cudaError_t error = cudaGetLastError();
+    if (error != cudaSuccess && !activeTuning())
+      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void qudaMemsetAsync_(void *ptr, int value, size_t count, const qudaStream_t &stream, const char *func,
+                        const char *file, const char *line)
+  {
+    if (count == 0) return;
+    QudaMem copy(ptr, value, count, stream, true, func, file, line);
+    cudaError_t error = cudaGetLastError();
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void qudaMemset2D_(void *ptr, size_t pitch, int value, size_t width, size_t height, const char *func,
+                     const char *file, const char *line)
+  {
+    cudaError_t error = cudaMemset2D(ptr, pitch, value, width, height);
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void qudaMemset2DAsync_(void *ptr, size_t pitch, int value, size_t width, size_t height, const qudaStream_t &stream,
+                          const char *func, const char *file, const char *line)
+  {
+    cudaError_t error = cudaMemset2DAsync(ptr, pitch, value, width, height, device::get_cuda_stream(stream));
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void qudaMemPrefetchAsync_(void *ptr, size_t count, QudaFieldLocation mem_space, const qudaStream_t &stream,
+                             const char *func, const char *file, const char *line)
+  {ompwip("doing nothing");/*
+    int dev_id = 0;
+    if (mem_space == QUDA_CUDA_FIELD_LOCATION)
+      dev_id = comm_gpuid();
+    else if (mem_space == QUDA_CPU_FIELD_LOCATION)
+      dev_id = cudaCpuDeviceId;
+    else
+      errorQuda("Invalid QudaFieldLocation.");
+
+    cudaError_t error = cudaMemPrefetchAsync(ptr, count, dev_id, device::get_cuda_stream(stream));
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  */}
+
+  bool qudaEventQuery_(cudaEvent_t &event, const char *func, const char *file, const char *line)
+  {
+#ifdef USE_DRIVER_API
+    PROFILE(CUresult error = cuEventQuery(event), QUDA_PROFILE_EVENT_QUERY);
+    switch (error) {
+    case CUDA_SUCCESS: return true;
+    case CUDA_ERROR_NOT_READY: return false;
+    default: {
+      const char *str;
+      cuGetErrorName(error, &str);
+      errorQuda("cuEventQuery returned error %s\n (%s:%s in %s())", str, file, line, func);
+    }
+    }
+#else
+    PROFILE(cudaError_t error = cudaEventQuery(event), QUDA_PROFILE_EVENT_QUERY);
+    switch (error) {
+    case cudaSuccess: return true;
+    case cudaErrorNotReady: return false;
+    default: errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    }
+#endif
+    return false;
+  }
+
+  void qudaEventRecord_(cudaEvent_t &event, qudaStream_t stream, const char *func, const char *file, const char *line)
+  {
+#ifdef USE_DRIVER_API
+    PROFILE(CUresult error = cuEventRecord(event, device::get_cuda_stream(stream)), QUDA_PROFILE_EVENT_RECORD);
+    if (error != CUDA_SUCCESS) {
+      const char *str;
+      cuGetErrorName(error, &str);
+      errorQuda("cuEventRecord returned error %s\n (%s:%s in %s())", str, file, line, func);
+    }
+#else
+    PROFILE(cudaError_t error = cudaEventRecord(event, device::get_cuda_stream(stream)), QUDA_PROFILE_EVENT_RECORD);
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+#endif
+  }
+
+  void qudaStreamWaitEvent_(qudaStream_t stream, cudaEvent_t event, unsigned int flags, const char *func,
+                            const char *file, const char *line)
+  {
+#ifdef USE_DRIVER_API
+    PROFILE(CUresult error = cuStreamWaitEvent(device::get_cuda_stream(stream), event, flags), QUDA_PROFILE_STREAM_WAIT_EVENT);
+    if (error != CUDA_SUCCESS) {
+      const char *str;
+      cuGetErrorName(error, &str);
+      errorQuda("cuStreamWaitEvent returned error %s\n (%s:%s in %s())", str, file, line, func);
+    }
+#else
+    PROFILE(cudaError_t error = cudaStreamWaitEvent(device::get_cuda_stream(stream), event, flags), QUDA_PROFILE_STREAM_WAIT_EVENT);
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+#endif
+  }
+
+  void qudaEventSynchronize_(const cudaEvent_t &event, const char *func, const char *file, const char *line)
+  {
+#ifdef USE_DRIVER_API
+    PROFILE(CUresult error = cuEventSynchronize(event), QUDA_PROFILE_EVENT_SYNCHRONIZE);
+    if (error != CUDA_SUCCESS) {
+      const char *str;
+      cuGetErrorName(error, &str);
+      errorQuda("cuEventSynchronize returned error %s\n (%s:%s in %s())", str, file, line, func);
+    }
+#else
+    PROFILE(cudaError_t error = cudaEventSynchronize(event), QUDA_PROFILE_EVENT_SYNCHRONIZE);
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+#endif
+  }
+
+  void qudaStreamSynchronize_(const qudaStream_t &stream, const char *func, const char *file, const char *line)
+  {
+#ifdef USE_DRIVER_API
+    PROFILE(CUresult error = cuStreamSynchronize(device::get_cuda_stream(stream)), QUDA_PROFILE_STREAM_SYNCHRONIZE);
+    if (error != CUDA_SUCCESS) {
+      const char *str;
+      cuGetErrorName(error, &str);
+      errorQuda("(CUDA) cuStreamSynchronize returned error %s\n (%s:%s in %s())\n", str, file, line, func);
+    }
+#else
+    PROFILE(cudaError_t error = cudaStreamSynchronize(device::get_cuda_stream(stream)), QUDA_PROFILE_STREAM_SYNCHRONIZE);
+    if (error != cudaSuccess && !activeTuning())
+      errorQuda("(CUDA) %s\n (%s:%s in %s())", cudaGetErrorString(error), file, line, func);
+#endif
+  }
+
+  void qudaDeviceSynchronize_(const char *func, const char *file, const char *line)
+  {
+#ifdef USE_DRIVER_API
+    PROFILE(CUresult error = cuCtxSynchronize(), QUDA_PROFILE_DEVICE_SYNCHRONIZE);
+    if (error != CUDA_SUCCESS) {
+      const char *str;
+      cuGetErrorName(error, &str);
+      errorQuda("cuCtxSynchronize returned error %s (%s:%s in %s())\n", str, file, line, func);
+    }
+#else
+    PROFILE(cudaError_t error = cudaDeviceSynchronize(), QUDA_PROFILE_DEVICE_SYNCHRONIZE);
+    if (error != cudaSuccess)
+      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+#endif
+  }
+
+  void* qudaGetSymbolAddress_(const char *symbol, const char *func, const char *file, const char *line)
+  {
+    void *ptr;
+    cudaError_t error = cudaGetSymbolAddress(&ptr, symbol);
+    if (error != cudaSuccess)
+      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    return ptr;
+  }
+
+  void qudaFuncSetAttribute_(const void *kernel, cudaFuncAttribute attr, int value, const char *func, const char *file,
+                             const char *line)
+  {
+    // no driver API variant here since we have C++ functions
+    PROFILE(cudaError_t error = cudaFuncSetAttribute(kernel, attr, value), QUDA_PROFILE_FUNC_SET_ATTRIBUTE);
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void qudaFuncGetAttributes_(cudaFuncAttributes &attr, const void *kernel, const char *func, const char *file,
+                              const char *line)
+  {
+    // no driver API variant here since we have C++ functions
+    PROFILE(cudaError_t error = cudaFuncGetAttributes(&attr, kernel), QUDA_PROFILE_FUNC_SET_ATTRIBUTE);
+    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+  }
+
+  void printAPIProfile() {
+#ifdef API_PROFILE
+    apiTimer.Print();
+#endif
+  }
+
+} // namespace quda
diff --git a/lib/targets/omptarget/tune.cpp b/lib/targets/omptarget/tune.cpp
new file mode 100644
index 0000000000..3dd11fe565
--- /dev/null
+++ b/lib/targets/omptarget/tune.cpp
@@ -0,0 +1,876 @@
+#include <tune_quda.h>
+#include <comm_quda.h>
+#include <quda.h>     // for QUDA_VERSION_STRING
+#include <sys/stat.h> // for stat()
+#include <fcntl.h>
+#include <cfloat> // for FLT_MAX
+#include <ctime>
+#include <fstream>
+#include <typeinfo>
+#include <map>
+#include <list>
+#include <unistd.h>
+#include <uint_to_char.h>
+
+#include <deque>
+#include <queue>
+#include <functional>
+
+//#define LAUNCH_TIMER
+extern char *gitversion;
+
+namespace quda
+{
+  static TuneKey last_key;
+}
+
+// intentionally leave this outside of the namespace for now
+quda::TuneKey getLastTuneKey() { return quda::last_key; }
+
+namespace quda
+{
+  typedef std::map<TuneKey, TuneParam> map;
+
+  struct TraceKey {
+
+    TuneKey key;
+    float time;
+
+    long device_bytes;
+    long pinned_bytes;
+    long mapped_bytes;
+    long host_bytes;
+
+    TraceKey() {}
+
+    TraceKey(const TuneKey &key, float time) :
+      key(key),
+      time(time),
+      device_bytes(device_allocated_peak()),
+      pinned_bytes(pinned_allocated_peak()),
+      mapped_bytes(mapped_allocated_peak()),
+      host_bytes(host_allocated_peak())
+    {
+    }
+
+    TraceKey(const TraceKey &trace) :
+      key(trace.key),
+      time(trace.time),
+      device_bytes(trace.device_bytes),
+      pinned_bytes(trace.pinned_bytes),
+      mapped_bytes(trace.mapped_bytes),
+      host_bytes(trace.host_bytes)
+    {
+    }
+
+    TraceKey &operator=(const TraceKey &trace)
+    {
+      if (&trace != this) {
+        key = trace.key;
+        time = trace.time;
+        device_bytes = trace.device_bytes;
+        pinned_bytes = trace.pinned_bytes;
+        mapped_bytes = trace.mapped_bytes;
+        host_bytes = trace.host_bytes;
+      }
+      return *this;
+    }
+  };
+
+  // linked list that is augmented each time we call a kernel
+  static std::list<TraceKey> trace_list;
+  static int enable_trace = 0;
+
+  int traceEnabled()
+  {
+    static bool init = false;
+
+    if (!init) {
+      char *enable_trace_env = getenv("QUDA_ENABLE_TRACE");
+      if (enable_trace_env) {
+        if (strcmp(enable_trace_env, "1") == 0) {
+          // only explicitly posted trace events are included
+          enable_trace = 1;
+        } else if (strcmp(enable_trace_env, "2") == 0) {
+          // enable full kernel trace and posted trace events
+          enable_trace = 2;
+        }
+      }
+      init = true;
+    }
+    return enable_trace;
+  }
+
+  void postTrace_(const char *func, const char *file, int line)
+  {
+    if (traceEnabled() >= 1) {
+      char aux[TuneKey::aux_n];
+      strcpy(aux, file);
+      strcat(aux, ":");
+      char tmp[TuneKey::aux_n];
+      i32toa(tmp, line);
+      strcat(aux, tmp);
+      TuneKey key("", func, aux);
+      TraceKey trace_entry(key, 0.0);
+      trace_list.push_back(trace_entry);
+    }
+  }
+
+  static const std::string quda_hash = QUDA_HASH; // defined in lib/Makefile
+  static std::string resource_path;
+  static map tunecache;
+  static map::iterator it;
+  static size_t initial_cache_size = 0;
+
+#define STR_(x) #x
+#define STR(x) STR_(x)
+  static const std::string quda_version
+    = STR(QUDA_VERSION_MAJOR) "." STR(QUDA_VERSION_MINOR) "." STR(QUDA_VERSION_SUBMINOR);
+#undef STR
+#undef STR_
+
+  /** tuning in progress? */
+  static bool tuning = false;
+
+  bool activeTuning() { return tuning; }
+
+  static bool profile_count = true;
+
+  void disableProfileCount() { profile_count = false; }
+  void enableProfileCount() { profile_count = true; }
+
+  const map &getTuneCache() { return tunecache; }
+
+  /**
+   * Deserialize tunecache from an istream, useful for reading a file or receiving from other nodes.
+   */
+  static void deserializeTuneCache(std::istream &in)
+  {
+    std::string line;
+    std::stringstream ls;
+
+    TuneKey key;
+    TuneParam param;
+
+    std::string v;
+    std::string n;
+    std::string a;
+
+    int check;
+
+    while (in.good()) {
+      getline(in, line);
+      if (!line.length()) continue; // skip blank lines (e.g., at end of file)
+      ls.clear();
+      ls.str(line);
+      ls >> v >> n >> a >> param.block.x >> param.block.y >> param.block.z;
+      check = snprintf(key.volume, key.volume_n, "%s", v.c_str());
+      if (check < 0 || check >= key.volume_n) errorQuda("Error writing volume string (check = %d)", check);
+      check = snprintf(key.name, key.name_n, "%s", n.c_str());
+      if (check < 0 || check >= key.name_n) errorQuda("Error writing name string (check=%d)", check);
+      check = snprintf(key.aux, key.aux_n, "%s", a.c_str());
+      if (check < 0 || check >= key.aux_n) errorQuda("Error writing aux string (check=%d)", check);
+      ls >> param.grid.x >> param.grid.y >> param.grid.z >> param.shared_bytes >> param.aux.x >> param.aux.y
+        >> param.aux.z >> param.aux.w >> param.time;
+      ls.ignore(1);               // throw away tab before comment
+      getline(ls, param.comment); // assume anything remaining on the line is a comment
+      param.comment += "\n";      // our convention is to include the newline, since ctime() likes to do this
+      tunecache[key] = param;
+    }
+  }
+
+  /**
+   * Serialize tunecache to an ostream, useful for writing to a file or sending to other nodes.
+   */
+  static void serializeTuneCache(std::ostream &out)
+  {
+    map::iterator entry;
+
+    for (entry = tunecache.begin(); entry != tunecache.end(); entry++) {
+      TuneKey key = entry->first;
+      TuneParam param = entry->second;
+
+      out << std::setw(16) << key.volume << "\t" << key.name << "\t" << key.aux << "\t";
+      out << param.block.x << "\t" << param.block.y << "\t" << param.block.z << "\t";
+      out << param.grid.x << "\t" << param.grid.y << "\t" << param.grid.z << "\t";
+      out << param.shared_bytes << "\t" << param.aux.x << "\t" << param.aux.y << "\t" << param.aux.z << "\t"
+          << param.aux.w << "\t";
+      out << param.time << "\t" << param.comment; // param.comment ends with a newline
+    }
+  }
+
+  template <class T> struct less_significant : std::binary_function<T, T, bool> {
+    inline bool operator()(const T &lhs, const T &rhs)
+    {
+      return lhs.second.time * lhs.second.n_calls < rhs.second.time * rhs.second.n_calls;
+    }
+  };
+
+  /**
+   * Serialize tunecache to an ostream, useful for writing to a file or sending to other nodes.
+   */
+  static void serializeProfile(std::ostream &out, std::ostream &async_out)
+  {
+    map::iterator entry;
+    double total_time = 0.0;
+    double async_total_time = 0.0;
+
+    // first let's sort the entries in decreasing order of significance
+    typedef std::pair<TuneKey, TuneParam> profile_t;
+    typedef std::priority_queue<profile_t, std::deque<profile_t>, less_significant<profile_t>> queue_t;
+    queue_t q(tunecache.begin(), tunecache.end());
+
+    // now compute total time spent in kernels so we can give each kernel a significance
+    for (entry = tunecache.begin(); entry != tunecache.end(); entry++) {
+      TuneKey key = entry->first;
+      TuneParam param = entry->second;
+
+      char tmp[TuneKey::aux_n] = {};
+      strncpy(tmp, key.aux, TuneKey::aux_n);
+      bool is_policy_kernel = strncmp(tmp, "policy_kernel", 13) == 0 ? true : false;
+      bool is_policy = (strncmp(tmp, "policy", 6) == 0 && !is_policy_kernel) ? true : false;
+      if (param.n_calls > 0 && !is_policy) total_time += param.n_calls * param.time;
+      if (param.n_calls > 0 && is_policy) async_total_time += param.n_calls * param.time;
+    }
+
+    while (!q.empty()) {
+      TuneKey key = q.top().first;
+      TuneParam param = q.top().second;
+
+      char tmp[TuneKey::aux_n] = {};
+      strncpy(tmp, key.aux, TuneKey::aux_n);
+      bool is_policy_kernel = strncmp(tmp, "policy_kernel", 13) == 0 ? true : false;
+      bool is_policy = (strncmp(tmp, "policy", 6) == 0 && !is_policy_kernel) ? true : false;
+      bool is_nested_policy = (strncmp(tmp, "nested_policy", 6) == 0) ? true : false; // nested policies not included
+
+      // synchronous profile
+      if (param.n_calls > 0 && !is_policy && !is_nested_policy) {
+        double time = param.n_calls * param.time;
+
+        out << std::setw(12) << param.n_calls * param.time << "\t" << std::setw(12) << (time / total_time) * 100 << "\t";
+        out << std::setw(12) << param.n_calls << "\t" << std::setw(12) << param.time << "\t" << std::setw(16)
+            << key.volume << "\t";
+        out << key.name << "\t" << key.aux << "\t" << param.comment; // param.comment ends with a newline
+      }
+
+      // async policy profile
+      if (param.n_calls > 0 && is_policy) {
+        double time = param.n_calls * param.time;
+
+        async_out << std::setw(12) << param.n_calls * param.time << "\t" << std::setw(12)
+                  << (time / async_total_time) * 100 << "\t";
+        async_out << std::setw(12) << param.n_calls << "\t" << std::setw(12) << param.time << "\t" << std::setw(16)
+                  << key.volume << "\t";
+        async_out << key.name << "\t" << key.aux << "\t" << param.comment; // param.comment ends with a newline
+      }
+
+      q.pop();
+    }
+
+    out << std::endl << "# Total time spent in kernels = " << total_time << " seconds" << std::endl;
+    async_out << std::endl
+              << "# Total time spent in asynchronous execution = " << async_total_time << " seconds" << std::endl;
+  }
+
+  /**
+   * Serialize trace to an ostream, useful for writing to a file or sending to other nodes.
+   */
+  static void serializeTrace(std::ostream &out)
+  {
+    for (auto it = trace_list.begin(); it != trace_list.end(); it++) {
+
+      TuneKey &key = it->key;
+
+      // special case kernel members of a policy
+      char tmp[TuneKey::aux_n] = {};
+      strncpy(tmp, key.aux, TuneKey::aux_n);
+      bool is_policy_kernel = strcmp(tmp, "policy_kernel") == 0 ? true : false;
+
+      out << std::setw(12) << it->time << "\t";
+      out << std::setw(12) << it->device_bytes << "\t";
+      out << std::setw(12) << it->pinned_bytes << "\t";
+      out << std::setw(12) << it->mapped_bytes << "\t";
+      out << std::setw(12) << it->host_bytes << "\t";
+      out << std::setw(16) << key.volume << "\t";
+      if (is_policy_kernel) out << "\t";
+      out << key.name << "\t";
+      if (!is_policy_kernel) out << "\t";
+      out << key.aux << std::endl;
+    }
+  }
+
+  /**
+   * Distribute the tunecache from node 0 to all other nodes.
+   */
+  static void broadcastTuneCache()
+  {
+#ifdef MULTI_GPU
+    std::stringstream serialized;
+    size_t size;
+
+    if (comm_rank() == 0) {
+      serializeTuneCache(serialized);
+      size = serialized.str().length();
+    }
+    comm_broadcast(&size, sizeof(size_t));
+
+    if (size > 0) {
+      if (comm_rank() == 0) {
+        comm_broadcast(const_cast<char *>(serialized.str().c_str()), size);
+      } else {
+        char *serstr = new char[size + 1];
+        comm_broadcast(serstr, size);
+        serstr[size] = '\0'; // null-terminate
+        serialized.str(serstr);
+        deserializeTuneCache(serialized);
+        delete[] serstr;
+      }
+    }
+#endif
+  }
+
+  /*
+   * Read tunecache from disk.
+   */
+  void loadTuneCache()
+  {
+    if (getTuning() == QUDA_TUNE_NO) {
+      warningQuda("Autotuning disabled");
+      return;
+    }
+
+    char *path;
+    struct stat pstat;
+    std::string cache_path, line, token;
+    std::ifstream cache_file;
+    std::stringstream ls;
+
+    path = getenv("QUDA_RESOURCE_PATH");
+
+    if (!path) {
+      warningQuda("Environment variable QUDA_RESOURCE_PATH is not set.");
+      warningQuda("Caching of tuned parameters will be disabled.");
+      return;
+    } else if (stat(path, &pstat) || !S_ISDIR(pstat.st_mode)) {
+      warningQuda("The path \"%s\" specified by QUDA_RESOURCE_PATH does not exist or is not a directory.", path);
+      warningQuda("Caching of tuned parameters will be disabled.");
+      return;
+    } else {
+      resource_path = path;
+    }
+
+    bool version_check = true;
+    char *override_version_env = getenv("QUDA_TUNE_VERSION_CHECK");
+    if (override_version_env && strcmp(override_version_env, "0") == 0) {
+      version_check = false;
+      warningQuda("Disabling QUDA tunecache version check");
+    }
+
+#ifdef MULTI_GPU
+    if (comm_rank() == 0) {
+#endif
+
+      cache_path = resource_path;
+      cache_path += "/tunecache.tsv";
+      cache_file.open(cache_path.c_str());
+
+      if (cache_file) {
+
+        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
+        getline(cache_file, line);
+        ls.str(line);
+        ls >> token;
+        if (token.compare("tunecache")) errorQuda("Bad format in %s", cache_path.c_str());
+        ls >> token;
+        if (version_check && token.compare(quda_version))
+          errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
+                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
+                    cache_path.c_str());
+        ls >> token;
+#ifdef GITVERSION
+        if (version_check && token.compare(gitversion))
+          errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
+                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
+                    cache_path.c_str());
+#else
+      if (version_check && token.compare(quda_version))
+        errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
+                  "QUDA_RESOURCE_PATH environment variable to point to a new path.",
+                  cache_path.c_str());
+#endif
+        ls >> token;
+        if (version_check && token.compare(quda_hash))
+          errorQuda("Cache file %s does not match current QUDA build. \nPlease delete this file or set the "
+                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
+                    cache_path.c_str());
+
+        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
+        getline(cache_file, line); // eat the blank line
+
+        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
+        getline(cache_file, line); // eat the description line
+
+        deserializeTuneCache(cache_file);
+
+        cache_file.close();
+        initial_cache_size = tunecache.size();
+
+        if (getVerbosity() >= QUDA_SUMMARIZE) {
+          printfQuda("Loaded %d sets of cached parameters from %s\n", static_cast<int>(initial_cache_size),
+                     cache_path.c_str());
+        }
+
+      } else {
+        warningQuda("Cache file not found.  All kernels will be re-tuned (if tuning is enabled).");
+      }
+
+#ifdef MULTI_GPU
+    }
+#endif
+
+    broadcastTuneCache();
+  }
+
+  /**
+   * Write tunecache to disk.
+   */
+  void saveTuneCache(bool error)
+  {
+    time_t now;
+    int lock_handle;
+    std::string lock_path, cache_path;
+    std::ofstream cache_file;
+
+    if (resource_path.empty()) return;
+
+      // FIXME: We should really check to see if any nodes have tuned a kernel that was not also tuned on node 0, since as things
+      //       stand, the corresponding launch parameters would never get cached to disk in this situation.  This will come up if we
+      //       ever support different subvolumes per GPU (as might be convenient for lattice volumes that don't divide evenly).
+
+#ifdef MULTI_GPU
+    if (comm_rank() == 0) {
+#endif
+
+      if (tunecache.size() == initial_cache_size && !error) return;
+
+      // Acquire lock.  Note that this is only robust if the filesystem supports flock() semantics, which is true for
+      // NFS on recent versions of linux but not Lustre by default (unless the filesystem was mounted with "-o flock").
+      lock_path = resource_path + "/tunecache.lock";
+      lock_handle = open(lock_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
+      if (lock_handle == -1) {
+        warningQuda("Unable to lock cache file.  Tuned launch parameters will not be cached to disk.  "
+                    "If you are certain that no other instances of QUDA are accessing this filesystem, "
+                    "please manually remove %s",
+                    lock_path.c_str());
+        return;
+      }
+      char msg[] = "If no instances of applications using QUDA are running,\n"
+                   "this lock file shouldn't be here and is safe to delete.";
+      int stat = write(lock_handle, msg, sizeof(msg)); // check status to avoid compiler warning
+      if (stat == -1) warningQuda("Unable to write to lock file for some bizarre reason");
+
+      cache_path = resource_path + (error ? "/tunecache_error.tsv" : "/tunecache.tsv");
+      cache_file.open(cache_path.c_str());
+
+      if (getVerbosity() >= QUDA_SUMMARIZE) {
+        printfQuda("Saving %d sets of cached parameters to %s\n", static_cast<int>(tunecache.size()), cache_path.c_str());
+      }
+
+      time(&now);
+      cache_file << "tunecache\t" << quda_version;
+#ifdef GITVERSION
+      cache_file << "\t" << gitversion;
+#else
+    cache_file << "\t" << quda_version;
+#endif
+      cache_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
+      cache_file << std::setw(16) << "volume"
+                 << "\tname\taux\tblock.x\tblock.y\tblock.z\tgrid.x\tgrid.y\tgrid.z\tshared_bytes\taux.x\taux.y\taux."
+                    "z\taux.w\ttime\tcomment"
+                 << std::endl;
+      serializeTuneCache(cache_file);
+      cache_file.close();
+
+      // Release lock.
+      close(lock_handle);
+      remove(lock_path.c_str());
+
+      initial_cache_size = tunecache.size();
+
+#ifdef MULTI_GPU
+    } else {
+      // give process 0 time to write out its tunecache if needed, but
+      // doesn't cause a hang if error is not triggered on process 0
+      if (error) sleep(10);
+    }
+#endif
+  }
+
+  static bool policy_tuning = false;
+  bool policyTuning() { return policy_tuning; }
+
+  void setPolicyTuning(bool policy_tuning_) { policy_tuning = policy_tuning_; }
+
+  // flush profile, setting counts to zero
+  void flushProfile()
+  {
+    for (map::iterator entry = tunecache.begin(); entry != tunecache.end(); entry++) {
+      // set all n_calls = 0
+      TuneParam &param = entry->second;
+      param.n_calls = 0;
+    }
+  }
+
+  // save profile
+  void saveProfile(const std::string label)
+  {
+    time_t now;
+    int lock_handle;
+    std::string lock_path, profile_path, async_profile_path, trace_path;
+    std::ofstream profile_file, async_profile_file, trace_file;
+
+    if (resource_path.empty()) return;
+
+#ifdef MULTI_GPU
+    if (comm_rank() == 0) {
+#endif
+
+      // Acquire lock.  Note that this is only robust if the filesystem supports flock() semantics, which is true for
+      // NFS on recent versions of linux but not Lustre by default (unless the filesystem was mounted with "-o flock").
+      lock_path = resource_path + "/profile.lock";
+      lock_handle = open(lock_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
+      if (lock_handle == -1) {
+        warningQuda("Unable to lock profile file.  Profile will not be saved to disk.  "
+                    "If you are certain that no other instances of QUDA are accessing this filesystem, "
+                    "please manually remove %s",
+                    lock_path.c_str());
+        return;
+      }
+      char msg[] = "If no instances of applications using QUDA are running,\n"
+                   "this lock file shouldn't be here and is safe to delete.";
+      int stat = write(lock_handle, msg, sizeof(msg)); // check status to avoid compiler warning
+      if (stat == -1) warningQuda("Unable to write to lock file for some bizarre reason");
+
+      // profile counter for writing out unique profiles
+      static int count = 0;
+
+      char *profile_fname = getenv("QUDA_PROFILE_OUTPUT_BASE");
+
+      if (!profile_fname) {
+        warningQuda(
+          "Environment variable QUDA_PROFILE_OUTPUT_BASE not set; writing to profile.tsv and profile_async.tsv");
+        profile_path = resource_path + "/profile_" + std::to_string(count) + ".tsv";
+        async_profile_path = resource_path + "/profile_async_" + std::to_string(count) + ".tsv";
+        if (traceEnabled()) trace_path = resource_path + "/trace_" + std::to_string(count) + ".tsv";
+      } else {
+        profile_path = resource_path + "/" + profile_fname + "_" + std::to_string(count) + ".tsv";
+        async_profile_path = resource_path + "/" + profile_fname + "_" + std::to_string(count) + "_async.tsv";
+        if (traceEnabled())
+          trace_path = resource_path + "/" + profile_fname + "_trace_" + std::to_string(count) + ".tsv";
+      }
+
+      count++;
+
+      profile_file.open(profile_path.c_str());
+      async_profile_file.open(async_profile_path.c_str());
+      if (traceEnabled()) trace_file.open(trace_path.c_str());
+
+      if (getVerbosity() >= QUDA_SUMMARIZE) {
+        // compute number of non-zero entries that will be output in the profile
+        int n_entry = 0;
+        int n_policy = 0;
+        for (map::iterator entry = tunecache.begin(); entry != tunecache.end(); entry++) {
+          // if a policy entry, then we can ignore
+          char tmp[TuneKey::aux_n] = {};
+          strncpy(tmp, entry->first.aux, TuneKey::aux_n);
+          TuneParam param = entry->second;
+          bool is_policy = strcmp(tmp, "policy") == 0 ? true : false;
+          if (param.n_calls > 0 && !is_policy) n_entry++;
+          if (param.n_calls > 0 && is_policy) n_policy++;
+        }
+
+        printfQuda("Saving %d sets of cached parameters to %s\n", n_entry, profile_path.c_str());
+        printfQuda("Saving %d sets of cached profiles to %s\n", n_policy, async_profile_path.c_str());
+        if (traceEnabled())
+          printfQuda("Saving trace list with %lu entries to %s\n", trace_list.size(), trace_path.c_str());
+      }
+
+      time(&now);
+
+      std::string Label = label.empty() ? "profile" : label;
+
+      profile_file << Label << "\t" << quda_version;
+#ifdef GITVERSION
+      profile_file << "\t" << gitversion;
+#else
+    profile_file << "\t" << quda_version;
+#endif
+      profile_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
+      profile_file << std::setw(12) << "total time"
+                   << "\t" << std::setw(12) << "percentage"
+                   << "\t" << std::setw(12) << "calls"
+                   << "\t" << std::setw(12) << "time / call"
+                   << "\t" << std::setw(16) << "volume"
+                   << "\tname\taux\tcomment" << std::endl;
+
+      async_profile_file << Label << "\t" << quda_version;
+#ifdef GITVERSION
+      async_profile_file << "\t" << gitversion;
+#else
+    async_profile_file << "\t" << quda_version;
+#endif
+      async_profile_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
+      async_profile_file << std::setw(12) << "total time"
+                         << "\t" << std::setw(12) << "percentage"
+                         << "\t" << std::setw(12) << "calls"
+                         << "\t" << std::setw(12) << "time / call"
+                         << "\t" << std::setw(16) << "volume"
+                         << "\tname\taux\tcomment" << std::endl;
+
+      serializeProfile(profile_file, async_profile_file);
+
+      profile_file.close();
+      async_profile_file.close();
+
+      if (traceEnabled()) {
+        trace_file << "trace"
+                   << "\t" << quda_version;
+#ifdef GITVERSION
+        trace_file << "\t" << gitversion;
+#else
+      trace_file << "\t" << quda_version;
+#endif
+        trace_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
+
+        trace_file << std::setw(12) << "time\t" << std::setw(12) << "device-mem\t" << std::setw(12) << "pinned-mem\t";
+        trace_file << std::setw(12) << "mapped-mem\t" << std::setw(12) << "host-mem\t";
+        trace_file << std::setw(16) << "volume"
+                   << "\tname\taux" << std::endl;
+
+        serializeTrace(trace_file);
+
+        trace_file.close();
+      }
+
+      // Release lock.
+      close(lock_handle);
+      remove(lock_path.c_str());
+
+#ifdef MULTI_GPU
+    }
+#endif
+  }
+
+  static TimeProfile launchTimer("tuneLaunch");
+
+  /**
+   * Return the optimal launch parameters for a given kernel, either
+   * by retrieving them from tunecache or autotuning on the spot.
+   */
+  TuneParam &tuneLaunch(Tunable &tunable, QudaTune enabled, QudaVerbosity verbosity)
+  {
+
+#ifdef LAUNCH_TIMER
+    launchTimer.TPSTART(QUDA_PROFILE_TOTAL);
+    launchTimer.TPSTART(QUDA_PROFILE_INIT);
+#endif
+
+    TuneKey key = tunable.tuneKey();
+    if (use_managed_memory()) strcat(key.aux, ",managed");
+    last_key = key;
+    static TuneParam param;
+
+#ifdef LAUNCH_TIMER
+    launchTimer.TPSTOP(QUDA_PROFILE_INIT);
+    launchTimer.TPSTART(QUDA_PROFILE_PREAMBLE);
+#endif
+
+    static const Tunable *active_tunable; // for error checking
+    it = tunecache.find(key);
+
+    // first check if we have the tuned value and return if we have it
+    if (enabled == QUDA_TUNE_YES && it != tunecache.end()) {
+
+#ifdef LAUNCH_TIMER
+      launchTimer.TPSTOP(QUDA_PROFILE_PREAMBLE);
+      launchTimer.TPSTART(QUDA_PROFILE_COMPUTE);
+#endif
+
+      TuneParam &param = it->second;
+
+      if (verbosity >= QUDA_DEBUG_VERBOSE) {
+        printfQuda("Launching %s with %s at vol=%s with %s\n", key.name, key.aux, key.volume,
+                   tunable.paramString(param).c_str());
+      }
+
+#ifdef LAUNCH_TIMER
+      launchTimer.TPSTOP(QUDA_PROFILE_COMPUTE);
+      launchTimer.TPSTART(QUDA_PROFILE_EPILOGUE);
+#endif
+
+      tunable.checkLaunchParam(param);
+
+      // we could be tuning outside of the current scope
+      if (!tuning && profile_count) param.n_calls++;
+
+#ifdef LAUNCH_TIMER
+      launchTimer.TPSTOP(QUDA_PROFILE_EPILOGUE);
+      launchTimer.TPSTOP(QUDA_PROFILE_TOTAL);
+#endif
+
+      if (traceEnabled() >= 2) {
+        TraceKey trace_entry(key, param.time);
+        trace_list.push_back(trace_entry);
+      }
+
+      return param;
+    }
+
+#ifdef LAUNCH_TIMER
+    launchTimer.TPSTOP(QUDA_PROFILE_PREAMBLE);
+    launchTimer.TPSTOP(QUDA_PROFILE_TOTAL);
+#endif
+
+    if (enabled == QUDA_TUNE_NO) {
+      tunable.defaultTuneParam(param);
+      tunable.checkLaunchParam(param);
+      if (verbosity >= QUDA_DEBUG_VERBOSE) {
+        printfQuda("Launching %s with %s at vol=%s with %s (untuned)\n", key.name, key.aux, key.volume,
+                   tunable.paramString(param).c_str());
+      }
+    } else if (!tuning) {
+
+      /* As long as global reductions are not disabled, only do the
+         tuning on node 0, else do the tuning on all nodes since we
+         can't guarantee that all nodes are partaking */
+      if (comm_rank() == 0 || !commGlobalReduction() || policyTuning()) {
+        TuneParam best_param;
+        cudaError_t error = cudaSuccess;
+        cudaEvent_t start, end;
+        float elapsed_time, best_time;
+        time_t now;
+
+        tuning = true;
+        active_tunable = &tunable;
+        best_time = FLT_MAX;
+
+        if (verbosity >= QUDA_DEBUG_VERBOSE) printfQuda("PreTune %s\n", key.name);
+        tunable.preTune();
+
+        cudaEventCreate(&start);
+        cudaEventCreate(&end);
+
+        if (verbosity >= QUDA_DEBUG_VERBOSE) {
+          printfQuda("Tuning %s with %s at vol=%s\n", key.name, key.aux, key.volume);
+        }
+
+        Timer tune_timer;
+        tune_timer.Start(__func__, __FILE__, __LINE__);
+
+        tunable.initTuneParam(param);
+        while (tuning) {
+          cudaDeviceSynchronize();
+          cudaGetLastError(); // clear error counter
+          tunable.checkLaunchParam(param);
+          if (verbosity >= QUDA_DEBUG_VERBOSE) {
+            printfQuda("About to call tunable.apply block=(%d,%d,%d) grid=(%d,%d,%d) shared_bytes=%d aux=(%d,%d,%d)\n",
+                       param.block.x, param.block.y, param.block.z, param.grid.x, param.grid.y, param.grid.z,
+                       param.shared_bytes, param.aux.x, param.aux.y, param.aux.z);
+          }
+          tunable.apply(); // do initial call in case we need to jit compile for these parameters or if policy tuning
+
+          cudaEventRecord(start, 0);
+          for (int i = 0; i < tunable.tuningIter(); i++) {
+            tunable.apply(); // calls tuneLaunch() again, which simply returns the currently active param
+          }
+          cudaEventRecord(end, 0);
+          cudaEventSynchronize(end);
+          cudaEventElapsedTime(&elapsed_time, start, end);
+          cudaDeviceSynchronize();
+          error = cudaGetLastError();
+
+          { // check that error state is cleared
+            cudaDeviceSynchronize();
+            cudaError_t error = cudaGetLastError();
+            if (error != cudaSuccess) errorQuda("Failed to clear error state %s\n", cudaGetErrorString(error));
+          }
+
+          elapsed_time /= (1e3 * tunable.tuningIter());
+          if ((elapsed_time < best_time) && (error == cudaSuccess) && (tunable.jitifyError() == CUDA_SUCCESS)) {
+            best_time = elapsed_time;
+            best_param = param;
+          }
+          if ((verbosity >= QUDA_DEBUG_VERBOSE)) {
+            if (error == cudaSuccess && tunable.jitifyError() == CUDA_SUCCESS) {
+              printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(),
+                         tunable.perfString(elapsed_time).c_str());
+            } else {
+              if (tunable.jitifyError() == CUDA_SUCCESS) {
+                // if not jitify error must be regular error
+                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), cudaGetErrorString(error));
+              } else {
+                // else is a jitify error
+                const char *str;
+                cuGetErrorString(tunable.jitifyError(), &str);
+                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), str);
+              }
+            }
+          }
+          tuning = tunable.advanceTuneParam(param);
+          tunable.jitifyError() = CUDA_SUCCESS;
+        }
+
+        tune_timer.Stop(__func__, __FILE__, __LINE__);
+
+        if (best_time == FLT_MAX) {
+          errorQuda("Auto-tuning failed for %s with %s at vol=%s", key.name, key.aux, key.volume);
+        }
+        if (verbosity >= QUDA_VERBOSE) {
+          printfQuda("Tuned %s giving %s for %s with %s\n", tunable.paramString(best_param).c_str(),
+                     tunable.perfString(best_time).c_str(), key.name, key.aux);
+        }
+        time(&now);
+        best_param.comment = "# " + tunable.perfString(best_time);
+        best_param.comment += ", tuning took " + std::to_string(tune_timer.Last()) + " seconds at ";
+        best_param.comment += ctime(&now); // includes a newline
+        best_param.time = best_time;
+
+        cudaEventDestroy(start);
+        cudaEventDestroy(end);
+
+        if (verbosity >= QUDA_DEBUG_VERBOSE) printfQuda("PostTune %s\n", key.name);
+        tuning = true;
+        tunable.postTune();
+        tuning = false;
+        param = best_param;
+        tunecache[key] = best_param;
+      }
+      if (commGlobalReduction() || policyTuning()) broadcastTuneCache();
+
+      // check this process is getting the key that is expected
+      if (tunecache.find(key) == tunecache.end()) {
+        errorQuda("Failed to find key entry (%s:%s:%s)", key.name, key.volume, key.aux);
+      }
+      param = tunecache[key]; // read this now for all processes
+
+      if (traceEnabled() >= 2) {
+        TraceKey trace_entry(key, param.time);
+        trace_list.push_back(trace_entry);
+      }
+
+    } else if (&tunable != active_tunable) {
+      errorQuda("Unexpected call to tuneLaunch() in %s::apply()", typeid(tunable).name());
+    }
+
+    param.n_calls = profile_count ? 1 : 0;
+
+    return param;
+  }
+
+  void printLaunchTimer()
+  {
+#ifdef LAUNCH_TIMER
+    launchTimer.Print();
+#endif
+  }
+} // namespace quda

From 76b0b3df8d0fbbb78bb6f13fa5fe492aa0aaa176 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 3 Dec 2020 16:05:01 +0000
Subject: [PATCH 003/198] omptarget: guard against some cuda specifics

---
 include/atomic.cuh        | 13 +++++++++++++
 include/comm_quda.h       |  1 +
 include/cub_helper.cuh    |  2 ++
 include/index_helper.cuh  |  7 +++++++
 include/matrix_field.h    |  4 ++++
 include/reduce_helper.h   | 10 ++++++++++
 include/register_traits.h |  2 ++
 include/tune_quda.h       |  4 ++++
 lib/comm_common.cpp       | 19 +++++++++++++++++--
 lib/gauge_fix_ovr.cu      | 10 ++++++++--
 10 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/include/atomic.cuh b/include/atomic.cuh
index 7a6d5b0064..274f3d91a0 100644
--- a/include/atomic.cuh
+++ b/include/atomic.cuh
@@ -10,6 +10,17 @@
    avoid confusion when resolving the native atomicAdd functions.
  */
 
+#ifdef QUDA_BACKEND_OMPTARGET
+template<typename T>
+static inline T atomicAdd(T*x,T v)
+{
+  T z;
+#pragma omp atomic capture
+  {z=*x; *x+=v;}
+  return z;
+}
+#endif
+
 #if defined(__CUDA_ARCH__) 
 
 #if __COMPUTE_CAPABILITY__ < 600
@@ -98,6 +109,7 @@ union uint32_short2 { unsigned int i; short2 s; };
    @param addr Address that stores the atomic variable to be updated
    @param val Value to be added to the atomic
 */
+#ifndef QUDA_BACKEND_OMPTARGET
 static inline __device__ short2 atomicAdd(short2 *addr, short2 val){
   uint32_short2 old, assumed, incremented;
   old.s = *addr;
@@ -167,3 +179,4 @@ static inline __device__ float atomicAbsMax(float *addr, float val){
   uint32_t *addr_ = reinterpret_cast<uint32_t*>(addr);
   return atomicMax(addr_, val_);
 }
+#endif
diff --git a/include/comm_quda.h b/include/comm_quda.h
index 1cb2e7a34f..7f99c5fe84 100644
--- a/include/comm_quda.h
+++ b/include/comm_quda.h
@@ -1,5 +1,6 @@
 #pragma once
 #include <cstdint>
+#include <cstdlib>
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/include/cub_helper.cuh b/include/cub_helper.cuh
index 4833a2d97d..d3c8e76de1 100644
--- a/include/cub_helper.cuh
+++ b/include/cub_helper.cuh
@@ -15,4 +15,6 @@ using namespace quda;
 #define CUB_USE_COOPERATIVE_GROUPS
 #endif
 
+#ifndef QUDA_BACKEND_OMPTARGET
 #include <cub/block/block_reduce.cuh>
+#endif
diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index 493fd14fdf..2d4d10b434 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -1,6 +1,9 @@
 #pragma once
 
 #include <enum_quda.h>
+#ifdef QUDA_BACKEND_OMPTARGET
+#include <omp.h>
+#endif
 
 namespace quda {
   /**
@@ -889,7 +892,11 @@ namespace quda {
 #else
   template <typename T> __device__ inline int block_idx(const T &)
   {
+#ifdef QUDA_BACKEND_OMPTARGET
+    return omp_get_thread_num();
+#else
     return blockIdx.x;
+#endif
   }
 #endif
 
diff --git a/include/matrix_field.h b/include/matrix_field.h
index 2875c3efe6..4c8ecf1970 100644
--- a/include/matrix_field.h
+++ b/include/matrix_field.h
@@ -7,11 +7,15 @@
  */
 
 // trove requires the warp shuffle instructions introduced with Kepler
+#ifdef QUDA_BACKEND_OMPTARGET
+#define DISABLE_TROVE
+#else
 #if __COMPUTE_CAPABILITY__ >= 300
 #include <trove/ptr.h>
 #else
 #define DISABLE_TROVE
 #endif
+#endif
 
 #include <quda_matrix.h>
 
diff --git a/include/reduce_helper.h b/include/reduce_helper.h
index 13c6fa8e59..487fc61890 100644
--- a/include/reduce_helper.h
+++ b/include/reduce_helper.h
@@ -1,7 +1,9 @@
 #pragma once
 
 #include <float_vector.h>
+#ifndef QUDA_BACKEND_OMPTARGET
 #include <cub_helper.cuh>
+#endif
 #include <target_device.h>
 
 #ifdef QUAD_SUM
@@ -282,6 +284,9 @@ namespace quda
     template <int block_size_x, int block_size_y = 1, typename Reducer, typename Arg, typename T>
     __device__ inline void reduce(Arg &arg, const T &in, const int idx = 0)
     {
+#ifdef QUDA_BACKEND_OMPTARGET
+      ompwip("unimplemented");
+#else
       using BlockReduce = cub::BlockReduce<T, block_size_x, cub::BLOCK_REDUCE_WARP_REDUCTIONS, block_size_y>;
       __shared__ typename BlockReduce::TempStorage cub_tmp;
       __shared__ bool isLastBlockDone;
@@ -319,6 +324,7 @@ namespace quda
           arg.count[idx] = 0; // set to zero for next time
         }
       }
+#endif
     }
 #endif
 
@@ -328,12 +334,16 @@ namespace quda
      */
     template <typename T, int width> struct WarpReduce
     {
+#ifdef QUDA_BACKEND_OMPTARGET
+      WarpReduce() {ompwip("unimplemented");}
+#else
       static_assert(width <= device::warp_size(), "WarpReduce logical width must not be greater than the warp size");
       cub::WarpReduce<T, width> warp_reduce;
       typename decltype(warp_reduce)::TempStorage dummy;
 
       __device__ WarpReduce() : warp_reduce(dummy) {}
       __device__ T Sum(const T &value) { return warp_reduce.Sum(value); }
+#endif
     };
 
 } // namespace quda
diff --git a/include/register_traits.h b/include/register_traits.h
index e9b3cbb206..c92f118e56 100644
--- a/include/register_traits.h
+++ b/include/register_traits.h
@@ -9,7 +9,9 @@
 
 #include <quda_internal.h>
 #include <complex_quda.h>
+#ifndef QUDA_BACKEND_OMPTARGET
 #include <inline_ptx.h>
+#endif
 
 namespace quda {
 
diff --git a/include/tune_quda.h b/include/tune_quda.h
index 0259794b3f..c5690fd51f 100644
--- a/include/tune_quda.h
+++ b/include/tune_quda.h
@@ -16,6 +16,10 @@
 #include <iomanip>
 #include <typeinfo>
 #include <map>
+#ifdef QUDA_BACKEND_OMPTARGET
+#define CUresult bool
+#define CUDA_SUCCESS true
+#endif
 #else
 #define CUresult bool
 #define CUDA_SUCCESS true
diff --git a/lib/comm_common.cpp b/lib/comm_common.cpp
index c7b549c59e..df8e603a6f 100644
--- a/lib/comm_common.cpp
+++ b/lib/comm_common.cpp
@@ -6,6 +6,21 @@
 #include <comm_quda.h>
 #include <csignal>
 
+#ifdef QUDA_BACKEND_OMPTARGET
+#define CUDA_SUCCESS QUDA_SUCCESS
+struct cudaDeviceProp{int unifiedAddressing;};
+enum {cudaDevP2PAttrPerformanceRank};
+enum CUpointer_attribute {CU_POINTER_ATTRIBUTE_MEMORY_TYPE};
+using CUdeviceptr = void *;
+#define cudaGetDeviceProperties(a,b) ompwip([&](){(a)->unifiedAddressing=0;})
+#define cudaDeviceCanAccessPeer(a,b,c) ompwip([&](){(*(a))=0;})
+#define cudaDeviceGetP2PAttribute(a,b,c,d) ompwip([&](){(*(a))=0;})
+#define cudaGetDeviceCount(a) ompwip([&](){(*(a))=omp_get_num_devices();})
+enum CUmemorytype {CU_MEMORYTYPE_DEVICE,CU_MEMORYTYPE_ARRAY,CU_MEMORYTYPE_UNIFIED,CU_MEMORYTYPE_HOST};
+#define cuPointerGetAttributes(a,b,c,d) ompwip([=](){*((CUmemorytype *)(c[0]))=CU_MEMORYTYPE_HOST;})
+
+#endif  // QUDA_BACKEND_OMPTARGET
+
 #ifdef QUDA_BACKWARDSCPP
 #include "backward.hpp"
 namespace backward {
@@ -730,14 +745,14 @@ void comm_init_common(int ndim, const int *dims, QudaCommsMap rank_from_coords,
 
   int device_count;
   cudaGetDeviceCount(&device_count);
-  if (device_count == 0) { errorQuda("No CUDA devices found"); }
+  if (device_count == 0) { warningQuda("No CUDA devices found"); }
   if (gpuid >= device_count) {
     char *enable_mps_env = getenv("QUDA_ENABLE_MPS");
     if (enable_mps_env && strcmp(enable_mps_env, "1") == 0) {
       gpuid = gpuid % device_count;
       printf("MPS enabled, rank=%d -> gpu=%d\n", comm_rank(), gpuid);
     } else {
-      errorQuda("Too few GPUs available on %s", comm_hostname());
+      warningQuda("Too few GPUs available on %s", comm_hostname());
     }
   }
 
diff --git a/lib/gauge_fix_ovr.cu b/lib/gauge_fix_ovr.cu
index 229692f4a6..793822e0bf 100644
--- a/lib/gauge_fix_ovr.cu
+++ b/lib/gauge_fix_ovr.cu
@@ -4,7 +4,9 @@
 #include <comm_quda.h>
 #include <gauge_fix_ovr_hit_devf.cuh>
 #include <reduce_helper.h>
+#ifndef QUDA_BACKEND_OMPTARGET
 #include <thrust_helper.cuh>
+#endif
 #include <instantiate.h>
 #include <tunable_reduction.h>
 #include <tunable_nd.h>
@@ -468,6 +470,9 @@ public:
    */
   void PreCalculateLatticeIndices(size_t faceVolume[4], int X[4], int border[4], int &threads, int *borderpoints[2])
   {
+#ifdef QUDA_BACKEND_OMPTARGET
+    ompwip("unimplemented");
+#else
     BorderIdArg arg(X, border);
     int nlinksfaces = 0;
     for (int dir = 0; dir < 4; dir++)
@@ -505,6 +510,7 @@ public:
     }
     if (size[0] == size[1]) threads = size[0];
     else errorQuda("BORDER: Even and Odd sizes does not match, not supported!!!!, %d:%d", size[0], size[1]);
+#endif
   }
 
   /**
@@ -622,8 +628,8 @@ public:
         sendg_d[d] = device_malloc(bytes[d]);
         recvg_d[d] = device_malloc(bytes[d]);
         hostbuffer_h[d] = (void*)pinned_malloc(4 * bytes[d]);
-        tp[d].block = make_uint3(128, 1, 1);
-        tp[d].grid = make_uint3((faceVolumeCB[d] + tp[d].block.x - 1) / tp[d].block.x, 1, 1);
+        tp[d].block = dim3(128, 1, 1);
+        tp[d].grid = dim3((faceVolumeCB[d] + tp[d].block.x - 1) / tp[d].block.x, 1, 1);
       }
       for ( int d = 0; d < 4; d++ ) {
         if ( !commDimPartitioned(d)) continue;

From b98da988589dbca607ad674d79bd503da4e7d966 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 3 Dec 2020 16:31:21 +0000
Subject: [PATCH 004/198] omptarget: add QUDA_RT_CONSTS and guard more
 unimplemented funcs

---
 include/blas_magma.h                              |  2 ++
 include/dslash_helper.cuh                         |  1 +
 include/kernels/block_orthogonalize.cuh           |  4 ++++
 include/kernels/coarse_op_kernel.cuh              | 15 +++++++++++++++
 include/kernels/coarse_op_preconditioned.cuh      |  1 +
 include/kernels/dslash_coarse.cuh                 |  1 +
 include/kernels/dslash_domain_wall_m5.cuh         |  2 ++
 include/kernels/dslash_mobius_eofa.cuh            |  2 ++
 .../dslash_ndeg_twisted_mass_preconditioned.cuh   |  1 +
 include/kernels/dslash_pack.cuh                   |  7 +++++++
 include/kernels/gauge_fix_ovr.cuh                 |  6 ++++++
 include/quda_api.h                                |  2 ++
 include/shared_memory_cache_helper.cuh            |  2 ++
 include/transform_reduce.h                        |  1 +
 lib/block_orthogonalize.cu                        |  1 +
 lib/gauge_fix_fft.cu                              |  4 ++++
 lib/gauge_fix_ovr_hit_devf.cuh                    |  6 ++++++
 lib/random.cu                                     |  1 +
 18 files changed, 59 insertions(+)

diff --git a/include/blas_magma.h b/include/blas_magma.h
index 2bb88a4916..d8afcfbc84 100644
--- a/include/blas_magma.h
+++ b/include/blas_magma.h
@@ -1,6 +1,7 @@
 #ifndef _BLAS_MAGMA_H
 #define _BLAS_MAGMA_H
 
+#ifndef QUDA_BACKEND_OMPTARGET
 #include <cuda.h>
 #include <cuda_runtime.h>
 #include <string>
@@ -8,6 +9,7 @@
 #include <cuComplex.h>
 #include <stdio.h>
 #include <enum_quda.h>
+#endif
 
 //MAGMA library interface
 
diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index 6208c7b6d9..f183bd98bb 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -417,6 +417,7 @@ namespace quda
             KernelType kernel_type, typename Arg>
   __global__ void dslashGPU(Arg arg)
   {
+    QUDA_RT_CONSTS;
     D<nParity, dagger, xpay, kernel_type, Arg> dslash(arg);
 
     int s = blockDim.y * blockIdx.y + threadIdx.y;
diff --git a/include/kernels/block_orthogonalize.cuh b/include/kernels/block_orthogonalize.cuh
index eafe6b9e29..8eac9262ef 100644
--- a/include/kernels/block_orthogonalize.cuh
+++ b/include/kernels/block_orthogonalize.cuh
@@ -229,6 +229,9 @@ namespace quda {
             int nVec, typename Arg>
   __launch_bounds__(2 * block_size) __global__ void blockOrthoGPU(Arg arg)
   {
+#ifdef QUDA_BACKEND_OMPTARGET
+    ompwip("unimplemented");
+#else
     int x_coarse = blockIdx.x;
 #ifdef SWIZZLE
     // the portion of the grid that is exactly divisible by the number of SMs
@@ -343,6 +346,7 @@ namespace quda {
 
       } // j
     }   // n
+#endif
   }
 
 } // namespace quda
diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 351cf9deaf..7d1073dcdf 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -255,6 +255,7 @@ namespace quda {
   template<int dim, QudaDirection dir, typename Arg>
   __global__ void ComputeUVGPU(Arg arg)
   {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.fineVolumeCB) return;
 
@@ -338,6 +339,7 @@ namespace quda {
   template <typename Arg>
   __global__ void ComputeAVGPU(Arg arg)
   {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.fineVolumeCB) return;
 
@@ -392,6 +394,7 @@ namespace quda {
 
   template<typename Arg>
   __global__ void ComputeTMAVGPU(Arg arg) {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.fineVolumeCB) return;
 
@@ -558,6 +561,7 @@ namespace quda {
   template <typename Arg>
   __global__ void ComputeTMCAVGPU(Arg arg)
   {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.fineVolumeCB) return;
 
@@ -577,6 +581,7 @@ namespace quda {
 
   template<typename Arg>
   __device__ __host__ inline int virtualThreadIdx(const Arg &arg) {
+    QUDA_RT_CONSTS;
     int warp_id = threadIdx.x / device::warp_size();
     int warp_lane = threadIdx.x % device::warp_size();
     int tx = warp_id * (device::warp_size() / arg.aggregates_per_block) + warp_lane / arg.aggregates_per_block;
@@ -585,12 +590,14 @@ namespace quda {
 
   template<typename Arg>
   __device__ __host__ inline int virtualBlockDim(const Arg &arg) {
+    QUDA_RT_CONSTS;
     int block_dim_x = blockDim.x / arg.aggregates_per_block;
     return block_dim_x;
   }
 
   template<typename Arg>
   __device__ __host__ inline int coarseIndex(const Arg &arg) {
+    QUDA_RT_CONSTS;
     int warp_lane = threadIdx.x % device::warp_size();
     int x_coarse = (arg.coarse_color_wave ? blockIdx.y : blockIdx.x)*arg.aggregates_per_block + warp_lane % arg.aggregates_per_block;
     return x_coarse;
@@ -955,6 +962,7 @@ namespace quda {
   __device__ inline void getIndices(const Arg &arg, int &parity, int &x_cb, int &parity_coarse,
                                     int &x_coarse_cb, int &c_row, int &c_col)
   {
+    QUDA_RT_CONSTS;
     if (arg.coarse_color_wave) {
       int parity_c_row_block_idx_z = blockDim.y*blockIdx.x + threadIdx.y;
       int c_row_block_idx_z = parity_flip ? (parity_c_row_block_idx_z % arg.coarse_color_grid_z ) : (parity_c_row_block_idx_z / 2); // coarse color row index
@@ -1076,6 +1084,7 @@ namespace quda {
 
   template<typename Arg>
   __global__ void ComputeYReverseGPU(Arg arg) {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.coarseVolumeCB) return;
 
@@ -1178,6 +1187,7 @@ namespace quda {
 
   template <typename Arg>
   __global__ void ComputeCoarseCloverGPU(Arg arg) {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.fineVolumeCB) return;
 
@@ -1214,6 +1224,7 @@ namespace quda {
   //Adds the identity matrix to the coarse local term.
   template<typename Arg>
   __global__ void AddCoarseDiagonalGPU(Arg arg) {
+    QUDA_RT_CONSTS;
     using Float = typename Arg::Float;
 
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
@@ -1245,6 +1256,7 @@ namespace quda {
   // Adds the staggered mass to the coarse local term.
   template<typename Arg>
   __global__ void AddCoarseStaggeredMassGPU(Arg arg) {
+    QUDA_RT_CONSTS;
     using Float = typename Arg::Float;
 
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
@@ -1285,6 +1297,7 @@ namespace quda {
   //Adds the twisted-mass term to the coarse local term.
   template<typename Arg>
   __global__ void AddCoarseTmDiagonalGPU(Arg arg) {
+    QUDA_RT_CONSTS;
     using Float = typename Arg::Float;
 
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
@@ -1362,6 +1375,7 @@ namespace quda {
 
   template<typename Arg>
   __global__ void ConvertGPU(Arg arg) {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.coarseVolumeCB) return;
 
@@ -1411,6 +1425,7 @@ namespace quda {
 
   template<typename Arg>
   __global__ void RescaleYGPU(Arg arg) {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.coarseVolumeCB) return;
 
diff --git a/include/kernels/coarse_op_preconditioned.cuh b/include/kernels/coarse_op_preconditioned.cuh
index bd7b41de2a..7a285b3229 100644
--- a/include/kernels/coarse_op_preconditioned.cuh
+++ b/include/kernels/coarse_op_preconditioned.cuh
@@ -139,6 +139,7 @@ namespace quda {
 
   template <bool compute_max_only, typename Arg> __global__ void CalculateYhatGPU(Arg arg)
   {
+    QUDA_RT_CONSTS;
     int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
     if (x_cb >= arg.Y.VolumeCB()) return;
     int i_parity = blockDim.y*blockIdx.y + threadIdx.y;
diff --git a/include/kernels/dslash_coarse.cuh b/include/kernels/dslash_coarse.cuh
index a409b961fa..26d5602ce8 100644
--- a/include/kernels/dslash_coarse.cuh
+++ b/include/kernels/dslash_coarse.cuh
@@ -371,6 +371,7 @@ namespace quda {
   template <typename Float, int nDim, int Ns, int Nc, int Mc, int color_stride, int dim_thread_split, bool dslash, bool clover, bool dagger, DslashType type, typename Arg>
   __global__ void coarseDslashKernel(Arg arg)
   {
+    QUDA_RT_CONSTS;
     constexpr int warp_size = device::warp_size();
     const int lane_id = threadIdx.x % warp_size;
     const int warp_id = threadIdx.x / warp_size;
diff --git a/include/kernels/dslash_domain_wall_m5.cuh b/include/kernels/dslash_domain_wall_m5.cuh
index b5bf9056b6..e7bb7ca336 100644
--- a/include/kernels/dslash_domain_wall_m5.cuh
+++ b/include/kernels/dslash_domain_wall_m5.cuh
@@ -252,6 +252,7 @@ namespace quda
   template <typename Vector, typename Arg>
   __device__ __host__ inline Vector constantInv(Arg &arg, int parity, int x_cb, int s_)
   {
+    QUDA_RT_CONSTS;
     using real = typename Arg::real;
     const auto k = arg.b;
     const auto inv = arg.c;
@@ -307,6 +308,7 @@ namespace quda
   template <typename Vector, typename Arg>
   __device__ __host__ inline Vector variableInv(Arg &arg, int parity, int x_cb, int s_)
   {
+    QUDA_RT_CONSTS;
     constexpr int nSpin = 4;
     using real = typename Arg::real;
     typedef ColorSpinor<real, Arg::nColor, nSpin / 2> HalfVector;
diff --git a/include/kernels/dslash_mobius_eofa.cuh b/include/kernels/dslash_mobius_eofa.cuh
index 6f8eb6d301..6859f70a69 100644
--- a/include/kernels/dslash_mobius_eofa.cuh
+++ b/include/kernels/dslash_mobius_eofa.cuh
@@ -104,6 +104,7 @@ namespace quda
 
       __device__ __host__ inline void operator()(int x_cb, int s, int parity)
       {
+        QUDA_RT_CONSTS;
         using real = typename Arg::real;
         typedef ColorSpinor<real, Arg::nColor, 4> Vector;
 
@@ -181,6 +182,7 @@ namespace quda
 
       __device__ __host__ inline void operator()(int x_cb, int s, int parity)
       {
+        QUDA_RT_CONSTS;
         using real = typename Arg::real;
         typedef ColorSpinor<real, Arg::nColor, 4> Vector;
 
diff --git a/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh b/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
index 0fce821387..75bda914c3 100644
--- a/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
+++ b/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
@@ -53,6 +53,7 @@ namespace quda
     */
     __device__ __host__ inline void operator()(int idx, int flavor, int parity)
     {
+      QUDA_RT_CONSTS;
       typedef typename mapper<typename Arg::Float>::type real;
       typedef ColorSpinor<real, Arg::nColor, 4> Vector;
 
diff --git a/include/kernels/dslash_pack.cuh b/include/kernels/dslash_pack.cuh
index 3241c56096..323cb80205 100644
--- a/include/kernels/dslash_pack.cuh
+++ b/include/kernels/dslash_pack.cuh
@@ -183,6 +183,7 @@ namespace quda
 
   template <bool dagger, int twist, QudaPCType pc, typename Arg> __global__ void packKernel(Arg arg)
   {
+    QUDA_RT_CONSTS;
     const int sites_per_block = arg.sites_per_block;
     int local_tid = threadIdx.x;
     int tid = sites_per_block * blockIdx.x + local_tid;
@@ -223,6 +224,7 @@ namespace quda
 
     template <int twist> __device__ inline void operator()(Arg &arg, int s, int parity)
     {
+      QUDA_RT_CONSTS;
       // (active_dims * 2 + dir) * blocks_per_dir + local_block_idx
       int local_block_idx = blockIdx.x % arg.blocks_per_dir;
       int dim_dir = blockIdx.x / arg.blocks_per_dir;
@@ -293,6 +295,8 @@ namespace quda
 
   template <bool dagger, int twist, QudaPCType pc, typename Arg> __global__ void packShmemKernel(Arg arg)
   {
+    QUDA_RT_CONSTS;
+
     int s = blockDim.y * blockIdx.y + threadIdx.y;
     if (s >= arg.dc.Ls) return;
 
@@ -306,6 +310,7 @@ namespace quda
 
   template <typename Arg> __global__ void packStaggeredKernel(Arg arg)
   {
+    QUDA_RT_CONSTS;
     const int sites_per_block = arg.sites_per_block;
     int local_tid = threadIdx.x;
     int tid = sites_per_block * blockIdx.x + local_tid;
@@ -345,6 +350,7 @@ namespace quda
 
     __device__ inline void operator()(Arg &arg, int s, int parity, int = 0)
     {
+      QUDA_RT_CONSTS;
       // (active_dims * 2 + dir) * blocks_per_dir + local_block_idx
       int local_block_idx = blockIdx.x % arg.blocks_per_dir;
       int dim_dir = blockIdx.x / arg.blocks_per_dir;
@@ -406,6 +412,7 @@ namespace quda
 
   template <typename Arg> __global__ void packStaggeredShmemKernel(Arg arg)
   {
+    QUDA_RT_CONSTS;
 
     int s = blockDim.y * blockIdx.y + threadIdx.y;
     if (s >= arg.dc.Ls) return;
diff --git a/include/kernels/gauge_fix_ovr.cuh b/include/kernels/gauge_fix_ovr.cuh
index 3dec04f70c..316cc3d217 100644
--- a/include/kernels/gauge_fix_ovr.cuh
+++ b/include/kernels/gauge_fix_ovr.cuh
@@ -125,6 +125,7 @@ namespace quda {
   template<int ImplementationType, int blockSize, typename Float, typename Gauge, int gauge_dir>
   __global__ void computeFix(GaugeFixArg<Float, Gauge> arg, int parity)
   {
+    QUDA_RT_CONSTS;
     typedef complex<Float> Cmplx;
     int tid = (threadIdx.x + blockSize) % blockSize;
     int idx = blockIdx.x * blockSize + tid;
@@ -226,6 +227,7 @@ namespace quda {
    */
   template<int ImplementationType, int blockSize, typename Float, typename Gauge, int gauge_dir>
   __global__ void computeFixInteriorPoints(GaugeFixInteriorPointsArg<Float, Gauge> arg, int parity){
+    QUDA_RT_CONSTS;
     int tid = (threadIdx.x + blockSize) % blockSize;
     int idx = blockIdx.x * blockSize + tid;
     if ( idx >= arg.threads ) return;
@@ -331,6 +333,7 @@ namespace quda {
   */
   template<int ImplementationType, int blockSize, typename Float, typename Gauge, int gauge_dir>
   __global__ void computeFixBorderPoints(GaugeFixBorderPointsArg<Float, Gauge> arg, int parity){
+    QUDA_RT_CONSTS;
     typedef complex<Float> Cmplx;
 
     int tid = (threadIdx.x + blockSize) % blockSize;
@@ -409,6 +412,7 @@ namespace quda {
   template <typename Float, bool pack, typename Arg>
   __global__ void Kernel_UnPackGhost(int size, Arg arg, complex<Float> *array, int parity, int face, int dir)
   {
+    QUDA_RT_CONSTS;
     int idx = blockIdx.x * blockDim.x + threadIdx.x;
     if ( idx >= size ) return;
     int X[4];
@@ -481,6 +485,7 @@ namespace quda {
   template <typename Float, bool pack, typename Arg>
   __global__ void Kernel_UnPackTop(int size, Arg arg, complex<Float> *array, int parity, int face, int dir)
   {
+    QUDA_RT_CONSTS;
     int idx = blockIdx.x * blockDim.x + threadIdx.x;
     if ( idx >= size ) return;
     int X[4];
@@ -557,6 +562,7 @@ namespace quda {
 
   __global__ void ComputeBorderPointsActiveFaceIndex(BorderIdArg arg, int *faceindices, int facesize, int faceid, int parity)
   {
+    QUDA_RT_CONSTS;
     int idd = blockDim.x * blockIdx.x + threadIdx.x;
     if ( idd < facesize ) {
       int borderid = 0;
diff --git a/include/quda_api.h b/include/quda_api.h
index 2bf284386a..d3680ba169 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -211,6 +211,8 @@ static inline double4 make_double4(double x,double y,double z,double w){return d
 
 #else  // QUDA_BACKEND_OMPTARGET
 
+#define QUDA_RT_CONSTS
+
 #ifndef __CUDACC_RTC__
 #include <cuda.h>
 #include <cuda_runtime.h>
diff --git a/include/shared_memory_cache_helper.cuh b/include/shared_memory_cache_helper.cuh
index 53074da90d..5acdf5c8f5 100644
--- a/include/shared_memory_cache_helper.cuh
+++ b/include/shared_memory_cache_helper.cuh
@@ -41,6 +41,7 @@ namespace quda
      */
     __device__ inline void save(const Vector &a)
     {
+      QUDA_RT_CONSTS;
       int j = (threadIdx.z * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x;
 #pragma unroll
       for (int i = 0; i < 2 * a.size; i++) {
@@ -58,6 +59,7 @@ namespace quda
      */
     __device__ inline Vector load(int x, int y, int z)
     {
+      QUDA_RT_CONSTS;
       Vector a;
       int j = (z * blockDim.y + y) * blockDim.x + x;
 #pragma unroll
diff --git a/include/transform_reduce.h b/include/transform_reduce.h
index ca8e9d1e2f..97a445a186 100644
--- a/include/transform_reduce.h
+++ b/include/transform_reduce.h
@@ -57,6 +57,7 @@ namespace quda
 
   template <typename Arg> __launch_bounds__(Arg::block_size) __global__ void transform_reduce_kernel(Arg arg)
   {
+    QUDA_RT_CONSTS;
     using count_t = decltype(arg.n_items);
 
     count_t i = blockIdx.x * blockDim.x + threadIdx.x;
diff --git a/lib/block_orthogonalize.cu b/lib/block_orthogonalize.cu
index 8a59520029..d5de3fe165 100644
--- a/lib/block_orthogonalize.cu
+++ b/lib/block_orthogonalize.cu
@@ -104,6 +104,7 @@ namespace quda {
      */
     template <typename Rotator, typename Vector, std::size_t... S>
     void GPU(const TuneParam &tp, const qudaStream_t &stream, const std::vector<ColorSpinorField*> &B, std::index_sequence<S...>) {
+      QUDA_RT_CONSTS;
       typedef typename mapper<vFloat>::type RegType; // need to redeclare typedef (WAR for CUDA 7 and 8)
       typedef BlockOrthoArg<Rotator,Vector,nSpin,spinBlockSize,coarseSpin,nVec> Arg;
       Arg arg(V, fine_to_coarse, coarse_to_fine, QUDA_INVALID_PARITY, geo_bs, n_block_ortho, V, B[S]...);
diff --git a/lib/gauge_fix_fft.cu b/lib/gauge_fix_fft.cu
index bb114da61b..9919ba256c 100644
--- a/lib/gauge_fix_fft.cu
+++ b/lib/gauge_fix_fft.cu
@@ -185,6 +185,9 @@ namespace quda {
   void gaugeFixingFFT(GaugeField& data, int Nsteps, int verbose_interval,
                       double alpha0, int autotune, double tolerance, int stopWtheta)
   {
+#ifdef QUDA_BACKEND_OMPTARGET
+    ompwip("unimplemented");
+#else
     TimeProfile profileInternalGaugeFixFFT("InternalGaugeFixQudaFFT", false);
 
     profileInternalGaugeFixFFT.TPSTART(QUDA_PROFILE_COMPUTE);
@@ -363,6 +366,7 @@ namespace quda {
       gbytes = gbytes / (secs * 1e9);
       printfQuda("Time: %6.6f s, Gflop/s = %6.1f, GB/s = %6.1f\n", secs, gflops, gbytes);
     }
+#endif
   }
 
   template<typename Float, int nColors, QudaReconstructType recon> struct GaugeFixingFFT {
diff --git a/lib/gauge_fix_ovr_hit_devf.cuh b/lib/gauge_fix_ovr_hit_devf.cuh
index a5eb55696d..9135f98aa8 100644
--- a/lib/gauge_fix_ovr_hit_devf.cuh
+++ b/lib/gauge_fix_ovr_hit_devf.cuh
@@ -67,6 +67,7 @@ namespace quda {
    */
   template<int blockSize, typename Float, int gauge_dir, int NCOLORS>
   __forceinline__ __device__ void GaugeFixHit_AtomicAdd(Matrix<complex<Float>,NCOLORS> &link, const Float relax_boost, const int tid){
+    QUDA_RT_CONSTS;
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     //__shared__ Float elems[blockSize * 4];
@@ -157,6 +158,7 @@ namespace quda {
    */
   template<int blockSize, typename Float, int gauge_dir, int NCOLORS>
   __forceinline__ __device__ void GaugeFixHit_NoAtomicAdd(Matrix<complex<Float>,NCOLORS> &link, const Float relax_boost, const int tid){
+    QUDA_RT_CONSTS;
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     //__shared__ Float elems[blockSize * 4 * 8];
@@ -252,6 +254,7 @@ namespace quda {
    */
   template<int blockSize, typename Float, int gauge_dir, int NCOLORS>
   __forceinline__ __device__ void GaugeFixHit_NoAtomicAdd_LessSM(Matrix<complex<Float>,NCOLORS> &link, const Float relax_boost, const int tid){
+    QUDA_RT_CONSTS;
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     //__shared__ Float elems[blockSize * 4 * 8];
@@ -391,6 +394,7 @@ namespace quda {
   template<int blockSize, typename Float, int gauge_dir, int NCOLORS>
   __forceinline__ __device__ void GaugeFixHit_AtomicAdd(Matrix<complex<Float>,NCOLORS> &link, Matrix<complex<Float>,NCOLORS> &link1,
 							const Float relax_boost, const int tid){
+    QUDA_RT_CONSTS;
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     //__shared__ Float elems[blockSize * 4];
@@ -485,6 +489,7 @@ namespace quda {
   template<int blockSize, typename Float, int gauge_dir, int NCOLORS>
   __forceinline__ __device__ void GaugeFixHit_NoAtomicAdd(Matrix<complex<Float>,NCOLORS> &link, Matrix<complex<Float>,NCOLORS> &link1,
 							  const Float relax_boost, const int tid){
+    QUDA_RT_CONSTS;
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     //__shared__ Float elems[blockSize * 4 * 8];
@@ -561,6 +566,7 @@ namespace quda {
    */
   template<int blockSize, typename Float, int gauge_dir, int NCOLORS>
   __forceinline__ __device__ void GaugeFixHit_NoAtomicAdd_LessSM(Matrix<complex<Float>,NCOLORS> &link, Matrix<complex<Float>,NCOLORS> &link1, const Float relax_boost, const int tid){
+    QUDA_RT_CONSTS;
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     //__shared__ Float elems[blockSize * 4 * 8];
diff --git a/lib/random.cu b/lib/random.cu
index 26f88575c7..fb6ae74f9e 100644
--- a/lib/random.cu
+++ b/lib/random.cu
@@ -34,6 +34,7 @@ namespace quda {
   */
   __global__ void kernel_random(RNGState *state, unsigned long long seed, int size_cb, rngArg arg)
   {
+    QUDA_RT_CONSTS;
     int id = blockIdx.x * blockDim.x + threadIdx.x;
     int parity = blockIdx.y * blockDim.y + threadIdx.y;
     if (id < size_cb) {

From eb6bc78413b4dd1d77d20f12f1a2b34fb2750d85 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 3 Dec 2020 16:32:25 +0000
Subject: [PATCH 005/198] omptarget: inline static fast_intdiv

---
 include/fast_intdiv.h | 58 +++++++++++++++++++++----------------------
 1 file changed, 28 insertions(+), 30 deletions(-)

diff --git a/include/fast_intdiv.h b/include/fast_intdiv.h
index 75a0a93138..de89f2872f 100644
--- a/include/fast_intdiv.h
+++ b/include/fast_intdiv.h
@@ -111,28 +111,26 @@ class int_fastdiv
   }
 
   __host__ __device__ __forceinline__
-    friend int operator/(const int divident, const int_fastdiv& divisor);
-};
+  friend int operator/(const int n, const int_fastdiv& divisor)
+  {
+    int q;
+  #ifdef __CUDA_ARCH__
+    asm("mul.hi.s32 %0, %1, %2;" : "=r"(q) : "r"(divisor.M), "r"(n));
+  #else
+    q = (((unsigned long long)((long long)divisor.M * (long long)n)) >> 32);
+  #endif
+    q += n * divisor.n_add_sign;
+    if (divisor.s >= 0)
+      {
+        q >>= divisor.s; // we rely on this to be implemented as arithmetic shift
+        q += (((unsigned int)q) >> 31);
+      }
+    return q;
+  }
 
-__host__ __device__ __forceinline__
-int operator/(const int n, const int_fastdiv& divisor)
-{
-  int q;
-#ifdef __CUDA_ARCH__
-  asm("mul.hi.s32 %0, %1, %2;" : "=r"(q) : "r"(divisor.M), "r"(n));
-#else
-  q = (((unsigned long long)((long long)divisor.M * (long long)n)) >> 32);
-#endif
-  q += n * divisor.n_add_sign;
-  if (divisor.s >= 0)
-    {
-      q >>= divisor.s; // we rely on this to be implemented as arithmetic shift
-      q += (((unsigned int)q) >> 31);
-    }
-  return q;
-}
+};
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator%(const int n, const int_fastdiv& divisor)
 {
   int quotient = n / divisor;
@@ -140,61 +138,61 @@ int operator%(const int n, const int_fastdiv& divisor)
   return remainder;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator/(const unsigned int n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator%(const unsigned int n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator/(const short n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator%(const short n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator/(const unsigned short n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator%(const unsigned short n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator/(const char n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator%(const char n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator/(const unsigned char n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__
+__host__ __device__ __forceinline__ static
 int operator%(const unsigned char n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;

From 50c5aa607756eeb12fd6114c81584680c5fc2969 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 3 Dec 2020 16:33:09 +0000
Subject: [PATCH 006/198] omptarget: workaround rsqrt, sincos, sincosf

---
 include/gauge_field_order.h      |  4 ++--
 include/kernels/gauge_random.cuh | 10 ++++++----
 include/kernels/spinor_noise.cuh |  4 +++-
 include/register_traits.h        | 10 +++++++---
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/include/gauge_field_order.h b/include/gauge_field_order.h
index 57aa6f1cd7..6afc2baae1 100644
--- a/include/gauge_field_order.h
+++ b/include/gauge_field_order.h
@@ -1586,7 +1586,7 @@ namespace quda {
           real row_sum_inv = static_cast<real>(1.0) / row_sum;
 
           real diff = u0_inv * u0_inv - row_sum;
-          real U00_mag = diff > 0.0 ? diff * rsqrt(diff) : static_cast<real>(0.0);
+          real U00_mag = diff > 0.0 ? diff / sqrt(diff) : static_cast<real>(0.0);
 
           out[0] *= U00_mag;
 
@@ -1597,7 +1597,7 @@ namespace quda {
           column_sum += out[3].imag() * out[3].imag();
 
           diff = u0_inv * u0_inv - column_sum;
-          real U20_mag = diff > 0.0 ? diff * rsqrt(diff) : static_cast<real>(0.0);
+          real U20_mag = diff > 0.0 ? diff / sqrt(diff) : static_cast<real>(0.0);
 
           out[6] *= U20_mag;
 
diff --git a/include/kernels/gauge_random.cuh b/include/kernels/gauge_random.cuh
index 7c29642b8e..1f6be55c14 100644
--- a/include/kernels/gauge_random.cuh
+++ b/include/kernels/gauge_random.cuh
@@ -52,15 +52,17 @@ namespace quda {
     for (int i = 0; i < 4; ++i) {
       phi[i] = 2.0 * M_PI * rand1[i];
       radius[i] = sqrt(-log(rand2[i]));
-      sincos(phi[i], &temp2[i], &temp1[i]);
+      // sincos(phi[i], &temp2[i], &temp1[i]);
+      temp2[i] = sin(phi[i]);
+      temp1[i] = cos(phi[i]);
       temp1[i] *= radius[i];
       temp2[i] *= radius[i];
     }
 
     // construct Anti-Hermitian matrix
-    ret(0, 0) = complex<real>(0.0, temp1[2] + rsqrt(3.0) * temp2[3]);
-    ret(1, 1) = complex<real>(0.0, -temp1[2] + rsqrt(3.0) * temp2[3]);
-    ret(2, 2) = complex<real>(0.0, -2.0 * rsqrt(3.0) * temp2[3]);
+    ret(0, 0) = complex<real>(0.0, temp1[2] + sqrt(1.0/3.0) * temp2[3]);
+    ret(1, 1) = complex<real>(0.0, -temp1[2] + sqrt(1.0/3.0) * temp2[3]);
+    ret(2, 2) = complex<real>(0.0, -2.0 * sqrt(1.0/3.0) * temp2[3]);
     ret(0, 1) = complex<real>(temp1[0], temp1[1]);
     ret(1, 0) = complex<real>(-temp1[0], temp1[1]);
     ret(0, 2) = complex<real>(temp1[3], temp2[0]);
diff --git a/include/kernels/spinor_noise.cuh b/include/kernels/spinor_noise.cuh
index ffc1d21836..152ff105e5 100644
--- a/include/kernels/spinor_noise.cuh
+++ b/include/kernels/spinor_noise.cuh
@@ -28,7 +28,9 @@ namespace quda {
     real radius = uniform<real>::rand(localState);
     radius = sqrt(-log(radius));
     real phi_sin, phi_cos;
-    sincos(phi, &phi_sin, &phi_cos);
+    // sincos(phi, &phi_sin, &phi_cos);
+    phi_sin = sin(phi);
+    phi_cos = cos(phi);
     arg.v(parity, x_cb, s, c) = radius * complex<real>(phi_cos, phi_sin);
   }
 
diff --git a/include/register_traits.h b/include/register_traits.h
index c92f118e56..ce0469d657 100644
--- a/include/register_traits.h
+++ b/include/register_traits.h
@@ -369,7 +369,7 @@ namespace quda {
       __device__ __host__ static T Atan2( const T &a, const T &b) { return atan2(a,b); }
       __device__ __host__ static T Sin( const T &a ) { return sin(a); }
       __device__ __host__ static T Cos( const T &a ) { return cos(a); }
-      __device__ __host__ static void SinCos(const T &a, T *s, T *c) { sincos(a, s, c); }
+      __device__ __host__ static void SinCos(const T &a, T *s, T *c) { /*sincos(a, s, c);*/ *s=sin(a);*c=cos(a); }
     };
   
   /**
@@ -400,7 +400,9 @@ namespace quda {
 #ifdef __CUDA_ARCH__
        __sincosf(a, s, c);
 #else
-       sincosf(a, s, c);
+       // sincosf(a, s, c);
+       *s = sinf(a);
+       *c = cosf(a);
 #endif
     }
   };
@@ -433,7 +435,9 @@ namespace quda {
 #ifdef __CUDA_ARCH__
       __sincosf(a * static_cast<float>(M_PI), s, c);
 #else
-      sincosf(a * static_cast<float>(M_PI), s, c);
+      // sincosf(a * static_cast<float>(M_PI), s, c);
+      *s = sinf(a);
+      *c = cosf(a);
 #endif
     }
   };

From 8940fbb0167772405c31692128942e9e5a808f80 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 3 Dec 2020 16:33:28 +0000
Subject: [PATCH 007/198] omptarget: cuda* -> quda*

---
 lib/clover_field.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/clover_field.cpp b/lib/clover_field.cpp
index 50a2b6388b..cef284ae7f 100644
--- a/lib/clover_field.cpp
+++ b/lib/clover_field.cpp
@@ -79,11 +79,11 @@ namespace quda {
     if (norm_bytes) backup_norm_h = static_cast<char*>(safe_malloc(2 * norm_bytes));
 
     if (Location() == QUDA_CUDA_FIELD_LOCATION) {
-      cudaMemcpy(backup_h, clover, bytes, cudaMemcpyDeviceToHost);
-      cudaMemcpy(backup_h + bytes, cloverInv, bytes, cudaMemcpyDeviceToHost);
+      qudaMemcpy(backup_h, clover, bytes, cudaMemcpyDeviceToHost);
+      qudaMemcpy(backup_h + bytes, cloverInv, bytes, cudaMemcpyDeviceToHost);
       if (norm_bytes) {
-        cudaMemcpy(backup_norm_h, norm, norm_bytes, cudaMemcpyDeviceToHost);
-        cudaMemcpy(backup_norm_h + norm_bytes, invNorm, norm_bytes, cudaMemcpyDeviceToHost);
+        qudaMemcpy(backup_norm_h, norm, norm_bytes, cudaMemcpyDeviceToHost);
+        qudaMemcpy(backup_norm_h + norm_bytes, invNorm, norm_bytes, cudaMemcpyDeviceToHost);
       }
       checkCudaError();
     } else {
@@ -101,11 +101,11 @@ namespace quda {
   */
   void CloverField::restore() const {
     if (Location() == QUDA_CUDA_FIELD_LOCATION) {
-      cudaMemcpy(clover, backup_h, bytes, cudaMemcpyHostToDevice);
-      cudaMemcpy(cloverInv, backup_h + bytes, bytes, cudaMemcpyHostToDevice);
+      qudaMemcpy(clover, backup_h, bytes, cudaMemcpyHostToDevice);
+      qudaMemcpy(cloverInv, backup_h + bytes, bytes, cudaMemcpyHostToDevice);
       if (norm_bytes) {
-        cudaMemcpy(norm, backup_norm_h, norm_bytes, cudaMemcpyHostToDevice);
-        cudaMemcpy(invNorm, backup_norm_h + norm_bytes, norm_bytes, cudaMemcpyHostToDevice);
+        qudaMemcpy(norm, backup_norm_h, norm_bytes, cudaMemcpyHostToDevice);
+        qudaMemcpy(invNorm, backup_norm_h + norm_bytes, norm_bytes, cudaMemcpyHostToDevice);
       }
       checkCudaError();
     } else {

From efa80944fad1835d812037d59931886f3e7c1c69 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 9 Dec 2020 17:42:13 -0600
Subject: [PATCH 008/198] omptarget: fix last merge

---
 include/kernels/block_orthogonalize.cuh       |  4 +++
 .../omptarget/block_reduction_kernel.h        | 33 ++++++-------------
 include/targets/omptarget/target_device.h     |  2 +-
 lib/targets/omptarget/device.cpp              | 20 ++++++++---
 4 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/include/kernels/block_orthogonalize.cuh b/include/kernels/block_orthogonalize.cuh
index 65ebb2e2fb..c7d928b1ff 100644
--- a/include/kernels/block_orthogonalize.cuh
+++ b/include/kernels/block_orthogonalize.cuh
@@ -124,6 +124,9 @@ namespace quda {
 
     __device__ __host__ inline void operator()(dim3 block, dim3 thread)
     {
+#ifdef QUDA_BACKEND_OMPTARGET
+      ompwip("unimplemented");
+#else
       int x_coarse = block.x;
       int x_fine_offset = thread.x;
       int chirality = block.y;
@@ -231,6 +234,7 @@ namespace quda {
           }
         } // j
       }   // n
+#endif
     }
   };
 
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 2a825cae8b..83018deb30 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -35,36 +35,23 @@ namespace quda {
   }
 
   /**
-     @brief Generic block reduction kernel.  Here, we ensure that each
-     thread block maps exactly to a logical block to be reduced, with
-     number of threads equal to the number of sites per block.  The y
+     @brief Generic block kernel.  Here, we split the block and thread
+     indices in the x and y dimension and pass these indices
+     separately to the transform functor.  The x thread dimension is
+     templated, e.g., for efficient reductions, and typically the y
      thread dimension is a trivial vectorizable dimension.
-
-     TODO: add a Reducer class for non summation reductions
   */
-  template <int block_size, template <typename> class Transformer, typename Arg>
-  __global__ void BlockReductionKernel2D(Arg arg)
+  template <int block_size, template <int, typename> class Transformer, typename Arg>
+  __launch_bounds__(Arg::launch_bounds ? block_size : 0) __global__ void BlockKernel2D(Arg arg)
   {
     QUDA_RT_CONSTS;
-    using reduce_t = typename Transformer<Arg>::reduce_t;
-    Transformer<Arg> t(arg);
-
-    const int block = virtual_block_idx(arg);
-    const int i = threadIdx.x;
+    const dim3 block_idx(virtual_block_idx(arg), blockIdx.y, 0);
+    const dim3 thread_idx(threadIdx.x, threadIdx.y, 0);
     const int j = blockDim.y*blockIdx.y + threadIdx.y;
-    const int j_local = threadIdx.y;
     if (j >= arg.threads.y) return;
 
-    reduce_t value; // implicitly we assume here that default constructor zeros reduce_t
-    // only active threads call the transformer
-    if (i < arg.threads.x) value = t(block, i, j);
-/*
-    // but all threads take part in the reduction
-    using BlockReduce = cub::BlockReduce<reduce_t, block_size, cub::BLOCK_REDUCE_WARP_REDUCTIONS>;
-    __shared__ typename BlockReduce::TempStorage temp_storage[Arg::n_vector_y];
-    value = BlockReduce(temp_storage[j_local]).Sum(value);
-*/
-    if (i == 0) t.store(value, block, j);
+    Transformer<block_size, Arg> t(arg);
+    t(block_idx, thread_idx);
   }
 
 }
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index b514595aa7..d4c13f4c8c 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -40,7 +40,7 @@ namespace quda {
 #ifdef __CUDA_ARCH__
       return dim3(blockDim.x, blockDim.y, blockDim.z);
 #else
-      return dim3(0, 0, 0);
+      return dim3(1, 1, 1);
 #endif
 */
     }
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index a2cbf0d681..1587d7911d 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -340,16 +340,26 @@ namespace quda
       switch (deviceProp.major) {
       case 2:
 	return 8;
+	break;
       case 3:
 	return 16;
+	break;
       case 5:
-      case 6: return 32;
+	 return 32;
+	 break;
+      case 6: 
+	 return 32;
+	 break;
       case 7:
-        switch (deviceProp.minor) {
-        case 0: return 32;
-        case 2: return 32;
-        case 5: return 16;
+	{
+          switch (deviceProp.minor) {
+            case 0: return 32; break;
+            case 2: return 32; break;
+            case 5: return 16; break;
+	    default: return 32; break;
+	  };
         }
+	break;
       default:
         warningQuda("Unknown SM architecture %d.%d - assuming limit of 32 blocks per SM\n",
                     deviceProp.major, deviceProp.minor);

From 1fc7a8f48c9589d09b12dfc7a8e25dc66584c7f0 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 9 Dec 2020 17:43:01 -0600
Subject: [PATCH 009/198] omptarget: fix errors for clang libc++ build

---
 lib/block_orthogonalize.cu                |  2 ++
 lib/dslash_policy.cuh                     |  1 +
 lib/eig_block_trlm.cpp                    |  5 +++--
 lib/restrictor.cu                         |  2 ++
 lib/targets/generic/blas_lapack_eigen.cpp | 19 ++++++++++---------
 lib/targets/omptarget/malloc.cpp          |  2 +-
 6 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/lib/block_orthogonalize.cu b/lib/block_orthogonalize.cu
index bc33e29f0f..35dcce8fcd 100644
--- a/lib/block_orthogonalize.cu
+++ b/lib/block_orthogonalize.cu
@@ -1,3 +1,5 @@
+#include <array>
+
 #include <color_spinor_field.h>
 #include <uint_to_char.h>
 #include <vector>
diff --git a/lib/dslash_policy.cuh b/lib/dslash_policy.cuh
index ce84e1b06b..e39bea6fbe 100644
--- a/lib/dslash_policy.cuh
+++ b/lib/dslash_policy.cuh
@@ -1,3 +1,4 @@
+#include <array>
 #include <tune_quda.h>
 #include <index_helper.cuh>
 
diff --git a/lib/eig_block_trlm.cpp b/lib/eig_block_trlm.cpp
index 2dfb275cb9..09c6c26596 100644
--- a/lib/eig_block_trlm.cpp
+++ b/lib/eig_block_trlm.cpp
@@ -1,6 +1,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
+#include <complex>
 #include <iostream>
 #include <vector>
 #include <algorithm>
@@ -460,8 +461,8 @@ namespace quda
     for (int i = 0; i < blocks; i++) {
       for (int b = 0; b < block_size; b++) {
         idx = b * (block_size + 1);
-        residua[i * block_size + b + num_locked] = fabs(block_beta[n_kr * block_size - block_data_length + idx]
-                                                        * block_ritz_mat[dim * (i * block_size + b + 1) - 1]);
+        residua[i * block_size + b + num_locked] = std::abs(block_beta[n_kr * block_size - block_data_length + idx]
+                                                            * block_ritz_mat[dim * (i * block_size + b + 1) - 1]);
       }
     }
 
diff --git a/lib/restrictor.cu b/lib/restrictor.cu
index 8c5beb03f7..1da207a8a8 100644
--- a/lib/restrictor.cu
+++ b/lib/restrictor.cu
@@ -1,3 +1,5 @@
+#include <array>
+
 #include <color_spinor_field.h>
 #include <tunable_block_reduction.h>
 #include <kernels/restrictor.cuh>
diff --git a/lib/targets/generic/blas_lapack_eigen.cpp b/lib/targets/generic/blas_lapack_eigen.cpp
index 15c8dc503a..ce52f9aea1 100644
--- a/lib/targets/generic/blas_lapack_eigen.cpp
+++ b/lib/targets/generic/blas_lapack_eigen.cpp
@@ -1,3 +1,4 @@
+#include <complex>
 #include <blas_lapack.h>
 #include <eigen_helper.h>
 
@@ -53,7 +54,7 @@ namespace quda
                                   QudaFieldLocation location)
       {
         if (getVerbosity() >= QUDA_VERBOSE)
-          printfQuda("BatchInvertMatrix (generic - Eigen): Nc = %d, batch = %lu\n", n, batch);
+          printfQuda("BatchInvertMatrix (generic - Eigen): Nc = %d, batch = %llu\n", n, batch);
 
         size_t size = 2 * n * n * batch * prec;
         void *A_h = (location == QUDA_CUDA_FIELD_LOCATION ? pool_pinned_malloc(size) : A);
@@ -330,29 +331,29 @@ namespace quda
         if (blas_param.data_type == QUDA_BLAS_DATATYPE_Z) {
 
           typedef std::complex<double> Z;
-          const Z alpha = blas_param.alpha;
-          const Z beta = blas_param.beta;
+          const Z alpha = reinterpret_cast<std::complex<double>&>(blas_param.alpha);
+          const Z beta = reinterpret_cast<std::complex<double>&>(blas_param.beta);
           GEMM<MatrixXcd, Z>(A_h, B_h, C_h, alpha, beta, max_stride, blas_param);
 
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_C) {
 
           typedef std::complex<float> C;
-          const C alpha = blas_param.alpha;
-          const C beta = blas_param.beta;
+          const C alpha(reinterpret_cast<std::complex<double>&>(blas_param.alpha).real(),reinterpret_cast<std::complex<double>&>(blas_param.alpha).imag());
+          const C beta(reinterpret_cast<std::complex<double>&>(blas_param.beta).real(),reinterpret_cast<std::complex<double>&>(blas_param.beta).imag());
           GEMM<MatrixXcf, C>(A_h, B_h, C_h, alpha, beta, max_stride, blas_param);
 
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_D) {
 
           typedef double D;
-          const D alpha = (D)(static_cast<std::complex<double>>(blas_param.alpha).real());
-          const D beta = (D)(static_cast<std::complex<double>>(blas_param.beta).real());
+          const D alpha = (D)(static_cast<std::complex<double>>(reinterpret_cast<std::complex<double>&>(blas_param.alpha)).real());
+          const D beta = (D)(static_cast<std::complex<double>>(reinterpret_cast<std::complex<double>&>(blas_param.beta)).real());
           GEMM<MatrixXd, D>(A_h, B_h, C_h, alpha, beta, max_stride, blas_param);
 
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_S) {
 
           typedef float S;
-          const S alpha = (S)(static_cast<std::complex<float>>(blas_param.alpha).real());
-          const S beta = (S)(static_cast<std::complex<float>>(blas_param.beta).real());
+          const S alpha = (S)(static_cast<std::complex<float>>(reinterpret_cast<std::complex<double>&>(blas_param.alpha)).real());
+          const S beta = (S)(static_cast<std::complex<float>>(reinterpret_cast<std::complex<double>&>(blas_param.beta)).real());
           GEMM<MatrixXf, S>(A_h, B_h, C_h, alpha, beta, max_stride, blas_param);
 
         } else {
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index ad357f8ed5..0983021c87 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -215,7 +215,7 @@ namespace quda
       ptr = omp_target_alloc(size, omp_get_default_device());
     else{
       warningQuda("%s:%d %s() allocate on host instead of device", file, line, func);
-      ptr = std::aligned_alloc(64, size);
+      ptr = aligned_alloc(64, size);
     }
     if (!ptr) {
       errorQuda("Failed to allocate device memory of size %zu (%s:%d in %s())\n", size, file, line, func);

From d08eb5b75f8d4b77351197dc06054e04140d5648 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 11 Dec 2020 13:46:22 -0600
Subject: [PATCH 010/198] omptarget: match cuda changes

---
 include/targets/omptarget/target_device.h | 29 +++++++++++++++++------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index d4c13f4c8c..f7b22a7667 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -30,8 +30,8 @@ namespace quda {
 
     /**
        @brief Helper function that returns the thread block
-       dimensions.  On CUDA this returns the intrinsic blockDim.
-       @param[in] arg Kernel argument struct
+       dimensions.  On CUDA this returns the intrinsic blockDim,
+       whereas on the host this returns (1, 1, 1).
     */
     __device__ __host__ inline dim3 block_dim()
     {
@@ -46,11 +46,26 @@ namespace quda {
     }
 
     /**
-       @brief Helper function that returns the thread block
-       dimensions.  On CUDA this returns the intrinsic blockDim,
-       whereas on the host this grabs the dimensions from the kernel
-       argument struct
-       @param[in] arg Kernel argument struct
+       @brief Helper function that returns the thread indices within a
+       thread block.  On CUDA this returns the intrinsic
+       blockIdx, whereas on the host this just returns (0, 0, 0).
+    */
+    __device__ __host__ inline dim3 block_idx()
+    {
+      return dim3(omp_get_team_num(),0,0);
+/*
+#ifdef __CUDA_ARCH__
+      return dim3(blockIdx.x, blockIdx.y, blockIdx.z);
+#else
+      return dim3(0, 0, 0);
+#endif
+*/
+    }
+
+    /**
+       @brief Helper function that returns the thread indices within a
+       thread block.  On CUDA this returns the intrinsic
+       threadIdx, whereas on the host this just returns (0, 0, 0).
     */
     __device__ __host__ inline dim3 thread_idx()
     {

From c1b7ce5560e38bfb1340e03e0f71c7929df6ec52 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 16 Dec 2020 12:04:13 -0600
Subject: [PATCH 011/198] omptarget: incorporate targets/cuda changes from last
 merge

---
 lib/targets/omptarget/malloc.cpp   |  7 +++--
 lib/targets/omptarget/quda_api.cpp | 12 ++++++++
 lib/targets/omptarget/tune.cpp     | 49 +++++++-----------------------
 3 files changed, 27 insertions(+), 41 deletions(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index 0983021c87..8bcf248247 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -43,10 +43,11 @@ namespace quda
 #endif
     }
 
-    MemAlloc(const MemAlloc &a) = default;
-    MemAlloc(MemAlloc &&a) = default;
+    MemAlloc(const MemAlloc &) = default;
+    MemAlloc(MemAlloc &&) = default;
     virtual ~MemAlloc() = default;
-    MemAlloc &operator=(const MemAlloc &a) = default;
+    MemAlloc &operator=(const MemAlloc &) = default;
+    MemAlloc &operator=(MemAlloc &&) = default;
   };
 
   static std::map<void *, MemAlloc> alloc[N_ALLOC_TYPE];
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index c18088cbd6..964b0348ef 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -540,6 +540,18 @@ namespace quda {
     if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
+  static std::string error_str("CUDA_SUCCESS");
+
+  void qudaSetErrorString(const std::string &error_str_)
+  {
+    error_str = error_str_;
+  }
+
+  std::string qudaGetLastErrorString()
+  {
+    return error_str;
+  }
+
   void printAPIProfile() {
 #ifdef API_PROFILE
     apiTimer.Print();
diff --git a/lib/targets/omptarget/tune.cpp b/lib/targets/omptarget/tune.cpp
index 3dd11fe565..5b6fced824 100644
--- a/lib/targets/omptarget/tune.cpp
+++ b/lib/targets/omptarget/tune.cpp
@@ -22,13 +22,9 @@ extern char *gitversion;
 namespace quda
 {
   static TuneKey last_key;
-}
 
-// intentionally leave this outside of the namespace for now
-quda::TuneKey getLastTuneKey() { return quda::last_key; }
+  TuneKey getLastTuneKey() { return quda::last_key; }
 
-namespace quda
-{
   typedef std::map<TuneKey, TuneParam> map;
 
   struct TraceKey {
@@ -53,28 +49,10 @@ namespace quda
     {
     }
 
-    TraceKey(const TraceKey &trace) :
-      key(trace.key),
-      time(trace.time),
-      device_bytes(trace.device_bytes),
-      pinned_bytes(trace.pinned_bytes),
-      mapped_bytes(trace.mapped_bytes),
-      host_bytes(trace.host_bytes)
-    {
-    }
-
-    TraceKey &operator=(const TraceKey &trace)
-    {
-      if (&trace != this) {
-        key = trace.key;
-        time = trace.time;
-        device_bytes = trace.device_bytes;
-        pinned_bytes = trace.pinned_bytes;
-        mapped_bytes = trace.mapped_bytes;
-        host_bytes = trace.host_bytes;
-      }
-      return *this;
-    }
+    TraceKey(const TraceKey &) = default;
+    TraceKey(TraceKey &&) = default;
+    TraceKey &operator=(const TraceKey &) = default;
+    TraceKey &operator=(TraceKey &&) = default;
   };
 
   // linked list that is augmented each time we call a kernel
@@ -669,7 +647,6 @@ namespace quda
    */
   TuneParam &tuneLaunch(Tunable &tunable, QudaTune enabled, QudaVerbosity verbosity)
   {
-
 #ifdef LAUNCH_TIMER
     launchTimer.TPSTART(QUDA_PROFILE_TOTAL);
     launchTimer.TPSTART(QUDA_PROFILE_INIT);
@@ -796,28 +773,24 @@ namespace quda
           }
 
           elapsed_time /= (1e3 * tunable.tuningIter());
-          if ((elapsed_time < best_time) && (error == cudaSuccess) && (tunable.jitifyError() == CUDA_SUCCESS)) {
+          if ((elapsed_time < best_time) && (error == cudaSuccess) && (tunable.launchError() == QUDA_SUCCESS)) {
             best_time = elapsed_time;
             best_param = param;
           }
           if ((verbosity >= QUDA_DEBUG_VERBOSE)) {
-            if (error == cudaSuccess && tunable.jitifyError() == CUDA_SUCCESS) {
+            if (error == cudaSuccess && tunable.launchError() == QUDA_SUCCESS) {
               printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(),
                          tunable.perfString(elapsed_time).c_str());
             } else {
-              if (tunable.jitifyError() == CUDA_SUCCESS) {
-                // if not jitify error must be regular error
+              if (tunable.launchError() == QUDA_SUCCESS) { // must be regular error
                 printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), cudaGetErrorString(error));
-              } else {
-                // else is a jitify error
-                const char *str;
-                cuGetErrorString(tunable.jitifyError(), &str);
-                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), str);
+              } else { // else must be a manually thrown error
+                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), qudaGetLastErrorString().c_str());
               }
             }
           }
           tuning = tunable.advanceTuneParam(param);
-          tunable.jitifyError() = CUDA_SUCCESS;
+          tunable.launchError() = QUDA_SUCCESS;
         }
 
         tune_timer.Stop(__func__, __FILE__, __LINE__);

From a82a09bc3ea9d897e3333788703cbaa8ba37ca69 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 16 Dec 2020 12:17:21 -0600
Subject: [PATCH 012/198] omptarget: define away the reintroduced CUDA_SUCCESS

---
 include/quda_api.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/include/quda_api.h b/include/quda_api.h
index 83f05de412..2555d0134b 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -64,6 +64,8 @@ T * to_device(const T& x) {
 #define __launch_bounds__(x)
 #define __syncthreads() _Pragma("omp barrier")
 
+#define CUDA_SUCCESS QUDA_SUCCESS
+
 using size_t = std::size_t;
 
 #define __forceinline__ inline __attribute__((always_inline))

From 614ae66cc7d02866481d274d5e8cca806806392c Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 16 Dec 2020 12:22:00 -0600
Subject: [PATCH 013/198] omptarget: make dim3 constexpr

---
 include/quda_api.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/include/quda_api.h b/include/quda_api.h
index 2555d0134b..242d74ff36 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -176,10 +176,10 @@ using cudaIpcEventHandle_t = int;
 using cudaEvent_t = int;
 
 struct dim3 {unsigned int x,y,z;
-  dim3():x(1u),y(1u),z(1u){}
-  dim3(unsigned int x):x(x),y(1u),z(1u){}
-  dim3(unsigned int x,unsigned int y):x(x),y(y),z(1u){}
-  dim3(unsigned int x,unsigned int y,unsigned int z):x(x),y(y),z(z){}
+  constexpr dim3():x(1u),y(1u),z(1u){}
+  constexpr dim3(unsigned int x):x(x),y(1u),z(1u){}
+  constexpr dim3(unsigned int x,unsigned int y):x(x),y(y),z(1u){}
+  constexpr dim3(unsigned int x,unsigned int y,unsigned int z):x(x),y(y),z(z){}
 };
 struct int2 {int x,y;};
 struct int4 {int x,y,z,w;};

From 7acd76a4549fb42036f1a598322462e5a5551f14 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 16 Dec 2020 12:09:15 -0600
Subject: [PATCH 014/198] place alignas proper

---
 include/kernel_helper.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/kernel_helper.h b/include/kernel_helper.h
index 167292b55d..5551e53af0 100644
--- a/include/kernel_helper.h
+++ b/include/kernel_helper.h
@@ -7,7 +7,7 @@ namespace quda {
   struct constant_param_t {
     static constexpr size_t max_size = device::max_constant_param_size();
     size_t bytes;
-    char host[max_size] alignas(16);
+    char host alignas(16) [max_size];
     void *device_ptr;
     char device_name[128];
   };

From 53ea655f091eda21ffac248f30148f0024be3bfd Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 17 Dec 2020 18:09:29 +0000
Subject: [PATCH 015/198] omptarget: fix omp_target_associate_ptr

---
 lib/targets/omptarget/malloc.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index 8bcf248247..60d4cd5b2a 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -331,7 +331,7 @@ namespace quda
       ompwip("require special memcpy",[=](){printfQuda("mapped_malloc_ device: %p\n",dp);});
       if(!dp)
         errorQuda("%s:%d %s() Failed to allocate device memory of size %zu for mapped malloc\n", file, line, func, size);
-      if(!omp_target_associate_ptr(ptr, dp, a.base_size, 0, d))
+      if(omp_target_associate_ptr(ptr, dp, a.base_size, 0, d))
         errorQuda("%s:%d %s() Failed to assocaite device memory to host pointer\n", file, line, func);
     }else
       warningQuda("%s:%d %s() mapped malloc without a device", file, line, func);

From 99e5b0de7a57deb5f2d7aff1920324e90dc08b4b Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 17 Dec 2020 12:11:02 -0600
Subject: [PATCH 016/198] omptarget: reinstate omp target in reduce_nd

---
 include/quda_api.h                 |  1 +
 include/tunable_nd.h               | 27 +++++++++++++++++++++++++++
 lib/targets/omptarget/quda_api.cpp | 23 ++++++++++++++++++++++-
 3 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/include/quda_api.h b/include/quda_api.h
index 242d74ff36..23e09233e7 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -262,6 +262,7 @@ namespace quda
   template <typename T, typename... Arg>
   qudaError_t qudaLaunchKernel(T *func, const TuneParam &tp, qudaStream_t stream, const Arg &...arg)
   {
+    ompwip("directly calling qudaLaunchKernel is unsupported");
     const void *args[] = {&arg...};
     return qudaLaunchKernel(reinterpret_cast<const void *>(func), tp, const_cast<void **>(args), stream);
   }
diff --git a/include/tunable_nd.h b/include/tunable_nd.h
index 42606bb1d3..e70496f799 100644
--- a/include/tunable_nd.h
+++ b/include/tunable_nd.h
@@ -32,6 +32,7 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
 #ifdef JITIFY
       launch_error = launch_jitify<Functor>("quda::Kernel1D", tp, stream, arg, param);
 #else
@@ -44,6 +45,7 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_cuda(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
 #ifdef JITIFY
       launch_error = launch_jitify<Functor>("quda::raw_kernel", tp, stream, arg, param);
 #else
@@ -56,6 +58,7 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_host(const TuneParam &, const qudaStream_t &, const Arg &arg, const std::vector<constant_param_t> & = dummy_param)
     {
+      ompwip();
       Functor<Arg> f(const_cast<Arg &>(arg));
       for (int i = 0; i < (int)arg.threads.x; i++) {
         f(i);
@@ -67,6 +70,7 @@ namespace quda {
       launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg,
              const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
       if (location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg, param);
       } else {
@@ -79,6 +83,7 @@ namespace quda {
       launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg,
              const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
       if (location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg, param);
       } else {
@@ -147,18 +152,34 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip("OFFLOADING...");
 #ifdef JITIFY
       launch_error = launch_jitify<Functor>("quda::Kernel2D", tp, stream, arg, param);
 #else
       for (unsigned int i = 0; i < param.size(); i++)
         qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, cudaMemcpyHostToDevice, stream);
+#ifdef QUDA_BACKEND_OMPTARGET
+      const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
+      const int ld = tp.block.x*tp.block.y*tp.grid.z;
+      const int tx = arg.threads.x;
+      const int ty = arg.threads.y;
+      Functor<Arg> f(const_cast<Arg &>(arg));
+#pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld)
+      for (int i = 0; i < tx; i++) {
+        for (int j = 0; j < ty; j++) {
+          f(i, j);
+        }
+      }
+#else
       qudaLaunchKernel(Kernel2D<Functor, Arg, grid_stride>, tp, stream, arg);
+#endif
 #endif
     }
 
     template <template <typename> class Functor, typename Arg>
     void launch_host(const TuneParam &, const qudaStream_t &, const Arg &arg, const std::vector<constant_param_t> & = dummy_param)
     {
+      ompwip();
       Functor<Arg> f(const_cast<Arg &>(arg));
       for (int i = 0; i < (int)arg.threads.x; i++) {
         for (int j = 0; j < (int)arg.threads.y; j++) {
@@ -172,6 +193,7 @@ namespace quda {
       launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg,
              const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
       const_cast<Arg &>(arg).threads.y = vector_length_y;
       if (TunableKernel1D_base<grid_stride>::location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg, param);
@@ -185,6 +207,7 @@ namespace quda {
       launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg,
              const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
       const_cast<Arg &>(arg).threads.y = vector_length_y;
       if (TunableKernel1D_base<grid_stride>::location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg, param);
@@ -302,6 +325,7 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
 #ifdef JITIFY
       launch_error = launch_jitify<Functor>("quda::Kernel3D", tp, stream, arg, param);
 #else
@@ -314,6 +338,7 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_host(const TuneParam &, const qudaStream_t &, const Arg &arg, const std::vector<constant_param_t> & = dummy_param)
     {
+      ompwip();
       Functor<Arg> f(const_cast<Arg &>(arg));
       for (int i = 0; i < (int)arg.threads.x; i++) {
         for (int j = 0; j < (int)arg.threads.y; j++) {
@@ -329,6 +354,7 @@ namespace quda {
       launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg,
              const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
       const_cast<Arg &>(arg).threads.y = vector_length_y;
       const_cast<Arg &>(arg).threads.z = vector_length_z;
       if (TunableKernel2D_base<grid_stride>::location == QUDA_CUDA_FIELD_LOCATION) {
@@ -343,6 +369,7 @@ namespace quda {
       launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg,
              const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip();
       const_cast<Arg &>(arg).threads.y = vector_length_y;
       const_cast<Arg &>(arg).threads.z = vector_length_z;
       if (TunableKernel2D_base<grid_stride>::location == QUDA_CUDA_FIELD_LOCATION) {
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 964b0348ef..dcb442bcc2 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -203,7 +203,28 @@ namespace quda {
           default: errorQuda("Unsupported cudaMemcpyType %d", kind);
           }
 #else
-          cudaMemcpy(dst, src, count, kind);
+          printfQuda("memcpy %p <- %p\n",dst,src);
+          switch (kind) {
+          case cudaMemcpyHostToHost: memcpy(dst,(void*)src,count); break;
+          case cudaMemcpyHostToDevice:
+            if(0<omp_get_num_devices()) omp_target_memcpy(dst,(void*)src,count,0,0,omp_get_default_device(),omp_get_initial_device());
+            else{warningQuda("cudaMemcpyHostToDevice without a device"); memcpy(dst,(void*)src,count);}
+            break;
+          case cudaMemcpyDeviceToHost:
+            if(0<omp_get_num_devices()) omp_target_memcpy(dst,(void*)src,count,0,0,omp_get_initial_device(),omp_get_default_device());
+            else{warningQuda("cudaMemcpyDeviceToHost without a device"); memcpy(dst,(void*)src,count);}
+            break;
+          case cudaMemcpyDeviceToDevice:
+            warningQuda("unimplemented for cudaMemcpyDeviceToDevice");
+            break;
+          case cudaMemcpyDefault:
+            warningQuda("OMP treating cudaMemcpyDefault as cudaMemcpyHostToDevice");
+            if(0<omp_get_num_devices()) omp_target_memcpy(dst,(void*)src,count,0,0,omp_get_default_device(),omp_get_initial_device());
+            else{warningQuda("cudaMemcpyHostToDevice without a device"); memcpy(dst,(void*)src,count);}
+            break;
+          default: errorQuda("Unsupported cudaMemcpyType %d", kind);
+          }
+          // cudaMemcpy(dst, src, count, kind);
 #endif
         }
       } else {

From 84a8b8858ef2453aeb2ad2e94e36f537da6668cc Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 18 Dec 2020 15:51:49 -0600
Subject: [PATCH 017/198] omptarget: copy cuda changes over

---
 lib/targets/omptarget/blas_lapack_cublas.cpp | 315 ++++++++++++++++++-
 lib/targets/omptarget/quda_api.cpp           |  56 ++--
 2 files changed, 344 insertions(+), 27 deletions(-)

diff --git a/lib/targets/omptarget/blas_lapack_cublas.cpp b/lib/targets/omptarget/blas_lapack_cublas.cpp
index 94f1cf7f9f..36b2672c70 100644
--- a/lib/targets/omptarget/blas_lapack_cublas.cpp
+++ b/lib/targets/omptarget/blas_lapack_cublas.cpp
@@ -1,3 +1,4 @@
+#include <complex.h>
 #include <blas_lapack.h>
 #ifdef NATIVE_LAPACK_LIB
 #include <cublas_v2.h>
@@ -30,6 +31,8 @@ namespace quda
 #ifdef NATIVE_LAPACK_LIB
           cublasStatus_t error = cublasCreate(&handle);
           if (error != CUBLAS_STATUS_SUCCESS) errorQuda("cublasCreate failed with error %d", error);
+          else
+            printfQuda("cublasCreated successfully\n");
           cublas_init = true;
 #endif
         }
@@ -68,11 +71,11 @@ namespace quda
       }
 #endif
 
+#ifdef NATIVE_LAPACK_LIB
       // FIXME do this in pipelined fashion to reduce memory overhead.
       long long BatchInvertMatrix(void *Ainv, void *A, const int n, const uint64_t batch, QudaPrecision prec,
                                   QudaFieldLocation location)
       {
-#ifdef NATIVE_LAPACK_LIB
         init();
         if (getVerbosity() >= QUDA_VERBOSE)
           printfQuda("BatchInvertMatrix (native - cuBLAS): Nc = %d, batch = %lu\n", n, batch);
@@ -84,14 +87,14 @@ namespace quda
         size_t size = 2 * n * n * prec * batch;
         void *A_d = location == QUDA_CUDA_FIELD_LOCATION ? A : pool_device_malloc(size);
         void *Ainv_d = location == QUDA_CUDA_FIELD_LOCATION ? Ainv : pool_device_malloc(size);
-        if (location == QUDA_CPU_FIELD_LOCATION) qudaMemcpy(A_d, A, size, cudaMemcpyHostToDevice);
+        if (location == QUDA_CPU_FIELD_LOCATION) qudaMemcpy(A_d, A, size, qudaMemcpyHostToDevice);
 
 #ifdef _DEBUG
         // Debug code: Copy original A matrix to host
         std::complex<float> *A_h
           = (location == QUDA_CUDA_FIELD_LOCATION ? static_cast<std::complex<float> *>(pool_pinned_malloc(size)) :
                                                     static_cast<std::complex<float> *>(A_d));
-        if (location == QUDA_CUDA_FIELD_LOCATION) qudaMemcpy((void *)A_h, A_d, size, cudaMemcpyDeviceToHost);
+        if (location == QUDA_CUDA_FIELD_LOCATION) qudaMemcpy((void *)A_h, A_d, size, qudaMemcpyDeviceToHost);
 #endif
 
         int *dipiv = static_cast<int *>(pool_device_malloc(batch * n * sizeof(int)));
@@ -109,7 +112,7 @@ namespace quda
             A_array_h[i] = static_cast<C *>(A_d) + i * n * n;
             Ainv_array_h[i] = static_cast<C *>(Ainv_d) + i * n * n;
           }
-          qudaMemcpy(A_array, A_array_h, 2 * batch * sizeof(C *), cudaMemcpyHostToDevice);
+          qudaMemcpy(A_array, A_array_h, 2 * batch * sizeof(C *), qudaMemcpyHostToDevice);
 
           cublasStatus_t error = cublasCgetrfBatched(handle, n, A_array, n, dipiv, dinfo_array, batch);
           flops += batch * FLOPS_CGETRF(n, n);
@@ -117,7 +120,7 @@ namespace quda
           if (error != CUBLAS_STATUS_SUCCESS)
             errorQuda("\nError in LU decomposition (cublasCgetrfBatched), error code = %d\n", error);
 
-          qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), cudaMemcpyDeviceToHost);
+          qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), qudaMemcpyDeviceToHost);
           for (uint64_t i = 0; i < batch; i++) {
             if (info_array[i] < 0) {
               errorQuda("%lu argument had an illegal value or another error occured, such as memory allocation failed",
@@ -133,7 +136,7 @@ namespace quda
           if (error != CUBLAS_STATUS_SUCCESS)
             errorQuda("\nError in matrix inversion (cublasCgetriBatched), error code = %d\n", error);
 
-          qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), cudaMemcpyDeviceToHost);
+          qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), qudaMemcpyDeviceToHost);
 
           for (uint64_t i = 0; i < batch; i++) {
             if (info_array[i] < 0) {
@@ -150,7 +153,7 @@ namespace quda
 #ifdef _DEBUG
           // Debug code: Copy computed Ainv to host
           std::complex<float> *Ainv_h = static_cast<std::complex<float> *>(pool_pinned_malloc(size));
-          qudaMemcpy((void *)Ainv_h, Ainv_d, size, cudaMemcpyDeviceToHost);
+          qudaMemcpy((void *)Ainv_h, Ainv_d, size, qudaMemcpyDeviceToHost);
 
           for (uint64_t i = 0; i < batch; i++) { checkEigen<MatrixXcf, float>(A_h, Ainv_h, n, i); }
           pool_pinned_free(Ainv_h);
@@ -161,7 +164,7 @@ namespace quda
         }
 
         if (location == QUDA_CPU_FIELD_LOCATION) {
-          qudaMemcpy(Ainv, Ainv_d, size, cudaMemcpyDeviceToHost);
+          qudaMemcpy(Ainv, Ainv_d, size, qudaMemcpyDeviceToHost);
           pool_device_free(Ainv_d);
           pool_device_free(A_d);
         }
@@ -180,11 +183,307 @@ namespace quda
           printfQuda("Batched matrix inversion completed in %f seconds with GFLOPS = %f\n", time, 1e-9 * flops / time);
 
         return flops;
+      }
 #else
+      long long BatchInvertMatrix(void *, void *, const int, const uint64_t, QudaPrecision, QudaFieldLocation)
+      {
         errorQuda("Native BLAS not built. Please build and use native BLAS or use generic BLAS");
         return 0; // Stops a compiler warning
+      }
 #endif
+
+#ifdef NATIVE_LAPACK_LIB
+      long long stridedBatchGEMM(void *A_data, void *B_data, void *C_data, QudaBLASParam blas_param,
+                                 QudaFieldLocation location)
+      {
+        long long flops = 0;
+        timeval start, stop;
+        gettimeofday(&start, NULL);
+
+        // Sanity checks on parameters
+        //-------------------------------------------------------------------------
+        // If the user passes non positive M,N, or K, we error out
+        int min_dim = std::min(blas_param.m, std::min(blas_param.n, blas_param.k));
+        if (min_dim <= 0) {
+          errorQuda("BLAS dims must be positive: m=%d, n=%d, k=%d", blas_param.m, blas_param.n, blas_param.k);
+        }
+
+        // If the user passes a negative stride, we error out as this has no meaning.
+        int min_stride = std::min(std::min(blas_param.a_stride, blas_param.b_stride), blas_param.c_stride);
+        if (min_stride < 0) {
+          errorQuda("BLAS strides must be positive or zero: a_stride=%d, b_stride=%d, c_stride=%d", blas_param.a_stride,
+                    blas_param.b_stride, blas_param.c_stride);
+        }
+
+        // If the user passes a negative offset, we error out as this has no meaning.
+        int min_offset = std::min(std::min(blas_param.a_offset, blas_param.b_offset), blas_param.c_offset);
+        if (min_offset < 0) {
+          errorQuda("BLAS offsets must be positive or zero: a_offset=%d, b_offset=%d, c_offset=%d", blas_param.a_offset,
+                    blas_param.b_offset, blas_param.c_offset);
+        }
+
+        // If the batch value is non-positve, we error out
+        if (blas_param.batch_count <= 0) { errorQuda("Batches must be positive: batches=%d", blas_param.batch_count); }
+
+        // Leading dims are dependendent on the matrix op type.
+        if (blas_param.data_order == QUDA_BLAS_DATAORDER_COL) {
+          if (blas_param.trans_a == QUDA_BLAS_OP_N) {
+            if (blas_param.lda < std::max(1, blas_param.m))
+              errorQuda("lda=%d must be >= max(1,m=%d)", blas_param.lda, blas_param.m);
+          } else {
+            if (blas_param.lda < std::max(1, blas_param.k))
+              errorQuda("lda=%d must be >= max(1,k=%d)", blas_param.lda, blas_param.k);
+          }
+
+          if (blas_param.trans_b == QUDA_BLAS_OP_N) {
+            if (blas_param.ldb < std::max(1, blas_param.k))
+              errorQuda("ldb=%d must be >= max(1,k=%d)", blas_param.ldb, blas_param.k);
+          } else {
+            if (blas_param.ldb < std::max(1, blas_param.n))
+              errorQuda("ldb=%d must be >= max(1,n=%d)", blas_param.ldb, blas_param.n);
+          }
+          if (blas_param.ldc < std::max(1, blas_param.m))
+            errorQuda("ldc=%d must be >= max(1,m=%d)", blas_param.ldc, blas_param.m);
+        } else {
+          if (blas_param.trans_a == QUDA_BLAS_OP_N) {
+            if (blas_param.lda < std::max(1, blas_param.k))
+              errorQuda("lda=%d must be >= max(1,k=%d)", blas_param.lda, blas_param.k);
+          } else {
+            if (blas_param.lda < std::max(1, blas_param.m))
+              errorQuda("lda=%d must be >= max(1,m=%d)", blas_param.lda, blas_param.m);
+          }
+          if (blas_param.trans_b == QUDA_BLAS_OP_N) {
+            if (blas_param.ldb < std::max(1, blas_param.n))
+              errorQuda("ldb=%d must be >= max(1,n=%d)", blas_param.ldb, blas_param.n);
+          } else {
+            if (blas_param.ldb < std::max(1, blas_param.k))
+              errorQuda("ldb=%d must be >= max(1,k=%d)", blas_param.ldb, blas_param.k);
+          }
+          if (blas_param.ldc < std::max(1, blas_param.n))
+            errorQuda("ldc=%d must be >= max(1,n=%d)", blas_param.ldc, blas_param.n);
+        }
+        //-------------------------------------------------------------------------
+
+        // Parse parameters for CUBLAS
+        //-------------------------------------------------------------------------
+        // Swap A and B if in row order
+        if (blas_param.data_order == QUDA_BLAS_DATAORDER_ROW) {
+          std::swap(blas_param.m, blas_param.n);
+          std::swap(blas_param.lda, blas_param.ldb);
+          std::swap(blas_param.trans_a, blas_param.trans_b);
+          std::swap(blas_param.a_offset, blas_param.b_offset);
+          std::swap(blas_param.a_stride, blas_param.b_stride);
+          std::swap(A_data, B_data);
+        }
+
+        // Get maximum stride length to deduce the number of batches in the
+        // computation
+        int max_stride = std::max(std::max(blas_param.a_stride, blas_param.b_stride), blas_param.c_stride);
+
+        // If the user gives strides of 0 for all arrays, we are essentially performing
+        // a GEMM on the first matrices in the array N_{batch} times.
+        // Give them what they ask for, YMMV...
+        // If the strides have not been set, we are just using strides of 1.
+        if (max_stride == 0) max_stride = 1;
+
+        // The number of GEMMs to compute
+        const uint64_t batch = blas_param.batch_count / max_stride;
+
+        uint64_t data_size
+          = (blas_param.data_type == QUDA_BLAS_DATATYPE_S || blas_param.data_type == QUDA_BLAS_DATATYPE_C) ? 4 : 8;
+
+        if (blas_param.data_type == QUDA_BLAS_DATATYPE_C || blas_param.data_type == QUDA_BLAS_DATATYPE_Z) {
+          data_size *= 2;
+        }
+
+        // Number of data between batches
+        unsigned int A_batch_size = blas_param.lda * blas_param.k;
+        if (blas_param.trans_a != QUDA_BLAS_OP_N) A_batch_size = blas_param.lda * blas_param.m;
+        unsigned int B_batch_size = blas_param.ldb * blas_param.n;
+        if (blas_param.trans_b != QUDA_BLAS_OP_N) B_batch_size = blas_param.ldb * blas_param.k;
+        unsigned int C_batch_size = blas_param.ldc * blas_param.n;
+
+        // Strides in the cublas param are defaulted to -1. If that remains unchanged,
+        // the stride will be the regular batch size, else the user specified value
+        // is used.
+        unsigned int a_stride = blas_param.a_stride == 0 ? A_batch_size : A_batch_size * blas_param.a_stride;
+        unsigned int b_stride = blas_param.b_stride == 0 ? B_batch_size : B_batch_size * blas_param.b_stride;
+        unsigned int c_stride = blas_param.c_stride == 0 ? C_batch_size : C_batch_size * blas_param.c_stride;
+
+        // Data size of the entire array
+        size_t sizeAarr = A_batch_size * data_size * batch;
+        size_t sizeBarr = B_batch_size * data_size * batch;
+        size_t sizeCarr = C_batch_size * data_size * batch;
+
+        // If already on the device, just use the given pointer. If the data is on
+        // the host, allocate device memory and transfer
+        void *A_d = location == QUDA_CUDA_FIELD_LOCATION ? A_data : pool_device_malloc(sizeAarr);
+        void *B_d = location == QUDA_CUDA_FIELD_LOCATION ? B_data : pool_device_malloc(sizeBarr);
+        void *C_d = location == QUDA_CUDA_FIELD_LOCATION ? C_data : pool_device_malloc(sizeCarr);
+        if (location == QUDA_CPU_FIELD_LOCATION) {
+          qudaMemcpy(A_d, A_data, sizeAarr, qudaMemcpyHostToDevice);
+          qudaMemcpy(B_d, B_data, sizeBarr, qudaMemcpyHostToDevice);
+          qudaMemcpy(C_d, C_data, sizeCarr, qudaMemcpyHostToDevice);
+        }
+
+        cublasOperation_t trans_a = CUBLAS_OP_N;
+        switch (blas_param.trans_a) {
+        case QUDA_BLAS_OP_N: trans_a = CUBLAS_OP_N; break;
+        case QUDA_BLAS_OP_T: trans_a = CUBLAS_OP_T; break;
+        case QUDA_BLAS_OP_C: trans_a = CUBLAS_OP_C; break;
+        default: errorQuda("Unknown QUDA_BLAS_OP type %d\n", blas_param.trans_a);
+        }
+
+        cublasOperation_t trans_b = CUBLAS_OP_N;
+        switch (blas_param.trans_b) {
+        case QUDA_BLAS_OP_N: trans_b = CUBLAS_OP_N; break;
+        case QUDA_BLAS_OP_T: trans_b = CUBLAS_OP_T; break;
+        case QUDA_BLAS_OP_C: trans_b = CUBLAS_OP_C; break;
+        default: errorQuda("Unknown QUDA_BLAS_OP type %d\n", blas_param.trans_b);
+        }
+        //-------------------------------------------------------------------------
+
+        // Call CUBLAS
+        //-------------------------------------------------------------------------
+        if (blas_param.data_type == QUDA_BLAS_DATATYPE_Z) {
+
+          typedef cuDoubleComplex Z;
+
+          const Z alpha = make_double2((double)(static_cast<std::complex<double>>(blas_param.alpha).real()),
+                                       (double)(static_cast<std::complex<double>>(blas_param.alpha).imag()));
+
+          const Z beta = make_double2((double)(static_cast<std::complex<double>>(blas_param.beta).real()),
+                                      (double)(static_cast<std::complex<double>>(blas_param.beta).imag()));
+
+          cublasStatus_t error;
+          if (batch > 1) {
+            error = cublasZgemmStridedBatched(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                              &alpha, (Z *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                              (Z *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
+                                              (Z *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
+
+            if (error != CUBLAS_STATUS_SUCCESS)
+              errorQuda("\nError in cuBLASZGEMMStridedBatched, error code = %d\n", error);
+          } else {
+            error = cublasZgemm(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
+                                (Z *)A_d + blas_param.a_offset, blas_param.lda, (Z *)B_d + blas_param.b_offset,
+                                blas_param.ldb, &beta, (Z *)C_d + blas_param.c_offset, blas_param.ldc);
+
+            if (error != CUBLAS_STATUS_SUCCESS) errorQuda("\nError in cuBLASZGEMM, error code = %d\n", error);
+          }
+        } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_C) {
+
+          typedef cuFloatComplex C;
+
+          const C alpha = make_float2((float)(static_cast<std::complex<double>>(blas_param.alpha).real()),
+                                      (float)(static_cast<std::complex<double>>(blas_param.alpha).imag()));
+
+          const C beta = make_float2((float)(static_cast<std::complex<double>>(blas_param.beta).real()),
+                                     (float)(static_cast<std::complex<double>>(blas_param.beta).imag()));
+
+          cublasStatus_t error;
+          if (batch > 1) {
+            error = cublasCgemmStridedBatched(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                              &alpha, (C *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                              (C *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
+                                              (C *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
+
+            if (error != CUBLAS_STATUS_SUCCESS)
+              errorQuda("\nError in cuBLASCGEMMStridedBatched, error code = %d\n", error);
+          } else {
+            error = cublasCgemm(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
+                                (C *)A_d + blas_param.a_offset, blas_param.lda, (C *)B_d + blas_param.b_offset,
+                                blas_param.ldb, &beta, (C *)C_d + blas_param.c_offset, blas_param.ldc);
+
+            if (error != CUBLAS_STATUS_SUCCESS) errorQuda("\nError in cuBLASCGEMMBatched, error code = %d\n", error);
+          }
+        } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_D) {
+
+          typedef double D;
+
+          const D alpha = (D)(static_cast<std::complex<double>>(blas_param.alpha).real());
+          const D beta = (D)(static_cast<std::complex<double>>(blas_param.beta).real());
+
+          cublasStatus_t error;
+          if (batch > 1) {
+            error = cublasDgemmStridedBatched(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                              &alpha, (D *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                              (D *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
+                                              (D *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
+
+            if (error != CUBLAS_STATUS_SUCCESS)
+              errorQuda("\nError in cuBLASDGEMMStridedBatched, error code = %d\n", error);
+          } else {
+            error = cublasDgemm(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
+                                (D *)A_d + blas_param.a_offset, blas_param.lda, (D *)B_d + blas_param.b_offset,
+                                blas_param.ldb, &beta, (D *)C_d + blas_param.c_offset, blas_param.ldc);
+
+            if (error != CUBLAS_STATUS_SUCCESS) errorQuda("\nError in cuBLASDGEMMBatched, error code = %d\n", error);
+          }
+        } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_S) {
+
+          typedef float S;
+
+          const S alpha = (S)(static_cast<std::complex<float>>(blas_param.alpha).real());
+          const S beta = (S)(static_cast<std::complex<float>>(blas_param.beta).real());
+
+          cublasStatus_t error;
+          if (batch > 1) {
+            error = cublasSgemmStridedBatched(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                              &alpha, (S *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                              (S *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
+                                              (S *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
+
+            if (error != CUBLAS_STATUS_SUCCESS)
+              errorQuda("\nError in cuBLASSGEMMStridedBatched, error code = %d\n", error);
+          } else {
+            error = cublasSgemm(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
+                                (S *)A_d + blas_param.a_offset, blas_param.lda, (S *)B_d + blas_param.b_offset,
+                                blas_param.ldb, &beta, (S *)C_d + blas_param.c_offset, blas_param.ldc);
+
+            if (error != CUBLAS_STATUS_SUCCESS) errorQuda("\nError in cuBLASSGEMMBatched, error code = %d\n", error);
+          }
+        } else {
+          errorQuda("cublasGEMM type %d not implemented\n", blas_param.data_type);
+        }
+        //-------------------------------------------------------------------------
+
+        // Clean up
+        //-------------------------------------------------------------------------
+        if (blas_param.data_order == QUDA_BLAS_DATAORDER_ROW) {
+          std::swap(blas_param.m, blas_param.n);
+          std::swap(blas_param.lda, blas_param.ldb);
+          std::swap(blas_param.trans_a, blas_param.trans_b);
+          std::swap(blas_param.a_offset, blas_param.b_offset);
+          std::swap(blas_param.a_stride, blas_param.b_stride);
+          std::swap(A_data, B_data);
+        }
+
+        if (location == QUDA_CPU_FIELD_LOCATION) {
+          qudaMemcpy(C_data, C_d, sizeCarr, qudaMemcpyDeviceToHost);
+          pool_device_free(A_d);
+          pool_device_free(B_d);
+          pool_device_free(C_d);
+        }
+
+        qudaDeviceSynchronize();
+        gettimeofday(&stop, NULL);
+        long ds = stop.tv_sec - start.tv_sec;
+        long dus = stop.tv_usec - start.tv_usec;
+        double time = ds + 0.000001 * dus;
+        if (getVerbosity() >= QUDA_DEBUG_VERBOSE)
+          printfQuda("Batched matrix GEMM completed in %f seconds with GFLOPS = %f\n", time, 1e-9 * flops / time);
+        //-------------------------------------------------------------------------
+
+        return flops;
       }
+#else
+      long long stridedBatchGEMM(void *, void *, void *, QudaBLASParam, QudaFieldLocation)
+      {
+        errorQuda("Native BLAS not built. Please build and use native BLAS or use generic BLAS");
+        return 0;
+      }
+#endif
 
     } // namespace native
   }   // namespace blas_lapack
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index dcb442bcc2..6b902944d1 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -28,6 +28,24 @@ using CUdeviceptr = void*;
 
 namespace quda {
 
+  // Agnostic way to return a cuda API flag
+  namespace {
+    inline
+    cudaMemcpyKind qudaMemcpyKindToAPI( const qudaMemcpyKind& k)
+    {
+      switch(k) {
+      case qudaMemcpyHostToHost : return cudaMemcpyHostToHost; break;
+      case qudaMemcpyHostToDevice : return cudaMemcpyHostToDevice; break;
+      case qudaMemcpyDeviceToHost : return cudaMemcpyDeviceToHost; break;
+      case qudaMemcpyDeviceToDevice : return cudaMemcpyDeviceToDevice; break;
+      case qudaMemcpyDefault : return cudaMemcpyDefault; break;
+      default:
+	errorQuda(" unknown value for qudaMemcpyKind %d", static_cast<int>(k));
+	return cudaMemcpyDefault; // keep warnings away
+      }
+    }
+  }
+  
   // No need to abstract these across the library so keep these definitions local to CUDA target
 
   /**
@@ -255,43 +273,43 @@ namespace quda {
     long long bytes() const { return kind == cudaMemcpyDeviceToDevice ? 2*count : count; }
   };
 
-  void qudaMemcpy_(void *dst, const void *src, size_t count, cudaMemcpyKind kind,
+  void qudaMemcpy_(void *dst, const void *src, size_t count, qudaMemcpyKind kind,
                    const char *func, const char *file, const char *line) {
     if (count == 0) return;
-    QudaMem copy(dst, src, count, kind, device::get_default_stream(), false, func, file, line);
+    QudaMem copy(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_default_stream(), false, func, file, line);
     cudaError_t error = cudaGetLastError();
     if (error != cudaSuccess)
       errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
-  void qudaMemcpyAsync_(void *dst, const void *src, size_t count, cudaMemcpyKind kind, const qudaStream_t &stream,
+  void qudaMemcpyAsync_(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const qudaStream_t &stream,
                         const char *func, const char *file, const char *line)
   {
     if (count == 0) return;
 
-    if (kind == cudaMemcpyDeviceToDevice) {
-      QudaMem copy(dst, src, count, kind, stream, true, func, file, line);
+    if (kind == qudaMemcpyDeviceToDevice) {
+      QudaMem copy(dst, src, count, qudaMemcpyKindToAPI(kind), stream, true, func, file, line);
     } else {
 #ifdef USE_DRIVER_API
       switch (kind) {
-      case cudaMemcpyDeviceToHost:
+      case qudaMemcpyDeviceToHost:
         PROFILE(cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
         break;
-      case cudaMemcpyHostToDevice:
+      case qudaMemcpyHostToDevice:
         PROFILE(cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
         break;
-      case cudaMemcpyDeviceToDevice:
+      case qudaMemcpyDeviceToDevice:
         PROFILE(cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
         break;
-      case cudaMemcpyDefault:
+      case qudaMemcpyDefault:
         PROFILE(cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
         break;
       default:
         errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
       }
 #else
-      PROFILE(cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)),
-              kind == cudaMemcpyDeviceToHost ? QUDA_PROFILE_MEMCPY_D2H_ASYNC : QUDA_PROFILE_MEMCPY_H2D_ASYNC);
+      PROFILE(cudaMemcpyAsync(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)),
+              kind == qudaMemcpyDeviceToHost ? QUDA_PROFILE_MEMCPY_D2H_ASYNC : QUDA_PROFILE_MEMCPY_H2D_ASYNC);
 #endif
     }
   }
@@ -306,7 +324,7 @@ namespace quda {
   }
 
   void qudaMemcpy2D_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
-                     cudaMemcpyKind kind, const char *func, const char *file, const char *line)
+                     qudaMemcpyKind kind, const char *func, const char *file, const char *line)
   {
 #ifdef USE_DRIVER_API
     CUDA_MEMCPY2D param;
@@ -320,13 +338,13 @@ namespace quda {
     param.Height = height;
 
     switch (kind) {
-    case cudaMemcpyDeviceToHost:
+    case qudaMemcpyDeviceToHost:
       param.srcDevice = (CUdeviceptr)src;
       param.srcMemoryType = CU_MEMORYTYPE_DEVICE;
       param.dstHost = dst;
       param.dstMemoryType = CU_MEMORYTYPE_HOST;
       break;
-    default: errorQuda("Unsupported cuMemcpyType2DAsync %d", kind);
+    default: errorQuda("Unsupported cuMemcpyType2DAsync %d", qudaMemcpyKindToAPI(kind));
     }
     PROFILE(auto error = cuMemcpy2D(&param), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
     if (error != CUDA_SUCCESS) {
@@ -335,14 +353,14 @@ namespace quda {
       errorQuda("cuMemcpy2D returned error %s\n (%s:%s in %s())", str, file, line, func);
     }
 #else
-    PROFILE(auto error = cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
+    PROFILE(auto error = cudaMemcpy2D(dst, dpitch, src, spitch, width, height, qudaMemcpyKindToAPI(kind)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
     if (error != cudaSuccess)
       errorQuda("cudaMemcpy2D returned error %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
 #endif
   }
 
   void qudaMemcpy2DAsync_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
-                          cudaMemcpyKind kind, const qudaStream_t &stream, const char *func, const char *file,
+                          qudaMemcpyKind kind, const qudaStream_t &stream, const char *func, const char *file,
                           const char *line)
   {
 #ifdef USE_DRIVER_API
@@ -357,14 +375,14 @@ namespace quda {
     param.Height = height;
 
     switch (kind) {
-    case cudaMemcpyDeviceToHost:
+    case qudaMemcpyDeviceToHost:
       param.srcDevice = (CUdeviceptr)src;
       param.srcMemoryType = CU_MEMORYTYPE_DEVICE;
       param.dstHost = dst;
       param.dstMemoryType = CU_MEMORYTYPE_HOST;
       break;
     default:
-      errorQuda("Unsupported cuMemcpyType2DAsync %d", kind);
+      errorQuda("Unsupported cuMemcpyType2DAsync %d", qudaMemcpyKindToAPI(kind));
     }
     PROFILE(auto error = cuMemcpy2DAsync(&param, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
     if (error != CUDA_SUCCESS) {
@@ -373,7 +391,7 @@ namespace quda {
       errorQuda("cuMemcpy2DAsync returned error %s\n (%s:%s in %s())", str, file, line, func);
     }
 #else
-    PROFILE(auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
+    PROFILE(auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
     if (error != cudaSuccess)
       errorQuda("cudaMemcpy2DAsync returned error %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
 #endif

From cd30620f771e5afc7ca3485aa6d5afd01a85de83 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 10 Feb 2021 22:01:31 +0000
Subject: [PATCH 018/198] local tmp

---
 include/kernels/random_init.cuh      |  55 ++
 include/kernels/reduce_init.cuh      |  25 +
 include/kernels/transform_reduce.cuh |  59 ++
 include/reducer.h                    |  79 +++
 include/targets/cuda/aos.h           |  69 +++
 include/targets/cuda/quda_cuda_api.h |  22 +
 include/targets/hip/aos.h            |  27 +
 lib/interface/fortran_interface.cpp  | 253 ++++++++
 lib/targets/cuda/comm_target.cpp     | 156 +++++
 lib/transform_reduce.cu              | 184 ++++++
 lib/tune.cpp                         | 864 +++++++++++++++++++++++++++
 11 files changed, 1793 insertions(+)
 create mode 100644 include/kernels/random_init.cuh
 create mode 100644 include/kernels/reduce_init.cuh
 create mode 100644 include/kernels/transform_reduce.cuh
 create mode 100644 include/reducer.h
 create mode 100644 include/targets/cuda/aos.h
 create mode 100644 include/targets/cuda/quda_cuda_api.h
 create mode 100644 include/targets/hip/aos.h
 create mode 100644 lib/interface/fortran_interface.cpp
 create mode 100644 lib/targets/cuda/comm_target.cpp
 create mode 100644 lib/transform_reduce.cu
 create mode 100644 lib/tune.cpp

diff --git a/include/kernels/random_init.cuh b/include/kernels/random_init.cuh
new file mode 100644
index 0000000000..921cca29f5
--- /dev/null
+++ b/include/kernels/random_init.cuh
@@ -0,0 +1,55 @@
+#pragma once
+
+#include <random_helper.h>
+#include <lattice_field.h>
+#include <index_helper.cuh>
+#include <comm_quda.h>
+#include <kernel.h>
+
+namespace quda {
+
+  struct rngArg {
+    int commCoord[QUDA_MAX_DIM];
+    int X[QUDA_MAX_DIM];
+    int X_global[QUDA_MAX_DIM];
+    RNGState *state;
+    unsigned long long seed;
+    dim3 threads;
+    rngArg(RNGState *state, unsigned long long seed, const LatticeField &meta) :
+      state(state),
+      seed(seed),
+      threads(meta.VolumeCB(), meta.SiteSubset(), 1)
+    {
+      for (int i=0; i<4; i++) {
+        commCoord[i] = comm_coord(i);
+        X[i] = meta.X()[i];
+        X_global[i] = X[i] * comm_dim(i);
+      }
+    }
+  };
+
+  /**
+     @brief CUDA kernel to initialize CURAND RNG states
+     @param state CURAND RNG state array
+     @param seed initial seed for RNG
+     @param size size of the CURAND RNG state array
+     @param arg Metadata needed for computing multi-gpu offsets
+  */
+  template <typename Arg>
+  struct init_random {
+    Arg &arg;
+    __device__ constexpr init_random(Arg &arg) : arg(arg) {}
+    static constexpr const char *filename() { return KERNEL_FILE; }
+
+    __device__ inline void operator()(int id, int parity)
+    {
+      // Each thread gets same seed, a different sequence number, no offset
+      int x[4];
+      getCoords(x, id, arg.X, parity);
+      for (int i = 0; i < 4; i++) x[i] += arg.commCoord[i] * arg.X[i];
+      int idd = (((x[3] * arg.X_global[2] + x[2]) * arg.X_global[1]) + x[1]) * arg.X_global[0] + x[0];
+      random_init(arg.seed, idd, 0, arg.state[parity * arg.threads.x + id]);
+    }
+  };
+
+}
diff --git a/include/kernels/reduce_init.cuh b/include/kernels/reduce_init.cuh
new file mode 100644
index 0000000000..40427c8dd9
--- /dev/null
+++ b/include/kernels/reduce_init.cuh
@@ -0,0 +1,25 @@
+#include <reduce_helper.h>
+#include <kernel.h>
+
+namespace quda {
+
+  namespace reducer {
+
+    template <typename T_> struct init_arg {
+      using T = T_;
+      T *count;
+      dim3 threads;
+      init_arg(T *count) :
+        count(count),
+        threads(max_n_reduce(), 1, 1) {}
+    };
+
+    template <typename Arg> struct init_count {
+      Arg &arg;
+      static constexpr const char *filename() { return KERNEL_FILE; }
+      constexpr init_count(Arg &arg) : arg(arg) {}
+      __device__ void operator()(int i) { new (arg.count + i) typename Arg::T {0}; }
+    };
+
+  }
+}
diff --git a/include/kernels/transform_reduce.cuh b/include/kernels/transform_reduce.cuh
new file mode 100644
index 0000000000..eab0326120
--- /dev/null
+++ b/include/kernels/transform_reduce.cuh
@@ -0,0 +1,59 @@
+#pragma once
+
+#include <reduction_kernel.h>
+#include <limits>
+
+namespace quda {
+
+  template <typename reduce_t, typename T, typename count_t, typename transformer, typename reducer_>
+  struct TransformReduceArg : public ReduceArg<reduce_t> {
+    using reducer = reducer_;
+    static constexpr int n_batch_max = 8;
+    const T *v[n_batch_max];
+    count_t n_items;
+    int n_batch;
+    reduce_t init_value;
+    transformer h;
+    reducer r;
+    dim3 threads;
+
+    TransformReduceArg(const std::vector<T *> &v, count_t n_items, transformer h, reduce_t init_value, reducer r) :
+      ReduceArg<reduce_t>(v.size()),
+      n_items(n_items),
+      n_batch(v.size()),
+      init_value(init_value),
+      h(h),
+      r(r),
+      threads(n_items, n_batch, 1)
+    {
+      if (n_batch > n_batch_max) errorQuda("Requested batch %d greater than max supported %d", n_batch, n_batch_max);
+      if (n_items > std::numeric_limits<count_t>::max())
+        errorQuda("Requested size %lu greater than max supported %lu",
+                  (uint64_t)n_items, (uint64_t)std::numeric_limits<count_t>::max());
+      memcpy(this->v, v.data(), v.size() * sizeof(T*));
+    }
+
+    __device__ __host__ reduce_t init() const { return init_value; }
+  };
+
+  template <typename Arg> struct transform_reducer {
+    using count_t = decltype(Arg::n_items);
+    using reduce_t = decltype(Arg::init_value);
+
+    Arg &arg;
+    static constexpr const char *filename() { return KERNEL_FILE; }
+    constexpr transform_reducer(Arg &arg) : arg(arg) {}
+
+    static constexpr bool do_sum = Arg::reducer::do_sum;
+
+    __device__ __host__ inline reduce_t operator()(reduce_t a, reduce_t b) const { return arg.r(a, b); }
+
+    __device__ __host__ inline reduce_t operator()(reduce_t &value, count_t i, int j, int)
+    {
+      auto v = arg.v[j];
+      auto t = arg.h(v[i]);
+      return arg.r(t, value);
+    }
+  };
+
+}
diff --git a/include/reducer.h b/include/reducer.h
new file mode 100644
index 0000000000..d59c33c1f9
--- /dev/null
+++ b/include/reducer.h
@@ -0,0 +1,79 @@
+#pragma once
+
+namespace quda {
+
+  template <typename T> struct plus {
+    static constexpr bool do_sum = true;
+    __device__ __host__ T operator()(T a, T b) const { return a + b; }
+  };
+
+  template <typename T> struct maximum {
+    static constexpr bool do_sum = false;
+    __device__ __host__ T operator()(T a, T b) const { return a > b ? a : b; }
+  };
+
+  template <typename T> struct minimum {
+    static constexpr bool do_sum = false;
+    __device__ __host__ T operator()(T a, T b) const { return a < b ? a : b; }
+  };
+
+  template <typename T> struct identity {
+    static constexpr bool do_sum = false;
+    __device__ __host__ T operator()(T a) const { return a; }
+  };
+
+
+  template<typename ReduceType, typename Float> struct square_ {
+    square_(ReduceType = 1.0) { }
+    __host__ __device__ inline ReduceType operator()(const quda::complex<Float> &x)
+    { return static_cast<ReduceType>(norm(x)); }
+  };
+
+  template <typename ReduceType> struct square_<ReduceType, int8_t> {
+    const ReduceType scale;
+  square_(const ReduceType scale) : scale(scale) { }
+    __host__ __device__ inline ReduceType operator()(const quda::complex<int8_t> &x)
+    { return norm(scale * complex<ReduceType>(x.real(), x.imag())); }
+  };
+
+  template<typename ReduceType> struct square_<ReduceType,short> {
+    const ReduceType scale;
+    square_(const ReduceType scale) : scale(scale) { }
+    __host__ __device__ inline ReduceType operator()(const quda::complex<short> &x)
+    { return norm(scale * complex<ReduceType>(x.real(), x.imag())); }
+  };
+
+  template<typename ReduceType> struct square_<ReduceType,int> {
+    const ReduceType scale;
+    square_(const ReduceType scale) : scale(scale) { }
+    __host__ __device__ inline ReduceType operator()(const quda::complex<int> &x)
+    { return norm(scale * complex<ReduceType>(x.real(), x.imag())); }
+  };
+
+  template<typename Float, typename storeFloat> struct abs_ {
+    abs_(const Float = 1.0) { }
+    __host__ __device__ Float operator()(const quda::complex<storeFloat> &x) { return abs(x); }
+  };
+
+  template <typename Float> struct abs_<Float, int8_t> {
+    Float scale;
+    abs_(const Float scale) : scale(scale) { }
+    __host__ __device__ Float operator()(const quda::complex<int8_t> &x)
+    { return abs(scale * complex<Float>(x.real(), x.imag())); }
+  };
+
+  template<typename Float> struct abs_<Float,short> {
+    Float scale;
+    abs_(const Float scale) : scale(scale) { }
+    __host__ __device__ Float operator()(const quda::complex<short> &x)
+    { return abs(scale * complex<Float>(x.real(), x.imag())); }
+  };
+
+  template<typename Float> struct abs_<Float,int> {
+    Float scale;
+    abs_(const Float scale) : scale(scale) { }
+    __host__ __device__ Float operator()(const quda::complex<int> &x)
+    { return abs(scale * complex<Float>(x.real(), x.imag())); }
+  };
+
+}
diff --git a/include/targets/cuda/aos.h b/include/targets/cuda/aos.h
new file mode 100644
index 0000000000..bbd207f117
--- /dev/null
+++ b/include/targets/cuda/aos.h
@@ -0,0 +1,69 @@
+#pragma once
+
+// trove requires CUDA and has issues with device debug
+#if !defined(DEVICE_DEBUG)
+#include <trove/ptr.h>
+#else
+#define DISABLE_TROVE
+#endif
+
+namespace quda {
+
+  /**
+     @brief This is just a dummy structure we use for trove to define the
+     required structure size
+     @param real Real number type
+     @param length Number of elements in the structure
+  */
+  template <typename T, int n> struct S {
+    T v[n];
+    __host__ __device__ const T &operator[](int i) const { return v[i]; }
+    __host__ __device__ T &operator[](int i) { return v[i]; }
+  };
+
+  template <typename T, int n> __host__ __device__ void block_load(T out[n], const T *in)
+  {
+    using struct_t = S<T, n>;
+#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
+    struct_t v = *(trove::coalesced_ptr<const struct_t>(reinterpret_cast<const struct_t*>(in)));
+#else
+    struct_t v = *(reinterpret_cast<const struct_t*>(in));
+#endif
+
+#pragma unroll
+    for (int i = 0; i < n; i++) out[i] = v[i];
+  }
+
+  template <typename T, int n> __host__ __device__ void block_store(T *out, const T in[n])
+  {
+    using struct_t = S<T, n>;
+    struct_t v;
+#pragma unroll
+    for (int i = 0; i < n; i++) v[i] = in[i];
+
+#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
+    *(trove::coalesced_ptr<struct_t>(reinterpret_cast<struct_t *>(out))) = v;
+#else
+    *(reinterpret_cast<struct_t*>(out)) = v;
+#endif
+  }
+
+  template <typename T> __host__ __device__ void block_load(T &out, const T *in)
+  {
+#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
+    out = *(trove::coalesced_ptr<const T>(in));
+#else
+    out = *(in);
+#endif
+  }
+
+  template <typename T> __host__ __device__ void block_store(T *out, const T &in)
+  {
+#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
+    *(trove::coalesced_ptr<T>(out)) = in;
+#else
+    *out = in;
+#endif
+  }
+
+}
diff --git a/include/targets/cuda/quda_cuda_api.h b/include/targets/cuda/quda_cuda_api.h
new file mode 100644
index 0000000000..3ca628cee1
--- /dev/null
+++ b/include/targets/cuda/quda_cuda_api.h
@@ -0,0 +1,22 @@
+#include <cuda.h>
+#include <cuda_runtime.h>
+
+namespace quda {
+
+  /**
+     @file quda_cuda_api.h
+     @brief Header file that declares some functions that will be called from within the CUDA target
+  */
+
+  namespace cuda {
+
+    void set_runtime_error(cudaError_t error, const char *api_func, const char *func, const char *file, const char *line,
+                           bool allow_error = false);
+
+    // defined in quda_api.cpp
+    void set_driver_error(CUresult error, const char *api_func, const char *func, const char *file, const char *line,
+                          bool allow_error = false);
+
+
+  }
+}
diff --git a/include/targets/hip/aos.h b/include/targets/hip/aos.h
new file mode 100644
index 0000000000..262c46e51d
--- /dev/null
+++ b/include/targets/hip/aos.h
@@ -0,0 +1,27 @@
+#pragma once
+
+namespace quda {
+
+  template <typename T, int n> __host__ __device__ void block_load(T out[n], const T *in)
+  {
+#pragma unroll
+    for (int i = 0; i < n; i++) out[i] = in[i];
+  }
+
+  template <typename T, int n> __host__ __device__ void block_store(T *out, const T in[n])
+  {
+#pragma unroll
+    for (int i = 0; i < n; i++) out[i] = in[i];
+  }
+
+  template <typename T> __host__ __device__ void block_load(T &out, const T *in)
+  {
+    out = *in;
+  }
+
+  template <typename T> __host__ __device__ void block_store(T *out, const T &in)
+  {
+    *out = in;
+  }
+
+}
diff --git a/lib/interface/fortran_interface.cpp b/lib/interface/fortran_interface.cpp
new file mode 100644
index 0000000000..8ce312d44b
--- /dev/null
+++ b/lib/interface/fortran_interface.cpp
@@ -0,0 +1,253 @@
+#include <quda.h>
+#include <quda_fortran.h>
+#include <quda_api.h>
+#include <malloc_quda.h>
+#include <util_quda.h>
+#include <gauge_field.h>
+
+/*
+  @file fortran_interface.cpp
+
+  @brief The following functions are for the Fortran interface, which
+  is utilized by the TIFR and BQCD applications.
+*/
+
+// local function declarations from interface_quda.cpp that are not exported
+namespace quda {
+  GaugeField* getResidentGauge();
+}
+void freeSloppyGaugeQuda();
+
+void init_quda_(int *dev) { initQuda(*dev); }
+void init_quda_device_(int *dev) { initQudaDevice(*dev); }
+void init_quda_memory_() { initQudaMemory(); }
+void end_quda_() { endQuda(); }
+void load_gauge_quda_(void *h_gauge, QudaGaugeParam *param) { loadGaugeQuda(h_gauge, param); }
+void free_gauge_quda_() { freeGaugeQuda(); }
+void free_sloppy_gauge_quda_() { freeSloppyGaugeQuda(); }
+void load_clover_quda_(void *h_clover, void *h_clovinv, QudaInvertParam *inv_param)
+{ loadCloverQuda(h_clover, h_clovinv, inv_param); }
+void free_clover_quda_(void) { freeCloverQuda(); }
+void dslash_quda_(void *h_out, void *h_in, QudaInvertParam *inv_param,
+    QudaParity *parity) { dslashQuda(h_out, h_in, inv_param, *parity); }
+void clover_quda_(void *h_out, void *h_in, QudaInvertParam *inv_param,
+    QudaParity *parity, int *inverse) { cloverQuda(h_out, h_in, inv_param, *parity, *inverse); }
+void mat_quda_(void *h_out, void *h_in, QudaInvertParam *inv_param)
+{ MatQuda(h_out, h_in, inv_param); }
+void mat_dag_mat_quda_(void *h_out, void *h_in, QudaInvertParam *inv_param)
+{ MatDagMatQuda(h_out, h_in, inv_param); }
+void invert_quda_(void *hp_x, void *hp_b, QudaInvertParam *param) {
+  fflush(stdout);
+  // ensure that fifth dimension is set to 1
+  if (param->dslash_type == QUDA_ASQTAD_DSLASH || param->dslash_type == QUDA_STAGGERED_DSLASH) param->Ls = 1;
+  invertQuda(hp_x, hp_b, param);
+  fflush(stdout);
+}
+
+void invert_multishift_quda_(void *h_x, void *hp_b, QudaInvertParam *param)
+{
+  if (!quda::getResidentGauge()) errorQuda("Resident gauge field not allocated");
+
+  auto &U = *quda::getResidentGauge();
+
+  // ensure that fifth dimension is set to 1
+  if (param->dslash_type == QUDA_ASQTAD_DSLASH || param->dslash_type == QUDA_STAGGERED_DSLASH) param->Ls = 1;
+
+  // get data into array of pointers
+  int nSpin = (param->dslash_type == QUDA_STAGGERED_DSLASH || param->dslash_type == QUDA_ASQTAD_DSLASH) ? 1 : 4;
+
+  // compute offset assuming TIFR padded ordering (FIXME)
+  if (param->dirac_order != QUDA_TIFR_PADDED_DIRAC_ORDER)
+    errorQuda("Fortran multi-shift solver presently only supports QUDA_TIFR_PADDED_DIRAC_ORDER");
+
+  const int *X = U.X();
+  size_t cb_offset = (X[0]/2) * X[1] * (X[2] + 4) * X[3] * U.Ncolor() * nSpin * 2 * param->cpu_prec;
+  void *hp_x[QUDA_MAX_MULTI_SHIFT];
+  for (int i=0; i<param->num_offset; i++) hp_x[i] = static_cast<char*>(h_x) + i*cb_offset;
+
+  invertMultiShiftQuda(hp_x, hp_b, param);
+}
+
+void flush_chrono_quda_(int *index) { flushChronoQuda(*index); }
+
+void register_pinned_quda_(void *ptr, size_t *bytes) {
+  register_pinned(ptr, *bytes);
+}
+
+void unregister_pinned_quda_(void *ptr) {
+  unregister_pinned(ptr);
+}
+
+void new_quda_gauge_param_(QudaGaugeParam *param) {
+  *param = newQudaGaugeParam();
+}
+void new_quda_invert_param_(QudaInvertParam *param) {
+  *param = newQudaInvertParam();
+}
+
+void update_gauge_field_quda_(void *gauge, void *momentum, double *dt,
+    bool *conj_mom, bool *exact,
+    QudaGaugeParam *param) {
+  updateGaugeFieldQuda(gauge, momentum, *dt, (int)*conj_mom, (int)*exact, param);
+}
+
+static inline int opp(int dir) { return 7-dir; }
+
+static void createGaugeForcePaths(int **paths, int dir, int num_loop_types){
+
+  int index=0;
+  // Plaquette paths
+  if (num_loop_types >= 1)
+    for(int i=0; i<4; ++i){
+      if(i==dir) continue;
+      paths[index][0] = i;        paths[index][1] = opp(dir);   paths[index++][2] = opp(i);
+      paths[index][0] = opp(i);   paths[index][1] = opp(dir);   paths[index++][2] = i;
+    }
+
+  // Rectangle Paths
+  if (num_loop_types >= 2)
+    for(int i=0; i<4; ++i){
+      if(i==dir) continue;
+      paths[index][0] = paths[index][1] = i;       paths[index][2] = opp(dir); paths[index][3] = paths[index][4] = opp(i);
+      index++;
+      paths[index][0] = paths[index][1] = opp(i);  paths[index][2] = opp(dir); paths[index][3] = paths[index][4] = i;
+      index++;
+      paths[index][0] = dir; paths[index][1] = i; paths[index][2] = paths[index][3] = opp(dir); paths[index][4] = opp(i);
+      index++;
+      paths[index][0] = dir; paths[index][1] = opp(i); paths[index][2] = paths[index][3] = opp(dir); paths[index][4] = i;
+      index++;
+      paths[index][0] = i;  paths[index][1] = paths[index][2] = opp(dir); paths[index][3] = opp(i); paths[index][4] = dir;
+      index++;
+      paths[index][0] = opp(i);  paths[index][1] = paths[index][2] = opp(dir); paths[index][3] = i; paths[index][4] = dir;
+      index++;
+    }
+
+  if (num_loop_types >= 3) {
+    // Staple paths
+    for(int i=0; i<4; ++i){
+      for(int j=0; j<4; ++j){
+	if(i==dir || j==dir || i==j) continue;
+	paths[index][0] = i; paths[index][1] = j; paths[index][2] = opp(dir); paths[index][3] = opp(i), paths[index][4] = opp(j);
+	index++;
+	paths[index][0] = i; paths[index][1] = opp(j); paths[index][2] = opp(dir); paths[index][3] = opp(i), paths[index][4] = j;
+	index++;
+	paths[index][0] = opp(i); paths[index][1] = j; paths[index][2] = opp(dir); paths[index][3] = i, paths[index][4] = opp(j);
+	index++;
+	paths[index][0] = opp(i); paths[index][1] = opp(j); paths[index][2] = opp(dir); paths[index][3] = i, paths[index][4] = j;
+	index++;
+     }
+    }
+  }
+
+}
+
+void compute_gauge_force_quda_(void *mom, void *gauge, int *num_loop_types, double *coeff, double *dt,
+			       QudaGaugeParam *param) {
+
+  int numPaths = 0;
+  switch (*num_loop_types) {
+  case 1:
+    numPaths = 6;
+    break;
+  case 2:
+    numPaths = 24;
+    break;
+  case 3:
+    numPaths = 48;
+    break;
+  default:
+    errorQuda("Invalid num_loop_types = %d\n", *num_loop_types);
+  }
+
+  auto *loop_coeff = static_cast<double*>(safe_malloc(numPaths*sizeof(double)));
+  int *path_length = static_cast<int*>(safe_malloc(numPaths*sizeof(int)));
+
+  if (*num_loop_types >= 1) for(int i= 0; i< 6; ++i) {
+      loop_coeff[i] = coeff[0];
+      path_length[i] = 3;
+    }
+  if (*num_loop_types >= 2) for(int i= 6; i<24; ++i) {
+      loop_coeff[i] = coeff[1];
+      path_length[i] = 5;
+    }
+  if (*num_loop_types >= 3) for(int i=24; i<48; ++i) {
+      loop_coeff[i] = coeff[2];
+      path_length[i] = 5;
+    }
+
+  int** input_path_buf[4];
+  for(int dir=0; dir<4; ++dir){
+    input_path_buf[dir] = static_cast<int**>(safe_malloc(numPaths*sizeof(int*)));
+    for(int i=0; i<numPaths; ++i){
+      input_path_buf[dir][i] = static_cast<int*>(safe_malloc(path_length[i]*sizeof(int)));
+    }
+    createGaugeForcePaths(input_path_buf[dir], dir, *num_loop_types);
+  }
+
+  int max_length = 6;
+
+  computeGaugeForceQuda(mom, gauge, input_path_buf, path_length, loop_coeff, numPaths, max_length, *dt, param);
+
+  for(auto & dir : input_path_buf){
+    for(int i=0; i<numPaths; ++i) host_free(dir[i]);
+    host_free(dir);
+  }
+
+  host_free(path_length);
+  host_free(loop_coeff);
+}
+
+void compute_staggered_force_quda_(void* h_mom, double *dt, double *delta, void *gauge, void *x, QudaGaugeParam *gauge_param, QudaInvertParam *inv_param) {
+  computeStaggeredForceQuda(h_mom, *dt, *delta, gauge, (void**)x, gauge_param, inv_param);
+}
+
+// apply the staggered phases
+void apply_staggered_phase_quda_() {
+  if (getVerbosity() >= QUDA_VERBOSE) printfQuda("applying staggered phase\n");
+  if (quda::getResidentGauge()) {
+    quda::getResidentGauge()->applyStaggeredPhase();
+  } else {
+    errorQuda("No persistent gauge field");
+  }
+}
+
+// remove the staggered phases
+void remove_staggered_phase_quda_() {
+  if (getVerbosity() >= QUDA_VERBOSE) printfQuda("removing staggered phase\n");
+  if (quda::getResidentGauge()) {
+    quda::getResidentGauge()->removeStaggeredPhase();
+  } else {
+    errorQuda("No persistent gauge field");
+  }
+  qudaDeviceSynchronize();
+}
+
+// evaluate the kinetic term
+void kinetic_quda_(double *kin, void* momentum, QudaGaugeParam* param) {
+  *kin = momActionQuda(momentum, param);
+}
+
+
+/**
+ * BQCD wants a node mapping with x varying fastest.
+ */
+static int bqcd_rank_from_coords(const int *coords, void *fdata)
+{
+  int *dims = static_cast<int *>(fdata);
+
+  int rank = coords[3];
+  for (int i = 2; i >= 0; i--) {
+    rank = dims[i] * rank + coords[i];
+  }
+  return rank;
+}
+
+void comm_set_gridsize_(int *grid)
+{
+  initCommsGridQuda(4, grid, bqcd_rank_from_coords, static_cast<void *>(grid));
+}
+
+void plaq_quda_(double plaq[3]) {
+  plaqQuda(plaq);
+}
+
diff --git a/lib/targets/cuda/comm_target.cpp b/lib/targets/cuda/comm_target.cpp
new file mode 100644
index 0000000000..88773d61e4
--- /dev/null
+++ b/lib/targets/cuda/comm_target.cpp
@@ -0,0 +1,156 @@
+#include <comm_quda.h>
+#include <quda_api.h>
+#include <quda_cuda_api.h>
+#include <algorithm>
+
+#define CHECK_CUDA_ERROR(func)                                          \
+  quda::cuda::set_runtime_error(func, #func, __func__, __FILE__, __STRINGIFY__(__LINE__));
+
+bool comm_peer2peer_possible(int local_gpuid, int neighbor_gpuid)
+{
+  int canAccessPeer[2];
+  CHECK_CUDA_ERROR(cudaDeviceCanAccessPeer(&canAccessPeer[0], local_gpuid, neighbor_gpuid));
+  CHECK_CUDA_ERROR(cudaDeviceCanAccessPeer(&canAccessPeer[1], neighbor_gpuid, local_gpuid));
+
+  // require symmetric peer-to-peer access to enable peer-to-peer
+  return canAccessPeer[0] && canAccessPeer[1];
+}
+
+int comm_peer2peer_performance(int local_gpuid, int neighbor_gpuid)
+{
+  int accessRank[2] = { };
+  if (comm_peer2peer_possible(local_gpuid, neighbor_gpuid)) {
+    CHECK_CUDA_ERROR(cudaDeviceGetP2PAttribute(&accessRank[0], cudaDevP2PAttrPerformanceRank, local_gpuid, neighbor_gpuid));
+    CHECK_CUDA_ERROR(cudaDeviceGetP2PAttribute(&accessRank[1], cudaDevP2PAttrPerformanceRank, neighbor_gpuid, local_gpuid));
+  }
+
+  // return the slowest direction of access (lower is faster)
+  return std::max(accessRank[0], accessRank[1]);
+}
+
+void comm_create_neighbor_memory(void *remote[QUDA_MAX_DIM][2], void *local)
+{
+  // handles for obtained ghost pointers
+  cudaIpcMemHandle_t remote_handle[2][QUDA_MAX_DIM];
+
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    for (int dir=0; dir<2; ++dir) {
+      MsgHandle* sendHandle = nullptr;
+      MsgHandle* receiveHandle = nullptr;
+      int disp = (dir == 1) ? +1 : -1;
+
+      // first set up receive
+      if (comm_peer2peer_enabled(1-dir,dim)) {
+        receiveHandle = comm_declare_receive_relative(&remote_handle[1-dir][dim],
+                                                      dim, -disp, sizeof(remote_handle));
+      }
+      // now send
+      cudaIpcMemHandle_t local_handle;
+      CHECK_CUDA_ERROR(cudaIpcGetMemHandle(&local_handle, local));
+      if (comm_peer2peer_enabled(dir,dim)) {
+        sendHandle = comm_declare_send_relative(&local_handle, dim, disp, sizeof(local_handle));
+      }
+      if (receiveHandle) comm_start(receiveHandle);
+      if (sendHandle) comm_start(sendHandle);
+
+      if (receiveHandle) comm_wait(receiveHandle);
+      if (sendHandle) comm_wait(sendHandle);
+
+      if (sendHandle) comm_free(sendHandle);
+      if (receiveHandle) comm_free(receiveHandle);
+    }
+  }
+
+  // open the remote memory handles and set the send ghost pointers
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    // even if comm_dim(2) == 2, we might not have p2p enabled in both directions, so check this
+    const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
+    for (int dir=0; dir<num_dir; ++dir) {
+      remote[dim][dir] = nullptr;
+      if (comm_peer2peer_enabled(dir,dim)) CHECK_CUDA_ERROR(cudaIpcOpenMemHandle(&remote[dim][dir], remote_handle[dir][dim], cudaIpcMemLazyEnablePeerAccess));
+    }
+    if (num_dir == 1) remote[dim][1] = remote[dim][0];
+  }
+}
+
+void comm_destroy_neighbor_memory(void *remote[QUDA_MAX_DIM][2])
+{
+  for (int dim=0; dim<4; ++dim) {
+
+    if (comm_dim(dim)==1) continue;
+    const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
+
+    if (comm_peer2peer_enabled(1,dim)) {
+      // only close this handle if it doesn't alias the back ghost
+      if (num_dir == 2 && remote[dim][1]) CHECK_CUDA_ERROR(cudaIpcCloseMemHandle(remote[dim][1]));
+    }
+
+    if (comm_peer2peer_enabled(0,dim)) {
+      if (remote[dim][0]) CHECK_CUDA_ERROR(cudaIpcCloseMemHandle(remote[dim][0]));
+    }
+  } // iterate over dim
+}
+  
+void comm_create_neighbor_event(qudaEvent_t remote[2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
+{
+  // handles for obtained events
+  cudaIpcEventHandle_t ipcRemoteEventHandle[2][QUDA_MAX_DIM];
+
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    for (int dir=0; dir<2; ++dir) {
+      MsgHandle* sendHandle = nullptr;
+      MsgHandle* receiveHandle = nullptr;
+      int disp = (dir == 1) ? +1 : -1;
+
+      // first set up receive
+      if (comm_peer2peer_enabled(1-dir,dim)) {
+        receiveHandle = comm_declare_receive_relative(&ipcRemoteEventHandle[1-dir][dim], dim, -disp,
+                                                      sizeof(ipcRemoteEventHandle[1-dir][dim]));
+      }
+
+      cudaIpcEventHandle_t handle;
+
+      // now send
+      if (comm_peer2peer_enabled(dir,dim)) {
+        cudaEvent_t event;
+        CHECK_CUDA_ERROR(cudaEventCreate(&event, cudaEventDisableTiming | cudaEventInterprocess));
+        local[dir][dim].event = reinterpret_cast<void*>(event);
+        CHECK_CUDA_ERROR(cudaIpcGetEventHandle(&handle, event));
+        sendHandle = comm_declare_send_relative(&handle, dim, disp, sizeof(handle));
+      }
+
+      if (receiveHandle) comm_start(receiveHandle);
+      if (sendHandle) comm_start(sendHandle);
+
+      if (receiveHandle) comm_wait(receiveHandle);
+      if (sendHandle) comm_wait(sendHandle);
+
+      if (sendHandle) comm_free(sendHandle);
+      if (receiveHandle) comm_free(receiveHandle);
+    }
+  }
+
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    for (int dir=0; dir<2; ++dir) {
+      if (!comm_peer2peer_enabled(dir,dim)) continue;
+      cudaEvent_t event;
+      CHECK_CUDA_ERROR(cudaIpcOpenEventHandle(&event, ipcRemoteEventHandle[dir][dim]));
+      remote[dir][dim].event = reinterpret_cast<void*>(event);
+    }
+  }
+}
+
+void comm_destroy_neighbor_event(qudaEvent_t [2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
+{
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    for (int dir = 0; dir <2; dir++) {
+      cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(local[dir][dim].event);
+      if (comm_peer2peer_enabled(dir, dim)) CHECK_CUDA_ERROR(cudaEventDestroy(event));
+    } 
+  } // iterate over dim
+}
diff --git a/lib/transform_reduce.cu b/lib/transform_reduce.cu
new file mode 100644
index 0000000000..41d4d4d6e7
--- /dev/null
+++ b/lib/transform_reduce.cu
@@ -0,0 +1,184 @@
+#include <reduce_helper.h>
+#include <transform_reduce.h>
+#include <tunable_reduction.h>
+#include <kernels/transform_reduce.cuh>
+
+namespace quda
+{
+
+  template <typename reduce_t, typename T, typename count_t, typename transformer, typename reducer>
+  class TransformReduce : TunableMultiReduction<1>
+  {
+    using Arg = TransformReduceArg<reduce_t, T, count_t, transformer, reducer>;
+    QudaFieldLocation location;
+    std::vector<reduce_t> &result;
+    const std::vector<T *> &v;
+    count_t n_items;
+    transformer &h;
+    reduce_t init;
+    reducer &r;
+
+    bool tuneSharedBytes() const { return false; }
+
+    bool advanceTuneParam(TuneParam &param) const // only do autotuning if we have device fields
+    {
+      return location == QUDA_CUDA_FIELD_LOCATION ? Tunable::advanceTuneParam(param) : false;
+    }
+
+    void initTuneParam(TuneParam &param) const
+    {
+      Tunable::initTuneParam(param);
+      param.grid.y = v.size();
+    }
+
+  public:
+    TransformReduce(QudaFieldLocation location, std::vector<reduce_t> &result, const std::vector<T *> &v, count_t n_items,
+                    transformer &h, reduce_t init, reducer &r) :
+      TunableMultiReduction(n_items, v.size(), location),
+      location(location),
+      result(result),
+      v(v),
+      n_items(n_items),
+      h(h),
+      init(init),
+      r(r)
+    {
+      strcpy(aux, "batch_size=");
+      u32toa(aux + 11, v.size());
+      apply(device::get_default_stream());
+    }
+
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      Arg arg(v, n_items, h, init, r);
+      launch<transform_reducer, true>(result, tp, stream, arg);
+    }
+
+    long long bytes() const { return v.size() * n_items * sizeof(T); }
+  };
+
+  template <typename reduce_t, typename T, typename count_t, typename transformer, typename reducer>
+  void transform_reduce(QudaFieldLocation location, std::vector<reduce_t> &result, const std::vector<T *> &v, count_t n_items,
+                        transformer h, reduce_t init, reducer r)
+  {
+    if (result.size() != v.size()) errorQuda("result %lu and input %lu set sizes do not match", result.size(), v.size());
+    TransformReduce<reduce_t, T, count_t, transformer, reducer> reduce(location, result, v, n_items, h, init, r);
+  }
+
+  template <typename reduce_t, typename T, typename count_t, typename transformer, typename reducer>
+  reduce_t transform_reduce(QudaFieldLocation location, const T *v, count_t n_items, transformer h, reduce_t init, reducer r)
+  {
+    std::vector<reduce_t> result = {0.0};
+    std::vector<const T *> v_ = {v};
+    transform_reduce(location, result, v_, n_items, h, init, r);
+    return result[0];
+  }
+
+  template <typename reduce_t, typename T, typename count_t, typename transformer, typename reducer>
+  void reduce(QudaFieldLocation location, std::vector<reduce_t> &result, const std::vector<T *> &v, count_t n_items,
+              reduce_t init, reducer r)
+  {
+    transform_reduce(location, result, v, n_items, identity<T>(), init, r);
+  }
+
+  template <typename reduce_t, typename T, typename count_t, typename reducer>
+  reduce_t reduce(QudaFieldLocation location, const T *v, count_t n_items, reduce_t init, reducer r)
+  {
+    std::vector<reduce_t> result = {0.0};
+    std::vector<const T *> v_ = {v};
+    transform_reduce(location, result, v_, n_items, identity<T>(), init, r);
+    return result[0];
+  }
+
+  // explicit instantiation list for transform_reduce
+
+  template void transform_reduce<double, complex<float>, int, square_<double, float>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<float> *> const &, int, square_<double, float>,
+    double, plus<double>);
+  template void transform_reduce<double, complex<int>, int, abs_<double, int>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<quda::complex<int> *> const &, int, abs_<double, int>, double,
+    plus<double>);
+  template void transform_reduce<double, complex<double>, int, abs_<double, double>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<double> *> const &, int, abs_<double, double>, double,
+    plus<double>);
+  template void transform_reduce<double, complex<float>, int, abs_<float, float>, minimum<float>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<float> *> const &, int, abs_<float, float>, double,
+    minimum<float>);
+  template void transform_reduce<double, complex<int>, int, square_<double, int>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<int> *> const &, int, square_<double, int>, double,
+    plus<double>);
+  template void transform_reduce<double, complex<int>, int, abs_<float, int>, maximum<float>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<int> *> const &, int, abs_<float, int>, double,
+    maximum<float>);
+  template void transform_reduce<double, complex<double>, int, abs_<double, double>, minimum<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<double> *> const &, int, abs_<double, double>, double,
+    minimum<double>);
+  template void transform_reduce<double, complex<float>, int, abs_<float, float>, maximum<float>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<float> *> const &, int, abs_<float, float>, double,
+    maximum<float>);
+  template void transform_reduce<double, complex<double>, int, square_<double, double>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<double> *> const &, int, square_<double, double>,
+    double, plus<double>);
+  template void transform_reduce<double, complex<int>, int, abs_<float, int>, minimum<float>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<int> *> const &, int, abs_<float, int>, double,
+    minimum<float>);
+  template void transform_reduce<double, complex<double>, int, abs_<double, double>, maximum<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<double> *> const &, int, abs_<double, double>, double,
+    maximum<double>);
+  template void transform_reduce<double, complex<float>, int, abs_<double, float>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<float> *> const &, int, abs_<double, float>, double,
+    plus<double>);
+  template void transform_reduce<double, complex<signed char>, int, square_<double, signed char>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<signed char> *> const &, int,
+    square_<double, signed char>, double, plus<double>);
+  template void transform_reduce<double, complex<short>, int, abs_<float, short>, minimum<float>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<short> *> const &, int, abs_<float, short>, double,
+    minimum<float>);
+  template void transform_reduce<double, complex<short>, int, square_<double, short>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<short> *> const &, int, square_<double, short>,
+    double, plus<double>);
+  template void transform_reduce<double, complex<signed char>, int, abs_<float, signed char>, maximum<float>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<signed char> *> const &, int,
+    abs_<float, signed char>, double, maximum<float>);
+  template void transform_reduce<double, complex<signed char>, int, abs_<float, signed char>, minimum<float>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<signed char> *> const &, int,
+    abs_<float, signed char>, double, minimum<float>);
+  template void transform_reduce<double, complex<signed char>, int, abs_<double, signed char>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<signed char> *> const &, int,
+    abs_<double, signed char>, double, plus<double>);
+  template void transform_reduce<double, complex<short>, int, abs_<double, short>, plus<double>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<short> *> const &, int, abs_<double, short>, double,
+    plus<double>);
+  template void transform_reduce<double, complex<short>, int, abs_<float, short>, maximum<float>>(
+    QudaFieldLocation, std::vector<double> &, std::vector<complex<short> *> const &, int, abs_<float, short>, double,
+    maximum<float>);
+
+  template double transform_reduce<double, complex<double>, unsigned long, abs_<double, double>, minimum<double>>(
+    QudaFieldLocation, complex<double> const *, unsigned long, abs_<double, double>, double, minimum<double>);
+  template double transform_reduce<double, complex<float>, int, abs_<double, float>, maximum<double>>(
+    QudaFieldLocation, complex<float> const *, int, abs_<double, float>, double, maximum<double>);
+  template double transform_reduce<double, complex<double>, unsigned long, square_<double, double>, plus<double>>(
+    QudaFieldLocation, complex<double> const *, unsigned long, square_<double, double>, double, plus<double>);
+  template double transform_reduce<double, complex<double>, int, abs_<double, double>, maximum<double>>(
+    QudaFieldLocation, complex<double> const *, int, abs_<double, double>, double, maximum<double>);
+  template double transform_reduce<double, complex<float>, unsigned long, abs_<double, float>, plus<double>>(
+    QudaFieldLocation, complex<float> const *, unsigned long, abs_<double, float>, double, plus<double>);
+  template double transform_reduce<double, complex<double>, unsigned long, abs_<double, double>, plus<double>>(
+    QudaFieldLocation, complex<double> const *, unsigned long, abs_<double, double>, double, plus<double>);
+  template double transform_reduce<double, complex<float>, unsigned long, abs_<float, float>, maximum<float>>(
+    QudaFieldLocation, complex<float> const *, unsigned long, abs_<float, float>, double, maximum<float>);
+  template double transform_reduce<double, complex<float>, unsigned long, square_<double, float>, plus<double>>(
+    QudaFieldLocation, complex<float> const *, unsigned long, square_<double, float>, double, plus<double>);
+  template double transform_reduce<double, complex<float>, unsigned long, abs_<float, float>, minimum<float>>(
+    QudaFieldLocation, complex<float> const *, unsigned long, abs_<float, float>, double, minimum<float>);
+  template double transform_reduce<double, complex<double>, unsigned long, abs_<double, double>, maximum<double>>(
+    QudaFieldLocation, complex<double> const *, unsigned long, abs_<double, double>, double, maximum<double>);
+  template double transform_reduce<double, complex<float>, int, square_<double, float>, plus<double>>(
+    QudaFieldLocation, complex<float> const *, int, square_<double, float>, double, plus<double>);
+  template double transform_reduce<double, complex<short>, int, square_<double, short>, plus<double>>(
+    QudaFieldLocation, complex<short> const *, int, square_<double, short>, double, plus<double>);
+
+  template float reduce<float, float, int, maximum<float>>(QudaFieldLocation, float const *, int, float, maximum<float>);
+
+} // namespace quda
diff --git a/lib/tune.cpp b/lib/tune.cpp
new file mode 100644
index 0000000000..f3e94ea782
--- /dev/null
+++ b/lib/tune.cpp
@@ -0,0 +1,864 @@
+#include <tune_quda.h>
+#include <comm_quda.h>
+#include <quda.h>     // for QUDA_VERSION_STRING
+#include <timer.h>
+#include <sys/stat.h> // for stat()
+#include <fcntl.h>
+#include <cfloat> // for FLT_MAX
+#include <ctime>
+#include <fstream>
+#include <typeinfo>
+#include <map>
+#include <list>
+#include <unistd.h>
+#include <uint_to_char.h>
+#include <target_device.h> // for device::warp_size
+
+#include <deque>
+#include <queue>
+#include <functional>
+
+//#define LAUNCH_TIMER
+extern char *gitversion;
+
+namespace quda
+{
+  static TuneKey last_key;
+
+  TuneKey getLastTuneKey() { return quda::last_key; }
+
+  typedef std::map<TuneKey, TuneParam> map;
+
+  struct TraceKey {
+
+    TuneKey key;
+    float time;
+
+    long device_bytes;
+    long pinned_bytes;
+    long mapped_bytes;
+    long host_bytes;
+
+    TraceKey() {}
+
+    TraceKey(const TuneKey &key, float time) :
+      key(key),
+      time(time),
+      device_bytes(device_allocated_peak()),
+      pinned_bytes(pinned_allocated_peak()),
+      mapped_bytes(mapped_allocated_peak()),
+      host_bytes(host_allocated_peak())
+    {
+    }
+
+    TraceKey(const TraceKey &) = default;
+    TraceKey(TraceKey &&) = default;
+    TraceKey &operator=(const TraceKey &) = default;
+    TraceKey &operator=(TraceKey &&) = default;
+  };
+
+  // linked list that is augmented each time we call a kernel
+  static std::list<TraceKey> trace_list;
+  static int enable_trace = 0;
+
+  int traceEnabled()
+  {
+    static bool init = false;
+
+    if (!init) {
+      char *enable_trace_env = getenv("QUDA_ENABLE_TRACE");
+      if (enable_trace_env) {
+        if (strcmp(enable_trace_env, "1") == 0) {
+          // only explicitly posted trace events are included
+          enable_trace = 1;
+        } else if (strcmp(enable_trace_env, "2") == 0) {
+          // enable full kernel trace and posted trace events
+          enable_trace = 2;
+        }
+      }
+      init = true;
+    }
+    return enable_trace;
+  }
+
+  void postTrace_(const char *func, const char *file, int line)
+  {
+    if (traceEnabled() >= 1) {
+      char aux[TuneKey::aux_n];
+      strcpy(aux, file);
+      strcat(aux, ":");
+      char tmp[TuneKey::aux_n];
+      i32toa(tmp, line);
+      strcat(aux, tmp);
+      TuneKey key("", func, aux);
+      TraceKey trace_entry(key, 0.0);
+      trace_list.push_back(trace_entry);
+    }
+  }
+
+  static const std::string quda_hash = QUDA_HASH; // defined in lib/Makefile
+  static std::string resource_path;
+  static map tunecache;
+  static map::iterator it;
+  static size_t initial_cache_size = 0;
+
+#define STR_(x) #x
+#define STR(x) STR_(x)
+  static const std::string quda_version
+    = STR(QUDA_VERSION_MAJOR) "." STR(QUDA_VERSION_MINOR) "." STR(QUDA_VERSION_SUBMINOR);
+#undef STR
+#undef STR_
+
+  /** tuning in progress? */
+  static bool tuning = false;
+
+  bool activeTuning() { return tuning; }
+
+  static bool profile_count = true;
+
+  void disableProfileCount() { profile_count = false; }
+  void enableProfileCount() { profile_count = true; }
+
+  const map &getTuneCache() { return tunecache; }
+
+  /**
+   * Deserialize tunecache from an istream, useful for reading a file or receiving from other nodes.
+   */
+  static void deserializeTuneCache(std::istream &in)
+  {
+    std::string line;
+    std::stringstream ls;
+
+    TuneKey key;
+    TuneParam param;
+
+    std::string v;
+    std::string n;
+    std::string a;
+
+    int check;
+
+    while (in.good()) {
+      getline(in, line);
+      if (!line.length()) continue; // skip blank lines (e.g., at end of file)
+      ls.clear();
+      ls.str(line);
+      ls >> v >> n >> a >> param.block.x >> param.block.y >> param.block.z;
+      check = snprintf(key.volume, key.volume_n, "%s", v.c_str());
+      if (check < 0 || check >= key.volume_n) errorQuda("Error writing volume string (check = %d)", check);
+      check = snprintf(key.name, key.name_n, "%s", n.c_str());
+      if (check < 0 || check >= key.name_n) errorQuda("Error writing name string (check=%d)", check);
+      check = snprintf(key.aux, key.aux_n, "%s", a.c_str());
+      if (check < 0 || check >= key.aux_n) errorQuda("Error writing aux string (check=%d)", check);
+      ls >> param.grid.x >> param.grid.y >> param.grid.z >> param.shared_bytes >> param.aux.x >> param.aux.y
+        >> param.aux.z >> param.aux.w >> param.time;
+      ls.ignore(1);               // throw away tab before comment
+      getline(ls, param.comment); // assume anything remaining on the line is a comment
+      param.comment += "\n";      // our convention is to include the newline, since ctime() likes to do this
+      tunecache[key] = param;
+    }
+  }
+
+  /**
+   * Serialize tunecache to an ostream, useful for writing to a file or sending to other nodes.
+   */
+  static void serializeTuneCache(std::ostream &out)
+  {
+    map::iterator entry;
+
+    for (entry = tunecache.begin(); entry != tunecache.end(); entry++) {
+      TuneKey key = entry->first;
+      TuneParam param = entry->second;
+
+      out << std::setw(16) << key.volume << "\t" << key.name << "\t" << key.aux << "\t";
+      out << param.block.x << "\t" << param.block.y << "\t" << param.block.z << "\t";
+      out << param.grid.x << "\t" << param.grid.y << "\t" << param.grid.z << "\t";
+      out << param.shared_bytes << "\t" << param.aux.x << "\t" << param.aux.y << "\t" << param.aux.z << "\t"
+          << param.aux.w << "\t";
+      out << param.time << "\t" << param.comment; // param.comment ends with a newline
+    }
+  }
+
+  template <class T> struct less_significant : std::binary_function<T, T, bool> {
+    inline bool operator()(const T &lhs, const T &rhs)
+    {
+      return lhs.second.time * lhs.second.n_calls < rhs.second.time * rhs.second.n_calls;
+    }
+  };
+
+  /**
+   * Serialize tunecache to an ostream, useful for writing to a file or sending to other nodes.
+   */
+  static void serializeProfile(std::ostream &out, std::ostream &async_out)
+  {
+    map::iterator entry;
+    double total_time = 0.0;
+    double async_total_time = 0.0;
+
+    // first let's sort the entries in decreasing order of significance
+    typedef std::pair<TuneKey, TuneParam> profile_t;
+    typedef std::priority_queue<profile_t, std::deque<profile_t>, less_significant<profile_t>> queue_t;
+    queue_t q(tunecache.begin(), tunecache.end());
+
+    // now compute total time spent in kernels so we can give each kernel a significance
+    for (entry = tunecache.begin(); entry != tunecache.end(); entry++) {
+      TuneKey key = entry->first;
+      TuneParam param = entry->second;
+
+      char tmp[TuneKey::aux_n] = {};
+      strncpy(tmp, key.aux, TuneKey::aux_n);
+      bool is_policy_kernel = strncmp(tmp, "policy_kernel", 13) == 0 ? true : false;
+      bool is_policy = (strncmp(tmp, "policy", 6) == 0 && !is_policy_kernel) ? true : false;
+      if (param.n_calls > 0 && !is_policy) total_time += param.n_calls * param.time;
+      if (param.n_calls > 0 && is_policy) async_total_time += param.n_calls * param.time;
+    }
+
+    while (!q.empty()) {
+      TuneKey key = q.top().first;
+      TuneParam param = q.top().second;
+
+      char tmp[TuneKey::aux_n] = {};
+      strncpy(tmp, key.aux, TuneKey::aux_n);
+      bool is_policy_kernel = strncmp(tmp, "policy_kernel", 13) == 0 ? true : false;
+      bool is_policy = (strncmp(tmp, "policy", 6) == 0 && !is_policy_kernel) ? true : false;
+      bool is_nested_policy = (strncmp(tmp, "nested_policy", 6) == 0) ? true : false; // nested policies not included
+
+      // synchronous profile
+      if (param.n_calls > 0 && !is_policy && !is_nested_policy) {
+        double time = param.n_calls * param.time;
+
+        out << std::setw(12) << param.n_calls * param.time << "\t" << std::setw(12) << (time / total_time) * 100 << "\t";
+        out << std::setw(12) << param.n_calls << "\t" << std::setw(12) << param.time << "\t" << std::setw(16)
+            << key.volume << "\t";
+        out << key.name << "\t" << key.aux << "\t" << param.comment; // param.comment ends with a newline
+      }
+
+      // async policy profile
+      if (param.n_calls > 0 && is_policy) {
+        double time = param.n_calls * param.time;
+
+        async_out << std::setw(12) << param.n_calls * param.time << "\t" << std::setw(12)
+                  << (time / async_total_time) * 100 << "\t";
+        async_out << std::setw(12) << param.n_calls << "\t" << std::setw(12) << param.time << "\t" << std::setw(16)
+                  << key.volume << "\t";
+        async_out << key.name << "\t" << key.aux << "\t" << param.comment; // param.comment ends with a newline
+      }
+
+      q.pop();
+    }
+
+    out << std::endl << "# Total time spent in kernels = " << total_time << " seconds" << std::endl;
+    async_out << std::endl
+              << "# Total time spent in asynchronous execution = " << async_total_time << " seconds" << std::endl;
+  }
+
+  /**
+   * Serialize trace to an ostream, useful for writing to a file or sending to other nodes.
+   */
+  static void serializeTrace(std::ostream &out)
+  {
+    for (auto it = trace_list.begin(); it != trace_list.end(); it++) {
+
+      TuneKey &key = it->key;
+
+      // special case kernel members of a policy
+      char tmp[TuneKey::aux_n] = {};
+      strncpy(tmp, key.aux, TuneKey::aux_n);
+      bool is_policy_kernel = strcmp(tmp, "policy_kernel") == 0 ? true : false;
+
+      out << std::setw(12) << it->time << "\t";
+      out << std::setw(12) << it->device_bytes << "\t";
+      out << std::setw(12) << it->pinned_bytes << "\t";
+      out << std::setw(12) << it->mapped_bytes << "\t";
+      out << std::setw(12) << it->host_bytes << "\t";
+      out << std::setw(16) << key.volume << "\t";
+      if (is_policy_kernel) out << "\t";
+      out << key.name << "\t";
+      if (!is_policy_kernel) out << "\t";
+      out << key.aux << std::endl;
+    }
+  }
+
+  /**
+   * Distribute the tunecache from node 0 to all other nodes.
+   */
+  static void broadcastTuneCache()
+  {
+#ifdef MULTI_GPU
+    std::stringstream serialized;
+    size_t size;
+
+    if (comm_rank() == 0) {
+      serializeTuneCache(serialized);
+      size = serialized.str().length();
+    }
+    comm_broadcast(&size, sizeof(size_t));
+
+    if (size > 0) {
+      if (comm_rank() == 0) {
+        comm_broadcast(const_cast<char *>(serialized.str().c_str()), size);
+      } else {
+        char *serstr = new char[size + 1];
+        comm_broadcast(serstr, size);
+        serstr[size] = '\0'; // null-terminate
+        serialized.str(serstr);
+        deserializeTuneCache(serialized);
+        delete[] serstr;
+      }
+    }
+#endif
+  }
+
+  /*
+   * Read tunecache from disk.
+   */
+  void loadTuneCache()
+  {
+    if (getTuning() == QUDA_TUNE_NO) {
+      warningQuda("Autotuning disabled");
+      return;
+    }
+
+    char *path;
+    struct stat pstat;
+    std::string cache_path, line, token;
+    std::ifstream cache_file;
+    std::stringstream ls;
+
+    path = getenv("QUDA_RESOURCE_PATH");
+
+    if (!path) {
+      warningQuda("Environment variable QUDA_RESOURCE_PATH is not set.");
+      warningQuda("Caching of tuned parameters will be disabled.");
+      return;
+    } else if (stat(path, &pstat) || !S_ISDIR(pstat.st_mode)) {
+      warningQuda("The path \"%s\" specified by QUDA_RESOURCE_PATH does not exist or is not a directory.", path);
+      warningQuda("Caching of tuned parameters will be disabled.");
+      return;
+    } else {
+      resource_path = path;
+    }
+
+    bool version_check = true;
+    char *override_version_env = getenv("QUDA_TUNE_VERSION_CHECK");
+    if (override_version_env && strcmp(override_version_env, "0") == 0) {
+      version_check = false;
+      warningQuda("Disabling QUDA tunecache version check");
+    }
+
+#ifdef MULTI_GPU
+    if (comm_rank() == 0) {
+#endif
+
+      cache_path = resource_path;
+      cache_path += "/tunecache.tsv";
+      cache_file.open(cache_path.c_str());
+
+      if (cache_file) {
+
+        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
+        getline(cache_file, line);
+        ls.str(line);
+        ls >> token;
+        if (token.compare("tunecache")) errorQuda("Bad format in %s", cache_path.c_str());
+        ls >> token;
+        if (version_check && token.compare(quda_version))
+          errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
+                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
+                    cache_path.c_str());
+        ls >> token;
+#ifdef GITVERSION
+        if (version_check && token.compare(gitversion))
+          errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
+                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
+                    cache_path.c_str());
+#else
+      if (version_check && token.compare(quda_version))
+        errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
+                  "QUDA_RESOURCE_PATH environment variable to point to a new path.",
+                  cache_path.c_str());
+#endif
+        ls >> token;
+        if (version_check && token.compare(quda_hash))
+          errorQuda("Cache file %s does not match current QUDA build. \nPlease delete this file or set the "
+                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
+                    cache_path.c_str());
+
+        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
+        getline(cache_file, line); // eat the blank line
+
+        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
+        getline(cache_file, line); // eat the description line
+
+        deserializeTuneCache(cache_file);
+
+        cache_file.close();
+        initial_cache_size = tunecache.size();
+
+        if (getVerbosity() >= QUDA_SUMMARIZE) {
+          printfQuda("Loaded %d sets of cached parameters from %s\n", static_cast<int>(initial_cache_size),
+                     cache_path.c_str());
+        }
+
+      } else {
+        warningQuda("Cache file not found.  All kernels will be re-tuned (if tuning is enabled).");
+      }
+
+#ifdef MULTI_GPU
+    }
+#endif
+
+    broadcastTuneCache();
+  }
+
+  /**
+   * Write tunecache to disk.
+   */
+  void saveTuneCache(bool error)
+  {
+    time_t now;
+    int lock_handle;
+    std::string lock_path, cache_path;
+    std::ofstream cache_file;
+
+    if (resource_path.empty()) return;
+
+      // FIXME: We should really check to see if any nodes have tuned a kernel that was not also tuned on node 0, since as things
+      //       stand, the corresponding launch parameters would never get cached to disk in this situation.  This will come up if we
+      //       ever support different subvolumes per GPU (as might be convenient for lattice volumes that don't divide evenly).
+
+#ifdef MULTI_GPU
+    if (comm_rank() == 0) {
+#endif
+
+      if (tunecache.size() == initial_cache_size && !error) return;
+
+      // Acquire lock.  Note that this is only robust if the filesystem supports flock() semantics, which is true for
+      // NFS on recent versions of linux but not Lustre by default (unless the filesystem was mounted with "-o flock").
+      lock_path = resource_path + "/tunecache.lock";
+      lock_handle = open(lock_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
+      if (lock_handle == -1) {
+        warningQuda("Unable to lock cache file.  Tuned launch parameters will not be cached to disk.  "
+                    "If you are certain that no other instances of QUDA are accessing this filesystem, "
+                    "please manually remove %s",
+                    lock_path.c_str());
+        return;
+      }
+      char msg[] = "If no instances of applications using QUDA are running,\n"
+                   "this lock file shouldn't be here and is safe to delete.";
+      int stat = write(lock_handle, msg, sizeof(msg)); // check status to avoid compiler warning
+      if (stat == -1) warningQuda("Unable to write to lock file for some bizarre reason");
+
+      cache_path = resource_path + (error ? "/tunecache_error.tsv" : "/tunecache.tsv");
+      cache_file.open(cache_path.c_str());
+
+      if (getVerbosity() >= QUDA_SUMMARIZE) {
+        printfQuda("Saving %d sets of cached parameters to %s\n", static_cast<int>(tunecache.size()), cache_path.c_str());
+      }
+
+      time(&now);
+      cache_file << "tunecache\t" << quda_version;
+#ifdef GITVERSION
+      cache_file << "\t" << gitversion;
+#else
+    cache_file << "\t" << quda_version;
+#endif
+      cache_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
+      cache_file << std::setw(16) << "volume"
+                 << "\tname\taux\tblock.x\tblock.y\tblock.z\tgrid.x\tgrid.y\tgrid.z\tshared_bytes\taux.x\taux.y\taux."
+                    "z\taux.w\ttime\tcomment"
+                 << std::endl;
+      serializeTuneCache(cache_file);
+      cache_file.close();
+
+      // Release lock.
+      close(lock_handle);
+      remove(lock_path.c_str());
+
+      initial_cache_size = tunecache.size();
+
+#ifdef MULTI_GPU
+    } else {
+      // give process 0 time to write out its tunecache if needed, but
+      // doesn't cause a hang if error is not triggered on process 0
+      if (error) sleep(10);
+    }
+#endif
+  }
+
+  static bool policy_tuning = false;
+  bool policyTuning() { return policy_tuning; }
+
+  void setPolicyTuning(bool policy_tuning_) { policy_tuning = policy_tuning_; }
+
+  // flush profile, setting counts to zero
+  void flushProfile()
+  {
+    for (map::iterator entry = tunecache.begin(); entry != tunecache.end(); entry++) {
+      // set all n_calls = 0
+      TuneParam &param = entry->second;
+      param.n_calls = 0;
+    }
+  }
+
+  // save profile
+  void saveProfile(const std::string label)
+  {
+    time_t now;
+    int lock_handle;
+    std::string lock_path, profile_path, async_profile_path, trace_path;
+    std::ofstream profile_file, async_profile_file, trace_file;
+
+    if (resource_path.empty()) return;
+
+#ifdef MULTI_GPU
+    if (comm_rank() == 0) {
+#endif
+
+      // Acquire lock.  Note that this is only robust if the filesystem supports flock() semantics, which is true for
+      // NFS on recent versions of linux but not Lustre by default (unless the filesystem was mounted with "-o flock").
+      lock_path = resource_path + "/profile.lock";
+      lock_handle = open(lock_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
+      if (lock_handle == -1) {
+        warningQuda("Unable to lock profile file.  Profile will not be saved to disk.  "
+                    "If you are certain that no other instances of QUDA are accessing this filesystem, "
+                    "please manually remove %s",
+                    lock_path.c_str());
+        return;
+      }
+      char msg[] = "If no instances of applications using QUDA are running,\n"
+                   "this lock file shouldn't be here and is safe to delete.";
+      int stat = write(lock_handle, msg, sizeof(msg)); // check status to avoid compiler warning
+      if (stat == -1) warningQuda("Unable to write to lock file for some bizarre reason");
+
+      // profile counter for writing out unique profiles
+      static int count = 0;
+
+      char *profile_fname = getenv("QUDA_PROFILE_OUTPUT_BASE");
+
+      if (!profile_fname) {
+        warningQuda(
+          "Environment variable QUDA_PROFILE_OUTPUT_BASE not set; writing to profile.tsv and profile_async.tsv");
+        profile_path = resource_path + "/profile_" + std::to_string(count) + ".tsv";
+        async_profile_path = resource_path + "/profile_async_" + std::to_string(count) + ".tsv";
+        if (traceEnabled()) trace_path = resource_path + "/trace_" + std::to_string(count) + ".tsv";
+      } else {
+        profile_path = resource_path + "/" + profile_fname + "_" + std::to_string(count) + ".tsv";
+        async_profile_path = resource_path + "/" + profile_fname + "_" + std::to_string(count) + "_async.tsv";
+        if (traceEnabled())
+          trace_path = resource_path + "/" + profile_fname + "_trace_" + std::to_string(count) + ".tsv";
+      }
+
+      count++;
+
+      profile_file.open(profile_path.c_str());
+      async_profile_file.open(async_profile_path.c_str());
+      if (traceEnabled()) trace_file.open(trace_path.c_str());
+
+      if (getVerbosity() >= QUDA_SUMMARIZE) {
+        // compute number of non-zero entries that will be output in the profile
+        int n_entry = 0;
+        int n_policy = 0;
+        for (map::iterator entry = tunecache.begin(); entry != tunecache.end(); entry++) {
+          // if a policy entry, then we can ignore
+          char tmp[TuneKey::aux_n] = {};
+          strncpy(tmp, entry->first.aux, TuneKey::aux_n);
+          TuneParam param = entry->second;
+          bool is_policy = strcmp(tmp, "policy") == 0 ? true : false;
+          if (param.n_calls > 0 && !is_policy) n_entry++;
+          if (param.n_calls > 0 && is_policy) n_policy++;
+        }
+
+        printfQuda("Saving %d sets of cached parameters to %s\n", n_entry, profile_path.c_str());
+        printfQuda("Saving %d sets of cached profiles to %s\n", n_policy, async_profile_path.c_str());
+        if (traceEnabled())
+          printfQuda("Saving trace list with %lu entries to %s\n", trace_list.size(), trace_path.c_str());
+      }
+
+      time(&now);
+
+      std::string Label = label.empty() ? "profile" : label;
+
+      profile_file << Label << "\t" << quda_version;
+#ifdef GITVERSION
+      profile_file << "\t" << gitversion;
+#else
+    profile_file << "\t" << quda_version;
+#endif
+      profile_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
+      profile_file << std::setw(12) << "total time"
+                   << "\t" << std::setw(12) << "percentage"
+                   << "\t" << std::setw(12) << "calls"
+                   << "\t" << std::setw(12) << "time / call"
+                   << "\t" << std::setw(16) << "volume"
+                   << "\tname\taux\tcomment" << std::endl;
+
+      async_profile_file << Label << "\t" << quda_version;
+#ifdef GITVERSION
+      async_profile_file << "\t" << gitversion;
+#else
+    async_profile_file << "\t" << quda_version;
+#endif
+      async_profile_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
+      async_profile_file << std::setw(12) << "total time"
+                         << "\t" << std::setw(12) << "percentage"
+                         << "\t" << std::setw(12) << "calls"
+                         << "\t" << std::setw(12) << "time / call"
+                         << "\t" << std::setw(16) << "volume"
+                         << "\tname\taux\tcomment" << std::endl;
+
+      serializeProfile(profile_file, async_profile_file);
+
+      profile_file.close();
+      async_profile_file.close();
+
+      if (traceEnabled()) {
+        trace_file << "trace"
+                   << "\t" << quda_version;
+#ifdef GITVERSION
+        trace_file << "\t" << gitversion;
+#else
+      trace_file << "\t" << quda_version;
+#endif
+        trace_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
+
+        trace_file << std::setw(12) << "time\t" << std::setw(12) << "device-mem\t" << std::setw(12) << "pinned-mem\t";
+        trace_file << std::setw(12) << "mapped-mem\t" << std::setw(12) << "host-mem\t";
+        trace_file << std::setw(16) << "volume"
+                   << "\tname\taux" << std::endl;
+
+        serializeTrace(trace_file);
+
+        trace_file.close();
+      }
+
+      // Release lock.
+      close(lock_handle);
+      remove(lock_path.c_str());
+
+#ifdef MULTI_GPU
+    }
+#endif
+  }
+
+  TuneParam::TuneParam() :
+    block(device::warp_size(), 1, 1),
+    grid(1, 1, 1),
+    shared_bytes(0),
+    set_max_shared_bytes(false),
+    aux(),
+    time(FLT_MAX),
+    n_calls(0)
+    {
+      aux = make_int4(1,1,1,1);
+    }
+
+  int Tunable::blockStep() const { return device::warp_size(); }
+  int Tunable::blockMin() const { return device::warp_size(); }
+
+  static TimeProfile launchTimer("tuneLaunch");
+
+  /**
+   * Return the optimal launch parameters for a given kernel, either
+   * by retrieving them from tunecache or autotuning on the spot.
+   */
+  TuneParam &tuneLaunch(Tunable &tunable, QudaTune enabled, QudaVerbosity verbosity)
+  {
+#ifdef LAUNCH_TIMER
+    launchTimer.TPSTART(QUDA_PROFILE_TOTAL);
+    launchTimer.TPSTART(QUDA_PROFILE_INIT);
+#endif
+
+    TuneKey key = tunable.tuneKey();
+    if (use_managed_memory()) strcat(key.aux, ",managed");
+    last_key = key;
+    static TuneParam param;
+
+#ifdef LAUNCH_TIMER
+    launchTimer.TPSTOP(QUDA_PROFILE_INIT);
+    launchTimer.TPSTART(QUDA_PROFILE_PREAMBLE);
+#endif
+
+    static const Tunable *active_tunable; // for error checking
+    it = tunecache.find(key);
+
+    // first check if we have the tuned value and return if we have it
+    if (enabled == QUDA_TUNE_YES && it != tunecache.end()) {
+
+#ifdef LAUNCH_TIMER
+      launchTimer.TPSTOP(QUDA_PROFILE_PREAMBLE);
+      launchTimer.TPSTART(QUDA_PROFILE_COMPUTE);
+#endif
+
+      TuneParam &param = it->second;
+
+      if (verbosity >= QUDA_DEBUG_VERBOSE) {
+        printfQuda("Launching %s with %s at vol=%s with %s\n", key.name, key.aux, key.volume,
+                   tunable.paramString(param).c_str());
+      }
+
+#ifdef LAUNCH_TIMER
+      launchTimer.TPSTOP(QUDA_PROFILE_COMPUTE);
+      launchTimer.TPSTART(QUDA_PROFILE_EPILOGUE);
+#endif
+
+      tunable.checkLaunchParam(param);
+
+      // we could be tuning outside of the current scope
+      if (!tuning && profile_count) param.n_calls++;
+
+#ifdef LAUNCH_TIMER
+      launchTimer.TPSTOP(QUDA_PROFILE_EPILOGUE);
+      launchTimer.TPSTOP(QUDA_PROFILE_TOTAL);
+#endif
+
+      if (traceEnabled() >= 2) {
+        TraceKey trace_entry(key, param.time);
+        trace_list.push_back(trace_entry);
+      }
+
+      return param;
+    }
+
+#ifdef LAUNCH_TIMER
+    launchTimer.TPSTOP(QUDA_PROFILE_PREAMBLE);
+    launchTimer.TPSTOP(QUDA_PROFILE_TOTAL);
+#endif
+
+    if (enabled == QUDA_TUNE_NO) {
+      tunable.defaultTuneParam(param);
+      tunable.checkLaunchParam(param);
+      if (verbosity >= QUDA_DEBUG_VERBOSE) {
+        printfQuda("Launching %s with %s at vol=%s with %s (untuned)\n", key.name, key.aux, key.volume,
+                   tunable.paramString(param).c_str());
+      }
+    } else if (!tuning) {
+
+      /* As long as global reductions are not disabled, only do the
+         tuning on node 0, else do the tuning on all nodes since we
+         can't guarantee that all nodes are partaking */
+      if (comm_rank() == 0 || !commGlobalReduction() || policyTuning()) {
+        TuneParam best_param;
+        float best_time;
+        time_t now;
+
+        tuning = true;
+        active_tunable = &tunable;
+        best_time = FLT_MAX;
+
+        if (verbosity >= QUDA_DEBUG_VERBOSE) printfQuda("PreTune %s\n", key.name);
+        tunable.preTune();
+
+        if (verbosity >= QUDA_DEBUG_VERBOSE) {
+          printfQuda("Tuning %s with %s at vol=%s\n", key.name, key.aux, key.volume);
+        }
+
+        const auto &stream = device::get_default_stream();
+        device_timer_t timer(stream);
+
+        host_timer_t tune_timer;
+        tune_timer.start(__func__, __FILE__, __LINE__);
+
+        tunable.initTuneParam(param);
+        while (tuning) {
+          qudaDeviceSynchronize();
+          tunable.checkLaunchParam(param);
+          if (verbosity >= QUDA_DEBUG_VERBOSE) {
+            printfQuda("About to call tunable.apply block=(%d,%d,%d) grid=(%d,%d,%d) shared_bytes=%d aux=(%d,%d,%d)\n",
+                       param.block.x, param.block.y, param.block.z, param.grid.x, param.grid.y, param.grid.z,
+                       param.shared_bytes, param.aux.x, param.aux.y, param.aux.z);
+          }
+
+          tunable.apply(stream); // do initial call in case we need to jit compile for these parameters or if policy tuning
+
+          timer.start();
+          for (int i = 0; i < tunable.tuningIter(); i++) {
+            tunable.apply(stream); // calls tuneLaunch() again, which simply returns the currently active param
+          }
+          timer.stop();
+          qudaDeviceSynchronize();
+          auto error = qudaGetLastError();
+
+          if (error != QUDA_SUCCESS) { // check we don't have a sticky error
+            qudaDeviceSynchronize();
+            if (qudaGetLastError() != QUDA_SUCCESS)
+              errorQuda("Failed to clear error state %s\n", qudaGetLastErrorString().c_str());
+          }
+
+          float elapsed_time = timer.last() / tunable.tuningIter();
+          if ((elapsed_time < best_time) && (error == QUDA_SUCCESS) && (tunable.launchError() == QUDA_SUCCESS)) {
+            best_time = elapsed_time;
+            best_param = param;
+          }
+          if ((verbosity >= QUDA_DEBUG_VERBOSE)) {
+            if (error == QUDA_SUCCESS && tunable.launchError() == QUDA_SUCCESS) {
+              printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(),
+                         tunable.perfString(elapsed_time).c_str());
+            } else {
+              if (tunable.launchError() == QUDA_SUCCESS) { // must be regular error
+                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), qudaGetLastErrorString().c_str());
+              } else { // else must be a manually thrown error
+                printfQuda("    %s gives thrown error\n", tunable.paramString(param).c_str());
+              }
+            }
+          }
+          tuning = tunable.advanceTuneParam(param);
+          tunable.launchError() = QUDA_SUCCESS;
+        }
+
+        tune_timer.stop(__func__, __FILE__, __LINE__);
+
+        if (best_time == FLT_MAX) {
+          errorQuda("Auto-tuning failed for %s with %s at vol=%s", key.name, key.aux, key.volume);
+        }
+        if (verbosity >= QUDA_VERBOSE) {
+          printfQuda("Tuned %s giving %s for %s with %s\n", tunable.paramString(best_param).c_str(),
+                     tunable.perfString(best_time).c_str(), key.name, key.aux);
+        }
+        time(&now);
+        best_param.comment = "# " + tunable.perfString(best_time);
+        best_param.comment += ", tuning took " + std::to_string(tune_timer.last()) + " seconds at ";
+        best_param.comment += ctime(&now); // includes a newline
+        best_param.time = best_time;
+
+        if (verbosity >= QUDA_DEBUG_VERBOSE) printfQuda("PostTune %s\n", key.name);
+        tuning = true;
+        tunable.postTune();
+        tuning = false;
+        param = best_param;
+        tunecache[key] = best_param;
+      }
+      if (commGlobalReduction() || policyTuning()) broadcastTuneCache();
+
+      // check this process is getting the key that is expected
+      if (tunecache.find(key) == tunecache.end()) {
+
+        // if we can't find the key, and debugging, then print out the entire map
+        if (verbosity >= QUDA_DEBUG_VERBOSE)
+          for (auto elem : tunecache) std::cout << elem.first << ": " << elem.second << std::endl;
+
+        errorQuda("Failed to find key entry (%s:%s:%s)", key.name, key.volume, key.aux);
+      }
+      param = tunecache[key]; // read this now for all processes
+
+      if (traceEnabled() >= 2) {
+        TraceKey trace_entry(key, param.time);
+        trace_list.push_back(trace_entry);
+      }
+
+    } else if (&tunable != active_tunable) {
+      errorQuda("Unexpected call to tuneLaunch() in %s::apply()", typeid(tunable).name());
+    }
+
+    param.n_calls = profile_count ? 1 : 0;
+
+    return param;
+  }
+
+  void printLaunchTimer()
+  {
+#ifdef LAUNCH_TIMER
+    launchTimer.Print();
+#endif
+  }
+} // namespace quda

From cf83a611532a4b209cbf2ffcef8a0a7cd62e51d6 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 10 Feb 2021 22:02:30 +0000
Subject: [PATCH 019/198] local tmp

---
 include/blas_helper.cuh                       |   1 +
 include/clover_field_order.h                  |  44 +-
 include/color_spinor_field.h                  |   6 +-
 include/color_spinor_field_order.h            | 164 +---
 include/comm_quda.h                           |  66 +-
 include/complex_quda.h                        |  10 +-
 include/convert.h                             |  46 +-
 include/deflation.h                           |   7 +-
 include/device.h                              |   5 +
 include/dirac_quda.h                          |   1 +
 include/eigensolve_quda.h                     |   1 +
 include/gauge_field_order.h                   | 351 ++------
 include/gauge_tools.h                         |   1 +
 include/invert_quda.h                         |   1 +
 include/jitify_helper2.cuh                    |  34 +-
 include/kernel_helper.h                       |  15 +
 include/kernels/clover_invert.cuh             |   9 +-
 include/kernels/coarse_op_kernel.cuh          |   1 +
 include/kernels/coarse_op_preconditioned.cuh  |  79 +-
 .../kernels/coarse_op_preconditioned_mma.cuh  |  97 +-
 include/kernels/copy_gauge_extended.cuh       |   8 +-
 include/kernels/covDev.cuh                    |   3 -
 include/kernels/dslash_coarse.cuh             |   9 +-
 include/kernels/dslash_domain_wall_5d.cuh     |   7 +-
 include/kernels/dslash_wilson.cuh             |  12 +
 .../kernels/extract_gauge_ghost_extended.cuh  |   2 +-
 include/kernels/gauge_det_trace.cuh           |  10 +-
 include/kernels/gauge_fix_fft.cuh             |   9 +-
 include/kernels/gauge_fix_ovr.cuh             |   9 +-
 include/kernels/gauge_heatbath.cuh            |  10 +-
 include/kernels/gauge_plaq.cuh                |   9 +-
 include/kernels/gauge_qcharge.cuh             |  10 +-
 include/kernels/gauge_random.cuh              |   8 +-
 include/kernels/momentum.cuh                  |  39 +-
 include/kernels/multi_reduce_core.cuh         |   6 +-
 include/kernels/pgauge_init.cuh               |   8 +-
 include/kernels/reduce_core.cuh               |   6 +-
 include/kernels/restrictor.cuh                |   2 +-
 include/kernels/spinor_noise.cuh              |   8 +-
 include/lattice_field.h                       |  21 +-
 include/malloc_quda.h                         |  10 +-
 include/matrix_field.h                        |  32 +-
 include/quda.h                                |   9 +-
 include/quda_api.h                            |  71 +-
 include/quda_fortran.h                        |   5 -
 include/quda_internal.h                       |  91 +-
 include/quda_new_interace.h                   |  36 -
 include/random_quda.h                         |  31 +-
 include/reduce_helper.h                       |  33 +-
 include/register_traits.h                     |  20 +
 include/shared_memory_cache_helper.cuh        | 121 +--
 include/targets/cuda/block_reduction_kernel.h |  10 +-
 include/targets/cuda/kernel.h                 |   2 +-
 include/targets/cuda/random_helper.h          |   1 -
 include/targets/cuda/reduction_kernel.h       |  20 +-
 include/targets/cuda/target_device.h          |   2 +
 include/targets/hip/random_helper.h           |   1 -
 include/timer.h                               | 152 ++--
 include/transform_reduce.h                    | 160 +---
 include/trove_helper.cuh                      |  11 -
 include/tunable_block_reduction.h             |   2 +-
 include/tunable_nd.h                          | 131 ++-
 include/tunable_reduction.h                   | 131 ++-
 include/tune_key.h                            |   9 +
 include/tune_quda.h                           |  42 +-
 include/util_quda.h                           |  30 +-
 lib/CMakeLists.txt                            |  14 +-
 lib/block_orthogonalize.cu                    |  11 +-
 lib/clover_field.cpp                          |   8 +-
 lib/coarse_op.cuh                             |   2 +-
 lib/coarse_op_preconditioned.cu               | 245 ++---
 lib/coarse_op_preconditioned_mma_launch.h     | 127 ++-
 lib/comm_common.cpp                           | 147 +--
 lib/comm_mpi.cpp                              |   4 +-
 lib/comm_qmp.cpp                              |   4 +-
 lib/comm_single.cpp                           |   2 +-
 lib/copy_gauge_extended.cu                    |   2 +-
 lib/covDev.cu                                 |   2 +
 lib/cpu_gauge_field.cpp                       |   1 +
 lib/cuda_color_spinor_field.cpp               |   8 +-
 lib/cuda_gauge_field.cpp                      |   5 +-
 lib/dslash_domain_wall_5d.cu                  |   2 +
 lib/dslash_pack2.cu                           |   1 +
 lib/dslash_policy.cuh                         |  17 +-
 lib/dslash_quda.cu                            |  36 +-
 lib/eigensolve_quda.cpp                       |   5 +-
 lib/extract_gauge_ghost_mg.cu                 |   3 -
 lib/gauge_field.cpp                           |   3 +-
 lib/gauge_fix_fft.cu                          |  11 +-
 lib/gauge_fix_ovr.cu                          |   5 +-
 lib/gauge_random.cu                           |   7 +-
 lib/interface/CMakeLists.txt                  |   3 +
 lib/interface/blas_interface.cpp              |   1 +
 lib/interface_quda.cpp                        | 268 +-----
 lib/inv_eigcg_quda.cpp                        |  45 -
 lib/inv_gmresdr_quda.cpp                      |  57 --
 lib/lattice_field.cpp                         | 167 +---
 lib/momentum.cu                               |   2 +-
 lib/multigrid.cpp                             |   3 +-
 lib/pgauge_heatbath.cu                        |  15 +-
 lib/pgauge_init.cu                            |   2 +-
 lib/random.cu                                 | 153 +---
 lib/reduce_helper.cu                          |  41 +-
 lib/restrictor.cu                             |   8 +
 lib/spinor_noise.cu                           |   7 +-
 lib/targets/cuda/CMakeLists.txt               |   2 +-
 lib/targets/cuda/blas_lapack_cublas.cpp       |   1 +
 lib/targets/cuda/device.cpp                   |  71 +-
 lib/targets/cuda/jitify_helper.cpp            |  16 +-
 lib/targets/cuda/malloc.cpp                   |  18 +-
 lib/targets/cuda/quda_api.cpp                 | 236 +++--
 lib/targets/cuda/tune.cpp                     | 849 ------------------
 lib/targets/generic/blas_lapack_eigen.cpp     |   1 +
 lib/targets/hip/tune.cpp                      | 849 ------------------
 lib/timer.cpp                                 |   2 +-
 tests/blas_test.cpp                           |  19 +-
 tests/contract_test.cpp                       |  15 +-
 tests/covdev_test.cpp                         |  19 +-
 tests/dslash_ctest.cpp                        |  32 +-
 tests/dslash_test.cpp                         |  33 +-
 tests/eigensolve_test.cpp                     |  11 +-
 tests/gauge_alg_test.cpp                      |  55 +-
 tests/heatbath_test.cpp                       |  36 +-
 .../host_reference/gauge_force_reference.cpp  | 132 ++-
 .../host_reference/hisq_force_reference2.cpp  |   6 +-
 tests/invert_test.cpp                         |   4 +-
 tests/multigrid_benchmark_test.cpp            |  22 +-
 tests/multigrid_evolve_test.cpp               |   4 +-
 tests/pack_test.cpp                           |  42 +-
 tests/staggered_dslash_ctest.cpp              |  33 +-
 tests/staggered_dslash_test.cpp               |  33 +-
 tests/staggered_invert_test.cpp               |   4 +-
 tests/unitarize_link_test.cpp                 |   1 +
 tests/utils/face_gauge.cpp                    |   1 -
 tests/utils/host_utils.cpp                    |  17 +-
 tests/utils/host_utils.h                      |   2 -
 136 files changed, 1615 insertions(+), 4693 deletions(-)
 delete mode 100644 include/quda_new_interace.h
 delete mode 100644 include/trove_helper.cuh
 delete mode 100644 lib/targets/cuda/tune.cpp
 delete mode 100644 lib/targets/hip/tune.cpp

diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index 19b388dc3f..591171129f 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <color_spinor_field.h>
+#include <reduce_helper.h>
 
 //#define QUAD_SUM
 #ifdef QUAD_SUM
diff --git a/include/clover_field_order.h b/include/clover_field_order.h
index 94efd3b276..c3190863aa 100644
--- a/include/clover_field_order.h
+++ b/include/clover_field_order.h
@@ -6,13 +6,14 @@
  *
  */
 
+#include <limits>
 #include <register_traits.h>
 #include <convert.h>
 #include <clover_field.h>
 #include <complex_quda.h>
 #include <quda_matrix.h>
 #include <color_spinor.h>
-#include <trove_helper.cuh>
+#include <aos.h>
 #include <transform_reduce.h>
 
 namespace quda {
@@ -70,16 +71,6 @@ namespace quda {
 
   namespace clover {
 
-    template<typename ReduceType, typename Float> struct square_ {
-      __host__ __device__ inline ReduceType operator()(const quda::complex<Float> &x)
-      { return static_cast<ReduceType>(norm(x)); }
-    };
-
-    template<typename ReduceType, typename Float> struct abs_ {
-      __host__ __device__ inline ReduceType operator()(const quda::complex<Float> &x)
-      { return static_cast<ReduceType>(abs(x)); }
-    };
-
     /**
        The internal ordering for each clover matrix has chirality as the
        slowest running dimension, with the internal 36 degrees of
@@ -721,14 +712,6 @@ namespace quda {
         }
       };
 
-    /**
-       @brief This is just a dummy structure we use for trove to define the
-       required structure size
-       @tparam real Real number type
-       @tparam length Number of elements in the structure
-    */
-    template <typename real, int length> struct S { real v[length]; };
-
     /**
        QDP ordering for clover fields
     */
@@ -757,26 +740,15 @@ namespace quda {
 
 	__device__ __host__ inline void load(RegType v[length], int x, int parity) const {
 	  // factor of 0.5 comes from basis change
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	  typedef S<Float,length> structure;
-	  trove::coalesced_ptr<structure> clover_((structure*)clover);
-	  structure v_ = clover_[parity*volumeCB + x];
-	  for (int i=0; i<length; i++) v[i] = 0.5*(RegType)v_.v[i];
-#else
-	  for (int i=0; i<length; i++) v[i] = 0.5*clover[parity*offset + x*length+i];
-#endif
+          Float v_[length];
+          block_load<Float, length>(v_, &clover[parity*offset + x*length]);
+          for (int i=0; i<length; i++) v[i] = 0.5*v_[i];
 	}
   
 	__device__ __host__ inline void save(const RegType v[length], int x, int parity) {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	  typedef S<Float,length> structure;
-	  trove::coalesced_ptr<structure> clover_((structure*)clover);
-	  structure v_;
-	  for (int i=0; i<length; i++) v_.v[i] = 2.0*(Float)v[i];
-	  clover_[parity*volumeCB + x] = v_;
-#else
-	  for (int i=0; i<length; i++) clover[parity*offset + x*length+i] = 2.0*v[i];
-#endif
+          Float v_[length];
+          for (int i=0; i<length; i++) v_[i] = 2.0*v[i];
+          block_store<Float, length>(&clover[parity*offset + x*length], v_);
 	}
 
 	size_t Bytes() const { return length*sizeof(Float); }
diff --git a/include/color_spinor_field.h b/include/color_spinor_field.h
index fa6645bdfb..a5ba5a0c38 100644
--- a/include/color_spinor_field.h
+++ b/include/color_spinor_field.h
@@ -6,7 +6,6 @@
 #include <iostream>
 
 #include <lattice_field.h>
-#include <random_quda.h>
 #include <fast_intdiv.h>
 
 namespace quda {
@@ -1036,6 +1035,11 @@ namespace quda {
   void genericPackGhost(void **ghost, const ColorSpinorField &a, QudaParity parity,
 			int nFace, int dagger, MemoryLocation *destination=nullptr);
 
+  /**
+     @brief pre-declaration of RNG class (defined in non-device-safe random_quda.h)
+  */
+  class RNG;
+
   /**
      @brief Generate a random noise spinor.  This variant allows the user to manage the RNG state.
      @param src The colorspinorfield
diff --git a/include/color_spinor_field_order.h b/include/color_spinor_field_order.h
index 79684ec17a..df9b491c57 100644
--- a/include/color_spinor_field_order.h
+++ b/include/color_spinor_field_order.h
@@ -11,13 +11,14 @@
  *  also.
  */
 
+#include <limits>
 #include <register_traits.h>
 #include <convert.h>
 #include <complex_quda.h>
 #include <index_helper.cuh>
 #include <color_spinor.h>
 #include <color_spinor_field.h>
-#include <trove_helper.cuh>
+#include <aos.h>
 #include <transform_reduce.h>
 
 namespace quda {
@@ -184,45 +185,6 @@ namespace quda {
 
   namespace colorspinor {
 
-    template<typename ReduceType, typename Float> struct square_ {
-      square_(ReduceType) { }
-      __host__ __device__ inline ReduceType operator()(const quda::complex<Float> &x)
-      { return static_cast<ReduceType>(norm(x)); }
-    };
-
-    template<typename ReduceType> struct square_<ReduceType,short> {
-      const ReduceType scale;
-      square_(ReduceType scale) : scale(scale) { }
-      __host__ __device__ inline ReduceType operator()(const quda::complex<short> &x)
-      { return norm(scale * complex<ReduceType>(x.real(), x.imag())); }
-    };
-
-    template <typename ReduceType> struct square_<ReduceType, int8_t> {
-      const ReduceType scale;
-      square_(ReduceType scale) : scale(scale) { }
-      __host__ __device__ inline ReduceType operator()(const quda::complex<int8_t> &x)
-      { return norm(scale * complex<ReduceType>(x.real(), x.imag())); }
-    };
-
-    template<typename Float, typename storeFloat> struct abs_ {
-      abs_(const Float) { }
-      __host__ __device__ Float operator()(const quda::complex<storeFloat> &x) { return abs(x); }
-    };
-
-    template<typename Float> struct abs_<Float,short> {
-      Float scale;
-      abs_(const Float scale) : scale(scale) { }
-      __host__ __device__ Float operator()(const quda::complex<short> &x)
-      { return abs(scale * complex<Float>(x.real(), x.imag())); }
-    };
-
-    template <typename Float> struct abs_<Float, int8_t> {
-      Float scale;
-      abs_(const Float scale) : scale(scale) { }
-      __host__ __device__ Float operator()(const quda::complex<int8_t> &x)
-      { return abs(scale * complex<Float>(x.real(), x.imag())); }
-    };
-
     template <typename Float, int nSpin, int nColor, int nVec, QudaFieldOrder order> struct AccessorCB {
       AccessorCB(const ColorSpinorField &) { errorQuda("Not implemented"); }
       AccessorCB() { errorQuda("Not implemented"); }
@@ -1224,14 +1186,6 @@ namespace quda {
   }
     };
 
-    /**
-       @brief This is just a dummy structure we use for trove to define the
-       required structure size
-       @tparam real Real number type
-       @tparam length Number of elements in the structure
-    */
-    template <typename real, int length> struct S { real v[length]; };
-
     template <typename Float, int Ns, int Nc>
       struct SpaceColorSpinorOrder {
       using Accessor = SpaceColorSpinorOrder<Float, Ns, Nc>;
@@ -1259,44 +1213,28 @@ namespace quda {
 
   __device__ __host__ inline void load(complex v[length / 2], int x, int parity = 0) const
   {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-    typedef S<Float,length> structure;
-    trove::coalesced_ptr<structure> field_((structure*)field);
-    structure v_ = field_[parity*volumeCB + x];
-    for (int s=0; s<Ns; s++) {
-      for (int c = 0; c < Nc; c++) { v[s * Nc + c] = complex(v_.v[(c * Ns + s) * 2 + 0], v_.v[(c * Ns + s) * 2 + 1]); }
-    }
-#else
+    auto in = &field[(parity * volumeCB + x) * length];
+    complex v_[length/2];
+    block_load<complex, length/2>(v_, reinterpret_cast<const complex*>(in));
+
     for (int s=0; s<Ns; s++) {
       for (int c=0; c<Nc; c++) {
-        v[s * Nc + c] = complex(field[parity * offset + ((x * Nc + c) * Ns + s) * 2 + 0],
-                                field[parity * offset + ((x * Nc + c) * Ns + s) * 2 + 1]);
+        v[s * Nc + c] = v_[c * Ns + s];
       }
     }
-#endif
   }
 
   __device__ __host__ inline void save(const complex v[length / 2], int x, int parity = 0)
   {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-    typedef S<Float,length> structure;
-    trove::coalesced_ptr<structure> field_((structure*)field);
-    structure v_;
-    for (int s=0; s<Ns; s++) {
-      for (int c=0; c<Nc; c++) {
-        v_.v[(c*Ns + s)*2 + 0] = (Float)v[s*Nc+c].real();
-        v_.v[(c*Ns + s)*2 + 1] = (Float)v[s*Nc+c].imag();
-      }
-    }
-    field_[parity*volumeCB + x] = v_;
-#else
+    auto out = &field[(parity * volumeCB + x) * length];
+    complex v_[length/2];
     for (int s=0; s<Ns; s++) {
       for (int c=0; c<Nc; c++) {
-        field[parity*offset + ((x*Nc + c)*Ns + s)*2 + 0] = v[s*Nc+c].real();
-        field[parity*offset + ((x*Nc + c)*Ns + s)*2 + 1] = v[s*Nc+c].imag();
+        v_[c * Ns + s] = v[s * Nc + c];
       }
     }
-#endif
+
+    block_store<complex, length/2>(reinterpret_cast<complex*>(out), v_);
   }
 
   /**
@@ -1377,44 +1315,14 @@ namespace quda {
 
   __device__ __host__ inline void load(complex v[length / 2], int x, int parity = 0) const
   {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-    typedef S<Float,length> structure;
-    trove::coalesced_ptr<structure> field_((structure*)field);
-    structure v_ = field_[parity*volumeCB + x];
-    for (int s=0; s<Ns; s++) {
-      for (int c = 0; c < Nc; c++) { v[s * Nc + c] = complex(v_.v[(s * Nc + c) * 2 + 0], v_.v[(s * Nc + c) * 2 + 1]); }
-    }
-#else
-    for (int s=0; s<Ns; s++) {
-      for (int c=0; c<Nc; c++) {
-        v[s * Nc + c] = complex(field[parity * offset + ((x * Ns + s) * Nc + c) * 2 + 0],
-                                field[parity * offset + ((x * Ns + s) * Nc + c) * 2 + 1]);
-      }
-    }
-#endif
+    auto in = &field[(parity * volumeCB + x) * length];
+    block_load<complex, length/2>(v, reinterpret_cast<const complex*>(in));
   }
 
   __device__ __host__ inline void save(const complex v[length / 2], int x, int parity = 0)
   {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-    typedef S<Float,length> structure;
-    trove::coalesced_ptr<structure> field_((structure*)field);
-    structure v_;
-    for (int s=0; s<Ns; s++) {
-      for (int c=0; c<Nc; c++) {
-        v_.v[(s * Nc + c) * 2 + 0] = v[s * Nc + c].real();
-        v_.v[(s * Nc + c) * 2 + 1] = v[s * Nc + c].imag();
-      }
-    }
-    field_[parity*volumeCB + x] = v_;
-#else
-    for (int s=0; s<Ns; s++) {
-      for (int c=0; c<Nc; c++) {
-        field[parity * offset + ((x * Ns + s) * Nc + c) * 2 + 0] = v[s * Nc + c].real();
-        field[parity * offset + ((x * Ns + s) * Nc + c) * 2 + 1] = v[s * Nc + c].imag();
-      }
-    }
-#endif
+    auto out = &field[(parity * volumeCB + x) * length];
+    block_store<complex, length/2>(reinterpret_cast<complex*>(out), v);
   }
 
   /**
@@ -1521,47 +1429,15 @@ namespace quda {
   __device__ __host__ inline void load(complex v[length / 2], int x, int parity = 0) const
   {
     int y = getPaddedIndex(x, parity);
-
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-    typedef S<Float,length> structure;
-    trove::coalesced_ptr<structure> field_((structure*)field);
-    structure v_ = field_[parity*exVolumeCB + y];
-    for (int s=0; s<Ns; s++) {
-      for (int c = 0; c < Nc; c++) { v[s * Nc + c] = complex(v_.v[(s * Nc + c) * 2 + 0], v_.v[(s * Nc + c) * 2 + 1]); }
-    }
-#else
-    for (int s=0; s<Ns; s++) {
-      for (int c=0; c<Nc; c++) {
-        v[s * Nc + c] = complex(field[parity * offset + ((y * Ns + s) * Nc + c) * 2 + 0],
-                                field[parity * offset + ((y * Ns + s) * Nc + c) * 2 + 1]);
-      }
-    }
-#endif
+    auto in = &field[(parity * exVolumeCB + y) * length];
+    block_load<complex, length/2>(v, reinterpret_cast<const complex *>(in));
   }
 
   __device__ __host__ inline void save(const complex v[length / 2], int x, int parity = 0)
   {
     int y = getPaddedIndex(x, parity);
-
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-    typedef S<Float,length> structure;
-    trove::coalesced_ptr<structure> field_((structure*)field);
-    structure v_;
-    for (int s=0; s<Ns; s++) {
-      for (int c=0; c<Nc; c++) {
-        v_.v[(s * Nc + c) * 2 + 0] = v[s * Nc + c].real();
-        v_.v[(s * Nc + c) * 2 + 1] = v[s * Nc + c].imag();
-      }
-    }
-    field_[parity*exVolumeCB + y] = v_;
-#else
-    for (int s=0; s<Ns; s++) {
-      for (int c=0; c<Nc; c++) {
-        field[parity * offset + ((y * Ns + s) * Nc + c) * 2 + 0] = v[s * Nc + c].real();
-        field[parity * offset + ((y * Ns + s) * Nc + c) * 2 + 1] = v[s * Nc + c].imag();
-      }
-    }
-#endif
+    auto out = &field[(parity * exVolumeCB + y) * length];
+    block_store<complex, length/2>(reinterpret_cast<complex *>(out), v);
   }
 
   /**
diff --git a/include/comm_quda.h b/include/comm_quda.h
index 7f99c5fe84..05e139b2e5 100644
--- a/include/comm_quda.h
+++ b/include/comm_quda.h
@@ -1,6 +1,10 @@
 #pragma once
 #include <cstdint>
 #include <cstdlib>
+#include <quda_constants.h>
+#include <quda_api.h>
+
+using namespace quda;
 
 #ifdef __cplusplus
 extern "C" {
@@ -165,7 +169,7 @@ extern "C" {
   /**
      @return Number of processes
   */
-  int comm_size(void);
+  size_t comm_size(void);
 
   /**
      @return GPU id associated with this process
@@ -198,6 +202,66 @@ extern "C" {
    */
   void comm_peer2peer_init(const char *hostname_recv_buf);
 
+  /**
+     @brief Query if peer-to-peer communication is possible between two GPUs
+     @param[in] local_gpuid GPU associated with this process
+     @param[in] neighbor_gpuid GPU associated with neighboring process
+     (assumed on same node)
+     @return True/false if peer-to-peer is possible
+  */
+  bool comm_peer2peer_possible(int local_gpuid, int neighbor_gpuid);
+
+  /**
+     @brief Query the performance of peer-to-peer communication between two GPUs
+     @param[in] local_gpuid GPU associated with this process
+     @param[in] neighbor_gpuid GPU associated with neighboring process
+     (assumed on same node)
+     @return Relative performance ranking between this pair of GPUs
+  */
+  int comm_peer2peer_performance(int local_gpuid, int neighbor_gpuid);
+
+   /**
+     @brief Symmetric exchange of local memory addresses between
+     logically neighboring processes on the lattice.  The remote
+     addresses that are returned are directly addressable by the local
+     process and can be read or written to by a kernel, or can be
+     copied to and from.  This exchange is only defined between
+     devices that are peer-to-peer enabled.
+     @param[out] remote Array of remote memory pointers to neighboring
+     pointers
+     @param[in] local The process-local memory pointer to be exchanged
+     from this process
+   */
+  void comm_create_neighbor_memory(void *remote[QUDA_MAX_DIM][2], void *local);
+
+  /**
+     @brief Deallocate the remote addresses to logically neighboring
+     processes on the on the lattice.
+     @param[in] remote Array of remote memory pointers to neighboring
+     pointers
+   */
+  void comm_destroy_neighbor_memory(void *remote[QUDA_MAX_DIM][2]);
+
+  /**
+     @brief Create unique events shared between each logical pair of
+     neighboring processes, e.g., the event in the forwards direction
+     in a given dimension on a given process aliases the event in the
+     backward direction in the same dimension, and is unique
+     between that process pair. This exchange is only defined between
+     devices that are peer-to-peer enabled.
+     @param[out] remote Array of remote events to neighboring processes
+     @param[in] local Array of local event to neighboring processes
+   */
+  void comm_create_neighbor_event(qudaEvent_t remote[2][QUDA_MAX_DIM],
+                                  qudaEvent_t local[2][QUDA_MAX_DIM]);
+
+  /**
+     @brief Destroy the coupled events
+     @param[out] remote Array of remote events to neighboring processes
+     @param[in] local Array of local event to neighboring processes
+   */
+  void comm_destroy_neighbor_event(qudaEvent_t remote[2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM]);
+
   /**
      @brief Returns true if any peer-to-peer capability is present on
      this system (regardless of whether it has been disabled or not.  We
diff --git a/include/complex_quda.h b/include/complex_quda.h
index f1609faa10..ee428d412c 100644
--- a/include/complex_quda.h
+++ b/include/complex_quda.h
@@ -20,12 +20,10 @@
 
 #pragma once
 
-//#include <math.h>
-
 #include <cmath>
 #include <complex>
 #include <sstream>
-//#include <cuComplex.h>
+#include <cstdint>
 
 namespace quda {
   namespace gauge {
@@ -820,6 +818,12 @@ struct complex <int> : public int2
     }
 
   __host__ __device__ inline complex<int>(const complex<int> & z) : int2(z){}
+  __host__ __device__ complex<int>& operator=(const complex<int> &z)
+    {
+      x = z.x;
+      y = z.y;
+      return *this;
+    }
 
   template <typename X>
   inline complex<int>(const std::complex<X> & z) : int2{static_cast<int>(z.x), static_cast<int>(z.y)} {}
diff --git a/include/convert.h b/include/convert.h
index 2721ab16ee..173d719ef0 100644
--- a/include/convert.h
+++ b/include/convert.h
@@ -9,12 +9,56 @@
  */
 
 #include <type_traits>
-#include <quda_internal.h> // for maximum short, char traits.
 #include <register_traits.h>
 
 namespace quda
 {
 
+  /**
+   * Traits for determining the maximum and inverse maximum
+   * value of a (signed) char and short. Relevant for
+   * fixed-precision types.
+   */
+  template< typename T > struct fixedMaxValue{ static constexpr float value = 0.0f; };
+  template<> struct fixedMaxValue<short>{ static constexpr float value = 32767.0f; };
+  template<> struct fixedMaxValue<short2>{ static constexpr float value = 32767.0f; };
+  template<> struct fixedMaxValue<short4>{ static constexpr float value = 32767.0f; };
+  template<> struct fixedMaxValue<short8>{ static constexpr float value = 32767.0f; };
+  template <> struct fixedMaxValue<int8_t> {
+    static constexpr float value = 127.0f;
+  };
+  template<> struct fixedMaxValue<char2>{ static constexpr float value = 127.0f; };
+  template<> struct fixedMaxValue<char4>{ static constexpr float value = 127.0f; };
+  template<> struct fixedMaxValue<char8>{ static constexpr float value = 127.0f; };
+
+  template <typename T> struct fixedInvMaxValue {
+    static constexpr float value = 3.402823e+38f;
+  };
+  template <> struct fixedInvMaxValue<short> {
+    static constexpr float value = 3.0518509476e-5f;
+  };
+  template <> struct fixedInvMaxValue<short2> {
+    static constexpr float value = 3.0518509476e-5f;
+  };
+  template <> struct fixedInvMaxValue<short4> {
+    static constexpr float value = 3.0518509476e-5f;
+  };
+  template <> struct fixedInvMaxValue<short8> {
+    static constexpr float value = 3.0518509476e-5f;
+  };
+  template <> struct fixedInvMaxValue<int8_t> {
+    static constexpr float value = 7.874015748031e-3f;
+  };
+  template <> struct fixedInvMaxValue<char2> {
+    static constexpr float value = 7.874015748031e-3f;
+  };
+  template <> struct fixedInvMaxValue<char4> {
+    static constexpr float value = 7.874015748031e-3f;
+  };
+  template <> struct fixedInvMaxValue<char8> {
+    static constexpr float value = 7.874015748031e-3f;
+  };
+
   template <typename T> __host__ __device__ inline float i2f(T a)
   {
 #if 1
diff --git a/include/deflation.h b/include/deflation.h
index f20f936dc9..7cce97fb66 100644
--- a/include/deflation.h
+++ b/include/deflation.h
@@ -56,18 +56,15 @@ namespace quda {
         tot_dim      = param.np;
         ld           = ((tot_dim+15) / 16) * tot_dim;
         //allocate deflation resources:
-        matProj      = new Complex[ld*tot_dim];
+        matProj      = static_cast<Complex*>(pool_pinned_malloc(ld * tot_dim * sizeof(Complex)));
         invRitzVals  = new double[tot_dim];
 
         //Check that RV is a composite field:
         if(RV->IsComposite() == false) errorQuda("\nRitz vectors must be contained in a composite field.\n");
-
-        cudaHostRegister(matProj,ld*tot_dim*sizeof(Complex),cudaHostRegisterDefault);
      }
 
      ~DeflationParam(){
-        cudaHostUnregister(matProj);
-        if(matProj) delete[]  matProj;
+       pool_pinned_free(matProj);
         if(invRitzVals)       delete[]  invRitzVals;
      }
   };
diff --git a/include/device.h b/include/device.h
index 811c8f797f..bb5fb787b3 100644
--- a/include/device.h
+++ b/include/device.h
@@ -10,6 +10,11 @@ namespace quda
      */
     void init(int dev);
 
+    /**
+       @brief Get number of devices present on node
+    */
+    int get_device_count();
+
     /**
        @brief Query and print to stdout device properties of all GPUs
     */
diff --git a/include/dirac_quda.h b/include/dirac_quda.h
index fa9fe5b902..25d2e7d0b6 100644
--- a/include/dirac_quda.h
+++ b/include/dirac_quda.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <quda_internal.h>
+#include <timer.h>
 #include <color_spinor_field.h>
 #include <gauge_field.h>
 #include <clover_field.h>
diff --git a/include/eigensolve_quda.h b/include/eigensolve_quda.h
index 4e182d406e..fcdffad453 100644
--- a/include/eigensolve_quda.h
+++ b/include/eigensolve_quda.h
@@ -2,6 +2,7 @@
 
 #include <quda.h>
 #include <quda_internal.h>
+#include <timer.h>
 #include <dirac_quda.h>
 #include <color_spinor_field.h>
 
diff --git a/include/gauge_field_order.h b/include/gauge_field_order.h
index a7d051eb61..6a49bfe448 100644
--- a/include/gauge_field_order.h
+++ b/include/gauge_field_order.h
@@ -6,9 +6,7 @@
  *
  */
 
-#ifndef __CUDACC_RTC__
-#include <assert.h>
-#endif
+#include <cassert>
 #include <type_traits>
 
 #include <register_traits.h>
@@ -22,7 +20,7 @@
 #include <atomic.cuh>
 #include <gauge_field.h>
 #include <index_helper.cuh>
-#include <trove_helper.cuh>
+#include <aos.h>
 #include <transform_reduce.h>
 
 namespace quda {
@@ -160,59 +158,6 @@ namespace quda {
 
   namespace gauge {
 
-    template<typename ReduceType, typename Float> struct square_ {
-      square_(ReduceType) { }
-      __host__ __device__ inline ReduceType operator()(const quda::complex<Float> &x)
-      { return static_cast<ReduceType>(norm(x)); }
-    };
-
-    template <typename ReduceType> struct square_<ReduceType, int8_t> {
-      const ReduceType scale;
-      square_(const ReduceType scale) : scale(scale) { }
-      __host__ __device__ inline ReduceType operator()(const quda::complex<int8_t> &x)
-      { return norm(scale * complex<ReduceType>(x.real(), x.imag())); }
-    };
-
-    template<typename ReduceType> struct square_<ReduceType,short> {
-      const ReduceType scale;
-      square_(const ReduceType scale) : scale(scale) { }
-      __host__ __device__ inline ReduceType operator()(const quda::complex<short> &x)
-      { return norm(scale * complex<ReduceType>(x.real(), x.imag())); }
-    };
-
-    template<typename ReduceType> struct square_<ReduceType,int> {
-      const ReduceType scale;
-      square_(const ReduceType scale) : scale(scale) { }
-      __host__ __device__ inline ReduceType operator()(const quda::complex<int> &x)
-      { return norm(scale * complex<ReduceType>(x.real(), x.imag())); }
-    };
-
-    template<typename Float, typename storeFloat> struct abs_ {
-      abs_(const Float) { }
-      __host__ __device__ Float operator()(const quda::complex<storeFloat> &x) { return abs(x); }
-    };
-
-    template <typename Float> struct abs_<Float, int8_t> {
-      Float scale;
-      abs_(const Float scale) : scale(scale) { }
-      __host__ __device__ Float operator()(const quda::complex<int8_t> &x)
-      { return abs(scale * complex<Float>(x.real(), x.imag())); }
-    };
-
-    template<typename Float> struct abs_<Float,short> {
-      Float scale;
-      abs_(const Float scale) : scale(scale) { }
-      __host__ __device__ Float operator()(const quda::complex<short> &x)
-      { return abs(scale * complex<Float>(x.real(), x.imag())); }
-    };
-
-    template<typename Float> struct abs_<Float,int> {
-      Float scale;
-      abs_(const Float scale) : scale(scale) { }
-      __host__ __device__ Float operator()(const quda::complex<int> &x)
-      { return abs(scale * complex<Float>(x.real(), x.imag())); }
-    };
-
     template <typename Float, typename storeFloat> __host__ __device__ inline constexpr bool fixed_point() { return false; }
     template <> __host__ __device__ inline constexpr bool fixed_point<float, int8_t>() { return true; }
     template<> __host__ __device__ inline constexpr bool fixed_point<float,short>() { return true; }
@@ -2092,18 +2037,6 @@ namespace quda {
       size_t Bytes() const { return reconLen * sizeof(Float); }
       };
 
-  /**
-     @brief This is just a dummy structure we use for trove to define the
-     required structure size
-     @param real Real number type
-     @param length Number of elements in the structure
-  */
-      template <typename real, int length> struct S {
-        real v[length];
-        __host__ __device__ const real &operator[](int i) const { return v[i]; }
-        __host__ __device__ real &operator[](int i) { return v[i]; }
-      };
-
       /**
          @brief The LegacyOrder defines the ghost zone storage and ordering for
          all cpuGaugeFields, which use the same ghost zone storage.
@@ -2150,34 +2083,14 @@ namespace quda {
 
         __device__ __host__ inline void loadGhost(complex v[length / 2], int x, int dir, int parity, real = 1.0) const
         {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-          typedef S<Float, length> structure;
-          trove::coalesced_ptr<structure> ghost_((structure *)ghost[dir]);
-          structure v_ = ghost_[parity * faceVolumeCB[dir] + x];
-#else
-          auto v_ = &ghost[dir][(parity * faceVolumeCB[dir] + x) * length];
-#endif
-          for (int i = 0; i < length / 2; i++) v[i] = complex(v_[2 * i + 0], v_[2 * i + 1]);
+          auto in = &ghost[dir][(parity * faceVolumeCB[dir] + x) * length];
+          block_load<complex, length/2>(v, reinterpret_cast<complex*>(in));
         }
 
         __device__ __host__ inline void saveGhost(const complex v[length / 2], int x, int dir, int parity)
         {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-          typedef S<Float, length> structure;
-          trove::coalesced_ptr<structure> ghost_((structure *)ghost[dir]);
-          structure v_;
-          for (int i = 0; i < length / 2; i++) {
-            v_[2 * i + 0] = (Float)v[i].real();
-            v_[2 * i + 1] = (Float)v[i].imag();
-          }
-          ghost_[parity * faceVolumeCB[dir] + x] = v_;
-#else
-          auto v_ = &ghost[dir][(parity * faceVolumeCB[dir] + x) * length];
-          for (int i = 0; i < length / 2; i++) {
-            v_[2 * i + 0] = (Float)v[i].real();
-            v_[2 * i + 1] = (Float)v[i].imag();
-          }
-#endif
+          auto out = &ghost[dir][(parity * faceVolumeCB[dir] + x) * length];
+          block_store<complex, length/2>(reinterpret_cast<complex*>(out), v);
         }
 
         /**
@@ -2215,35 +2128,15 @@ namespace quda {
         __device__ __host__ inline void loadGhostEx(complex v[length / 2], int x, int, int dir, int dim, int g,
                                                     int parity, const int R[]) const
         {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	typedef S<Float,length> structure;
-	trove::coalesced_ptr<structure> ghost_((structure*)ghost[dim]);
-	structure v_ = ghost_[((dir*2+parity)*R[dim]*faceVolumeCB[dim] + x)*geometry+g];
-#else
-          auto v_ = &ghost[dim][(((dir * 2 + parity) * R[dim] * faceVolumeCB[dim] + x) * geometry + g) * length];
-#endif
-        for (int i = 0; i < length / 2; i++) v[i] = complex(v_[2 * i + 0], v_[2 * i + 1]);
+          auto in = &ghost[dim][(((dir * 2 + parity) * R[dim] * faceVolumeCB[dim] + x) * geometry + g) * length];
+          block_load<complex, length/2>(v, reinterpret_cast<complex*>(in));
         }
 
         __device__ __host__ inline void saveGhostEx(const complex v[length / 2], int x, int, int dir, int dim,
                                                     int g, int parity, const int R[])
         {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-          typedef S<Float, length> structure;
-          trove::coalesced_ptr<structure> ghost_((structure *)ghost[dim]);
-          structure v_;
-          for (int i = 0; i < length / 2; i++) {
-            v_[2 * i + 0] = (Float)v[i].real();
-            v_[2 * i + 1] = (Float)v[i].imag();
-          }
-          ghost_[((dir * 2 + parity) * R[dim] * faceVolumeCB[dim] + x) * geometry + g] = v_;
-#else
-          auto v_ = &ghost[dim][(((dir * 2 + parity) * R[dim] * faceVolumeCB[dim] + x) * geometry + g) * length];
-          for (int i = 0; i < length / 2; i++) {
-            v_[2 * i + 0] = (Float)v[i].real();
-            v_[2 * i + 1] = (Float)v[i].imag();
-          }
-#endif
+          auto out = &ghost[dim][(((dir * 2 + parity) * R[dim] * faceVolumeCB[dim] + x) * geometry + g) * length];
+          block_store<complex, length/2>(reinterpret_cast<complex*>(out), v);
         }
       };
 
@@ -2266,34 +2159,14 @@ namespace quda {
 
       __device__ __host__ inline void load(complex v[length / 2], int x, int dir, int parity, real = 1.0) const
       {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	typedef S<Float,length> structure;
-	trove::coalesced_ptr<structure> gauge_((structure*)gauge[dir]);
-	structure v_ = gauge_[parity*volumeCB + x];
-#else
-        auto v_ = &gauge[dir][(parity * volumeCB + x) * length];
-#endif
-        for (int i = 0; i < length / 2; i++) v[i] = complex(v_[2 * i + 0], v_[2 * i + 1]);
+        auto in = &gauge[dir][(parity * volumeCB + x) * length];
+        block_load<complex, length/2>(v, reinterpret_cast<complex*>(in));
       }
 
       __device__ __host__ inline void save(const complex v[length / 2], int x, int dir, int parity)
       {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	typedef S<Float,length> structure;
-	trove::coalesced_ptr<structure> gauge_((structure*)gauge[dir]);
-	structure v_;
-        for (int i = 0; i < length / 2; i++) {
-          v_[2 * i + 0] = (Float)v[i].real();
-          v_[2 * i + 1] = (Float)v[i].imag();
-        }
-        gauge_[parity * volumeCB + x] = v_;
-#else
-        auto v_ = &gauge[dir][(parity * volumeCB + x) * length];
-        for (int i = 0; i < length / 2; i++) {
-          v_[2 * i + 0] = (Float)v[i].real();
-          v_[2 * i + 1] = (Float)v[i].imag();
-        }
-#endif
+        auto out = &gauge[dir][(parity * volumeCB + x) * length];
+        block_store<complex, length/2>(reinterpret_cast<complex*>(out), v);
       }
 
       /**
@@ -2413,36 +2286,16 @@ namespace quda {
       gauge(order.gauge), volumeCB(order.volumeCB), geometry(order.geometry)
       { ; }
 
-      __device__ __host__ inline void load(complex v[length / 2], int x, int dir, int parity, real = 1.0) const
-      {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-      typedef S<Float,length> structure;
-      trove::coalesced_ptr<structure> gauge_((structure*)gauge);
-      structure v_ = gauge_[(parity*volumeCB+x)*geometry + dir];
-#else
-        auto v_ = &gauge[((parity * volumeCB + x) * geometry + dir) * length];
-#endif
-      for (int i = 0; i < length / 2; i++) v[i] = complex(v_[2 * i + 0], v_[2 * i + 1]);
+    __device__ __host__ inline void load(complex v[length / 2], int x, int dir, int parity, real = 1.0) const
+    {
+      auto in = &gauge[((parity * volumeCB + x) * geometry + dir) * length];
+      block_load<complex, length/2>(v, reinterpret_cast<complex*>(in));
     }
 
     __device__ __host__ inline void save(const complex v[length / 2], int x, int dir, int parity)
     {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-      typedef S<Float,length> structure;
-      trove::coalesced_ptr<structure> gauge_((structure*)gauge);
-      structure v_;
-      for (int i = 0; i < length / 2; i++) {
-        v_[2 * i + 0] = v[i].real();
-        v_[2 * i + 1] = v[i].imag();
-      }
-      gauge_[(parity*volumeCB+x)*geometry + dir] = v_;
-#else
-      auto v_ = &gauge[((parity * volumeCB + x) * geometry + dir) * length];
-      for (int i = 0; i < length / 2; i++) {
-        v_[2 * i + 0] = v[i].real();
-        v_[2 * i + 1] = v[i].imag();
-      }
-#endif
+      auto out = &gauge[((parity * volumeCB + x) * geometry + dir) * length];
+      block_store<complex, length/2>(reinterpret_cast<complex*>(out), v);
     }
 
     /**
@@ -2526,38 +2379,15 @@ namespace quda {
     __device__ __host__ inline void load(complex v[length / 2], int x, int dir, int parity, real = 1.0) const
     {
       // get base pointer
-      const Float *gauge0 = reinterpret_cast<const Float*>(reinterpret_cast<const char*>(gauge) + (parity*volumeCB+x)*size + offset);
-
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-      typedef S<Float,length> structure;
-      trove::coalesced_ptr<structure> gauge_((structure*)gauge0);
-      structure v_ = gauge_[dir];
-#else
-      auto v_ = &gauge0[dir * length];
-#endif
-      for (int i = 0; i < length / 2; i++) v[i] = complex(v_[2 * i + 0], v_[2 * i + 1]);
+      auto in = reinterpret_cast<const Float*>(reinterpret_cast<const char*>(gauge) + (parity*volumeCB+x)*size + offset + dir * length);
+      block_load<complex, length/2>(v, reinterpret_cast<const complex*>(in));
     }
 
     __device__ __host__ inline void save(const complex v[length / 2], int x, int dir, int parity)
     {
       // get base pointer
-      Float *gauge0 = reinterpret_cast<Float*>(reinterpret_cast<char*>(gauge) + (parity*volumeCB+x)*size + offset);
-
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-      typedef S<Float,length> structure;
-      trove::coalesced_ptr<structure> gauge_((structure*)gauge0);
-      structure v_;
-      for (int i = 0; i < length / 2; i++) {
-        v_[2 * i + 0] = v[i].real();
-        v_[2 * i + 1] = v[i].imag();
-      }
-      gauge_[dir] = v_;
-#else
-      for (int i = 0; i < length / 2; i++) {
-        gauge0[dir * length + 2 * i + 0] = v[i].real();
-        gauge0[dir * length + 2 * i + 1] = v[i].imag();
-      }
-#endif
+      auto out = reinterpret_cast<Float*>(reinterpret_cast<char*>(gauge) + (parity*volumeCB+x)*size + offset + dir * length);
+      block_store<complex, length/2>(reinterpret_cast<complex*>(out), v);
     }
 
     /**
@@ -2632,41 +2462,28 @@ namespace quda {
     // we need to transpose and scale for CPS ordering
     __device__ __host__ inline void load(complex v[9], int x, int dir, int parity, Float = 1.0) const
     {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-      typedef S<Float,length> structure;
-      trove::coalesced_ptr<structure> gauge_((structure*)gauge);
-      structure v_ = gauge_[((parity*volumeCB+x)*geometry + dir)];
-#else
-      auto v_ = &gauge[((parity * volumeCB + x) * geometry + dir) * length];
-#endif
+      auto in = &gauge[((parity * volumeCB + x) * geometry + dir) * length];
+      complex v_[9];
+      block_load<complex, length/2>(v_, reinterpret_cast<complex*>(in));
+
       for (int i=0; i<Nc; i++) {
 	for (int j=0; j<Nc; j++) {
-          v[i * Nc + j] = complex(v_[(j * Nc + i) * 2 + 0], v_[(j * Nc + i) * 2 + 1]) * anisotropy_inv;
+          v[i * Nc + j] = v_[j * Nc + i] * anisotropy_inv;
         }
       }
     }
 
     __device__ __host__ inline void save(const complex v[9], int x, int dir, int parity)
     {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-      typedef S<Float,length> structure;
-      trove::coalesced_ptr<structure> gauge_((structure*)gauge);
-      structure v_;
-      for (int i=0; i<Nc; i++)
-        for (int j = 0; j < Nc; j++) {
-          v_[(j * Nc + i) * 2 + 0] = anisotropy * v[i * Nc + j].real();
-          v_[(j * Nc + i) * 2 + 1] = anisotropy * v[i * Nc + j].imag();
-        }
-      gauge_[((parity*volumeCB+x)*geometry + dir)] = v_;
-#else
-      auto v_ = &gauge[((parity * volumeCB + x) * geometry + dir) * length];
+      auto out = &gauge[((parity * volumeCB + x) * geometry + dir) * length];
+      complex v_[9];
       for (int i=0; i<Nc; i++) {
 	for (int j=0; j<Nc; j++) {
-          v_[(j * Nc + i) * 2 + 0] = anisotropy * v[i * Nc + j].real();
-          v_[(j * Nc + i) * 2 + 1] = anisotropy * v[i * Nc + j].imag();
+          v_[i * Nc + j] = v[j * Nc + i] * anisotropy;
         }
       }
-#endif
+
+      block_store<complex, length/2>(reinterpret_cast<complex*>(out), v_);
     }
 
     /**
@@ -2739,39 +2556,28 @@ namespace quda {
       // we need to transpose for BQCD ordering
       __device__ __host__ inline void load(complex v[9], int x, int dir, int parity, real = 1.0) const
       {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-        typedef S<Float, length> structure;
-        trove::coalesced_ptr<structure> gauge_((structure *)gauge);
-        structure v_ = gauge_[(dir * 2 + parity) * exVolumeCB + x];
-#else
-        auto v_ = &gauge[((dir * 2 + parity) * exVolumeCB + x) * length];
-#endif
+        auto in = &gauge[((dir * 2 + parity) * exVolumeCB + x) * length];
+        complex v_[9];
+        block_load<complex, 9>(v_, reinterpret_cast<complex*>(in));
+
         for (int i = 0; i < Nc; i++) {
-          for (int j = 0; j < Nc; j++) { v[i * Nc + j] = complex(v_[(j * Nc + i) * 2 + 0], v_[(j * Nc + i) * 2 + 1]); }
+          for (int j = 0; j < Nc; j++) {
+            v[i * Nc + j] = v_[j * Nc + i];
+          }
         }
       }
 
       __device__ __host__ inline void save(const complex v[9], int x, int dir, int parity)
       {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	typedef S<Float,length> structure;
-	trove::coalesced_ptr<structure> gauge_((structure*)gauge);
-	structure v_;
-	for (int i=0; i<Nc; i++)
-          for (int j = 0; j < Nc; j++) {
-            v_[(j * Nc + i) * 2 + 0] = v[i * Nc + j].real();
-            v_[(j * Nc + i) * 2 + 1] = v[i * Nc + j].imag();
-          }
-        gauge_[(dir * 2 + parity) * exVolumeCB + x] = v_;
-#else
-        auto v_ = &gauge[((dir * 2 + parity) * exVolumeCB + x) * length];
+        auto out = &gauge[((dir * 2 + parity) * exVolumeCB + x) * length];
+        complex v_[9];
         for (int i = 0; i < Nc; i++) {
           for (int j = 0; j < Nc; j++) {
-            v_[(j * Nc + i) * 2 + 0] = v[i * Nc + j].real();
-            v_[(j * Nc + i) * 2 + 1] = v[i * Nc + j].imag();
+            v_[i * Nc + j] = v[j * Nc + i];
           }
         }
-#endif
+
+        block_store<complex, 9>(reinterpret_cast<complex*>(out), v_);
       }
 
       /**
@@ -2842,41 +2648,28 @@ namespace quda {
       // we need to transpose for TIFR ordering
       __device__ __host__ inline void load(complex v[9], int x, int dir, int parity, real = 1.0) const
       {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-        typedef S<Float, length> structure;
-        trove::coalesced_ptr<structure> gauge_((structure *)gauge);
-        structure v_ = gauge_[(dir * 2 + parity) * volumeCB + x];
-#else
-        auto v_ = &gauge[((dir * 2 + parity) * volumeCB + x) * length];
-#endif
+        auto in = &gauge[((dir * 2 + parity) * volumeCB + x) * length];
+        complex v_[9];
+        block_load<complex, 9>(v_, reinterpret_cast<complex*>(in));
+
         for (int i = 0; i < Nc; i++) {
           for (int j = 0; j < Nc; j++) {
-            v[i * Nc + j] = complex(v_[(j * Nc + i) * 2 + 0], v_[(j * Nc + i) * 2 + 1]) * scale_inv;
+            v[i * Nc + j] = v_[j * Nc + i] * scale_inv;
           }
         }
       }
 
       __device__ __host__ inline void save(const complex v[9], int x, int dir, int parity)
       {
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	typedef S<Float,length> structure;
-	trove::coalesced_ptr<structure> gauge_((structure*)gauge);
-	structure v_;
-	for (int i=0; i<Nc; i++)
-          for (int j = 0; j < Nc; j++) {
-            v_[(j * Nc + i) * 2 + 0] = v[i * Nc + j].real() * scale;
-            v_[(j * Nc + i) * 2 + 1] = v[i * Nc + j].imag() * scale;
-          }
-        gauge_[(dir * 2 + parity) * volumeCB + x] = v_;
-#else
-        auto v_ = &gauge[((dir * 2 + parity) * volumeCB + x) * length];
+        auto out = &gauge[((dir * 2 + parity) * volumeCB + x) * length];
+        complex v_[9];
         for (int i = 0; i < Nc; i++) {
           for (int j = 0; j < Nc; j++) {
-            v_[(j * Nc + i) * 2 + 0] = v[i * Nc + j].real() * scale;
-            v_[(j * Nc + i) * 2 + 1] = v[i * Nc + j].imag() * scale;
+            v_[i * Nc + j] = v[j * Nc + i] * scale;
           }
         }
-#endif
+
+        block_store<complex, 9>(reinterpret_cast<complex*>(out), v_);
       }
 
       /**
@@ -2976,17 +2769,13 @@ namespace quda {
       __device__ __host__ inline void load(complex v[9], int x, int dir, int parity, real = 1.0) const
       {
         int y = getPaddedIndex(x, parity);
+        auto in = &gauge[((dir * 2 + parity) * exVolumeCB + y) * length];
+        complex v_[9];
+        block_load<complex, 9>(v_, reinterpret_cast<complex*>(in));
 
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	typedef S<Float,length> structure;
-	trove::coalesced_ptr<structure> gauge_((structure*)gauge);
-	structure v_ = gauge_[(dir*2+parity)*exVolumeCB + y];
-#else
-        auto v_ = &gauge[((dir * 2 + parity) * exVolumeCB + y) * length];
-#endif
         for (int i = 0; i < Nc; i++) {
           for (int j = 0; j < Nc; j++) {
-            v[i * Nc + j] = complex(v_[(j * Nc + i) * 2 + 0], v_[(j * Nc + i) * 2 + 1]) * scale_inv;
+            v[i * Nc + j] = v_[j * Nc + i] * scale_inv;
           }
         }
       }
@@ -2994,26 +2783,16 @@ namespace quda {
       __device__ __host__ inline void save(const complex v[9], int x, int dir, int parity)
       {
         int y = getPaddedIndex(x, parity);
+        auto out = &gauge[((dir * 2 + parity) * exVolumeCB + y) * length];
 
-#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
-	typedef S<Float,length> structure;
-	trove::coalesced_ptr<structure> gauge_((structure*)gauge);
-	structure v_;
-	for (int i=0; i<Nc; i++)
-          for (int j = 0; j < Nc; j++) {
-            v_[(j * Nc + i) * 2 + 0] = v[i * Nc + j].real() * scale;
-            v_[(j * Nc + i) * 2 + 1] = v[i * Nc + j].imag() * scale;
-          }
-        gauge_[(dir * 2 + parity) * exVolumeCB + y] = v_;
-#else
-        auto v_ = &gauge[((dir * 2 + parity) * exVolumeCB + y) * length];
+        complex v_[9];
         for (int i = 0; i < Nc; i++) {
           for (int j = 0; j < Nc; j++) {
-            v_[(j * Nc + i) * 2 + 0] = v[i * Nc + j].real() * scale;
-            v_[(j * Nc + i) * 2 + 1] = v[i * Nc + j].imag() * scale;
+            v_[i * Nc + j] = v[j * Nc + i] * scale;
           }
         }
-#endif
+
+        block_store<complex, 9>(reinterpret_cast<complex*>(out), v_);
       }
 
       /**
diff --git a/include/gauge_tools.h b/include/gauge_tools.h
index 9c1654f483..0cdc575a5a 100644
--- a/include/gauge_tools.h
+++ b/include/gauge_tools.h
@@ -1,4 +1,5 @@
 #include <random_quda.h>
+#include <timer.h>
 
 namespace quda
 {
diff --git a/include/invert_quda.h b/include/invert_quda.h
index b6d45ede93..74df7d8a84 100644
--- a/include/invert_quda.h
+++ b/include/invert_quda.h
@@ -2,6 +2,7 @@
 
 #include <quda.h>
 #include <quda_internal.h>
+#include <timer.h>
 #include <dirac_quda.h>
 #include <color_spinor_field.h>
 #include <qio_field.h>
diff --git a/include/jitify_helper2.cuh b/include/jitify_helper2.cuh
index 29d8e3ebeb..1de58b3525 100644
--- a/include/jitify_helper2.cuh
+++ b/include/jitify_helper2.cuh
@@ -59,7 +59,7 @@ namespace quda {
                             const std::vector<constant_param_t> &param,
                             std::vector<void*> arg_ptrs, jitify::detail::vector<std::string> arg_types);
 
-  template <template <typename> class Functor, typename Arg>
+  template <template <typename> class Functor, bool grid_stride, typename Arg, bool template_block_size = false>
   qudaError_t launch_jitify(const std::string &kernel, const TuneParam &tp, const qudaStream_t &stream,
                             const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
   {
@@ -68,49 +68,33 @@ namespace quda {
     auto Functor_instance = reflect<Functor<Arg>>();
     auto Functor_naked = Functor_instance.substr(0, Functor_instance.find("<"));
     auto Arg_reflect = reflect<Arg>();
+    std::vector<std::string> template_args = template_block_size ?
+      std::vector<std::string>{reflect((int)tp.block.x), reflect((int)tp.block.y), Functor_naked, Arg_reflect, reflect(grid_stride)} :
+      std::vector<std::string>{Functor_naked, Arg_reflect, reflect(grid_stride)};
 
     std::vector<void*> arg_ptrs{(void*)&arg};
     jitify::detail::vector<std::string> arg_types{Arg_reflect};
 
-    return launch_jitify(Functor<Arg>::filename(), kernel, {Functor_naked, Arg_reflect},
-                         tp, stream, param, arg_ptrs, arg_types);    
-  }
-
-  template <template <typename> class Functor, template <typename> class Reducer, typename Arg>
-  qudaError_t launch_jitify(const std::string &kernel, const TuneParam &tp, const qudaStream_t &stream,
-                            const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
-  {
-    // we need this hackery to get the naked unbound template class parameters
-    using namespace jitify::reflection;
-    auto Functor_instance = reflect<Functor<Arg>>();
-    auto Functor_naked = Functor_instance.substr(0, Functor_instance.find("<"));
-    auto Reducer_instance = reflect<Reducer<Arg>>();
-    auto Reducer_naked = Reducer_instance.substr(0, Reducer_instance.find("<"));
-    auto Arg_reflect = reflect<Arg>();
-
-    std::vector<void*> arg_ptrs{(void*)&arg};
-    jitify::detail::vector<std::string> arg_types{Arg_reflect};
-
-    return launch_jitify(Functor<Arg>::filename(), kernel,
-                         {reflect((int)tp.block.x), reflect((int)tp.block.y), Functor_naked, Reducer_naked, Arg_reflect},
+    return launch_jitify(Functor<Arg>::filename(), kernel, template_args,
+                         //{reflect((int)tp.block.x), reflect((int)tp.block.y), Functor_naked, Arg_reflect, reflect(grid_stride)},
                          tp, stream, param, arg_ptrs, arg_types);
   }
 
   // FIXME merge the functionality of these launchers
-  template <template <typename> class Functor, typename Arg>
+  template <template <int, typename> class Functor, typename Arg>
   qudaError_t launch_jitify_block(const std::string &kernel, const TuneParam &tp, const qudaStream_t &stream,
                                   const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
   {
     // we need this hackery to get the naked unbound template class parameters
     using namespace jitify::reflection;
-    auto Functor_instance = reflect<Functor<Arg>>();
+    auto Functor_instance = reflect<Functor<1, Arg>>();
     auto Functor_naked = Functor_instance.substr(0, Functor_instance.find("<"));
     auto Arg_reflect = reflect<Arg>();
 
     std::vector<void*> arg_ptrs{(void*)&arg};
     jitify::detail::vector<std::string> arg_types{Arg_reflect};
 
-    return launch_jitify(Functor<Arg>::filename(), kernel,
+    return launch_jitify(Functor<1, Arg>::filename(), kernel,
                          {reflect((int)tp.block.x), Functor_naked, Arg_reflect},
                          tp, stream, param, arg_ptrs, arg_types);
   }
diff --git a/include/kernel_helper.h b/include/kernel_helper.h
index b22610b0d8..9f470542bc 100644
--- a/include/kernel_helper.h
+++ b/include/kernel_helper.h
@@ -14,4 +14,19 @@ namespace quda {
 
   static std::vector<constant_param_t> dummy_param;
 
+  struct kernel_t {
+    const void *func;
+    const std::string name;
+
+    kernel_t(const void *func, const char *name) :
+      func(func),
+      name(name) {}
+  };
+
+#ifdef JITIFY
+#define KERNEL(kernel) kernel_t(nullptr, #kernel)
+#else
+#define KERNEL(kernel) kernel_t(reinterpret_cast<const void *>(kernel<Functor, Arg, grid_stride>), #kernel)
+#endif
+
 }
diff --git a/include/kernels/clover_invert.cuh b/include/kernels/clover_invert.cuh
index e5d13f0de3..7806169704 100644
--- a/include/kernels/clover_invert.cuh
+++ b/include/kernels/clover_invert.cuh
@@ -34,9 +34,9 @@ namespace quda
     __device__ __host__ auto init() const { return zero<double2>(); }
   };
 
-  template <typename Arg> struct InvertClover {
-
+  template <typename Arg> struct InvertClover : plus<double2> {
     using reduce_t = double2;
+    using plus<reduce_t>::operator();
     Arg &arg;
     constexpr InvertClover(Arg &arg) : arg(arg) {}
     static constexpr const char *filename() { return KERNEL_FILE; }
@@ -44,8 +44,7 @@ namespace quda
     /**
        Use a Cholesky decomposition and invert the clover matrix
     */
-    template <typename Reducer>
-    __device__ __host__ inline reduce_t operator()(reduce_t &value, Reducer &r, int x_cb, int parity)
+    __device__ __host__ inline reduce_t operator()(reduce_t &value, int x_cb, int parity)
     {
       using real = typename Arg::real;
       constexpr int N = Arg::nColor * Arg::nSpin / 2;
@@ -74,7 +73,7 @@ namespace quda
 
       reduce_t result = zero<reduce_t>();
       parity ? result.y = trLogA : result.x = trLogA;
-      return r(result, value);
+      return operator()(result, value);
     }
 
   };
diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 0e85301f94..6fcd5acd5d 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -8,6 +8,7 @@
 #include <gamma.cuh>
 #include <linalg.cuh>
 #include <matrix_tile.cuh>
+#include <target_device.h>
 
 namespace quda {
 
diff --git a/include/kernels/coarse_op_preconditioned.cuh b/include/kernels/coarse_op_preconditioned.cuh
index 7a285b3229..8cc177bddd 100644
--- a/include/kernels/coarse_op_preconditioned.cuh
+++ b/include/kernels/coarse_op_preconditioned.cuh
@@ -6,7 +6,7 @@
 
 namespace quda {
 
-  template <typename Float_, typename PreconditionedGauge, typename Gauge, typename GaugeInv, int n_, int M_, int N_>
+  template <typename Float_, typename PreconditionedGauge, typename Gauge, typename GaugeInv, int n_, int M_, int N_, bool compute_max_>
   struct CalculateYhatArg {
     using Float = Float_;
     using yhatTileType = TileSize<n_, n_, n_, M_, N_, 1>;
@@ -17,6 +17,7 @@ namespace quda {
     static constexpr int K = n_;
 
     static constexpr bool is_mma_compatible = PreconditionedGauge::is_mma_compatible;
+    static constexpr bool compute_max = compute_max_;
 
     PreconditionedGauge Yhat;
     const Gauge Y;
@@ -28,18 +29,20 @@ namespace quda {
     Float *max_h;  // host scalar that stores the maximum element of Yhat. Pointer b/c pinned.
     Float *max_d; // device scalar that stores the maximum element of Yhat
 
-    CalculateYhatArg(const PreconditionedGauge &Yhat, const Gauge Y, const GaugeInv Xinv, const int *dim,
-                     const int *comm_dim, int nFace) :
-      Yhat(Yhat), Y(Y), Xinv(Xinv), nFace(nFace), max_h(nullptr), max_d(nullptr)
+    dim3 threads;
+
+    CalculateYhatArg(GaugeField &Yhat, const GaugeField &Y, const GaugeField &Xinv) :
+      Yhat(Yhat), Y(Y), Xinv(Xinv), nFace(1), max_h(nullptr), max_d(nullptr),
+      threads(Y.VolumeCB(), 2 * yhatTileType::M_tiles, 4 * yhatTileType::N_tiles)
     {
       for (int i=0; i<4; i++) {
-        this->comm_dim[i] = comm_dim[i];
-        this->dim[i] = dim[i];
+        this->comm_dim[i] = comm_dim_partitioned(i);
+        this->dim[i] = Xinv.X()[i];
       }
     }
   };
 
-  template <bool compute_max_only, typename Arg>
+  template <typename Arg>
   inline __device__ __host__ auto computeYhat(Arg &arg, int d, int x_cb, int parity, int i0, int j0)
   {
     using real = typename Arg::Float;
@@ -68,7 +71,7 @@ namespace quda {
         yHat.mma_nt(Y, X);
       }
 
-      if (compute_max_only) {
+      if (Arg::compute_max) {
         yHatMax = yHat.abs_max();
       } else {
         yHat.save(arg.Yhat, d, 1 - parity, ghost_idx, i0, j0);
@@ -89,7 +92,7 @@ namespace quda {
 
         yHat.mma_nt(Y, X);
       }
-      if (compute_max_only) {
+      if (Arg::compute_max) {
         yHatMax = yHat.abs_max();
       } else {
         yHat.save(arg.Yhat, d, 1 - parity, back_idx, i0, j0);
@@ -109,7 +112,7 @@ namespace quda {
 
         yHat.mma_nn(X, Y);
       }
-      if (compute_max_only) {
+      if (Arg::compute_max) {
         yHatMax = fmax(yHatMax, yHat.abs_max());
       } else {
         yHat.save(arg.Yhat, d + 4, parity, x_cb, i0, j0);
@@ -119,42 +122,26 @@ namespace quda {
     return yHatMax;
   }
 
-  template <bool compute_max_only, typename Arg> void CalculateYhatCPU(Arg &arg)
-  {
-    typename Arg::Float max = 0.0;
-    for (int d=0; d<4; d++) {
-      for (int parity=0; parity<2; parity++) {
-#pragma omp parallel for
-        for (int x_cb = 0; x_cb < arg.Y.VolumeCB(); x_cb++) {
-          for (int i = 0; i < Arg::yhatTileType::m; i += Arg::yhatTileType::M)
-            for (int j = 0; j < Arg::yhatTileType::n; j += Arg::yhatTileType::N) {
-              typename Arg::Float max_x = computeYhat<compute_max_only>(arg, d, x_cb, parity, i, j);
-              if (compute_max_only) max = max > max_x ? max : max_x;
-            }
-        }
-      } //parity
-    } // dimension
-    if (compute_max_only) *arg.max_h = max;
-  }
+  template <typename Arg> struct ComputeYhat {
+    Arg &arg;
+    constexpr ComputeYhat(Arg &arg) : arg(arg) {}
+    static constexpr const char *filename() { return KERNEL_FILE; }
 
-  template <bool compute_max_only, typename Arg> __global__ void CalculateYhatGPU(Arg arg)
-  {
-    QUDA_RT_CONSTS;
-    int x_cb = blockDim.x*blockIdx.x + threadIdx.x;
-    if (x_cb >= arg.Y.VolumeCB()) return;
-    int i_parity = blockDim.y*blockIdx.y + threadIdx.y;
-    if (i_parity >= 2 * Arg::yhatTileType::M_tiles) return;
-    int j_d = blockDim.z*blockIdx.z + threadIdx.z;
-    if (j_d >= 4 * Arg::yhatTileType::N_tiles) return;
-
-    int i = i_parity % Arg::yhatTileType::M_tiles;
-    int parity = i_parity / Arg::yhatTileType::M_tiles;
-    int j = j_d % Arg::yhatTileType::N_tiles;
-    int d = j_d / Arg::yhatTileType::N_tiles;
-
-    typename Arg::Float max
-      = computeYhat<compute_max_only>(arg, d, x_cb, parity, i * Arg::yhatTileType::M, j * Arg::yhatTileType::N);
-    if (compute_max_only) atomicAbsMax(arg.max_d, max);
-  }
+    __device__ __host__ void operator()(int x_cb, int i_parity, int j_d)
+    {
+      int i = i_parity % Arg::yhatTileType::M_tiles;
+      int parity = i_parity / Arg::yhatTileType::M_tiles;
+      int j = j_d % Arg::yhatTileType::N_tiles;
+      int d = j_d / Arg::yhatTileType::N_tiles;
+
+      auto max = computeYhat(arg, d, x_cb, parity, i * Arg::yhatTileType::M, j * Arg::yhatTileType::N);
+#ifdef __CUDA_ARCH__
+      if (Arg::compute_max) atomicAbsMax(arg.max_d, max);
+#else
+      // not omp safe (yet)
+      if (Arg::compute_max) *arg.max_h = std::max(max, *arg.max_h);
+#endif
+    }
+  };
 
 } // namespace quda
diff --git a/include/kernels/coarse_op_preconditioned_mma.cuh b/include/kernels/coarse_op_preconditioned_mma.cuh
index b058a80f48..500a7e2512 100644
--- a/include/kernels/coarse_op_preconditioned_mma.cuh
+++ b/include/kernels/coarse_op_preconditioned_mma.cuh
@@ -2,12 +2,12 @@
 
 #include <gauge_field_order.h>
 #include <index_helper.cuh>
-
 #include <mma_tensor_op/gemm.cuh>
-
-#include <type_traits>
-
 #include <cub_helper.cuh>
+#include <kernel.h>
+#include <kernels/coarse_op_preconditioned.cuh>
+
+// This is MMA implementation of the computeYhat kernels.
 
 namespace quda
 {
@@ -15,10 +15,21 @@ namespace quda
   namespace mma
   {
 
-    // This is MMA implementation of the computeYhat kernels.
-
-    template <bool compute_max_only, typename Arg, int bM, int bN, int bK, int block_y, int block_z>
-    inline __device__ auto computeYhat(Arg &arg, int d, int x_cb, int parity, int m, int n)
+    template <typename Arg, int bM_, int bN_, int bK_, int block_y_, int block_z_, int min_blocks_>
+    struct CalculateYhatMMAArg : Arg {
+      static constexpr int bM = bM_;
+      static constexpr int bN = bN_;
+      static constexpr int bK = bK_;
+      static constexpr int block_y = block_y_;
+      static constexpr int block_z = block_z_;
+      static constexpr int block_dim = block_y * block_z;
+      static constexpr int min_blocks = min_blocks_;
+
+      CalculateYhatMMAArg(const Arg &arg) : Arg(arg) {}
+    };
+
+    template <typename Arg>
+    inline __device__ auto computeYhatMMA(Arg &arg, int d, int x_cb, int parity, int m, int n)
     {
       using real = typename Arg::Float;
       constexpr int nDim = 4;
@@ -27,7 +38,8 @@ namespace quda
 
       real yHatMax = 0.0;
 
-      using Config = MmaConfig<Arg::M, Arg::N, Arg::K, Arg::M, Arg::N, Arg::K, bM, bN, bK, block_y, block_z>;
+      using Config = MmaConfig<Arg::M, Arg::N, Arg::K, Arg::M, Arg::N, Arg::K,
+                               Arg::bM, Arg::bN, Arg::bK, Arg::block_y, Arg::block_z>;
 
       {
         // first do the backwards links Y^{+\mu} * X^{-\dagger}
@@ -42,7 +54,7 @@ namespace quda
           auto b = arg.Xinv.wrap(0, parity, x_cb);
           auto c = arg.Yhat.wrap_ghost(d, 1 - parity, ghost_idx);
 
-          yHatMax = Config::template perform_mma<a_dagger, b_dagger, compute_max_only>(a, b, c, m, n);
+          yHatMax = Config::template perform_mma<a_dagger, b_dagger, Arg::compute_max>(a, b, c, m, n);
 
         } else {
 
@@ -52,7 +64,7 @@ namespace quda
           auto b = arg.Xinv.wrap(0, parity, x_cb);
           auto c = arg.Yhat.wrap(d, 1 - parity, back_idx);
 
-          yHatMax = Config::template perform_mma<a_dagger, b_dagger, compute_max_only>(a, b, c, m, n);
+          yHatMax = Config::template perform_mma<a_dagger, b_dagger, Arg::compute_max>(a, b, c, m, n);
         }
       }
 
@@ -64,48 +76,53 @@ namespace quda
         constexpr bool a_dagger = false;
         constexpr bool b_dagger = false;
 
-        real yHatMax_ = Config::template perform_mma<a_dagger, b_dagger, compute_max_only>(a, b, c, m, n);
+        real yHatMax_ = Config::template perform_mma<a_dagger, b_dagger, Arg::compute_max>(a, b, c, m, n);
         yHatMax = fmax(yHatMax, yHatMax_);
       }
 
       return yHatMax;
     }
 
-    template <bool compute_max_only, typename Arg, int bM, int bN, int bK, int block_y, int block_z, int min_block_cta>
-    __global__ void __launch_bounds__(block_y *block_z, min_block_cta) CalculateYhatGPU(Arg arg)
-    {
-      int index_x = blockDim.x * blockIdx.x + threadIdx.x;
+    template <typename Arg> struct CalculateYhatMMA {
+      Arg &arg;
+      constexpr CalculateYhatMMA(Arg &arg) : arg(arg) {}
+      static constexpr const char *filename() { return KERNEL_FILE; }
 
-      constexpr bool divide_b_no = bM < Arg::M && bK == Arg::K && bN == Arg::N;
-      constexpr int t_m = divide_b_no ? 1 : (Arg::M + bM - 1) / bM;
-      constexpr int t_n = divide_b_no ? 1 : (Arg::N + bN - 1) / bN;
+      __device__ __forceinline__ void operator()()
+      {
+        int index_x = blockDim.x * blockIdx.x + threadIdx.x;
 
-      int x_cb = index_x / (t_m * t_n);
-      int mn = index_x % (t_m * t_n);
+        constexpr bool divide_b_no = Arg::bM < Arg::M && Arg::bK == Arg::K && Arg::bN == Arg::N;
+        constexpr int t_m = divide_b_no ? 1 : (Arg::M + Arg::bM - 1) / Arg::bM;
+        constexpr int t_n = divide_b_no ? 1 : (Arg::N + Arg::bN - 1) / Arg::bN;
 
-      int n = (mn % t_n) * bN;
-      int m = (mn / t_n) * bM;
+        int x_cb = index_x / (t_m * t_n);
+        int mn = index_x % (t_m * t_n);
 
-      if (x_cb >= arg.Y.VolumeCB()) return;
+        int n = (mn % t_n) * Arg::bN;
+        int m = (mn / t_n) * Arg::bM;
 
-      int parity = blockIdx.y;
-      int d = blockIdx.z;
+        if (x_cb >= arg.Y.VolumeCB()) return;
 
-      using real = typename Arg::Float;
-      real max = 0.0;
-      switch (d) {
-      case 0: max = computeYhat<compute_max_only, Arg, bM, bN, bK, block_y, block_z>(arg, 0, x_cb, parity, m, n); break;
-      case 1: max = computeYhat<compute_max_only, Arg, bM, bN, bK, block_y, block_z>(arg, 1, x_cb, parity, m, n); break;
-      case 2: max = computeYhat<compute_max_only, Arg, bM, bN, bK, block_y, block_z>(arg, 2, x_cb, parity, m, n); break;
-      case 3: max = computeYhat<compute_max_only, Arg, bM, bN, bK, block_y, block_z>(arg, 3, x_cb, parity, m, n); break;
-      }
-      if (compute_max_only) {
-        using BlockReduce = cub::BlockReduce<unsigned, 1, cub::BLOCK_REDUCE_WARP_REDUCTIONS, block_y, block_z>;
-        __shared__ typename BlockReduce::TempStorage temp_storage;
-        unsigned aggregate = BlockReduce(temp_storage).Reduce(__float_as_uint(max), cub::Max());
-        if (threadIdx.y == 0 && threadIdx.z == 0) atomicAbsMax(arg.max_d, __uint_as_float(aggregate));
+        int parity = blockIdx.y;
+        int d = blockIdx.z;
+
+        using real = typename Arg::Float;
+        real max = 0.0;
+        switch (d) {
+        case 0: max = computeYhatMMA(arg, 0, x_cb, parity, m, n); break;
+        case 1: max = computeYhatMMA(arg, 1, x_cb, parity, m, n); break;
+        case 2: max = computeYhatMMA(arg, 2, x_cb, parity, m, n); break;
+        case 3: max = computeYhatMMA(arg, 3, x_cb, parity, m, n); break;
+        }
+        if (Arg::compute_max) {
+          using BlockReduce = cub::BlockReduce<unsigned, 1, cub::BLOCK_REDUCE_WARP_REDUCTIONS, Arg::block_y, Arg::block_z>;
+          __shared__ typename BlockReduce::TempStorage temp_storage;
+          unsigned aggregate = BlockReduce(temp_storage).Reduce(__float_as_uint(max), cub::Max());
+          if (threadIdx.y == 0 && threadIdx.z == 0) atomicAbsMax(arg.max_d, __uint_as_float(aggregate));
+        }
       }
-    }
+    };
 
   } // namespace mma
 
diff --git a/include/kernels/copy_gauge_extended.cuh b/include/kernels/copy_gauge_extended.cuh
index c13de70c7b..f7bb9cd8b3 100644
--- a/include/kernels/copy_gauge_extended.cuh
+++ b/include/kernels/copy_gauge_extended.cuh
@@ -28,12 +28,16 @@ namespace quda {
       out(out, Out),
       in(in, In),
       geometry(in.Geometry()),
-      threads(in.LocalVolumeCB(), 2, 1)
+      threads(in.VolumeCB() == out.VolumeCB() ? in.VolumeCB() : in.LocalVolumeCB(), 2, 1)
     {
       for (int d=0; d < in.Ndim(); d++) {
 	Xout[d] = out.X()[d];
 	Xin[d] = in.X()[d];
-        border[d] = regularToextended ? out.R()[d] : in.R()[d];
+        // if volumes match then both extended and just regular copy
+        border[d] = Xout[d] == Xin[d] ? 0 : regularToextended ? out.R()[d] : in.R()[d];
+        // sanity check that if the volumes match, we have the same border dimensions
+        if (in.VolumeCB() == out.VolumeCB() && out.R()[d] != in.R()[d])
+          errorQuda("Cannot copy extended fields of differing borders d = %d out = %d in = %d", d, out.R()[d], in.R()[d]);
       }
     }
   };
diff --git a/include/kernels/covDev.cuh b/include/kernels/covDev.cuh
index 0c7c036f5a..d76afc3ae4 100644
--- a/include/kernels/covDev.cuh
+++ b/include/kernels/covDev.cuh
@@ -48,10 +48,7 @@ namespace quda
       checkLocation(out, in, U);  // check all locations match
       if (!out.isNative() || !in.isNative() || !U.isNative())
         errorQuda("Unsupported field order colorspinor(in)=%d gauge=%d combination\n", in.FieldOrder(), U.FieldOrder());
-      pushKernelPackT(true); // non-spin projection requires kernel packing
     }
-
-    virtual ~CovDevArg() { popKernelPackT(); }
   };
 
   /**
diff --git a/include/kernels/dslash_coarse.cuh b/include/kernels/dslash_coarse.cuh
index 43010b86b8..683e663af1 100644
--- a/include/kernels/dslash_coarse.cuh
+++ b/include/kernels/dslash_coarse.cuh
@@ -3,6 +3,7 @@
 #include <index_helper.cuh>
 #include <float_vector.h>
 #include <shared_memory_cache_helper.cuh>
+#include <kernel.h>
 
 namespace quda {
 
@@ -309,8 +310,10 @@ namespace quda {
 	// reduce down to the first group of column-split threads
 	constexpr int warp_size = device::warp_size(); // FIXME - this is buggy when x-dim * color_stride < 32
 #pragma unroll
-	for (int offset = warp_size/2; offset >= warp_size/Arg::color_stride; offset /= 2)
-	  out[color_local] += __shfl_down_sync(device::warp_converged_mask(), out[color_local], offset);
+	for (int offset = warp_size/2; offset >= warp_size/Arg::color_stride; offset /= 2) {
+          out[color_local].real(out[color_local].real() + __shfl_down_sync(device::warp_converged_mask(), out[color_local].real(), offset));
+          out[color_local].imag(out[color_local].imag() + __shfl_down_sync(device::warp_converged_mask(), out[color_local].imag(), offset));
+        }
       }
 
 #pragma unroll
@@ -329,7 +332,7 @@ namespace quda {
   template <typename Arg> struct CoarseDslash {
     Arg &arg;
     constexpr CoarseDslash(Arg &arg) : arg(arg) {}
-    constexpr const char *filename() { return KERNEL_FILE; }
+    static constexpr const char *filename() { return KERNEL_FILE; }
 
     __device__ __host__ inline void operator()(int x_cb_color_offset, int parity, int sMd)
     {
diff --git a/include/kernels/dslash_domain_wall_5d.cuh b/include/kernels/dslash_domain_wall_5d.cuh
index e0f47af506..aafb490cd4 100644
--- a/include/kernels/dslash_domain_wall_5d.cuh
+++ b/include/kernels/dslash_domain_wall_5d.cuh
@@ -19,12 +19,7 @@ namespace quda
       WilsonArg<Float, nColor, nDim, reconstruct_>(out, in, U, xpay ? a : 0.0, x, parity, dagger, comm_override),
       Ls(in.X(4)),
       a(a),
-      m_f(m_f)
-    {
-      pushKernelPackT(true); // with 5-d checkerboarding we must use kernel packing
-    }
-
-    virtual ~DomainWall5DArg() { popKernelPackT(); }
+      m_f(m_f) {}
   };
 
   template <int nParity, bool dagger, bool xpay, KernelType kernel_type, typename Arg>
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index d425b2b243..ab57398726 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -54,6 +54,18 @@ namespace quda
       if (F::N != F::N_ghost) pushKernelPackT(true); // must use packing kernel is ghost vector length is different than bulk
     }
 
+    // defined the copy constructor to ensure we don't have an excess pop if the arg is copied
+    WilsonArg(const WilsonArg &arg) :
+      DslashArg<Float, nDim>(arg),
+      out(arg.out),
+      in(arg.in),
+      x(arg.x),
+      U(arg.U),
+      a(arg.a)
+    {
+      if (F::N != F::N_ghost) pushKernelPackT(true);
+    }
+
     virtual ~WilsonArg() { if (F::N != F::N_ghost) popKernelPackT(); }
   };
 
diff --git a/include/kernels/extract_gauge_ghost_extended.cuh b/include/kernels/extract_gauge_ghost_extended.cuh
index f6f08c8881..88bdd8f800 100644
--- a/include/kernels/extract_gauge_ghost_extended.cuh
+++ b/include/kernels/extract_gauge_ghost_extended.cuh
@@ -11,7 +11,7 @@ namespace quda {
     static constexpr int nDim = 4;
     static constexpr int dim = dim_;
     static constexpr bool extract = extract_;
-    static constexpr int nColor = Gauge::length;
+    static constexpr int nColor = gauge::Ncolor(Gauge::length);
     using real = typename Gauge::real;
     Gauge u;
     int E[nDim];
diff --git a/include/kernels/gauge_det_trace.cuh b/include/kernels/gauge_det_trace.cuh
index 3a79ad04ec..f49449a8f9 100644
--- a/include/kernels/gauge_det_trace.cuh
+++ b/include/kernels/gauge_det_trace.cuh
@@ -31,16 +31,15 @@ namespace quda {
     __device__ __host__ auto init() const { return zero<double2>(); }
   };
 
-  template <typename Arg> struct DetTrace {
-
+  template <typename Arg> struct DetTrace : plus<double2> {
     using reduce_t = double2;
+    using plus<reduce_t>::operator();
     Arg &arg;
     constexpr DetTrace(Arg &arg) : arg(arg) {}
     static constexpr const char *filename() { return KERNEL_FILE; }
 
     // return the determinant or trace at site (x_cb, parity)
-    template <typename Reducer>
-    __device__ __host__ inline reduce_t operator()(reduce_t &value, Reducer &r, int x_cb, int parity)
+    __device__ __host__ inline reduce_t operator()(reduce_t &value, int x_cb, int parity)
     {
       int X[4];
 #pragma unroll
@@ -57,8 +56,9 @@ namespace quda {
       for (int mu = 0; mu < 4; mu++) {
         Matrix<complex<typename Arg::real>, Arg::nColor> U = arg.u(mu, linkIndex(x, X), parity);
         complex<double> local = Arg::type == 0 ? getDeterminant(U) : getTrace(U);
-        value = r(value, static_cast<reduce_t&>(local));
+        value = plus::operator()(value, static_cast<reduce_t&>(local));
       }
+
       return value;
     }
   };
diff --git a/include/kernels/gauge_fix_fft.cuh b/include/kernels/gauge_fix_fft.cuh
index c7713dddd3..de63914be7 100644
--- a/include/kernels/gauge_fix_fft.cuh
+++ b/include/kernels/gauge_fix_fft.cuh
@@ -179,9 +179,9 @@ namespace quda {
     double getTheta() { return result.y; }
   };
 
-  template <typename Arg> struct FixQualityFFT {
-
+  template <typename Arg> struct FixQualityFFT : plus<double2> {
     using reduce_t = double2;
+    using plus<reduce_t>::operator();
     Arg &arg;
     static constexpr const char *filename() { return KERNEL_FILE; }
     constexpr FixQualityFFT(Arg &arg) : arg(arg) {}
@@ -189,8 +189,7 @@ namespace quda {
     /**
      * @brief Measure gauge fixing quality
      */
-    template <typename Reducer>
-    __device__ __host__ inline reduce_t operator()(reduce_t &value, Reducer &r, int x_cb, int parity)
+    __device__ __host__ inline reduce_t operator()(reduce_t &value, int x_cb, int parity)
     {
       reduce_t data;
       using matrix = Matrix<complex<typename Arg::real>, 3>;
@@ -230,7 +229,7 @@ namespace quda {
 
       //35
       //T=36*gauge_dir+65
-      return r(data, value);
+      return operator()(data, value);
     }
   };
 
diff --git a/include/kernels/gauge_fix_ovr.cuh b/include/kernels/gauge_fix_ovr.cuh
index 316cc3d217..daa5330bdc 100644
--- a/include/kernels/gauge_fix_ovr.cuh
+++ b/include/kernels/gauge_fix_ovr.cuh
@@ -40,9 +40,9 @@ namespace quda {
     double getTheta(){ return result.y; }
   };
 
-  template <typename Arg> struct FixQualityOVR {
-
+  template <typename Arg> struct FixQualityOVR : plus<double2> {
     using reduce_t = double2;
+    using plus<reduce_t>::operator();
     Arg &arg;
     static constexpr const char *filename() { return KERNEL_FILE; }
     constexpr FixQualityOVR(Arg &arg) : arg(arg) {}
@@ -50,8 +50,7 @@ namespace quda {
     /**
      * @brief Measure gauge fixing quality
      */
-    template <typename Reducer>
-    __device__ __host__ inline reduce_t operator()(reduce_t &value, Reducer &r, int x_cb, int parity)
+    __device__ __host__ inline reduce_t operator()(reduce_t &value, int x_cb, int parity)
     {
       reduce_t data;
       using Link = Matrix<complex<typename Arg::real>, 3>;
@@ -91,7 +90,7 @@ namespace quda {
       //35
       //T=36*gauge_dir+65
 
-      return r(data, value);
+      return operator()(data, value);
     }
   };
 
diff --git a/include/kernels/gauge_heatbath.cuh b/include/kernels/gauge_heatbath.cuh
index ae7adcd651..db60a19b0b 100644
--- a/include/kernels/gauge_heatbath.cuh
+++ b/include/kernels/gauge_heatbath.cuh
@@ -568,13 +568,13 @@ namespace quda {
     int border[4];
     Gauge dataOr;
     Float BetaOverNc;
-    RNG rngstate;
+    RNGState *rng;
     int mu;
     int parity;
     dim3 threads;   // number of active threads required
-    MonteArg(GaugeField &data, Float Beta, RNG &rngstate, int mu, int parity) :
+    MonteArg(GaugeField &data, Float Beta, RNGState *rng, int mu, int parity) :
       dataOr(data),
-      rngstate(rngstate),
+      rng(rng),
       mu(mu),
       parity(parity),
       threads(data.LocalVolumeCB(), 1, 1)
@@ -639,9 +639,9 @@ namespace quda {
         }
       U = arg.dataOr(mu, e_cb, parity);
       if (Arg::heatbath) {
-        RNGState localState = arg.rngstate.State()[x_cb];
+        RNGState localState = arg.rng[x_cb];
         heatBathSUN( U, conj(staple), localState, arg.BetaOverNc );
-        arg.rngstate.State()[x_cb] = localState;
+        arg.rng[x_cb] = localState;
       } else {
         overrelaxationSUN( U, conj(staple) );
       }
diff --git a/include/kernels/gauge_plaq.cuh b/include/kernels/gauge_plaq.cuh
index 80d06661f7..b5b220162b 100644
--- a/include/kernels/gauge_plaq.cuh
+++ b/include/kernels/gauge_plaq.cuh
@@ -56,16 +56,15 @@ namespace quda {
     return getTrace( U1 * U2 * conj(U3) * conj(U4) ).real();
   }
 
-  template <typename Arg> struct Plaquette {
-
+  template <typename Arg> struct Plaquette : plus<double2> {
     using reduce_t = double2;
+    using plus<reduce_t>::operator();
     Arg &arg;
     constexpr Plaquette(Arg &arg) : arg(arg) {}
     static constexpr const char *filename() { return KERNEL_FILE; }
 
     // return the plaquette at site (x_cb, parity)
-    template <typename Reducer>
-    __device__ __host__ inline reduce_t operator()(reduce_t &value, Reducer &r, int x_cb, int parity)
+    __device__ __host__ inline reduce_t operator()(reduce_t &value, int x_cb, int parity)
     {
       reduce_t plaq = zero<reduce_t>();
 
@@ -84,7 +83,7 @@ namespace quda {
         plaq.y += plaquette(arg, x, parity, mu, 3);
       }
 
-      return r(plaq, value);
+      return plus::operator()(plaq, value);
     }
 
   };
diff --git a/include/kernels/gauge_qcharge.cuh b/include/kernels/gauge_qcharge.cuh
index 072d85f59e..7281fe36e0 100644
--- a/include/kernels/gauge_qcharge.cuh
+++ b/include/kernels/gauge_qcharge.cuh
@@ -29,16 +29,15 @@ namespace quda
   };
 
   // Core routine for computing the topological charge from the field strength
-  template <typename Arg> struct qCharge {
-
+  template <typename Arg> struct qCharge : plus<double3> {
     using reduce_t = double3;
+    using plus<reduce_t>::operator();
     Arg &arg;
     constexpr qCharge(Arg &arg) : arg(arg) {}
     static constexpr const char *filename() { return KERNEL_FILE; }
 
     // return the qcharge and field strength at site (x_cb, parity)
-    template <typename Reducer>
-    __device__ __host__ inline reduce_t operator()(reduce_t &E, Reducer &r, int x_cb, int parity)
+    __device__ __host__ inline reduce_t operator()(reduce_t &E, int x_cb, int parity)
     {
       using real = typename Arg::Float;
       using Link = Matrix<complex<real>, Arg::nColor>;
@@ -80,9 +79,8 @@ namespace quda
       for (int i=0; i<3; i++) i % 2 == 0 ? Q_idx += Qi[i]: Q_idx -= Qi[i];
       Q = Q_idx * q_norm;
       if (Arg::density) arg.qDensity[x_cb + parity * arg.threads.x] = Q;
-      E = r(E, E_local);
 
-      return E;
+      return plus::operator()(E, E_local);
     }
 
   };
diff --git a/include/kernels/gauge_random.cuh b/include/kernels/gauge_random.cuh
index 1f6be55c14..a0122f8698 100644
--- a/include/kernels/gauge_random.cuh
+++ b/include/kernels/gauge_random.cuh
@@ -19,11 +19,11 @@ namespace quda {
     int X[4]; // true grid dimensions
     int border[4];
     Gauge U;
-    RNG rng;
+    RNGState *rng;
     real sigma; // where U = exp(sigma * H)
     dim3 threads; // number of active threads required
 
-    GaugeGaussArg(const GaugeField &U, RNG &rng, double sigma) :
+    GaugeGaussArg(const GaugeField &U, RNGState *rng, double sigma) :
       U(U),
       rng(rng),
       sigma(sigma),
@@ -95,7 +95,7 @@ namespace quda {
         for (int mu = 0; mu < 4; mu++) arg.U(mu, linkIndex(x, arg.E), parity) = I;
       } else {
         for (int mu = 0; mu < 4; mu++) {
-          RNGState localState = arg.rng.State()[parity * arg.threads.x + x_cb];
+          RNGState localState = arg.rng[parity * arg.threads.x + x_cb];
 
           // generate Gaussian distributed su(n) fiueld
           Link u = gauss_su3<real, Link>(localState);
@@ -105,7 +105,7 @@ namespace quda {
           }
           arg.U(mu, linkIndex(x, arg.E), parity) = u;
 
-          arg.rng.State()[parity * arg.threads.x + x_cb] = localState;
+          arg.rng[parity * arg.threads.x + x_cb] = localState;
         }
       }
     }
diff --git a/include/kernels/momentum.cuh b/include/kernels/momentum.cuh
index 72f32037fc..daed7fe8a9 100644
--- a/include/kernels/momentum.cuh
+++ b/include/kernels/momentum.cuh
@@ -27,8 +27,9 @@ namespace quda {
     __device__ __host__ double init() const { return 0.0; }
   };
 
-  template <typename Arg> struct MomAction {
+  template <typename Arg> struct MomAction : plus<double> {
     using reduce_t = double;
+    using plus<reduce_t>::operator();
     Arg &arg;
     constexpr MomAction(Arg &arg) : arg(arg) {}
     static constexpr const char *filename() { return KERNEL_FILE; }
@@ -36,8 +37,7 @@ namespace quda {
     // calculate the momentum contribution to the action.  This uses the
     // MILC convention where we subtract 4.0 from each matrix norm in
     // order to increase stability
-    template <typename Reducer>
-    __device__ __host__ inline reduce_t operator()(reduce_t &action, Reducer &r, int x_cb, int parity)
+    __device__ __host__ inline reduce_t operator()(reduce_t &action, int x_cb, int parity)
     {
       using matrix = Matrix<complex<typename Arg::Float>, Arg::nColor>;
 
@@ -57,7 +57,7 @@ namespace quda {
         local_sum += mom(1,2).imag() * mom(1,2).imag();
 	local_sum -= 4.0;
 
-	action = r(action, local_sum);
+	action = plus::operator()(action, local_sum);
       }
       return action;
     }
@@ -90,33 +90,30 @@ namespace quda {
     __device__ __host__ double2 init() const{ return zero<double2>(); }
   };
 
-  /**
-     @brief Functor for finding the maximum over a vec2 field.  Each
-     lane is evaluated separately.
-   */
-  template <typename T>
-  struct max_reducer2 {
+  template <typename Arg> struct MomUpdate {
+    using reduce_t = double2;
+    Arg &arg;
+    constexpr MomUpdate(Arg &arg) : arg(arg) {}
+    static constexpr const char *filename() { return KERNEL_FILE; }
+
     static constexpr bool do_sum = false;
 
-    __device__ __host__ inline T operator()(const T &a, const T &b) const {
+    /**
+       @brief Functor for finding the maximum over a vec2 field.  Each
+       lane is evaluated separately.
+    */
+    __device__ __host__ inline reduce_t operator()(const reduce_t &a, const reduce_t &b) const
+    {
       auto c = a;
       if (b.x > a.x) c.x = b.x;
       if (b.y > a.y) c.y = b.y;
       return c;
     }
-  };
-
-  template <typename Arg> struct MomUpdate {
-    using reduce_t = double2;
-    Arg &arg;
-    constexpr MomUpdate(Arg &arg) : arg(arg) {}
-    static constexpr const char *filename() { return KERNEL_FILE; }
 
     // calculate the momentum contribution to the action.  This uses the
     // MILC convention where we subtract 4.0 from each matrix norm in
     // order to increase stability
-    template <typename Reducer>
-    __device__ __host__ inline reduce_t operator()(reduce_t &norm, Reducer &r, int x_cb, int parity)
+    __device__ __host__ inline reduce_t operator()(reduce_t &norm, int x_cb, int parity)
     {
       int x[4];
       getCoords(x, x_cb, arg.X, parity);
@@ -132,7 +129,7 @@ namespace quda {
         makeAntiHerm(f);
 
         // compute force norms
-        norm = r(make_double2(f.L1(), f.L2()), norm);
+        norm = operator()(make_double2(f.L1(), f.L2()), norm);
 
         m = m + arg.coeff * f;
 
diff --git a/include/kernels/multi_reduce_core.cuh b/include/kernels/multi_reduce_core.cuh
index 749e2e5456..e06997e402 100644
--- a/include/kernels/multi_reduce_core.cuh
+++ b/include/kernels/multi_reduce_core.cuh
@@ -77,15 +77,15 @@ namespace quda
     /**
        Generic multi-reduction functor with up to four loads and saves
     */
-    template <typename Arg> struct MultiReduce_ {
+    template <typename Arg> struct MultiReduce_ : plus<vector_type<typename Arg::Reducer::reduce_t, Arg::NXZ>> {
       using vec = vector_type<complex<typename Arg::real>, Arg::n/2>;
       using reduce_t = vector_type<typename Arg::Reducer::reduce_t, Arg::NXZ>;
+      using plus<reduce_t>::operator();
       Arg &arg;
       constexpr MultiReduce_(Arg &arg) : arg(arg) {}
       static constexpr const char *filename() { return KERNEL_FILE; }
 
-      template <typename Reducer>
-      __device__ __host__ inline reduce_t operator()(reduce_t &sum, Reducer &, int tid, int k, int)
+      __device__ __host__ inline reduce_t operator()(reduce_t &sum, int tid, int k, int)
       {
         unsigned int parity = tid >= arg.length_cb ? 1 : 0;
         unsigned int i = tid - parity * arg.length_cb;
diff --git a/include/kernels/pgauge_init.cuh b/include/kernels/pgauge_init.cuh
index a3ed98ab08..37b7a3ae0d 100644
--- a/include/kernels/pgauge_init.cuh
+++ b/include/kernels/pgauge_init.cuh
@@ -51,10 +51,10 @@ namespace quda {
     using Gauge = typename gauge_mapper<real, recon>::type;
     int X[4]; // grid dimensions
     Gauge U;
-    RNG rng;
+    RNGState *rng;
     int border[4];
     dim3 threads; // number of active threads required
-    InitGaugeHotArg(const GaugeField &U, RNG &rng) :
+    InitGaugeHotArg(const GaugeField &U, RNGState *rng) :
       U(U),
       rng(rng),
       //the optimal number of RNG states in rngstate array must be equal to half the lattice volume
@@ -210,7 +210,7 @@ namespace quda {
       int X[4], x[4];
       for ( int dr = 0; dr < 4; ++dr ) X[dr] = arg.X[dr];
       for ( int dr = 0; dr < 4; ++dr ) X[dr] += 2 * arg.border[dr];
-      RNGState localState = arg.rng.State()[x_cb];
+      RNGState localState = arg.rng[x_cb];
       for (int parity = 0; parity < 2; parity++) {
         getCoords(x, x_cb, arg.X, parity);
         for (int dr = 0; dr < 4; dr++) x[dr] += arg.border[dr];
@@ -221,7 +221,7 @@ namespace quda {
           arg.U(d, e_cb, parity) = U;
         }
       }
-      arg.rng.State()[x_cb] = localState;
+      arg.rng[x_cb] = localState;
     }
   };
 
diff --git a/include/kernels/reduce_core.cuh b/include/kernels/reduce_core.cuh
index 6e4889f496..bbf1719913 100644
--- a/include/kernels/reduce_core.cuh
+++ b/include/kernels/reduce_core.cuh
@@ -47,14 +47,14 @@ namespace quda
     /**
        Generic reduction kernel with up to five loads and saves.
     */
-    template <typename Arg> struct Reduce_ {
+    template <typename Arg> struct Reduce_ : plus<typename Arg::Reducer::reduce_t> {
       using reduce_t = typename Arg::Reducer::reduce_t;
+      using plus<reduce_t>::operator();
       Arg &arg;
       constexpr Reduce_(Arg &arg) : arg(arg) {}
       static constexpr const char *filename() { return KERNEL_FILE; }
 
-      template <typename Reducer>
-      __device__ __host__ inline reduce_t operator()(reduce_t &sum, Reducer &, int tid, int)
+      __device__ __host__ inline reduce_t operator()(reduce_t &sum, int tid, int)
       {
         using vec = vector_type<complex<typename Arg::real>, Arg::n/2>;
 
diff --git a/include/kernels/restrictor.cuh b/include/kernels/restrictor.cuh
index b0790ffa1b..765343dbb5 100644
--- a/include/kernels/restrictor.cuh
+++ b/include/kernels/restrictor.cuh
@@ -146,7 +146,7 @@ namespace quda {
         }
       }
 
-      reduced = BlockReduce<vector, block_size, Arg::n_vector_y>(thread.y).Sum<true>(reduced);
+      reduced = BlockReduce<vector, block_size, Arg::n_vector_y>(thread.y). template Sum<true>(reduced);
 
       if (x_fine_offset == 0) {
         const int parity_coarse = x_coarse >= arg.out.VolumeCB() ? 1 : 0;
diff --git a/include/kernels/spinor_noise.cuh b/include/kernels/spinor_noise.cuh
index 152ff105e5..5fe1bd6e55 100644
--- a/include/kernels/spinor_noise.cuh
+++ b/include/kernels/spinor_noise.cuh
@@ -14,9 +14,9 @@ namespace quda {
     static constexpr QudaNoiseType noise = noise_;
     using V = typename colorspinor::FieldOrderCB<real, nSpin, nColor, 1, order>;
     V v;
-    RNG rng;
+    RNGState *rng;
     dim3 threads;
-    SpinorNoiseArg(ColorSpinorField &v, RNG &rng) :
+    SpinorNoiseArg(ColorSpinorField &v, RNGState *rng) :
       v(v),
       rng(rng),
       threads(v.VolumeCB(), v.SiteSubset(), 1) { }
@@ -48,14 +48,14 @@ namespace quda {
 
     __device__ __host__ void operator()(int x_cb, int parity)
     {
-      RNGState localState = arg.rng.State()[parity * arg.threads.x + x_cb];
+      RNGState localState = arg.rng[parity * arg.threads.x + x_cb];
       for (int s=0; s<Arg::nSpin; s++) {
         for (int c=0; c<Arg::nColor; c++) {
           if (Arg::noise == QUDA_NOISE_GAUSS) genGauss<typename Arg::real>(arg, localState, parity, x_cb, s, c);
           else if (Arg::noise == QUDA_NOISE_UNIFORM) genUniform<typename Arg::real>(arg, localState, parity, x_cb, s, c);
         }
       }
-      arg.rng.State()[parity * arg.threads.x + x_cb] = localState;
+      arg.rng[parity * arg.threads.x + x_cb] = localState;
     }
   };
 
diff --git a/include/lattice_field.h b/include/lattice_field.h
index 91d8f3455e..fe52d0ec2b 100644
--- a/include/lattice_field.h
+++ b/include/lattice_field.h
@@ -367,10 +367,10 @@ namespace quda {
     static int buffer_recv_p2p_back[2][QUDA_MAX_DIM];
 
     /** Local copy of event used for peer-to-peer synchronization */
-    static cudaEvent_t ipcCopyEvent[2][2][QUDA_MAX_DIM];
+    static qudaEvent_t ipcCopyEvent[2][2][QUDA_MAX_DIM];
 
     /** Remote copy of event used for peer-to-peer synchronization */
-    static cudaEvent_t ipcRemoteCopyEvent[2][2][QUDA_MAX_DIM];
+    static qudaEvent_t ipcRemoteCopyEvent[2][2][QUDA_MAX_DIM];
 
     /** Whether we have initialized communication for this field */
     bool initComms;
@@ -471,12 +471,12 @@ namespace quda {
     /**
        Handle to local copy event used for peer-to-peer synchronization
     */
-    const cudaEvent_t& getIPCCopyEvent(int dir, int dim) const;
+    const qudaEvent_t& getIPCCopyEvent(int dir, int dim) const;
 
     /**
        Handle to remote copy event used for peer-to-peer synchronization
     */
-    const cudaEvent_t& getIPCRemoteCopyEvent(int dir, int dim) const;
+    const qudaEvent_t& getIPCRemoteCopyEvent(int dir, int dim) const;
 
     /**
        Static variable that is determined which ghost buffer we are using
@@ -807,4 +807,17 @@ namespace quda {
 #endif
   }
 
+  /**
+     @brief Helper function for setting auxilary string
+     @return String containing location and compilation type
+   */
+  inline const char *compile_type_str(QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION)
+  {
+#ifdef JITIFY
+    return location == QUDA_CUDA_FIELD_LOCATION ? "GPU-jitify," : "CPU,";
+#else
+    return location == QUDA_CUDA_FIELD_LOCATION ? "GPU-offline," : "CPU,";
+#endif
+  }
+
 } // namespace quda
diff --git a/include/malloc_quda.h b/include/malloc_quda.h
index 05b0ff72d3..97e49929dd 100644
--- a/include/malloc_quda.h
+++ b/include/malloc_quda.h
@@ -53,12 +53,8 @@ namespace quda {
   void device_pinned_free_(const char *func, const char *file, int line, void *ptr);
   void managed_free_(const char *func, const char *file, int line, void *ptr);
   void host_free_(const char *func, const char *file, int line, void *ptr);
-
-  // strip path from __FILE__
-  inline constexpr const char* str_end(const char *str) { return *str ? str_end(str + 1) : str; }
-  inline constexpr bool str_slant(const char *str) { return *str == '/' ? true : (*str ? str_slant(str + 1) : false); }
-  inline constexpr const char* r_slant(const char* str) { return *str == '/' ? (str + 1) : r_slant(str - 1); }
-  inline constexpr const char* file_name(const char* str) { return str_slant(str) ? r_slant(str_end(str)) : str; }
+  void register_pinned_(const char *func, const char *file, int line, void *ptr, size_t bytes);
+  void unregister_pinned_(const char *func, const char *file, int line, void *ptr);
 
   QudaFieldLocation get_pointer_location(const void *ptr);
 
@@ -88,6 +84,8 @@ namespace quda {
 #define managed_free(ptr) quda::managed_free_(__func__, quda::file_name(__FILE__), __LINE__, ptr)
 #define host_free(ptr) quda::host_free_(__func__, quda::file_name(__FILE__), __LINE__, ptr)
 #define get_mapped_device_pointer(ptr) quda::get_mapped_device_pointer_(__func__, quda::file_name(__FILE__), __LINE__, ptr)
+#define register_pinned(ptr, bytes) quda::register_pinned_(__func__, quda::file_name(__FILE__), __LINE__, ptr, bytes)
+#define unregister_pinned(size) quda::unregister_pinned_(__func__, quda::file_name(__FILE__), __LINE__, ptr)
 
 namespace quda {
 
diff --git a/include/matrix_field.h b/include/matrix_field.h
index 4c8ecf1970..d4aab30026 100644
--- a/include/matrix_field.h
+++ b/include/matrix_field.h
@@ -6,17 +6,7 @@
  * site consists of an n x n matrix
  */
 
-// trove requires the warp shuffle instructions introduced with Kepler
-#ifdef QUDA_BACKEND_OMPTARGET
-#define DISABLE_TROVE
-#else
-#if __COMPUTE_CAPABILITY__ >= 300
-#include <trove/ptr.h>
-#else
-#define DISABLE_TROVE
-#endif
-#endif
-
+#include <aos.h>
 #include <quda_matrix.h>
 
 namespace quda
@@ -31,29 +21,13 @@ namespace quda
     __device__ __host__ inline void load(Matrix<T, n> &A, int x_cb, int parity) const
     {
       int idx = parity * volume_cb + x_cb;
-#ifdef __CUDA_ARCH__
-      const trove::coalesced_ptr<Matrix<T, n>> field_((Matrix<T, n> *)field);
-      A = field_[idx];
-#else
-#pragma unroll
-      for (int i = 0; i < n; i++)
-#pragma unroll
-        for (int j = 0; j < n; j++) A(i, j) = field[(n * idx + i) * n + j] = A(i, j);
-#endif
+      block_load(A, reinterpret_cast<const Matrix<T, n> *>(field) + idx);
     }
 
     __device__ __host__ inline void save(const Matrix<T, n> &A, int x_cb, int parity)
     {
       int idx = parity * volume_cb + x_cb;
-#ifdef __CUDA_ARCH__
-      trove::coalesced_ptr<Matrix<T, n>> field_((Matrix<T, n> *)field);
-      field_[idx] = A;
-#else
-#pragma unroll
-      for (int i = 0; i < n; i++)
-#pragma unroll
-        for (int j = 0; j < n; j++) field[(n * idx + i) * n + j] = A(i, j);
-#endif
+      block_store(reinterpret_cast<Matrix<T, n> *>(field) + idx, A);
     }
   };
 
diff --git a/include/quda.h b/include/quda.h
index c23701ff22..e1493e23a3 100644
--- a/include/quda.h
+++ b/include/quda.h
@@ -1109,7 +1109,7 @@ extern "C" {
    * @param inverse Whether to apply the inverse of the clover term
    */
   void cloverQuda(void *h_out, void *h_in, QudaInvertParam *inv_param,
-      QudaParity *parity, int inverse);
+                  QudaParity parity, int inverse);
 
   /**
    * Apply the full Dslash matrix, possibly even/odd preconditioned.
@@ -1279,7 +1279,7 @@ extern "C" {
    * @param gauge_param Gauge field meta data
    * @param invert_param Dirac and solver meta data
    */
-  void computeStaggeredForceQuda(void* mom, double dt, double delta, void **x, void *gauge,
+  void computeStaggeredForceQuda(void* mom, double dt, double delta, void *gauge, void **x,
 				 QudaGaugeParam *gauge_param, QudaInvertParam *invert_param);
 
   /**
@@ -1331,10 +1331,9 @@ extern "C" {
 
   /**
    * Performs a deep copy from the internal extendedGaugeResident field.
-   * @param Pointer to externalGaugeResident cudaGaugeField
-   * @param Location of gauge field
+   * @param Pointer to externally allocated GaugeField
    */
-  void copyExtendedResidentGaugeQuda(void* resident_gauge, QudaFieldLocation loc);
+  void copyExtendedResidentGaugeQuda(void* resident_gauge);
 
   /**
    * Performs Wuppertal smearing on a given spinor using the gauge field
diff --git a/include/quda_api.h b/include/quda_api.h
index 220a56ee0e..6823922881 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -222,17 +222,7 @@ static inline double4 make_double4(double x,double y,double z,double w){return d
 
 #endif  // QUDA_BACKEND_OMPTARGET
 
-// Target specific definitions in include/targets/XXX/quda_api_target.h
-// The correct targets/ directory is set by the build system
-// Eventually we want to shrink the contents of that
 #include <quda_define.h>
-
-enum qudaMemcpyKind { qudaMemcpyHostToHost, 
-	              qudaMemcpyHostToDevice,
-		      qudaMemcpyDeviceToHost,
-		      qudaMemcpyDeviceToDevice,
-		      qudaMemcpyDefault };
-
 #include <string>
 #include <enum_quda.h>
 
@@ -243,6 +233,12 @@ enum qudaMemcpyKind { qudaMemcpyHostToHost,
    profile and switch between using the CUDA runtime and driver APIs.
  */
 
+enum qudaMemcpyKind { qudaMemcpyHostToHost,
+	              qudaMemcpyHostToDevice,
+		      qudaMemcpyDeviceToHost,
+		      qudaMemcpyDeviceToDevice,
+		      qudaMemcpyDefault };
+
 namespace quda
 {
 
@@ -250,7 +246,10 @@ namespace quda
 
   struct qudaStream_t {
     int idx;
-    //qudaStream_t(int idx) : idx(idx) {}
+  };
+
+  struct qudaEvent_t {
+    void *event;
   };
 
   /**
@@ -401,12 +400,33 @@ namespace quda
   void qudaMemPrefetchAsync_(void *ptr, size_t count, QudaFieldLocation mem_space, const qudaStream_t &stream,
                              const char *func, const char *file, const char *line);
 
+  /**
+     @brief Return instance of an event.
+  */
+  qudaEvent_t qudaEventCreate_(const char *func, const char *file, const char *line);
+
+  /**
+     @brief Return instance of an event that can be used for timing.
+  */
+  qudaEvent_t qudaChronoEventCreate_(const char *func, const char *file, const char *line);
+
+  /**
+     @brief Return elapsed time in seconds between two events
+  */
+  float qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
+                              const char *func, const char *file, const char *line);
+
+  /**
+     @brief Destroy the event
+  */
+  void qudaEventDestroy_(qudaEvent_t &event, const char *func, const char *file, const char *line);
+
   /**
      @brief Wrapper around cudaEventQuery or cuEventQuery with built-in error checking
      @param[in] event Event we are querying
      @return true if event has been reached
    */
-  bool qudaEventQuery_(cudaEvent_t &event, const char *func, const char *file, const char *line);
+  bool qudaEventQuery_(qudaEvent_t &event, const char *func, const char *file, const char *line);
 
   /**
      @brief Wrapper around cudaEventRecord or cuEventRecord with
@@ -414,7 +434,7 @@ namespace quda
      @param[in,out] event Event we are recording
      @param[in,out] stream Stream where to record the event
    */
-  void qudaEventRecord_(cudaEvent_t &event, qudaStream_t stream, const char *func, const char *file, const char *line);
+  void qudaEventRecord_(qudaEvent_t &event, qudaStream_t stream, const char *func, const char *file, const char *line);
 
   /**
      @brief Wrapper around cudaStreamWaitEvent or cuStreamWaitEvent
@@ -423,7 +443,7 @@ namespace quda
      @param[in] event Event we are waiting on
      @param[in] flags Flags to pass to function
    */
-  void qudaStreamWaitEvent_(qudaStream_t stream, cudaEvent_t event, unsigned int flags, const char *func,
+  void qudaStreamWaitEvent_(qudaStream_t stream, qudaEvent_t event, unsigned int flags, const char *func,
                             const char *file, const char *line);
 
   /**
@@ -431,7 +451,7 @@ namespace quda
      with built-in error checking
      @param[in] event Event which we are synchronizing with respect to
    */
-  void qudaEventSynchronize_(const cudaEvent_t &event, const char *func, const char *file, const char *line);
+  void qudaEventSynchronize_(const qudaEvent_t &event, const char *func, const char *file, const char *line);
 
   /**
      @brief Wrapper around cudaStreamSynchronize or
@@ -457,7 +477,14 @@ namespace quda
   void* qudaGetSymbolAddress_(const char *symbol, const char *func, const char *file, const char *line);
 
   /**
-     @brief Get the last error string recorded
+     @brief Get the last error recorded by the target runtime.  By
+     calling this, we reset the last error.
+  */
+  qudaError_t qudaGetLastError();
+
+  /**
+     @brief Get the error string associated with the last error that
+     was thrown by the target runtime
   */
   std::string qudaGetLastErrorString();
 
@@ -509,6 +536,18 @@ namespace quda
   ::quda::qudaMemPrefetchAsync_(ptr, count, mem_space, stream, __func__, quda::file_name(__FILE__),                    \
                                 __STRINGIFY__(__LINE__))
 
+#define qudaEventCreate()                                               \
+  ::quda::qudaEventCreate_( __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
+
+#define qudaChronoEventCreate()                                               \
+  ::quda::qudaChronoEventCreate_( __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
+
+#define qudaEventElapsedTime(start, stop)                               \
+  ::quda::qudaEventElapsedTime_(start, stop, __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
+
+#define qudaEventDestroy(event)                                         \
+  ::quda::qudaEventDestroy_(event, __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
+
 #define qudaEventQuery(event)                                                                                          \
   ::quda::qudaEventQuery_(event, __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
 
diff --git a/include/quda_fortran.h b/include/quda_fortran.h
index 9a538b1b03..064663c63d 100644
--- a/include/quda_fortran.h
+++ b/include/quda_fortran.h
@@ -235,11 +235,6 @@ extern "C" {
    */
   void plaq_quda_(double plaq[3]);
 
-  /**
-   * @brief fTemporary function exposed for TIFR benchmarking
-   */
-  void set_kernel_pack_t_(int *pack);
-
   /**
    * @brief Flush the chronological history for the given index
    * @param[in] index Index for which we are flushing
diff --git a/include/quda_internal.h b/include/quda_internal.h
index 61628c18fa..bbdcf46407 100644
--- a/include/quda_internal.h
+++ b/include/quda_internal.h
@@ -1,6 +1,13 @@
 #pragma once
 
+#include <quda_define.h>
 #include <quda_api.h>
+
+#if defined(QUDA_TARGET_CUDA)
+#include <cuda.h>
+#include <cuda_runtime.h>
+#endif
+
 #include <string>
 #include <complex>
 #include <vector>
@@ -40,93 +47,15 @@
 
 #define TEX_ALIGN_REQ (512*2) //Fermi, factor 2 comes from even/odd
 #define ALIGNMENT_ADJUST(n) ( (n+TEX_ALIGN_REQ-1)/TEX_ALIGN_REQ*TEX_ALIGN_REQ)
-#include <enum_quda.h>
 #include <quda.h>
 #include <util_quda.h>
 #include <malloc_quda.h>
 #include <object.h>
 #include <device.h>
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-  
-  struct QUDA_DiracField{
-    void *field; /**< Pointer to a ColorSpinorField */
-  };
-
-#ifdef __cplusplus
-}
-#endif
-
 namespace quda {
 
-  struct alignas(8) char8 {
-    char4 x;
-    char4 y;
-  };
-
-  struct alignas(16) short8 {
-    short4 x;
-    short4 y;
-  };
-
-  struct alignas(32) float8 {
-    float4 x;
-    float4 y;
-  };
-
-  struct alignas(64) double8 {
-    double4 x;
-    double4 y;
-  };
-
-  typedef std::complex<double> Complex;
-
-  /**
-   * Traits for determining the maximum and inverse maximum
-   * value of a (signed) char and short. Relevant for
-   * fixed-precision types.
-   */
-  template< typename T > struct fixedMaxValue{ static constexpr float value = 0.0f; };
-  template<> struct fixedMaxValue<short>{ static constexpr float value = 32767.0f; };
-  template<> struct fixedMaxValue<short2>{ static constexpr float value = 32767.0f; };
-  template<> struct fixedMaxValue<short4>{ static constexpr float value = 32767.0f; };
-  template<> struct fixedMaxValue<short8>{ static constexpr float value = 32767.0f; };
-  template <> struct fixedMaxValue<int8_t> {
-    static constexpr float value = 127.0f;
-  };
-  template<> struct fixedMaxValue<char2>{ static constexpr float value = 127.0f; };
-  template<> struct fixedMaxValue<char4>{ static constexpr float value = 127.0f; };
-  template<> struct fixedMaxValue<char8>{ static constexpr float value = 127.0f; };
-
-  template <typename T> struct fixedInvMaxValue {
-    static constexpr float value = 3.402823e+38f;
-  };
-  template <> struct fixedInvMaxValue<short> {
-    static constexpr float value = 3.0518509476e-5f;
-  };
-  template <> struct fixedInvMaxValue<short2> {
-    static constexpr float value = 3.0518509476e-5f;
-  };
-  template <> struct fixedInvMaxValue<short4> {
-    static constexpr float value = 3.0518509476e-5f;
-  };
-  template <> struct fixedInvMaxValue<short8> {
-    static constexpr float value = 3.0518509476e-5f;
-  };
-  template <> struct fixedInvMaxValue<int8_t> {
-    static constexpr float value = 7.874015748031e-3f;
-  };
-  template <> struct fixedInvMaxValue<char2> {
-    static constexpr float value = 7.874015748031e-3f;
-  };
-  template <> struct fixedInvMaxValue<char4> {
-    static constexpr float value = 7.874015748031e-3f;
-  };
-  template <> struct fixedInvMaxValue<char8> {
-    static constexpr float value = 7.874015748031e-3f;
-  };
+  using Complex = std::complex<double>;
 
   /**
    * Check that the resident gauge field is compatible with the requested inv_param
@@ -134,7 +63,7 @@ namespace quda {
    */
   bool canReuseResidentGauge(QudaInvertParam *inv_param);
 
-}
+  class TimeProfile;
 
-#include <timer.h>
+}
 
diff --git a/include/quda_new_interace.h b/include/quda_new_interace.h
deleted file mode 100644
index a204cbd35c..0000000000
--- a/include/quda_new_interace.h
+++ /dev/null
@@ -1,36 +0,0 @@
-#ifndef _QUDA_NEW_INTERFACE_H
-#define _QUDA_NEW_INTERFACE_H
-
-/**
- * @file quda_new_interface.h
- * @brief Experimental new interace.  This will eventually evolve into
- * the new quda.h, likely to coincide with release 0.5.0.
- *
- */
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-  /**
-   * Parameters relating to a DiracField
-   */
-  typedef struct QUDA_DiracFieldParam_s {
-    int X[4];          /**< Lattice dimensions */ 
-    int Ls;            /**< Extent of the 5th dimension (for domain wall) */
-    int Nc;            /**< Number of colors */
-    int Ns;            /**< Number of spins */
-    QudaTwistFlavorType twist_flavor;  /**< Twisted mass flavor */
-    int pad; /**< Pad used on the field */
-  } QUDA_DiracFieldParam;  
-
-  typedef struct QUDA_DiracField_s QUDA_DiracField;
-
-  #ifdef __cplusplus
-}
-#endif
-
-#endif /* _QUDA_NEW_INTERFACE_H */
-
-
diff --git a/include/random_quda.h b/include/random_quda.h
index 596152265f..9799b1355f 100644
--- a/include/random_quda.h
+++ b/include/random_quda.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <memory>
 #include <quda_define.h>
 #include <lattice_field.h>
 
@@ -13,35 +14,20 @@ namespace quda {
   */
   class RNG {
 
-    RNGState *state;        /*! array with current curand rng state */
-    RNGState *backup_state; /*! array for backup of current curand rng state */
-    unsigned long long seed;  /*! initial rng seed */
-    int size;                 /*! @brief number of curand states */
-    int size_cb;        /*! @brief number of curand states checkerboarded (equal to size if we have a single parity) */
-    int X[4];           /*! @brief local lattice dimensions */
-    void AllocateRNG(); /*! @brief allocate curand rng states array in device memory */
+    size_t size;                     /*! @brief number of curand states */
+    std::shared_ptr<RNGState> state; /*! array with current curand rng state */
+    RNGState *backup_state;          /*! array for backup of current curand rng state */
+    unsigned long long seed;         /*! initial rng seed */
 
   public:
     /**
-       @brief Constructor that takes its metadata from a field
+       @brief Allocate and initialize RNG states.  Constructor that
+       takes its metadata from pre-existing field
        @param[in] meta The field whose data we use
        @param[in] seed Seed to initialize the RNG
     */
     RNG(const LatticeField &meta, unsigned long long seedin);
 
-    /**
-       @brief Constructor that takes its metadata from a param
-       @param[in] param The param whose data we use
-       @param[in] seed Seed to initialize the RNG
-    */
-    RNG(const LatticeFieldParam &param, unsigned long long seedin);
-
-    /*! free array */
-    void Release();
-
-    /*! initialize rng states with seed */
-    void Init();
-
     unsigned long long Seed() { return seed; };
 
     /*! @brief Restore rng array states initialization */
@@ -50,7 +36,8 @@ namespace quda {
     /*! @brief Backup rng array states initialization */
     void backup();
 
-    constexpr RNGState *State() { return state; };
+    /*! @brief Get pointer to RNGState */
+    RNGState *State() { return state.get(); };
   };
 
 }
diff --git a/include/reduce_helper.h b/include/reduce_helper.h
index d0505f1a1a..71539b2319 100644
--- a/include/reduce_helper.h
+++ b/include/reduce_helper.h
@@ -5,6 +5,7 @@
 #include <cub_helper.cuh>
 #endif
 #include <target_device.h>
+#include <reducer.h>
 
 #ifdef QUAD_SUM
 using device_reduce_t = doubledouble;
@@ -31,29 +32,9 @@ namespace quda
     void *get_mapped_buffer();
     void *get_host_buffer();
     count_t *get_count();
-    cudaEvent_t &get_event();
+    qudaEvent_t &get_event();
   } // namespace reducer
 
-  template <typename T> struct plus {
-    static constexpr bool do_sum = true;
-    __device__ __host__ T operator()(T a, T b) const { return a + b; }
-  };
-
-  template <typename T> struct maximum {
-    static constexpr bool do_sum = false;
-    __device__ __host__ T operator()(T a, T b) const { return a > b ? a : b; }
-  };
-
-  template <typename T> struct minimum {
-    static constexpr bool do_sum = false;
-    __device__ __host__ T operator()(T a, T b) const { return a < b ? a : b; }
-  };
-
-  template <typename T> struct identity {
-    static constexpr bool do_sum = false;
-    __device__ __host__ T operator()(T a) const { return a; }
-  };
-
   constexpr int max_n_reduce() { return QUDA_MAX_MULTI_REDUCE; }
 
   /**
@@ -81,8 +62,8 @@ namespace quda
 
   template <typename T> struct ReduceArg {
 
-    template <int, int, typename Reduce, typename Arg, typename I>
-    friend __device__  void reduce(Arg&, const I &, const int);
+    template <int, int, typename Reducer, typename Arg, typename I>
+    friend __device__  void reduce(Arg&, const Reducer &, const I &, const int);
     qudaError_t launch_error; // only do complete if no launch error to avoid hang
 
   private:
@@ -215,13 +196,12 @@ namespace quda
        will be constant along constant blockIdx.y and blockIdx.z.
     */
     template <int block_size_x, int block_size_y = 1, typename Reducer, typename Arg, typename T>
-    __device__ inline void reduce(Arg &arg, const T &in, const int idx = 0)
+    __device__ inline void reduce(Arg &arg, const Reducer &r, const T &in, const int idx = 0)
     {
       using BlockReduce = cub::BlockReduce<T, block_size_x, cub::BLOCK_REDUCE_WARP_REDUCTIONS, block_size_y>;
       __shared__ typename BlockReduce::TempStorage cub_tmp;
       __shared__ bool isLastBlockDone;
 
-      Reducer r;
       T aggregate = Reducer::do_sum ? BlockReduce(cub_tmp).Sum(in) : BlockReduce(cub_tmp).Reduce(in, r);
 
       if (threadIdx.x == 0 && threadIdx.y == 0) {
@@ -282,7 +262,7 @@ namespace quda
        will be constant along constant blockIdx.y and blockIdx.z.
     */
     template <int block_size_x, int block_size_y = 1, typename Reducer, typename Arg, typename T>
-    __device__ inline void reduce(Arg &arg, const T &in, const int idx = 0)
+    __device__ inline void reduce(Arg &arg, const Reducer &r, const T &in, const int idx = 0)
     {
 #ifdef QUDA_BACKEND_OMPTARGET
       ompwip("unimplemented");
@@ -291,7 +271,6 @@ namespace quda
       __shared__ typename BlockReduce::TempStorage cub_tmp;
       __shared__ bool isLastBlockDone;
 
-      Reducer r;
       T aggregate = Reducer::do_sum ? BlockReduce(cub_tmp).Sum(in) : BlockReduce(cub_tmp).Reduce(in, r);
 
       if (threadIdx.x == 0 && threadIdx.y == 0) {
diff --git a/include/register_traits.h b/include/register_traits.h
index ce0469d657..19297bf59a 100644
--- a/include/register_traits.h
+++ b/include/register_traits.h
@@ -15,6 +15,26 @@
 
 namespace quda {
 
+  struct alignas(8) char8 {
+    char4 x;
+    char4 y;
+  };
+
+  struct alignas(16) short8 {
+    short4 x;
+    short4 y;
+  };
+
+  struct alignas(32) float8 {
+    float4 x;
+    float4 y;
+  };
+
+  struct alignas(64) double8 {
+    double4 x;
+    double4 y;
+  };
+
   /*
     Here we use traits to define the greater type used for mixing types of computation involving these types
   */
diff --git a/include/shared_memory_cache_helper.cuh b/include/shared_memory_cache_helper.cuh
index 382bafce35..ba0a60e1c5 100644
--- a/include/shared_memory_cache_helper.cuh
+++ b/include/shared_memory_cache_helper.cuh
@@ -37,7 +37,7 @@ namespace quda
     static constexpr int max_block_size_x = device::max_block_size<block_size_y, block_size_z>();
 
     /** pad in the x dimension width if requested to ensure that it isn't a multiple of the bank width */
-    static constexpr int block_size_x = pad ? max_block_size_x :
+    static constexpr int block_size_x = !pad ? max_block_size_x :
       ((max_block_size_x + device::shared_memory_bank_width() - 1) /
        device::shared_memory_bank_width()) * device::shared_memory_bank_width();
 
@@ -177,123 +177,4 @@ namespace quda
     }
   };
 
-#if 1
-  template <typename T, int block_size_y = 1, int block_size_z = 1> class SharedMemoryCacheOld
-  {
-    // maximum number of threads in x given the y and z block sizes
-    static constexpr int max_block_size_x = device::max_block_size<block_size_y, block_size_z>();
-
-    // we pad in the x dimension width to ensure that it isn't a multiple of the bank width
-    static constexpr int thread_width_x = ((max_block_size_x + device::shared_memory_bank_width() - 1) /
-                                           device::shared_memory_bank_width()) * device::shared_memory_bank_width();
-
-    /**
-       @brief This is the handle to the shared memory
-       @return Shared memory pointer
-     */
-    __device__ __host__ inline T *cache()
-    {
-#ifdef __CUDA_ARCH__
-      extern __shared__ int cache_[];
-#else
-      // dummy code path to keep clang happy
-      static int *cache_;
-#endif
-      return reinterpret_cast<T*>(cache_);
-    }
-
-  public:
-    /**
-       @brief Grab the raw base address to shared memory
-    */
-    __device__ __host__ inline T* data()
-    {
-      return cache();
-    }
-
-    /**
-       @brief Save the value into the 3-d shared memory cache.
-       Implicitly store at coordinates given by thread_idx()
-       @param[in] a The value to store in the shared memory cache
-     */
-    __device__ __host__ inline void save(const T &a)
-    {
-      auto tid = device::thread_idx();
-      int j = (tid.z * block_size_y + tid.y) * thread_width_x + tid.x;
-      cache()[j] = a;
-    }
-
-    /**
-       @brief Load a vector from the shared memory cache
-       @param[in] x The x index to use
-       @param[in] y The y index to use
-       @param[in] z The z index to use
-       @return The value at coordinates (x,y,z)
-    */
-    __device__ __host__ inline T load(int x = -1, int y = -1, int z = -1)
-    {
-      auto tid = device::thread_idx();
-      x = (x == -1) ? tid.x : x;
-      y = (y == -1) ? tid.y : y;
-      z = (z == -1) ? tid.z : z;
-      int j = (z * block_size_y + y) * thread_width_x + x;
-      return cache()[j];
-    }
-
-    /**
-       @brief Load a vector from the shared memory cache
-       @param[in] x The x index to use
-       @param[in] y The y index to use
-       @param[in] z The z index to use
-       @return The value at coordinates (x,y,z)
-    */
-    __device__ __host__ inline T load_x(int x = -1)
-    {
-      auto tid = device::thread_idx();
-      x = (x == -1) ? tid.x : x;
-      int j = (tid.z * block_size_y + tid.y) * thread_width_x + x;
-      return cache()[j];
-    }
-
-    /**
-       @brief Load a vector from the shared memory cache
-       @param[in] x The x index to use
-       @param[in] y The y index to use
-       @param[in] z The z index to use
-       @return The value at coordinates (x,y,z)
-    */
-    __device__ __host__ inline T load_y(int y = -1)
-    {
-      auto tid = device::thread_idx();
-      y = (y == -1) ? tid.y : y;
-      int j = (tid.z * block_size_y + y) * thread_width_x + tid.x;
-      return cache()[j];
-    }
-
-    /**
-       @brief Load a vector from the shared memory cache
-       @param[in] x The x index to use
-       @param[in] y The y index to use
-       @param[in] z The z index to use
-       @return The value at coordinates (x,y,z)
-    */
-    __device__ __host__ inline T load_z(int z = -1)
-    {
-      auto tid = device::thread_idx();
-      z = (z == -1) ? tid.z : z;
-      int j = (z * block_size_y + tid.y) * thread_width_x + tid.x;
-      return cache()[j];
-    }
-
-    /**
-       @brief Synchronize the cache
-    */
-    __device__ __host__ inline void sync()
-    {
-#ifdef __CUDA_ARCH__
-      __syncthreads();
-#endif
-    }
-  };
-#endif
 } // namespace quda
diff --git a/include/targets/cuda/block_reduction_kernel.h b/include/targets/cuda/block_reduction_kernel.h
index f0f4e46405..b742862448 100644
--- a/include/targets/cuda/block_reduction_kernel.h
+++ b/include/targets/cuda/block_reduction_kernel.h
@@ -15,13 +15,13 @@ namespace quda {
    */
   template <typename Arg> __device__ constexpr int virtual_block_idx(const Arg &arg)
   {
-    int block_idx = blockIdx.x;
+    auto block_idx = blockIdx.x;
     if (arg.swizzle) {
       // the portion of the grid that is exactly divisible by the number of SMs
-      const int gridp = gridDim.x - gridDim.x % arg.swizzle_factor;
+      const auto gridp = gridDim.x - gridDim.x % arg.swizzle_factor;
 
       block_idx = blockIdx.x;
-      if (blockIdx.x < gridp) {
+      if (block_idx < gridp) {
         // this is the portion of the block that we are going to transpose
         const int i = blockIdx.x % arg.swizzle_factor;
         const int j = blockIdx.x / arg.swizzle_factor;
@@ -41,11 +41,11 @@ namespace quda {
      thread dimension is a trivial vectorizable dimension.
   */
   template <int block_size, template <int, typename> class Transformer, typename Arg>
-  __launch_bounds__(Arg::launch_bounds ? block_size : 0) __global__ void BlockKernel2D(Arg arg)
+  __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0) __global__ void BlockKernel2D(Arg arg)
   {
     const dim3 block_idx(virtual_block_idx(arg), blockIdx.y, 0);
     const dim3 thread_idx(threadIdx.x, threadIdx.y, 0);
-    const int j = blockDim.y*blockIdx.y + threadIdx.y;
+    auto j = blockDim.y*blockIdx.y + threadIdx.y;
     if (j >= arg.threads.y) return;
 
     Transformer<block_size, Arg> t(arg);
diff --git a/include/targets/cuda/kernel.h b/include/targets/cuda/kernel.h
index 0a53a06ce2..5abc625c13 100644
--- a/include/targets/cuda/kernel.h
+++ b/include/targets/cuda/kernel.h
@@ -47,7 +47,7 @@ namespace quda {
     }
   }
 
-  template <template <typename> class Functor, typename Arg>
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
     Functor<Arg> f(arg);
diff --git a/include/targets/cuda/random_helper.h b/include/targets/cuda/random_helper.h
index 456dadab0a..c628db1bc5 100644
--- a/include/targets/cuda/random_helper.h
+++ b/include/targets/cuda/random_helper.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <random_quda.h>
 #include <curand_kernel.h>
 
 namespace quda {
diff --git a/include/targets/cuda/reduction_kernel.h b/include/targets/cuda/reduction_kernel.h
index dba7ba837a..0d2a9bf01a 100644
--- a/include/targets/cuda/reduction_kernel.h
+++ b/include/targets/cuda/reduction_kernel.h
@@ -4,13 +4,11 @@
 
 namespace quda {
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer,
-            template <typename> class Reducer, typename Arg>
+  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __global__ void Reduction2D(Arg arg)
   {
     using reduce_t = typename Transformer<Arg>::reduce_t;
     Transformer<Arg> t(arg);
-    Reducer<reduce_t> r;
 
     auto idx = threadIdx.x + blockIdx.x * blockDim.x;
     auto j = threadIdx.y;
@@ -18,21 +16,19 @@ namespace quda {
     reduce_t value = arg.init();
 
     while (idx < arg.threads.x) {
-      value = t(value, r, idx, j);
-      idx += blockDim.x * gridDim.x;
+      value = t(value, idx, j);
+      if (grid_stride) idx += blockDim.x * gridDim.x; else break;
     }
 
     // perform final inter-block reduction and write out result
-    reduce<block_size_x, block_size_y, decltype(r)>(arg, value);
+    reduce<block_size_x, block_size_y>(arg, t, value);
   }
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer,
-            template <typename> class Reducer, typename Arg>
+  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __global__ void MultiReduction(Arg arg)
   {
     using reduce_t = typename Transformer<Arg>::reduce_t;
     Transformer<Arg> t(arg);
-    Reducer<reduce_t> r;
 
     auto idx = threadIdx.x + blockIdx.x * blockDim.x;
     auto j = threadIdx.y + blockIdx.y * blockDim.y;
@@ -43,12 +39,12 @@ namespace quda {
     reduce_t value = arg.init();
 
     while (idx < arg.threads.x) {
-      value = t(value, r, idx, j, k);
-      idx += blockDim.x * gridDim.x;
+      value = t(value, idx, j, k);
+      if (grid_stride) idx += blockDim.x * gridDim.x; else break;
     }
 
     // perform final inter-block reduction and write out result
-    reduce<block_size_x, block_size_y, decltype(r)>(arg, value, j);
+    reduce<block_size_x, block_size_y>(arg, t, value, j);
   }
 
 }
diff --git a/include/targets/cuda/target_device.h b/include/targets/cuda/target_device.h
index 676a594e13..eb8d5504dd 100644
--- a/include/targets/cuda/target_device.h
+++ b/include/targets/cuda/target_device.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <algorithm>
+
 namespace quda {
 
   namespace device {
diff --git a/include/targets/hip/random_helper.h b/include/targets/hip/random_helper.h
index e9976dc726..19b63bca59 100644
--- a/include/targets/hip/random_helper.h
+++ b/include/targets/hip/random_helper.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <random_quda.h>
 #include <hiprand_kernel.h>
 
 namespace quda {
diff --git a/include/timer.h b/include/timer.h
index 749a73ff7a..e30fd9d3a3 100644
--- a/include/timer.h
+++ b/include/timer.h
@@ -1,27 +1,5 @@
 #pragma once
 
-#ifdef __CUDACC_RTC__
-
-namespace quda {
-  // dummy Implementation that can safely be parsed by nvrtc
-  enum QudaProfileType { };
-
-  class TimeProfile {
-  public:
-    TimeProfile(std::string fname);
-    TimeProfile(std::string fname, bool use_global);
-    void Print();
-    void Start_(const char *func, const char *file, int line, QudaProfileType idx);
-    void Stop_(const char *func, const char *file, int line, QudaProfileType idx);
-    void Reset_(const char *func, const char *file, int line);
-    double Last(QudaProfileType idx);
-    void PrintGlobal();
-    bool isRunning(QudaProfileType idx);
-  };
-}
-
-#else
-
 #include <sys/time.h>
 
 #ifdef INTERFACE_NVTX
@@ -32,73 +10,125 @@ namespace quda {
 #endif
 #endif
 
+#include <quda_internal.h>
+#include <util_quda.h>
+#include <device.h>
+
 namespace quda {
 
   /**
    * Use this for recording a fine-grained profile of a QUDA
-   * algorithm.  This uses host-side measurement, so should be used
-   * for timing fully host-device synchronous algorithms.
+   * algorithm.  This can be used for either host-side or device side
+   * measurement.  When using host-side measurement, the execution
+   * should host-device synchronous.  For device-side measurement, the
+   * measurement will be done on the default stream unless specified
+   * otherwise in the constructor.
    */
+  template <bool device = false>
   struct Timer {
     /**< The cumulative sum of time */
     double time;
 
     /**< The last recorded time interval */
-    double last;
+    double last_interval;
+
+    /**< Used to store when the timer was last started */
+    timeval host_start;
+
+    /**< Used to store when the timer was last stopped */
+    timeval host_stop;
 
     /**< Used to store when the timer was last started */
-    timeval start;
+    qudaEvent_t device_start;
 
     /**< Used to store when the timer was last stopped */
-    timeval stop;
+    qudaEvent_t device_stop;
+
+    /**< Which stream are we recording on */
+    qudaStream_t stream;
 
     /**< Are we currently timing? */
     bool running;
-    
+
     /**< Keep track of number of calls */
     int count;
 
-  Timer() : time(0.0), last(0.0), running(false), count(0) { ; } 
+    Timer(qudaStream_t stream = device::get_default_stream()) :
+      time(0.0),
+      last_interval(0.0),
+      stream(stream),
+      running(false),
+      count(0)
+    {
+      if (device) {
+        device_start = qudaChronoEventCreate();
+        device_stop = qudaChronoEventCreate();
+      }
+    }
+
+    ~Timer()
+    {
+      if (device) {
+        qudaEventDestroy(device_start);
+        qudaEventDestroy(device_stop);
+      }
+    }
 
-    void Start(const char *func, const char *file, int line) {
+    void start(const char *func = nullptr, const char *file = nullptr, int line = 0)
+    {
       if (running) {
-	printfQuda("ERROR: Cannot start an already running timer (%s:%d in %s())\n", file, line, func);
+	printfQuda("ERROR: Cannot start an already running timer (%s:%d in %s())",
+                   file ? file : "", line, func ? func : "");
 	errorQuda("Aborting");
       }
-      gettimeofday(&start, NULL);
+      if (!device) {
+        gettimeofday(&host_start, NULL);
+      } else {
+        qudaEventRecord(device_start, stream);
+      }
       running = true;
     }
 
-    void Stop(const char *func, const char *file, int line) {
+    void stop(const char *func = nullptr, const char *file = nullptr, int line = 0)
+    {
       if (!running) {
-	printfQuda("ERROR: Cannot stop an unstarted timer (%s:%d in %s())\n", file, line, func);
+	printfQuda("ERROR: Cannot stop an unstarted timer (%s:%d in %s())",
+                   file ? file : "", line, func ? func : "");
 	errorQuda("Aborting");
       }
-      gettimeofday(&stop, NULL);
-
-      long ds = stop.tv_sec - start.tv_sec;
-      long dus = stop.tv_usec - start.tv_usec;
-      last = ds + 0.000001*dus;
-      time += last;
+      if (!device) {
+        gettimeofday(&host_stop, NULL);
+        long ds = host_stop.tv_sec - host_start.tv_sec;
+        long dus = host_stop.tv_usec - host_start.tv_usec;
+        last_interval = ds + 0.000001*dus;
+      } else {
+        qudaEventRecord(device_stop, stream);
+        qudaEventSynchronize(device_stop);
+        last_interval = qudaEventElapsedTime(device_start, device_stop);
+      }
+      time += last_interval;
       count++;
 
       running = false;
     }
 
-    double Last() { return last; }
+    double last() { return last_interval; }
 
-    void Reset(const char *func, const char *file, int line) {
+    void reset(const char *func, const char *file, int line) {
       if (running) {
-	printfQuda("ERROR: Cannot reset a started timer (%s:%d in %s())\n", file, line, func);
+	printfQuda("ERROR: Cannot reset a started timer (%s:%d in %s())",
+                   file ? file : "", line, func ? func : "");
 	errorQuda("Aborting");
       }
       time = 0.0;
-      last = 0.0;
+      last_interval = 0.0;
       count = 0;
     }
-
   };
 
+  using device_timer_t = Timer<true>;
+  using host_timer_t = Timer<false>;
+
   /**< Enumeration type used for writing a simple but extensible profiling framework. */
   enum QudaProfileType {
     QUDA_PROFILE_H2D,          /**< host -> device transfers */
@@ -177,26 +207,26 @@ namespace quda {
     static const uint32_t nvtx_colors[];// = { 0x0000ff00, 0x000000ff, 0x00ffff00, 0x00ff00ff, 0x0000ffff, 0x00ff0000, 0x00ffffff };
     static const int nvtx_num_colors;// = sizeof(nvtx_colors)/sizeof(uint32_t);
 #endif
-    Timer profile[QUDA_PROFILE_COUNT];
+    host_timer_t profile[QUDA_PROFILE_COUNT];
     static std::string pname[];
 
     bool switchOff;
     bool use_global;
 
     // global timer
-    static Timer global_profile[QUDA_PROFILE_COUNT];
+    static host_timer_t global_profile[QUDA_PROFILE_COUNT];
     static bool global_switchOff[QUDA_PROFILE_COUNT];
     static int global_total_level[QUDA_PROFILE_COUNT]; // zero initialize
 
     static void StopGlobal(const char *func, const char *file, int line, QudaProfileType idx) {
 
       global_total_level[idx]--;
-      if (global_total_level[idx]==0) global_profile[idx].Stop(func,file,line);
+      if (global_total_level[idx]==0) global_profile[idx].stop(func,file,line);
 
       // switch off total timer if we need to
       if (global_switchOff[idx]) {
         global_total_level[idx]--;
-        if (global_total_level[idx]==0) global_profile[idx].Stop(func,file,line);
+        if (global_total_level[idx]==0) global_profile[idx].stop(func,file,line);
         global_switchOff[idx] = false;
       }
     }
@@ -204,12 +234,12 @@ namespace quda {
     static void StartGlobal(const char *func, const char *file, int line, QudaProfileType idx) {
       // if total timer isn't running, then start it running
       if (!global_profile[idx].running) {
-        global_profile[idx].Start(func,file,line);
+        global_profile[idx].start(func,file,line);
         global_total_level[idx]++;
         global_switchOff[idx] = true;
       }
 
-      if (global_total_level[idx]==0) global_profile[idx].Start(func,file,line);
+      if (global_total_level[idx]==0) global_profile[idx].start(func,file,line);
       global_total_level[idx]++;
     }
 
@@ -221,26 +251,26 @@ namespace quda {
     /**< Print out the profile information */
     void Print();
 
-    void Start_(const char *func, const char *file, int line, QudaProfileType idx) { 
+    void Start_(const char *func, const char *file, int line, QudaProfileType idx) {
       // if total timer isn't running, then start it running
       if (!profile[QUDA_PROFILE_TOTAL].running && idx != QUDA_PROFILE_TOTAL) {
-	profile[QUDA_PROFILE_TOTAL].Start(func,file,line);
+	profile[QUDA_PROFILE_TOTAL].start(func,file,line);
         switchOff = true;
       }
 
-      profile[idx].Start(func, file, line); 
+      profile[idx].start(func, file, line);
       PUSH_RANGE(fname.c_str(),idx)
 	if (use_global) StartGlobal(func,file,line,idx);
     }
 
 
     void Stop_(const char *func, const char *file, int line, QudaProfileType idx) {
-      profile[idx].Stop(func, file, line); 
+      profile[idx].stop(func, file, line);
       POP_RANGE
 
       // switch off total timer if we need to
       if (switchOff && idx != QUDA_PROFILE_TOTAL) {
-        profile[QUDA_PROFILE_TOTAL].Stop(func,file,line);
+        profile[QUDA_PROFILE_TOTAL].stop(func,file,line);
         switchOff = false;
       }
       if (use_global) StopGlobal(func,file,line,idx);
@@ -248,11 +278,11 @@ namespace quda {
 
     void Reset_(const char *func, const char *file, int line) {
       for (int idx=0; idx<QUDA_PROFILE_COUNT; idx++)
-	profile[idx].Reset(func, file, line);
+	profile[idx].reset(func, file, line);
     }
 
-    double Last(QudaProfileType idx) { 
-      return profile[idx].last;
+    double Last(QudaProfileType idx) {
+      return profile[idx].last_interval;
     }
 
     static void PrintGlobal();
@@ -263,13 +293,9 @@ namespace quda {
 
 } // namespace quda
 
-#endif
-
-
 #undef PUSH_RANGE
 #undef POP_RANGE
 
 #define TPSTART(idx) Start_(__func__, __FILE__, __LINE__, idx)
 #define TPSTOP(idx) Stop_(__func__, __FILE__, __LINE__, idx)
 #define TPRESET() Reset_(__func__, __FILE__, __LINE__)
-
diff --git a/include/transform_reduce.h b/include/transform_reduce.h
index 97a445a186..0bbba31a3b 100644
--- a/include/transform_reduce.h
+++ b/include/transform_reduce.h
@@ -1,8 +1,8 @@
 #pragma once
 
-#include <reduce_helper.h>
-#include <uint_to_char.h>
-#include <tune_quda.h>
+#include <vector>
+#include <enum_quda.h>
+#include <reducer.h>
 
 /**
    @file transform_reduce.h
@@ -14,131 +14,6 @@
 namespace quda
 {
 
-  template <typename reduce_t, typename T, typename count_t, typename transformer, typename reducer>
-  struct TransformReduceArg : public ReduceArg<reduce_t> {
-    static constexpr int block_size = 512;
-    static constexpr int n_batch_max = 8;
-    const T *v[n_batch_max];
-    count_t n_items;
-    int n_batch;
-    reduce_t init_value;
-    reduce_t result[n_batch_max];
-    transformer h;
-    reducer r;
-    TransformReduceArg(const std::vector<T *> &v, count_t n_items, transformer h, reduce_t init_value, reducer r) :
-      ReduceArg<reduce_t>(v.size()),
-      n_items(n_items),
-      n_batch(v.size()),
-      init_value(init_value),
-      h(h),
-      r(r)
-    {
-      if (n_batch > n_batch_max) errorQuda("Requested batch %d greater than max supported %d", n_batch, n_batch_max);
-      for (size_t j = 0; j < v.size(); j++) this->v[j] = v[j];
-    }
-
-    __device__ __host__ reduce_t init() const { return init_value; }
-  };
-
-  template <typename Arg> void transform_reduce(Arg &arg)
-  {
-    using count_t = decltype(arg.n_items);
-
-    for (int j = 0; j < arg.n_batch; j++) {
-      auto v = arg.v[j];
-      auto r_ = arg.init();
-      for (count_t i = 0; i < arg.n_items; i++) {
-        auto v_ = arg.h(v[i]);
-        r_ = arg.r(r_, v_);
-      }
-      arg.result[j] = r_;
-    }
-  }
-
-  template <typename Arg> __launch_bounds__(Arg::block_size) __global__ void transform_reduce_kernel(Arg arg)
-  {
-    QUDA_RT_CONSTS;
-    using count_t = decltype(arg.n_items);
-
-    count_t i = blockIdx.x * blockDim.x + threadIdx.x;
-    int j = blockIdx.y;
-    auto v = arg.v[j];
-    auto r_ = arg.init();
-
-    while (i < arg.n_items) {
-      auto v_ = arg.h(v[i]);
-      r_ = arg.r(r_, v_);
-      i += blockDim.x * gridDim.x;
-    }
-
-    reduce<Arg::block_size, 1, decltype(arg.r)>(arg, r_, j);
-  }
-
-  template <typename reduce_t, typename T, typename I, typename transformer, typename reducer>
-  class TransformReduce : Tunable
-  {
-    using Arg = TransformReduceArg<reduce_t, T, I, transformer, reducer>;
-    QudaFieldLocation location;
-    std::vector<reduce_t> &result;
-    const std::vector<T *> &v;
-    I n_items;
-    transformer &h;
-    reduce_t init;
-    reducer &r;
-
-    bool tuneSharedBytes() const { return false; }
-    unsigned int sharedBytesPerThread() const { return 0; }
-    unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
-    int blockMin() const { return Arg::block_size; }
-    unsigned int maxBlockSize(const TuneParam &) const { return Arg::block_size; }
-
-    bool advanceTuneParam(TuneParam &param) const // only do autotuning if we have device fields
-    {
-      return location == QUDA_CUDA_FIELD_LOCATION ? Tunable::advanceTuneParam(param) : false;
-    }
-
-    void initTuneParam(TuneParam &param) const
-    {
-      Tunable::initTuneParam(param);
-      param.grid.y = v.size();
-    }
-
-  public:
-    TransformReduce(QudaFieldLocation location, std::vector<reduce_t> &result, const std::vector<T *> &v, I n_items,
-                    transformer &h, reduce_t init, reducer &r) :
-      location(location), result(result), v(v), n_items(n_items), h(h), init(init), r(r)
-    {
-      strcpy(aux, "batch_size=");
-      u32toa(aux + 11, v.size());
-      if (location == QUDA_CPU_FIELD_LOCATION) strcat(aux, ",cpu");
-      apply(device::get_default_stream());
-    }
-
-    void apply(const qudaStream_t &stream)
-    {
-      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Arg arg(v, n_items, h, init, r);
-
-      if (location == QUDA_CUDA_FIELD_LOCATION) {
-        arg.launch_error = qudaLaunchKernel(transform_reduce_kernel<Arg>, tp, stream, arg);
-        arg.complete(result, stream);
-      } else {
-        transform_reduce(arg);
-        for (size_t j = 0; j < result.size(); j++) result[j] = arg.result[j];
-      }
-    }
-
-    TuneKey tuneKey() const
-    {
-      char count[16];
-      u32toa(count, n_items);
-      return TuneKey(count, typeid(*this).name(), aux);
-    }
-
-    long long flops() const { return 0; } // just care about bandwidth
-    long long bytes() const { return v.size() * n_items * sizeof(T); }
-  };
-
   /**
      @brief QUDA implementation providing thrust::transform_reduce like
      functionality.  Improves upon thrust's implementation since a
@@ -154,12 +29,7 @@ namespace quda
    */
   template <typename reduce_t, typename T, typename I, typename transformer, typename reducer>
   void transform_reduce(QudaFieldLocation location, std::vector<reduce_t> &result, const std::vector<T *> &v, I n_items,
-                        transformer h, reduce_t init, reducer r)
-  {
-    if (result.size() != v.size())
-      errorQuda("result %lu and input %lu set sizes do not match", result.size(), v.size());
-    TransformReduce<reduce_t, T, I, transformer, reducer> reduce(location, result, v, n_items, h, init, r);
-  }
+                        transformer h, reduce_t init, reducer r);
 
   /**
      @brief QUDA implementation providing thrust::transform_reduce like
@@ -175,13 +45,7 @@ namespace quda
      @param[in] reducer Functor that applies the reduction to each transformed element
    */
   template <typename reduce_t, typename T, typename I, typename transformer, typename reducer>
-  reduce_t transform_reduce(QudaFieldLocation location, const T *v, I n_items, transformer h, reduce_t init, reducer r)
-  {
-    std::vector<reduce_t> result = {0.0};
-    std::vector<const T *> v_ = {v};
-    transform_reduce(location, result, v_, n_items, h, init, r);
-    return result[0];
-  }
+  reduce_t transform_reduce(QudaFieldLocation location, const T *v, I n_items, transformer h, reduce_t init, reducer r);
 
   /**
      @brief QUDA implementation providing thrust::reduce like
@@ -197,10 +61,7 @@ namespace quda
    */
   template <typename reduce_t, typename T, typename I, typename transformer, typename reducer>
   void reduce(QudaFieldLocation location, std::vector<reduce_t> &result, const std::vector<T *> &v, I n_items,
-              reduce_t init, reducer r)
-  {
-    transform_reduce(location, result, v, n_items, identity<T>(), init, r);
-  }
+              reduce_t init, reducer r);
 
   /**
      @brief QUDA implementation providing thrust::reduce like
@@ -215,11 +76,6 @@ namespace quda
      @param[in] reducer Functor that applies the reduction to each transformed element
    */
   template <typename reduce_t, typename T, typename I, typename reducer>
-  reduce_t reduce(QudaFieldLocation location, const T *v, I n_items, reduce_t init, reducer r)
-  {
-    std::vector<reduce_t> result = {0.0};
-    std::vector<const T *> v_ = {v};
-    transform_reduce(location, result, v_, n_items, identity<T>(), init, r);
-    return result[0];
-  }
+  reduce_t reduce(QudaFieldLocation location, const T *v, I n_items, reduce_t init, reducer r);
+
 } // namespace quda
diff --git a/include/trove_helper.cuh b/include/trove_helper.cuh
deleted file mode 100644
index c289b9a1f1..0000000000
--- a/include/trove_helper.cuh
+++ /dev/null
@@ -1,11 +0,0 @@
-#pragma once
-
-// ensure that we include the quda_define.h file to ensure that __COMPUTE_CAPABILITY__ is set
-#include <quda_define.h>
-
-// trove requires CUDA and has issues with device debug
-#if defined(QUDA_TARGET_CUDA) && !defined(DEVICE_DEBUG)
-#include <trove/ptr.h>
-#else
-#define DISABLE_TROVE
-#endif
diff --git a/include/tunable_block_reduction.h b/include/tunable_block_reduction.h
index 30aa2e973e..c3ae680416 100644
--- a/include/tunable_block_reduction.h
+++ b/include/tunable_block_reduction.h
@@ -62,7 +62,7 @@ namespace quda {
       arg.grid_dim = tp.grid;
       arg.block_dim = tp.block;
 #ifdef JITIFY
-      launch_error = launch_jitify_block<Transformer>("quda::BlockKernel2D", tp, stream, arg);
+      launch_error = launch_jitify_block<Transformer>("BlockKernel2D", tp, stream, arg);
 #else
       launch_device<Block::block.size() - 1, Block, Transformer>(arg, tp, stream);
 #endif
diff --git a/include/tunable_nd.h b/include/tunable_nd.h
index a851077565..2d741caf17 100644
--- a/include/tunable_nd.h
+++ b/include/tunable_nd.h
@@ -12,47 +12,62 @@
 
 namespace quda {
 
-  template <bool grid_stride>
-  class TunableKernel1D_base : public Tunable
+  class TunableKernel : public Tunable
   {
-  protected:
-    const LatticeField &field;
-    QudaFieldLocation location;
 
+  protected:
     virtual unsigned int sharedBytesPerThread() const { return 0; }
     virtual unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
 
-    /**
-       Kernel1D (and its derivations Kernel2D and Kernel3D) do not
-       use grid-size tuning, so disable this, and we mark as final to
-       prevent a derived class from accidentally switching it on.
-    */
-    bool tuneGridDim() const final { return grid_stride; }
-
-    template <template <typename> class Functor, typename Arg>
-    void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
+    template <template <typename> class Functor, bool grid_stride = false, typename Arg>
+    void launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg,
+                       const std::vector<constant_param_t> &param = dummy_param)
     {
       ompwip();
 #ifdef JITIFY
-      launch_error = launch_jitify<Functor>("quda::Kernel1D", tp, stream, arg, param);
+      launch_error = launch_jitify<Functor, grid_stride, Arg, false>(kernel.name, tp, stream, arg, param);
 #else
       for (unsigned int i = 0; i < param.size(); i++)
         qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
-      qudaLaunchKernel(Kernel1D<Functor, Arg, grid_stride>, tp, stream, arg);
+      launch_error = qudaLaunchKernel(kernel.func, tp, stream, arg);
 #endif
     }
 
+  public:
+    /**
+       @brief Special kernel launcher used for raw CUDA kernels with no
+       assumption made about shape of parallelism.  Kernels launched
+       using this must take responsibility of bounds checking and
+       assignment of threads.
+     */
     template <template <typename> class Functor, typename Arg>
-    void launch_cuda(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
+    void launch_cuda(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg,
+                     const std::vector<constant_param_t> &param = dummy_param) const
     {
       ompwip();
-#ifdef JITIFY
-      launch_error = launch_jitify<Functor>("quda::raw_kernel", tp, stream, arg, param);
-#else
-      for (unsigned int i = 0; i < param.size(); i++)
-        qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
-      qudaLaunchKernel(raw_kernel<Functor, Arg>, tp, stream, arg);
-#endif
+      constexpr bool grid_stride = false;
+      const_cast<TunableKernel*>(this)->launch_device<Functor, grid_stride>(KERNEL(raw_kernel), tp, stream, arg, param);
+    }
+
+  };
+
+  template <bool grid_stride>
+  class TunableKernel1D_base : public TunableKernel
+  {
+  protected:
+    QudaFieldLocation location;
+
+    /**
+       Kernel1D (and its derivations Kernel2D and Kernel3D) do not
+       use grid-size tuning, so disable this, and we mark as final to
+       prevent a derived class from accidentally switching it on.
+    */
+    bool tuneGridDim() const final { return grid_stride; }
+
+    template <template <typename> class Functor, typename Arg>
+    void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
+    {
+      TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel1D), tp, stream, arg, param);
     }
 
     template <template <typename> class Functor, typename Arg>
@@ -93,7 +108,6 @@ namespace quda {
 
   public:
     TunableKernel1D_base(const LatticeField &field, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
-      field(field),
       location(location != QUDA_INVALID_FIELD_LOCATION ? location : field.Location())
     {
       strcpy(vol, field.VolString());
@@ -101,6 +115,13 @@ namespace quda {
       strcat(aux, field.AuxString());
     }
 
+    TunableKernel1D_base(size_t n_items, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      location(location)
+    {
+      u64toa(vol, n_items);
+      strcpy(aux, compile_type_str(location));
+    }
+
     virtual bool advanceTuneParam(TuneParam &param) const
     {
       return location == QUDA_CPU_FIELD_LOCATION ? false : Tunable::advanceTuneParam(param);
@@ -122,6 +143,9 @@ namespace quda {
   public:
     TunableKernel1D(const LatticeField &field, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
       TunableKernel1D_base<false>(field, location) {}
+
+    TunableKernel1D(size_t n_items, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableKernel1D_base<false>(n_items, location) {}
   };
 
   class TunableGridStrideKernel1D : public TunableKernel1D_base<true> {
@@ -131,8 +155,10 @@ namespace quda {
   public:
     TunableGridStrideKernel1D(const LatticeField &field, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
       TunableKernel1D_base<true>(field, location) {}
-  };
 
+    TunableGridStrideKernel1D(size_t n_items, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableKernel1D_base<true>(n_items, location) {}
+  };
 
   /**
      @brief This derived class is for algorithms that deploy a vector
@@ -144,7 +170,6 @@ namespace quda {
   class TunableKernel2D_base : public TunableKernel1D_base<grid_stride>
   {
   protected:
-    using Tunable::launch_error;
     mutable unsigned int vector_length_y;
     mutable unsigned int step_y;
     bool tune_block_x;
@@ -153,11 +178,6 @@ namespace quda {
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
     {
       ompwip("OFFLOADING...");
-#ifdef JITIFY
-      launch_error = launch_jitify<Functor>("quda::Kernel2D", tp, stream, arg, param);
-#else
-      for (unsigned int i = 0; i < param.size(); i++)
-        qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
 #ifdef QUDA_BACKEND_OMPTARGET
       const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
       const int ld = tp.block.x*tp.block.y*tp.grid.z;
@@ -171,8 +191,7 @@ namespace quda {
         }
       }
 #else
-      qudaLaunchKernel(Kernel2D<Functor, Arg, grid_stride>, tp, stream, arg);
-#endif
+      TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel2D), tp, stream, arg, param);
 #endif
     }
 
@@ -225,6 +244,14 @@ namespace quda {
       tune_block_x(true)
     { }
 
+    TunableKernel2D_base(size_t n_items, unsigned int vector_length_y,
+                         QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableKernel1D_base<grid_stride>(n_items, location),
+      vector_length_y(vector_length_y),
+      step_y(1),
+      tune_block_x(true)
+    { }
+
     bool advanceBlockDim(TuneParam &param) const
     {
       dim3 block = param.block;
@@ -293,6 +320,10 @@ namespace quda {
     TunableKernel2D(const LatticeField &field, unsigned int vector_length_y,
                     QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
       TunableKernel2D_base<false>(field, vector_length_y, location) {}
+
+    TunableKernel2D(size_t n_items, unsigned int vector_length_y,
+                    QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableKernel2D_base<false>(n_items, vector_length_y, location) {}
   };
 
   class TunableGridStrideKernel2D : public TunableKernel2D_base<true> {
@@ -301,8 +332,12 @@ namespace quda {
 
   public:
     TunableGridStrideKernel2D(const LatticeField &field, unsigned int vector_length_y,
-                    QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+                              QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
       TunableKernel2D_base<true>(field, vector_length_y, location) {}
+
+    TunableGridStrideKernel2D(size_t n_items, unsigned int vector_length_y,
+                              QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableKernel2D_base<true>(n_items, vector_length_y, location) {}
   };
 
   /**
@@ -316,7 +351,6 @@ namespace quda {
   class TunableKernel3D_base : public TunableKernel2D_base<grid_stride>
   {
   protected:
-    using Tunable::launch_error;
     using TunableKernel2D_base<grid_stride>::vector_length_y;
     mutable unsigned vector_length_z;
     mutable unsigned step_z;
@@ -326,13 +360,7 @@ namespace quda {
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
     {
       ompwip();
-#ifdef JITIFY
-      launch_error = launch_jitify<Functor>("quda::Kernel3D", tp, stream, arg, param);
-#else
-      for (unsigned int i = 0; i < param.size(); i++)
-        qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
-      qudaLaunchKernel(Kernel3D<Functor, Arg, grid_stride>, tp, stream, arg);
-#endif
+      TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel3D), tp, stream, arg, param);
     }
 
     template <template <typename> class Functor, typename Arg>
@@ -387,6 +415,13 @@ namespace quda {
       step_z(1),
       tune_block_y(true) { }
 
+    TunableKernel3D_base(size_t n_items, unsigned int vector_length_y, unsigned int vector_length_z,
+                         QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableKernel2D_base<grid_stride>(n_items, vector_length_y, location),
+      vector_length_z(vector_length_z),
+      step_z(1),
+      tune_block_y(true) { }
+
     bool advanceBlockDim(TuneParam &param) const
     {
       dim3 block = param.block;
@@ -456,6 +491,10 @@ namespace quda {
     TunableKernel3D(const LatticeField &field, unsigned int vector_length_y, unsigned int vector_length_z,
                     QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
       TunableKernel3D_base<false>(field, vector_length_y, vector_length_z, location) {}
+
+    TunableKernel3D(size_t n_items, unsigned int vector_length_y, unsigned int vector_length_z,
+                    QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableKernel3D_base<false>(n_items, vector_length_y, vector_length_z, location) {}
   };
 
   class TunableGridStrideKernel3D : public TunableKernel3D_base<true> {
@@ -464,8 +503,12 @@ namespace quda {
 
   public:
     TunableGridStrideKernel3D(const LatticeField &field, unsigned int vector_length_y, unsigned int vector_length_z,
-                    QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+                              QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
       TunableKernel3D_base<true>(field, vector_length_y, vector_length_z, location) {}
+
+    TunableGridStrideKernel3D(size_t n_items, unsigned int vector_length_y, unsigned int vector_length_z,
+                              QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableKernel3D_base<true>(n_items, vector_length_y, vector_length_z, location) {}
   };
 
 }
diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index 95259ea437..8b560e42e5 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <tune_quda.h>
+#include <lattice_field.h>
 #include <device.h>
 #include <kernel_helper.h>
 #include <reduce_helper.h>
@@ -24,7 +25,6 @@ namespace quda {
   class TunableReduction2D : public Tunable
   {
   protected:
-    const LatticeField &field;
     QudaFieldLocation location;
 
     unsigned int sharedBytesPerThread() const { return 0; }
@@ -48,37 +48,37 @@ namespace quda {
      */
     virtual unsigned int maxBlockSize(const TuneParam &) const { return device::max_reduce_block_size<block_size_y>(); }
 
-    template <int block_size_x, template <typename> class Transformer, template <typename> class Reducer, typename Arg>
+    template <int block_size_x, template <typename> class Transformer, typename Arg>
     typename std::enable_if<block_size_x != device::warp_size(), qudaError_t>::type
       launch(Arg &arg, const TuneParam &tp, const qudaStream_t &stream)
     {
       if (tp.block.x == block_size_x)
-        return qudaLaunchKernel(Reduction2D<block_size_x, block_size_y, Transformer, Reducer, Arg>, tp, stream, arg);
+        return qudaLaunchKernel(Reduction2D<block_size_x, block_size_y, Transformer, Arg>, tp, stream, arg);
       else
-        return launch<block_size_x - device::warp_size(), Transformer, Reducer>(arg, tp, stream);
+        return launch<block_size_x - device::warp_size(), Transformer>(arg, tp, stream);
     }
 
-    template <int block_size_x, template <typename> class Transformer, template <typename> class Reducer, typename Arg>
+    template <int block_size_x, template <typename> class Transformer, typename Arg>
     typename std::enable_if<block_size_x == device::warp_size(), qudaError_t>::type
       launch(Arg &arg, const TuneParam &tp, const qudaStream_t &stream)
     {
       if (tp.block.x == block_size_x)
-        return qudaLaunchKernel(Reduction2D<block_size_x, block_size_y, Transformer, Reducer, Arg>, tp, stream, arg);
+        return qudaLaunchKernel(Reduction2D<block_size_x, block_size_y, Transformer, Arg>, tp, stream, arg);
       else
         errorQuda("Unexpected block size %d\n", tp.block.x);
       return QUDA_ERROR;
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus, typename Arg, typename T>
+    template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
                        const std::vector<constant_param_t> &param = dummy_param)
     {
 #ifdef JITIFY
-      arg.launch_error = launch_jitify<Transformer, Reducer>("quda::Reduction2D", tp, stream, arg, param);
+      arg.launch_error = launch_jitify<Transformer, true, Arg, true>("Reduction2D", tp, stream, arg, param);
 #else
       for (unsigned int i = 0; i < param.size(); i++)
         qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
-      arg.launch_error = launch<device::max_reduce_block_size<block_size_y>(), Transformer, Reducer>(arg, tp, stream);
+      arg.launch_error = launch<device::max_reduce_block_size<block_size_y>(), Transformer>(arg, tp, stream);
 #endif
 
       if (!commAsyncReduction()) {
@@ -91,28 +91,27 @@ namespace quda {
       }
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus, typename Arg, typename T>
+    template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(T &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
                        const std::vector<constant_param_t> &param = dummy_param)
     {
       std::vector<T> result_(1);
-      launch_device<Transformer, Reducer>(result_, tp, stream, arg, param);
+      launch_device<Transformer>(result_, tp, stream, arg, param);
       result = result_[0];
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus, typename Arg, typename T>
+    template <template <typename> class Transformer, typename Arg, typename T>
     void launch_host(std::vector<T> &result, const TuneParam &, const qudaStream_t &, Arg &arg,
                      const std::vector<constant_param_t> & = dummy_param)
     {
       using reduce_t = typename Transformer<Arg>::reduce_t;
       Transformer<Arg> t(arg);
-      Reducer<reduce_t> r;
 
       reduce_t value = arg.init();
 
       for (int j = 0; j < (int)arg.threads.y; j++) {
         for (int i = 0; i < (int)arg.threads.x; i++) {
-          value = t(value, r, i, j);
+          value = t(value, i, j);
         }
       }
 
@@ -132,38 +131,36 @@ namespace quda {
       }
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus, typename Arg, typename T>
+    template <template <typename> class Transformer, typename Arg, typename T>
     void launch_host(T &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
                      const std::vector<constant_param_t> &param = dummy_param)
     {
       std::vector<T> result_(1);
-      launch_host<Transformer, Reducer>(result_, tp, stream, arg, param);
+      launch_host<Transformer>(result_, tp, stream, arg, param);
       result = result_[0];
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus,
-              bool enable_host = false, typename T, typename Arg>
+    template <template <typename> class Transformer, bool enable_host = false, typename T, typename Arg>
     typename std::enable_if<!enable_host, void>::type
       launch(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
              const std::vector<constant_param_t> &param = dummy_param)
     {
       if (location == QUDA_CUDA_FIELD_LOCATION) {
-        launch_device<Transformer, Reducer>(result, tp, stream, arg, param);
+        launch_device<Transformer>(result, tp, stream, arg, param);
       } else {
 	errorQuda("CPU not supported yet");
       }
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus,
-              bool enable_host = false, typename T, typename Arg>
+    template <template <typename> class Transformer, bool enable_host = false, typename T, typename Arg>
     typename std::enable_if<enable_host, void>::type
       launch(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
              const std::vector<constant_param_t> &param = dummy_param)
     {
       if (location == QUDA_CUDA_FIELD_LOCATION) {
-        launch_device<Transformer, Reducer>(result, tp, stream, arg, param);
+        launch_device<Transformer>(result, tp, stream, arg, param);
       } else {
-        launch_host<Transformer, Reducer>(result, tp, stream, arg, param);
+        launch_host<Transformer>(result, tp, stream, arg, param);
       }
     }
 
@@ -175,18 +172,17 @@ namespace quda {
        @param[in] arg Kernel argument struct
        @param[in] param Constant kernel meta data
      */
-    template <template <typename> class Transformer, template <typename> class Reducer = plus, typename T, typename Arg>
+    template <template <typename> class Transformer, typename T, typename Arg>
     void launch(T &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
                 const std::vector<constant_param_t> &param = dummy_param)
     {
       std::vector<T> result_(1);
-      launch<Transformer, Reducer>(result_, tp, stream, arg, param);
+      launch<Transformer>(result_, tp, stream, arg, param);
       result = result_[0];
     }
 
   public:
     TunableReduction2D(const LatticeField &field, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
-      field(field),
       location(location != QUDA_INVALID_FIELD_LOCATION ? location : field.Location())
     {
       strcpy(vol, field.VolString());
@@ -194,6 +190,13 @@ namespace quda {
       strcat(aux, field.AuxString());
     }
 
+    TunableReduction2D(size_t n_items, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      location(location)
+    {
+      u64toa(vol, n_items);
+      strcpy(aux, compile_type_str(location));
+    }
+
     virtual bool advanceTuneParam(TuneParam &param) const
     {
       return location == QUDA_CPU_FIELD_LOCATION ? false : Tunable::advanceTuneParam(param);
@@ -231,7 +234,6 @@ namespace quda {
     // for now we do not support anything other than block_size_y = 1
     static_assert(block_size_y == 1, "only block_size_y = 1 supported");
     using Tunable::apply;
-    using TunableReduction2D<block_size_y>::field;
     using TunableReduction2D<block_size_y>::location;
 
   protected:
@@ -259,42 +261,44 @@ namespace quda {
      */
     unsigned int maxBlockSize(const TuneParam &) const { return device::max_multi_reduce_block_size(); }
 
-    template <int block_size_x, template <typename> class Transformer, template <typename> class Reducer, typename Arg>
+    template <int block_size_x, template <typename> class Transformer, typename Arg>
     typename std::enable_if<block_size_x != device::warp_size(), qudaError_t>::type
       launch(Arg &arg, const TuneParam &tp, const qudaStream_t &stream)
     {
       if (tp.block.x == block_size_x)
-        return qudaLaunchKernel(MultiReduction<block_size_x, block_size_y, Transformer, Reducer, Arg>, tp, stream, arg);
+        return qudaLaunchKernel(MultiReduction<block_size_x, block_size_y, Transformer, Arg>, tp, stream, arg);
       else
-        return launch<block_size_x - device::warp_size(), Transformer, Reducer>(arg, tp, stream);
+        return launch<block_size_x - device::warp_size(), Transformer>(arg, tp, stream);
     }
 
-    template <int block_size_x, template <typename> class Transformer, template <typename> class Reducer, typename Arg>
-      typename std::enable_if<block_size_x == device::warp_size(), qudaError_t>::type
+    template <int block_size_x, template <typename> class Transformer, typename Arg>
+    typename std::enable_if<block_size_x == device::warp_size(), qudaError_t>::type
       launch(Arg &arg, const TuneParam &tp, const qudaStream_t &stream)
     {
       if (tp.block.x == block_size_x)
-        return qudaLaunchKernel(MultiReduction<block_size_x, block_size_y, Transformer, Reducer, Arg>, tp, stream, arg);
+        return qudaLaunchKernel(MultiReduction<block_size_x, block_size_y, Transformer, Arg>, tp, stream, arg);
       else
         errorQuda("Unexpected block size %d\n", tp.block.x);
       return QUDA_ERROR;
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus, typename Arg, typename T>
+    template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
                        const std::vector<constant_param_t> &param = dummy_param)
     {
 #ifdef JITIFY
-      arg.launch_error = launch_jitify<Transformer, Reducer>("quda::MultiReduction", tp, stream, arg, param);
+      arg.launch_error = launch_jitify<Transformer, true, Arg, true>("MultiReduction", tp, stream, arg, param);
 #else
       for (unsigned int i = 0; i < param.size(); i++)
         qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
-      arg.launch_error = launch<device::max_multi_reduce_block_size(), Transformer, Reducer>(arg, tp, stream);
+      arg.launch_error = launch<device::max_multi_reduce_block_size(), Transformer>(arg, tp, stream);
 #endif
 
       if (!commAsyncReduction()) {
         arg.complete(result, stream);
-#if 0 // WAR for now since otherwise we do a double reduction in multi_reduce_quda
+#if 0 // WAR for now since otherwise we do a double reduction in
+      // multi_reduce_quda and the accessors which do not necsssarily
+      // use summation and do their global reduction elsewhere
         if (!activeTuning() && commGlobalReduction()) {
           // FIXME - this will break when we have non-double reduction types, e.g., double-double on the host
           comm_allreduce_array((double*)result.data(), result.size() * sizeof(T) / sizeof(double));
@@ -303,37 +307,63 @@ namespace quda {
       }
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus,
-              typename Arg, typename T>
-    void launch_host(std::vector<T> &, const TuneParam &, const qudaStream_t &, Arg &,
+    template <template <typename> class Transformer, typename Arg, typename T>
+    void launch_host(std::vector<T> &result, const TuneParam &, const qudaStream_t &, Arg &arg,
                      const std::vector<constant_param_t> & = dummy_param)
     {
-      errorQuda("CPU not supported yet");
+      using reduce_t = typename Transformer<Arg>::reduce_t;
+      Transformer<Arg> t(arg);
+
+      int input_size = vec_length<reduce_t>::value;
+      int output_size = vec_length<T>::value;
+      if (output_size != input_size) errorQuda("Input %d and output %d length do not match", input_size, output_size);
+
+      for (int j = 0; j < (int)arg.threads.y; j++) {
+        reduce_t value = arg.init();
+
+        for (int k = 0; k < (int)arg.threads.z; k++) {
+          for (int i = 0; i < (int)arg.threads.x; i++) {
+            value = t(value, i, j, k);
+          }
+        }
+
+        // copy element by element to output vector
+        for (int i = 0; i < output_size; i++) {
+          reinterpret_cast<typename scalar<T>::type*>(&result[j])[i] =
+            reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i];
+        }
+      }
+
+
+#if 0 // WAR for now to avoid double and/or wrong reduction type
+      if (!activeTuning() && commGlobalReduction()) {
+        // FIXME - this will break when we have non-double reduction types, e.g., double-double on the host
+        comm_allreduce_array((double*)result.data(), result.size() * sizeof(T) / sizeof(double));
+      }
+#endif
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus,
-              bool enable_host = false, typename T, typename Arg>
+    template <template <typename> class Transformer, bool enable_host = false, typename T, typename Arg>
     typename std::enable_if<!enable_host, void>::type
     launch(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
            const std::vector<constant_param_t> &param = dummy_param)
     {
       if (location == QUDA_CUDA_FIELD_LOCATION) {
-        launch_device<Transformer, Reducer>(result, tp, stream, arg, param);
+        launch_device<Transformer>(result, tp, stream, arg, param);
       } else {
 	errorQuda("CPU not supported yet");
       }
     }
 
-    template <template <typename> class Transformer, template <typename> class Reducer = plus,
-              bool enable_host, typename T, typename Arg>
+    template <template <typename> class Transformer, bool enable_host, typename T, typename Arg>
     typename std::enable_if<enable_host, void>::type
     launch(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
            const std::vector<constant_param_t> &param = dummy_param)
     {
       if (location == QUDA_CUDA_FIELD_LOCATION) {
-        launch_device<Transformer, Reducer>(result, tp, stream, arg, param);
+        launch_device<Transformer>(result, tp, stream, arg, param);
       } else {
-	launch_host<Transformer, Reducer>(result, tp, stream, arg, param);
+	launch_host<Transformer>(result, tp, stream, arg, param);
       }
     }
 
@@ -343,6 +373,11 @@ namespace quda {
       n_batch(n_batch)
     { }
 
+    TunableMultiReduction(size_t n_items, int n_batch, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      TunableReduction2D<block_size_y>(n_items, location),
+      n_batch(n_batch)
+    { }
+
     bool advanceBlockDim(TuneParam &param) const { return Tunable::advanceBlockDim(param); }
 
     void initTuneParam(TuneParam &param) const
diff --git a/include/tune_key.h b/include/tune_key.h
index c80f972384..f50a3d4dbc 100644
--- a/include/tune_key.h
+++ b/include/tune_key.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cstring>
+#include <ostream>
 
 namespace quda {
 
@@ -40,6 +41,14 @@ namespace quda {
       return false;
     }
   
+    friend std::ostream& operator<<(std::ostream& output, const TuneKey& key)
+    {
+      output << "volume = " << key.volume << ", ";
+      output << "name = " << key.name << ", ";
+      output << "aux = " << key.aux;
+      return output;
+    }
+
   };
 
   /** Return the key of the last kernel that has been tuned / called.*/
diff --git a/include/tune_quda.h b/include/tune_quda.h
index b8205f9785..650e9fe9cd 100644
--- a/include/tune_quda.h
+++ b/include/tune_quda.h
@@ -5,18 +5,15 @@
 #include <cstring>
 #include <cfloat>
 #include <algorithm>
-#include <tune_key.h>
-#include <quda_internal.h>
-#include <device.h>
-#include <target_device.h>
-
-// this file has some workarounds to allow compilation using nvrtc of kernels that include this file
-#ifndef __CUDACC_RTC__
 #include <cstdarg>
 #include <iomanip>
 #include <typeinfo>
 #include <map>
-#endif
+
+#include <tune_key.h>
+#include <quda_internal.h>
+#include <device.h>
+#include <uint_to_char.h>
 
 namespace quda {
 
@@ -33,18 +30,12 @@ namespace quda {
     float time;
     long long n_calls;
 
-    TuneParam() :
-      block(device::warp_size(), 1, 1), grid(1, 1, 1), shared_bytes(0), set_max_shared_bytes(false), aux(), time(FLT_MAX), n_calls(0)
-    {
-      aux = make_int4(1,1,1,1);
-    }
-
+    TuneParam();
     TuneParam(const TuneParam &) = default;
     TuneParam(TuneParam &&) = default;
     TuneParam& operator=(const TuneParam &) = default;
     TuneParam& operator=(TuneParam &&) = default;
 
-#ifndef __CUDACC_RTC__
     friend std::ostream& operator<<(std::ostream& output, const TuneParam& param) {
       output << "block=(" << param.block.x << "," << param.block.y << "," << param.block.z << "), ";
       output << "grid=(" << param.grid.x << "," << param.grid.y << "," << param.grid.z << "), ";
@@ -52,22 +43,20 @@ namespace quda {
       output << ", aux=(" << param.aux.x << "," << param.aux.y << "," << param.aux.z << "," << param.aux.w << ")";
       return output;
     }
-#endif
+
   };
 
-#ifndef __CUDACC_RTC__
   /**
    * @brief Returns a reference to the tunecache map
    * @return tunecache reference
    */
   const std::map<TuneKey, TuneParam> &getTuneCache();
-#endif
 
   class Tunable {
 
   protected:
-    virtual long long flops() const = 0;
-    virtual long long bytes() const { return 0; } // FIXME
+    virtual long long flops() const { return 0; }
+    virtual long long bytes() const { return 0; }
 
     // the minimum number of shared bytes per thread
     virtual unsigned int sharedBytesPerThread() const = 0;
@@ -108,8 +97,8 @@ namespace quda {
     */
     virtual int gridStep() const { return 1; }
 
-    virtual int blockStep() const { return device::warp_size(); }
-    virtual int blockMin() const { return device::warp_size(); }
+    virtual int blockStep() const;
+    virtual int blockMin() const;
 
     virtual void resetBlockDim(TuneParam &param) const {
       if (tuneGridDim()) {
@@ -206,12 +195,10 @@ namespace quda {
 
     int writeAuxString(const char *format, ...) {
       int n = 0;
-#ifndef __CUDACC_RTC__
       va_list arguments;
       va_start(arguments, format);
       n = vsnprintf(aux, TuneKey::aux_n, format, arguments);
       if (n < 0 || n >= TuneKey::aux_n) errorQuda("Error writing auxiliary string");
-#endif
       return n;
     }
 
@@ -226,7 +213,6 @@ namespace quda {
     */
     bool tuned()
     {
-#ifndef __CUDACC_RTC__
       // not tuning is equivalent to already tuned
       if (!getTuning()) return true;
 
@@ -234,9 +220,6 @@ namespace quda {
       if (use_managed_memory()) strcat(key.aux, ",managed");
       // if key is present in cache then already tuned
       return getTuneCache().find(key) != getTuneCache().end();
-#else
-      return true;
-#endif
     }
 
   public:
@@ -244,12 +227,10 @@ namespace quda {
     virtual ~Tunable() { }
     virtual TuneKey tuneKey() const = 0;
     virtual void apply(const qudaStream_t &stream) = 0;
-    void apply() { apply(device::get_default_stream()); }
     virtual void preTune() { }
     virtual void postTune() { }
     virtual int tuningIter() const { return 1; }
 
-#ifndef __CUDACC_RTC__
     virtual std::string paramString(const TuneParam &param) const
     {
       std::stringstream ps;
@@ -266,7 +247,6 @@ namespace quda {
       ss << gbytes << " GB/s";
       return ss.str();
     }
-#endif
 
     virtual void initTuneParam(TuneParam &param) const
     {
diff --git a/include/util_quda.h b/include/util_quda.h
index 4e70ee14dc..3e0c036c76 100644
--- a/include/util_quda.h
+++ b/include/util_quda.h
@@ -7,6 +7,14 @@
 #include <tune_key.h>
 #include <malloc_quda.h>
 
+namespace quda {
+  // strip path from __FILE__
+  constexpr const char* str_end(const char *str) { return *str ? str_end(str + 1) : str; }
+  constexpr bool str_slant(const char *str) { return *str == '/' ? true : (*str ? str_slant(str + 1) : false); }
+  constexpr const char* r_slant(const char* str) { return *str == '/' ? (str + 1) : r_slant(str - 1); }
+  constexpr const char* file_name(const char* str) { return str_slant(str) ? r_slant(str_end(str)) : str; }
+}
+
 /**
    @brief Query whether autotuning is enabled or not.  Default is enabled but can be overridden by setting QUDA_ENABLE_TUNING=0.
    @return If autotuning is enabled
@@ -116,25 +124,3 @@ void errorQuda_(const char *func, const char *file, int line, ...);
 } while (0)
 
 #endif // MULTI_GPU
-
-
-#define checkCudaErrorNoSync() do {                    \
-  cudaError_t error = cudaGetLastError();              \
-  if (error != cudaSuccess)                            \
-    errorQuda("(CUDA) %s", cudaGetErrorString(error))  \
-    ;\
-} while (0)
-
-
-#ifdef HOST_DEBUG
-
-#define checkCudaError() do {  \
-  cudaDeviceSynchronize();     \
-  checkCudaErrorNoSync();      \
-} while (0)
-
-#else
-
-#define checkCudaError() checkCudaErrorNoSync()
-
-#endif // HOST_DEBUG
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 2f732c49a3..11a3eebd02 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -88,13 +88,13 @@ set (QUDA_OBJS
   staggered_oprod.cu clover_trace_quda.cu
   hisq_paths_force_quda.cu
   unitarize_force_quda.cu unitarize_links_quda.cu milc_interface.cpp
-  blas_magma.cu
+  blas_magma.cu tune.cpp
   inv_mpcg_quda.cpp inv_mpbicgstab_quda.cpp inv_gmresdr_quda.cpp
   pgauge_exchange.cu pgauge_init.cu pgauge_heatbath.cu random.cu
   gauge_fix_fft.cu gauge_fix_ovr.cu
   pgauge_det_trace.cu clover_outer_product.cu
   clover_sigma_outer_product.cu momentum.cu gauge_qcharge.cu
-  deflation.cpp checksum.cu
+  deflation.cpp checksum.cu transform_reduce.cu
   instantiate.cpp version.cpp )
 # cmake-format: on
 
@@ -162,7 +162,7 @@ target_link_libraries(quda PUBLIC ${CUDA_cuda_driver_LIBRARY})
 # set up QUDA compile options
 target_compile_definitions(
   quda PRIVATE $<$<CONFIG:DEVEL>:DEVEL> $<$<CONFIG:HOSTDEBUG>:HOST_DEBUG> $<$<CONFIG:DEVICEDEBUG>:DEVICE_DEBUG>
-               $<$<CONFIG:DEBUG>:HOST_DEBUGDEVICE_DEBUG> $<$<CONFIG:SANITIZE>:HOST_DEBUG>)
+               $<$<CONFIG:DEBUG>:HOST_DEBUG> $<$<CONFIG:DEBUG>:DEVICE_DEBUG> $<$<CONFIG:SANITIZE>:HOST_DEBUG>)
 
 target_compile_options(
   quda
@@ -261,10 +261,12 @@ endif()
 
 # Specific config dependent warning suppressions and lineinfo forwarding
 target_compile_options(quda PRIVATE $<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:
-                            -lineinfo -Wreorder
+                            -Wreorder
                             -Xcompiler -Wno-unknown-pragmas,-Wno-unused-function
-                            $<$<CONFIG:STRICT>:-Werror=all-warnings>
-                            $<$<CONFIG:SANITIZE>:-Xcompiler -fsanitize=address,undefined>
+                            $<$<CONFIG:DEVEL>:-Xptxas -warn-lmem-usage,-warn-spills -lineinfo>
+                            $<$<CONFIG:STRICT>: -Werror=all-warnings -lineinfo>
+                            $<$<CONFIG:HOSTDEBUG>:-lineinfo>
+                            $<$<CONFIG:SANITIZE>:-Xcompiler -fsanitize=address,undefined -lineinfo>
                             >)
 
 target_compile_options(quda PRIVATE $<$<COMPILE_LANG_AND_ID:CUDA,Clang>:
diff --git a/lib/block_orthogonalize.cu b/lib/block_orthogonalize.cu
index 35dcce8fcd..b100d2322a 100644
--- a/lib/block_orthogonalize.cu
+++ b/lib/block_orthogonalize.cu
@@ -16,7 +16,11 @@ namespace quda {
     // size is equal to number of fine points per aggregate, rounded
     // up to a whole power of two.  So for example, 2x2x2x2 and
     // 3x3x3x1 aggregation would both use the same block size 32
+#ifndef QUDA_FAST_COMPILE_REDUCE
     static constexpr std::array<unsigned int, 6> block = {32, 64, 128, 256, 512, 1024};
+#else
+    static constexpr std::array<unsigned int, 1> block = {1024};
+#endif
 
     /**
        @brief Return the first power of two block that is larger than the required size
@@ -29,7 +33,11 @@ namespace quda {
     }
   };
 
+#ifndef QUDA_FAST_COMPILE_REDUCE
   constexpr std::array<unsigned int, 6> OrthoAggregates::block;
+#else
+  constexpr std::array<unsigned int, 1> OrthoAggregates::block;
+#endif
 
   using namespace quda::colorspinor;
 
@@ -220,9 +228,9 @@ namespace quda {
   {
     const int Nvec = B.size();
     if (V.Ncolor()/Nvec == 3) {
-      constexpr int nColor = 3;
 #ifdef NSPIN4
       if (V.Nspin() == 4) {
+        constexpr int nColor = 3;
         constexpr int nSpin = 4;
         if (spin_bs != 2) errorQuda("Unexpected spin block size = %d", spin_bs);
         constexpr int spinBlockSize = 2;
@@ -243,6 +251,7 @@ namespace quda {
 #endif // NSPIN4
 #ifdef NSPIN1
       if (V.Nspin() == 1) {
+        constexpr int nColor = 3;
         constexpr int nSpin = 1;
         if (spin_bs != 0) errorQuda("Unexpected spin block size = %d", spin_bs);
         constexpr int spinBlockSize = 0;
diff --git a/lib/clover_field.cpp b/lib/clover_field.cpp
index 6ae55d5900..77d3b335f6 100644
--- a/lib/clover_field.cpp
+++ b/lib/clover_field.cpp
@@ -73,7 +73,8 @@ namespace quda {
   /**
      @brief Backup the field to the host when tuning
   */
-  void CloverField::backup() const {
+  void CloverField::backup() const
+  {
     if (backup_h) errorQuda("Already allocated host backup");
     backup_h = static_cast<char*>(safe_malloc(2 * bytes));
     if (norm_bytes) backup_norm_h = static_cast<char*>(safe_malloc(2 * norm_bytes));
@@ -85,7 +86,6 @@ namespace quda {
         qudaMemcpy(backup_norm_h, norm, norm_bytes, qudaMemcpyDeviceToHost);
         qudaMemcpy(backup_norm_h + norm_bytes, invNorm, norm_bytes, qudaMemcpyDeviceToHost);
       }
-      checkCudaError();
     } else {
       memcpy(backup_h, clover, bytes);
       memcpy(backup_h + bytes, cloverInv, bytes);
@@ -99,7 +99,8 @@ namespace quda {
   /**
      @brief Restore the field from the host after tuning
   */
-  void CloverField::restore() const {
+  void CloverField::restore() const
+  {
     if (Location() == QUDA_CUDA_FIELD_LOCATION) {
       qudaMemcpy(clover, backup_h, bytes, qudaMemcpyHostToDevice);
       qudaMemcpy(cloverInv, backup_h + bytes, bytes, qudaMemcpyHostToDevice);
@@ -107,7 +108,6 @@ namespace quda {
         qudaMemcpy(norm, backup_norm_h, norm_bytes, qudaMemcpyHostToDevice);
         qudaMemcpy(invNorm, backup_norm_h + norm_bytes, norm_bytes, qudaMemcpyHostToDevice);
       }
-      checkCudaError();
     } else {
       memcpy(clover, backup_h, bytes);
       memcpy(cloverInv, backup_h + bytes, bytes);
diff --git a/lib/coarse_op.cuh b/lib/coarse_op.cuh
index ae925915a5..fd94e1e739 100644
--- a/lib/coarse_op.cuh
+++ b/lib/coarse_op.cuh
@@ -783,7 +783,7 @@ namespace quda {
 
       if (use_mma && (type == COMPUTE_UV || type == COMPUTE_VUV) && dir != QUDA_IN_PLACE) {
         constexpr bool query_max = true;
-        int max;
+        int max = 0;
         if (type == COMPUTE_UV) {
           max = mma::launch_compute_uv_kernel<from_coarse, query_max>(param, arg, 1, device::get_default_stream());
         } else if (type == COMPUTE_VUV) {
diff --git a/lib/coarse_op_preconditioned.cu b/lib/coarse_op_preconditioned.cu
index 1049d6973b..1409e1dae2 100644
--- a/lib/coarse_op_preconditioned.cu
+++ b/lib/coarse_op_preconditioned.cu
@@ -2,96 +2,28 @@
 #include <gauge_field.h>
 #include <blas_lapack.h>
 #include <blas_quda.h>
-#include <tune_quda.h>
-
-#include <jitify_helper.cuh>
+#include <tunable_nd.h>
 #include <kernels/coarse_op_preconditioned.cuh>
-
 #include <coarse_op_preconditioned_mma_launch.h>
 
 namespace quda
 {
 
-  /**
-     @brief Launcher for CPU instantiations of preconditioned coarse-link construction
-  */
-  template <QudaFieldLocation location, typename Arg>
-  struct Launch {
-    Launch(Arg &arg, qudaError_t &, bool compute_max_only, TuneParam &, bool, const qudaStream_t &)
-    {
-      if (compute_max_only)
-        CalculateYhatCPU<true, Arg>(arg);
-      else
-        CalculateYhatCPU<false, Arg>(arg);
-    }
-  };
-
-  /**
-     @brief Launcher for GPU instantiations of preconditioned coarse-link construction
-  */
-  template <typename Arg>
-  struct Launch<QUDA_CUDA_FIELD_LOCATION, Arg> {
-    Launch(Arg &arg, qudaError_t &qerror, bool compute_max_only, TuneParam &tp, bool use_mma, const qudaStream_t &stream)
-    {
-      CUresult error = CUDA_SUCCESS;
-      if (compute_max_only) {
-        if (!activeTuning()) {
-          qudaMemsetAsync(arg.max_d, 0, sizeof(typename Arg::Float), stream);
-        }
-      }
-#ifdef JITIFY
-      if (use_mma) {
-        errorQuda("MMA kernels haven't been jitify'ed.");
-      } else {
-        using namespace jitify::reflection;
-        error = program->kernel("quda::CalculateYhatGPU")
-                  .instantiate(compute_max_only, Type<Arg>())
-                  .configure(tp.grid, tp.block, tp.shared_bytes, device::get_cuda_stream(stream))
-                  .launch(arg);
-      }
-#else
-      if (use_mma) {
-        if (compute_max_only) {
-          mma::launch_yhat_kernel<true>(arg, arg.Y.VolumeCB(), tp, stream);
-        } else {
-          mma::launch_yhat_kernel<false>(arg, arg.Y.VolumeCB(), tp, stream);
-        }
-      } else {
-        if (compute_max_only) {
-          qudaLaunchKernel(CalculateYhatGPU<true, Arg>, tp, stream, arg);
-        } else {
-          qudaLaunchKernel(CalculateYhatGPU<false, Arg>, tp, stream, arg);
-        }
-      }
-#endif
-      if (compute_max_only) {
-        if (!activeTuning()) { // only do copy once tuning is done
-          qudaMemcpyAsync(arg.max_h, arg.max_d, sizeof(typename Arg::Float), qudaMemcpyDeviceToHost, stream);
-          qudaStreamSynchronize(const_cast<qudaStream_t&>(stream));
-        }
-      }
-      // convert Jitify return error into QUDA error
-      qerror = error == CUDA_SUCCESS ? QUDA_SUCCESS : QUDA_ERROR;
-    }
-  };
-
-  template <QudaFieldLocation location, typename Arg>
-  class CalculateYhat : public TunableVectorYZ {
-
-    using Float = typename Arg::Float;
-    Arg &arg;
-    const LatticeField &meta;
-    const int n;
-
-    bool compute_max_only;
-
+  template <QudaFieldLocation location, typename Float_, typename PreconditionedGauge,
+            typename Gauge, typename GaugeInv, int n, int M, int N, bool compute_max>
+  class CalculateYhat : public TunableKernel3D {
+    using Float = Float_;
+    using Arg = CalculateYhatArg<Float, PreconditionedGauge, Gauge, GaugeInv, n, M, N, compute_max>;
+    Arg arg;
+    GaugeField &Yhat;
+    const GaugeField &Y;
+    const GaugeField &Xinv;
     bool use_mma;
 
-    long long flops() const { return 2l * arg.Y.VolumeCB() * 8 * n * n * (8*n-2); } // 8 from dir, 8 from complexity,
-    long long bytes() const { return 2l * (arg.Xinv.Bytes() + 8*arg.Y.Bytes() + !compute_max_only * 8*arg.Yhat.Bytes()) * n; }
+    long long flops() const { return Y.Volume() * 8 * n * n * (8 * n - 2); } // 8 from dir, 8 from complexity,
+    long long bytes() const { return 2l * (arg.Xinv.Bytes() + 8*arg.Y.Bytes() + !Arg::compute_max * 8*arg.Yhat.Bytes()) * n; }
 
-    unsigned int minThreads() const { return arg.Y.VolumeCB(); }
-    bool tuneGridDim() const { return false; } // don't tune the grid dimension
+    unsigned int minThreads() const { return Y.VolumeCB(); }
 
     // all the tuning done is only in matrix tile size (Y/Z block.grid)
     int blockMin() const { return 8; }
@@ -99,79 +31,73 @@ namespace quda
     unsigned int maxBlockSize(const TuneParam &) const { return 8u; }
 
   public:
-    CalculateYhat(Arg &arg, const LatticeField &meta, bool use_mma) :
-      TunableVectorYZ(2 * arg.tile.M_tiles, 4 * arg.tile.N_tiles),
-      arg(arg),
-      meta(meta),
-      n(arg.tile.n),
-      compute_max_only(false),
+    CalculateYhat(GaugeField &Yhat, const GaugeField &Y, const GaugeField &Xinv, bool use_mma) :
+      TunableKernel3D(Y, 2 * arg.tile.M_tiles, 4 * arg.tile.N_tiles),
+      arg(Yhat, Y, Xinv),
+      Yhat(Yhat),
+      Y(Y),
+      Xinv(Xinv),
       use_mma(use_mma)
     {
-      if (meta.Location() == QUDA_CUDA_FIELD_LOCATION) {
-#ifdef JITIFY
-        create_jitify_program("kernels/coarse_op_preconditioned.cuh");
-#endif
+      arg.max_h = static_cast<Float*>(pool_pinned_malloc(sizeof(Float)));
+      if (Y.Location() == QUDA_CUDA_FIELD_LOCATION) {
         arg.max_d = static_cast<Float*>(pool_device_malloc(sizeof(Float)));
+        strcat(aux, Y.MemType() == QUDA_MEMORY_MAPPED ? ",GPU-mapped" : ",GPU-device");
       }
-      arg.max_h = static_cast<Float*>(pool_pinned_malloc(sizeof(Float)));
-      strcpy(aux, compile_type_str(meta));
       strcat(aux, comm_dim_partitioned_string());
+      if (use_mma) { strcat(aux, ",mma"); }
+      if (Arg::compute_max) strcat(aux, ",compute_max");
+
+      apply(device::get_default_stream());
+
+      double max_h_double = *arg.max_h;
+      comm_allreduce_max(&max_h_double);
+      *arg.max_h = static_cast<Float>(max_h_double);
+      if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Yhat Max = %e\n", *arg.max_h);
+      Yhat.Scale(*arg.max_h);
     }
 
-    virtual ~CalculateYhat() {
-      if (meta.Location() == QUDA_CUDA_FIELD_LOCATION) {
-        pool_device_free(arg.max_d);
-      }
+    ~CalculateYhat()
+    {
+      if (Y.Location() == QUDA_CUDA_FIELD_LOCATION) pool_device_free(arg.max_d);
       pool_pinned_free(arg.max_h);
     }
 
     void apply(const qudaStream_t &stream)
     {
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      Launch<location, Arg>(arg, launch_error, compute_max_only, tp, use_mma, stream);
-    }
+      if (Arg::compute_max && !activeTuning()) {
+        qudaMemsetAsync(arg.max_d, 0, sizeof(typename Arg::Float), stream);
+      }
+
+      if (use_mma) mma::launch_yhat_kernel(tp, stream, arg, *this);
+      else launch<ComputeYhat>(tp, stream, arg);
 
-    /**
-       Set if we're doing a max-only compute (fixed point only)
-    */
-    void setComputeMaxOnly(bool compute_max_only_) { compute_max_only = compute_max_only_; }
+      if (Arg::compute_max && !activeTuning()) { // only do copy once tuning is done
+        qudaMemcpyAsync(arg.max_h, arg.max_d, sizeof(typename Arg::Float), qudaMemcpyDeviceToHost, stream);
+        qudaStreamSynchronize(const_cast<qudaStream_t&>(stream));
+      }
+    }
 
     bool advanceSharedBytes(TuneParam &) const { return false; }
 
-    bool advanceTuneParam(TuneParam &param) const {
+    bool advanceTuneParam(TuneParam &param) const
+    {
       if (use_mma) {
-
-        constexpr bool compute_max_only_dummy = true;
         constexpr bool query_max = true;
-        int max = mma::template launch_yhat_kernel<compute_max_only_dummy, query_max>(arg, 1, param, device::get_default_stream());
+        int max = mma::launch_yhat_kernel<query_max>(param, device::get_default_stream(), arg, *this);
         if (param.aux.x < max) {
           param.aux.x++;
           return true;
         }
         return false;
-
       } else {
-
-        if (meta.Location() == QUDA_CUDA_FIELD_LOCATION && meta.MemType() == QUDA_MEMORY_DEVICE)
-          return Tunable::advanceTuneParam(param);
+        if (Y.Location() == QUDA_CUDA_FIELD_LOCATION && Y.MemType() == QUDA_MEMORY_DEVICE)
+          return TunableKernel3D::advanceTuneParam(param);
         else
           return false;
       }
     }
-
-    TuneKey tuneKey() const {
-      char Aux[TuneKey::aux_n];
-      strcpy(Aux,aux);
-      if (compute_max_only) strcat(Aux, ",compute_max_only");
-      if (meta.Location() == QUDA_CUDA_FIELD_LOCATION) {
-        strcat(Aux, meta.MemType() == QUDA_MEMORY_MAPPED ? ",GPU-mapped" : ",GPU-device");
-      } else if (meta.Location() == QUDA_CPU_FIELD_LOCATION) {
-        strcat(Aux, ",CPU");
-        strcat(Aux, getOmpThreadStr());
-      }
-      if (use_mma) { strcat(Aux, ",MMA"); }
-      return TuneKey(meta.VolString(), typeid(*this).name(), Aux);
-    }
   };
 
   /**
@@ -240,10 +166,6 @@ namespace quda
     // Yhat_back(x-\mu) = Y_back(x-\mu) * Xinv^dagger(x) (positive projector)
     // Yhat_fwd(x) = Xinv(x) * Y_fwd(x)                  (negative projector)
     {
-      int xc_size[5];
-      for (int i=0; i<4; i++) xc_size[i] = X.X()[i];
-      xc_size[4] = 1;
-
       if (use_mma) {
 
         auto create_gauge_copy = [](const GaugeField &X, QudaGaugeFieldOrder order, bool copy_content) -> auto
@@ -265,38 +187,19 @@ namespace quda
 
         constexpr bool use_native_ghosts = true;
         // use spin-ignorant accessor to make multiplication simpler
-        typedef typename gauge::FieldOrder<Float, N, 1, gOrder_milc, use_native_ghosts, storeFloat> gCoarse;
-        typedef typename gauge::FieldOrder<Float, N, 1, gOrder_milc, use_native_ghosts, storeFloat> gPreconditionedCoarse;
-        gCoarse yAccessor(*Y_aos);
-        gPreconditionedCoarse yHatAccessor(*Yhat_aos);
-
-        // XXX: This doesn't work for double precision.
+        using gCoarse = typename gauge::FieldOrder<Float, N, 1, gOrder_milc, use_native_ghosts, storeFloat>;
+        using gPreconditionedCoarse = typename gauge::FieldOrder<Float, N, 1, gOrder_milc, use_native_ghosts, storeFloat>;
+        // XXX: This doesn't work for double precision since hard-coded to single precision
         using gCoarseInv = gauge::FieldOrder<float, N, 1, gOrder_milc, use_native_ghosts, float>;
-        gCoarseInv xInvAccessor(*Xinv_aos);
-        if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Xinv = %e\n", Xinv_aos->norm2(0));
 
-        int comm_dim[4];
-        for (int i = 0; i < 4; i++) comm_dim[i] = comm_dim_partitioned(i);
-
-        using yHatArg = CalculateYhatArg<Float, gPreconditionedCoarse, gCoarse, gCoarseInv, N, 4, 2>;
-        yHatArg arg(yHatAccessor, yAccessor, xInvAccessor, xc_size, comm_dim, 1);
+        if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Xinv = %e\n", Xinv_aos->norm2(0));
 
-        CalculateYhat<location, yHatArg> yHat(arg, Y, use_mma);
         if (Yhat.Precision() == QUDA_HALF_PRECISION || Yhat.Precision() == QUDA_QUARTER_PRECISION) {
-          yHat.setComputeMaxOnly(true);
-          yHat.apply(device::get_default_stream());
-
-          double max_h_double = *arg.max_h;
-          comm_allreduce_max(&max_h_double);
-          *arg.max_h = static_cast<Float>(max_h_double);
-
-          if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Yhat Max = %e\n", *arg.max_h);
-
-          Yhat_aos->Scale(*arg.max_h);
-          arg.Yhat.resetScale(*arg.max_h);
+          CalculateYhat<location, Float, gPreconditionedCoarse, gCoarse, gCoarseInv, N, 4, 2, true>
+            (*Yhat_aos, *Y_aos, *Xinv_aos, use_mma);
         }
-        yHat.setComputeMaxOnly(false);
-        yHat.apply(device::get_default_stream());
+        CalculateYhat<location, Float, gPreconditionedCoarse, gCoarse, gCoarseInv, N, 4, 2, false>
+          (*Yhat_aos, *Y_aos, *Xinv_aos, use_mma);
 
         if (&Y != Y_aos) { delete Y_aos; }
 
@@ -310,34 +213,16 @@ namespace quda
       } else {
 
         // use spin-ignorant accessor to make multiplication simpler
-        typedef typename gauge::FieldOrder<Float, N, 1, gOrder, true, storeFloat> gCoarse;
-        typedef typename gauge::FieldOrder<Float, N, 1, gOrder, true, storeFloat> gPreconditionedCoarse;
-        gCoarse yAccessor(const_cast<GaugeField &>(Y));
-        gPreconditionedCoarse yHatAccessor(const_cast<GaugeField &>(Yhat));
-        gCoarse xInvAccessor(const_cast<GaugeField &>(Xinv));
+        using gCoarse = typename gauge::FieldOrder<Float, N, 1, gOrder, true, storeFloat>;
+        using gPreconditionedCoarse = typename gauge::FieldOrder<Float, N, 1, gOrder, true, storeFloat>;
         if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Xinv = %e\n", Xinv.norm2(0));
 
-        int comm_dim[4];
-        for (int i = 0; i < 4; i++) comm_dim[i] = comm_dim_partitioned(i);
-        typedef CalculateYhatArg<Float, gPreconditionedCoarse, gCoarse, gCoarse, N, 4, 2> yHatArg;
-        yHatArg arg(yHatAccessor, yAccessor, xInvAccessor, xc_size, comm_dim, 1);
-
-        CalculateYhat<location, yHatArg> yHat(arg, Y, use_mma);
         if (Yhat.Precision() == QUDA_HALF_PRECISION || Yhat.Precision() == QUDA_QUARTER_PRECISION) {
-          yHat.setComputeMaxOnly(true);
-          yHat.apply(device::get_default_stream());
-
-          double max_h_double = *arg.max_h;
-          comm_allreduce_max(&max_h_double);
-          *arg.max_h = static_cast<Float>(max_h_double);
-
-          if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Yhat Max = %e\n", *arg.max_h);
-
-          Yhat.Scale(*arg.max_h);
-          arg.Yhat.resetScale(*arg.max_h);
+          CalculateYhat<location, Float, gPreconditionedCoarse, gCoarse, gCoarse, N, 4, 2, true>
+            (Yhat, Y, Xinv, use_mma);
         }
-        yHat.setComputeMaxOnly(false);
-        yHat.apply(device::get_default_stream());
+        CalculateYhat<location, Float, gPreconditionedCoarse, gCoarse, gCoarse, N, 4, 2, false>
+          (Yhat, Y, Xinv, use_mma);
       }
 
       if (getVerbosity() >= QUDA_VERBOSE)
diff --git a/lib/coarse_op_preconditioned_mma_launch.h b/lib/coarse_op_preconditioned_mma_launch.h
index 933ccc5cf9..98d52437db 100644
--- a/lib/coarse_op_preconditioned_mma_launch.h
+++ b/lib/coarse_op_preconditioned_mma_launch.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <gauge_field.h>
-#include <tune_quda.h>
+#include <tunable_nd.h>
 
 #if (CUDA_VERSION >= 10010 && __COMPUTE_CAPABILITY__ >= 700)
 
@@ -22,130 +22,127 @@ namespace quda
 
 #if (CUDA_VERSION >= 10010 && __COMPUTE_CAPABILITY__ >= 700)
 
-    template <bool compute_max_only, int bM, int bN, int bK, int block_y, int block_z, int min_block_cta = 1, class Arg>
-    typename std::enable_if<!Arg::is_mma_compatible, void>::type launch_kernel(Arg &, int, TuneParam &, const qudaStream_t &)
+    template <int bM, int bN, int bK, int block_y, int block_z, int min_block_cta = 1, class Arg, typename Tunable>
+    typename std::enable_if<!Arg::is_mma_compatible, void>::type launch_kernel(TuneParam &, const qudaStream_t &,
+                                                                              Arg &, Tunable &)
     {
       errorQuda("MMA implementation is ONLY built for AoS order.");
     }
 
-    template <bool compute_max_only, int bM, int bN, int bK, int block_y, int block_z, int min_block_cta = 1, class Arg>
-    typename std::enable_if<Arg::is_mma_compatible, void>::type launch_kernel(Arg &arg, int min_threads, TuneParam &tp,
-                                                                              const qudaStream_t &stream)
+    template <int bM, int bN, int bK, int block_y, int block_z, int min_block_cta = 1, class Arg, typename Tunable>
+    typename std::enable_if<Arg::is_mma_compatible, void>::type launch_kernel(TuneParam &tp, const qudaStream_t &stream,
+                                                                              Arg &arg, Tunable &tunable)
     {
-      tp.block.x = 1;
-      tp.block.y = block_y;
-      tp.block.z = block_z;
-      constexpr int shared_bytes = shared_memory_bytes(bM, bN, bK);
-      tp.shared_bytes = shared_bytes;
+      tp.block = dim3(1, block_y, block_z);
 
       constexpr bool divide_b_no = bM < Arg::M && bK == Arg::K && bN == Arg::N;
-
       constexpr int t_m = divide_b_no ? 1 : (Arg::M + bM - 1) / bM;
       constexpr int t_n = divide_b_no ? 1 : (Arg::N + bN - 1) / bN;
+      tp.grid = dim3(arg.threads.x * t_m * t_n, 2, 4);
 
-      tp.grid = dim3(min_threads * t_m * t_n, 2, 4);
-
-      auto kernel = mma::CalculateYhatGPU<compute_max_only, Arg, bM, bN, bK, block_y, block_z, min_block_cta>;
+      tp.shared_bytes = shared_memory_bytes(bM, bN, bK);
       tp.set_max_shared_bytes = true;
-      qudaLaunchKernel(kernel, tp, stream, arg);
+
+      tunable.template launch_cuda<CalculateYhatMMA>
+        (tp, stream, CalculateYhatMMAArg<Arg, bM, bN, bK, block_y, block_z, min_block_cta>(arg));
     }
 
     /**
-        The following functions have switch's that list computeYhat MMA kernels instantiations.
-        if query_max = true, it will simply return how many instantiations there are; if query_max = false,
-        the MMA kernel is launched with the corresponding configuration.
-     */
-    template <bool compute_max_only, bool query_max = false, class Arg>
-    typename std::enable_if<Arg::N == 48, int>::type launch_yhat_kernel(Arg &arg, int min_threads, TuneParam &tp,
-                                                                        const qudaStream_t &stream)
+       The following functions have switch's that list computeYhat MMA kernels instantiations.
+       if query_max = true, it will simply return how many instantiations there are; if query_max = false,
+       the MMA kernel is launched with the corresponding configuration.
+    */
+    template <bool query_max = false, class Arg, typename Tunable>
+    typename std::enable_if<Arg::N == 48, int>::type launch_yhat_kernel(TuneParam &tp, const qudaStream_t &stream,
+                                                                        Arg &arg, Tunable &tunable)
     {
       if (query_max) return 2;
       // clang-format off
       switch (tp.aux.x) {
-      case   0: launch_kernel<compute_max_only,  48,  48,  48,  24,  12>(arg, min_threads, tp, stream); break;
-      case   1: launch_kernel<compute_max_only,  48,  48,  48,   6,  48>(arg, min_threads, tp, stream); break;
-      case   2: launch_kernel<compute_max_only,  48,  48,  48,  12,  24>(arg, min_threads, tp, stream); break;
+      case 0: launch_kernel<48, 48, 48, 24,  12>(tp, stream, arg, tunable); break;
+      case 1: launch_kernel<48, 48, 48,  6,  48>(tp, stream, arg, tunable); break;
+      case 2: launch_kernel<48, 48, 48, 12,  24>(tp, stream, arg, tunable); break;
       default: errorQuda("tp.aux.x(=%d) is NOT supported by N = 48", tp.aux.x);
       }
       // clang-format on
       return -1;
     }
 
-    template <bool compute_max_only, bool query_max = false, class Arg>
-    typename std::enable_if<Arg::N == 12, int>::type launch_yhat_kernel(Arg &arg, int min_threads, TuneParam &tp,
-                                                                        const qudaStream_t &stream)
+    template <bool query_max = false, class Arg, typename Tunable>
+    typename std::enable_if<Arg::N == 12, int>::type launch_yhat_kernel(TuneParam &tp, const qudaStream_t &stream,
+                                                                        Arg &arg, Tunable &tunable)
     {
       if (query_max) return 1;
       // clang-format off
       switch (tp.aux.x) {
-      case   0: launch_kernel<compute_max_only,  16,  16,  16,   4,   8>(arg, min_threads, tp, stream); break;
-      case   1: launch_kernel<compute_max_only,  16,  16,  16,   8,   4>(arg, min_threads, tp, stream); break;
+      case 0: launch_kernel<16, 16, 16, 4, 8>(tp, stream, arg, tunable); break;
+      case 1: launch_kernel<16, 16, 16, 8, 4>(tp, stream, arg, tunable); break;
       default: errorQuda("tp.aux.x(=%d) is NOT supported by N = 12", tp.aux.x);
       }
       // clang-format on
       return -1;
     }
 
-    template <bool compute_max_only, bool query_max = false, class Arg>
-    typename std::enable_if<Arg::N == 64, int>::type launch_yhat_kernel(Arg &arg, int min_threads, TuneParam &tp,
-                                                                        const qudaStream_t &stream)
+    template <bool query_max = false, class Arg, typename Tunable>
+    typename std::enable_if<Arg::N == 64, int>::type launch_yhat_kernel(TuneParam &tp, const qudaStream_t &stream,
+                                                                        Arg &arg, Tunable &tunable)
     {
       if (query_max) return 6;
       // clang-format off
       switch (tp.aux.x) {
-      case   0: launch_kernel<compute_max_only,  64,  64,  16,  32,   8>(arg, min_threads, tp, stream); break;
-      case   1: launch_kernel<compute_max_only,  64,  64,  16,  16,  16>(arg, min_threads, tp, stream); break;
-      case   2: launch_kernel<compute_max_only,  64,  64,  16,  32,  16>(arg, min_threads, tp, stream); break;
-      case   3: launch_kernel<compute_max_only,  64,  64,  32,  32,  16>(arg, min_threads, tp, stream); break;
-      case   4: launch_kernel<compute_max_only,  64,  64,  64,   8,  64>(arg, min_threads, tp, stream); break;
-      case   5: launch_kernel<compute_max_only,  64,  64,  64,  16,  32>(arg, min_threads, tp, stream); break;
-      case   6: launch_kernel<compute_max_only,  64,  64,  64,  32,  16>(arg, min_threads, tp, stream); break;
+      case 0: launch_kernel<64, 64, 16, 32,  8>(tp, stream, arg, tunable); break;
+      case 1: launch_kernel<64, 64, 16, 16, 16>(tp, stream, arg, tunable); break;
+      case 2: launch_kernel<64, 64, 16, 32, 16>(tp, stream, arg, tunable); break;
+      case 3: launch_kernel<64, 64, 32, 32, 16>(tp, stream, arg, tunable); break;
+      case 4: launch_kernel<64, 64, 64,  8, 64>(tp, stream, arg, tunable); break;
+      case 5: launch_kernel<64, 64, 64, 16, 32>(tp, stream, arg, tunable); break;
+      case 6: launch_kernel<64, 64, 64, 32, 16>(tp, stream, arg, tunable); break;
       default: errorQuda("tp.aux.x(=%d) is NOT supported by N = 64", tp.aux.x);
-      }
+        }
       // clang-format on
       return -1;
     }
 
-    template <bool compute_max_only, bool query_max = false, class Arg>
-    typename std::enable_if<Arg::N == 128, int>::type launch_yhat_kernel(Arg &arg, int min_threads, TuneParam &tp,
-                                                                         const qudaStream_t &stream)
+    template <bool query_max = false, class Arg, typename Tunable>
+    typename std::enable_if<Arg::N == 128, int>::type launch_yhat_kernel(TuneParam &tp, const qudaStream_t &stream,
+                                                                             Arg &arg, Tunable &tunable)
     {
       if (query_max) return 7;
       // clang-format off
       switch (tp.aux.x) {
-      case   0: launch_kernel<compute_max_only,  64,  64,  16,  32,  16,  2>(arg, min_threads, tp, stream); break;
+      case 0: launch_kernel< 64,  64,  16,  32,  16,  2>(tp, stream, arg, tunable); break;
 #if (__COMPUTE_CAPABILITY__ >= 750) // Turing or above
-      case   1: launch_kernel<compute_max_only,  16, 128, 128,  32,  16,  2>(arg, min_threads, tp, stream); break;
+      case 1: launch_kernel< 16, 128, 128,  32,  16,  2>(tp, stream, arg, tunable); break;
 #else
-      case   1: launch_kernel<compute_max_only,  32, 128, 128,  32,  16,  2>(arg, min_threads, tp, stream); break;
+      case 1: launch_kernel< 32, 128, 128,  32,  16,  2>(tp, stream, arg, tunable); break;
 #endif
-      case   2: launch_kernel<compute_max_only, 128, 128,  16,  64,   8    >(arg, min_threads, tp, stream); break;
-      case   3: launch_kernel<compute_max_only, 128, 128,  16,  32,  16    >(arg, min_threads, tp, stream); break;
-      case   4: launch_kernel<compute_max_only, 128, 128,  32,  16,  32    >(arg, min_threads, tp, stream); break;
-      case   5: launch_kernel<compute_max_only, 128, 128,  32,  64,   8    >(arg, min_threads, tp, stream); break;
-      case   6: launch_kernel<compute_max_only, 128, 128,  32,  32,  16    >(arg, min_threads, tp, stream); break;
-      case   7: launch_kernel<compute_max_only, 128, 128,  32,  32,  32    >(arg, min_threads, tp, stream); break;
+      case 2: launch_kernel<128, 128,  16,  64,   8    >(tp, stream, arg, tunable); break;
+      case 3: launch_kernel<128, 128,  16,  32,  16    >(tp, stream, arg, tunable); break;
+      case 4: launch_kernel<128, 128,  32,  16,  32    >(tp, stream, arg, tunable); break;
+      case 5: launch_kernel<128, 128,  32,  64,   8    >(tp, stream, arg, tunable); break;
+      case 6: launch_kernel<128, 128,  32,  32,  16    >(tp, stream, arg, tunable); break;
+      case 7: launch_kernel<128, 128,  32,  32,  32    >(tp, stream, arg, tunable); break;
       default: errorQuda("tp.aux.x(=%d) is NOT supported by N = 128", tp.aux.x);
       }
       // clang-format on
       return -1;
     }
 
-    template <bool compute_max_only, bool query_max = false, class Arg>
-    typename std::enable_if<Arg::N == 192, int>::type launch_yhat_kernel(Arg &arg, int min_threads, TuneParam &tp,
-                                                                         const qudaStream_t &stream)
+    template <bool query_max = false, class Arg, typename Tunable>
+    typename std::enable_if<Arg::N == 192, int>::type launch_yhat_kernel(TuneParam &tp, const qudaStream_t &stream,
+                                                                         Arg &arg, Tunable &tunable)
     {
       if (query_max) return 4;
       // clang-format off
       switch (tp.aux.x) {
-      case   0: launch_kernel<compute_max_only,  64,  64,  16,  16,  16,  2>(arg, min_threads, tp, stream); break;
-      case   1: launch_kernel<compute_max_only,  64,  64,  64,  16,  16,  2>(arg, min_threads, tp, stream); break;
-      case   2: launch_kernel<compute_max_only,  16, 192, 192,  24,  16    >(arg, min_threads, tp, stream); break;
-      case   3: launch_kernel<compute_max_only,  64,  64,  32,  16,  16,  2>(arg, min_threads, tp, stream); break;
+      case 0: launch_kernel<64,  64,  16,  16,  16,  2>(tp, stream, arg, tunable); break;
+      case 1: launch_kernel<64,  64,  64,  16,  16,  2>(tp, stream, arg, tunable); break;
+      case 2: launch_kernel<16, 192, 192,  24,  16    >(tp, stream, arg, tunable); break;
+      case 3: launch_kernel<64,  64,  32,  16,  16,  2>(tp, stream, arg, tunable); break;
 #if (__COMPUTE_CAPABILITY__ >= 750) // Turing or above
-      case   4: launch_kernel<compute_max_only,  16, 192, 192,  96,   8    >(arg, min_threads, tp, stream); break;
+      case 4: launch_kernel<16, 192, 192,  96,   8    >(tp, stream, arg, tunable); break;
 #else
-      case   4: launch_kernel<compute_max_only,  16, 192, 192,  48,   8    >(arg, min_threads, tp, stream); break;
+      case 4: launch_kernel<16, 192, 192,  48,   8    >(tp, stream, arg, tunable); break;
 #endif
       default: errorQuda("tp.aux.x(=%d) is NOT supported by N = 192", tp.aux.x);
       }
@@ -155,8 +152,8 @@ namespace quda
 
 #else
 
-    template <bool compute_max_only, bool query_max = false, class Arg>
-    int launch_yhat_kernel(Arg &arg, int min_threads, TuneParam &tp, const qudaStream_t &stream)
+    template <bool query_max = false, class Arg, typename Tunable>
+    int launch_yhat_kernel(TuneParam &, const qudaStream_t &, Arg &, Tunable &)
     {
       errorQuda("MMA multigrid is not available for this setup.");
       return -1;
diff --git a/lib/comm_common.cpp b/lib/comm_common.cpp
index 2c765e4058..8235f166be 100644
--- a/lib/comm_common.cpp
+++ b/lib/comm_common.cpp
@@ -248,9 +248,6 @@ void comm_peer2peer_init(const char* hostname_recv_buf)
 
     // first check that the local GPU supports UVA
     const int gpuid = comm_gpuid();
-    cudaDeviceProp prop;
-    cudaGetDeviceProperties(&prop, gpuid);
-    if (!prop.unifiedAddressing) return;
 
     comm_set_neighbor_ranks();
 
@@ -262,9 +259,9 @@ void comm_peer2peer_init(const char* hostname_recv_buf)
     for(int dir=0; dir<2; ++dir){ // forward/backward directions
       for(int dim=0; dim<4; ++dim){
 	int neighbor_rank = comm_neighbor_rank(dir,dim);
-	if(neighbor_rank == comm_rank()) continue;
+	if (neighbor_rank == comm_rank()) continue;
 
-        if (neighbor_rank >= comm_size()) errorQuda("neighbor rank %d > number of ranks %d", neighbor_rank, comm_size());
+        if ((unsigned)neighbor_rank >= comm_size()) errorQuda("neighbor rank %d > number of ranks %lu", neighbor_rank, comm_size());
 
         // disable peer-to-peer comms in one direction
 	if ( ((comm_rank() > neighbor_rank && dir == 0) || (comm_rank() < neighbor_rank && dir == 1)) &&
@@ -273,25 +270,17 @@ void comm_peer2peer_init(const char* hostname_recv_buf)
 	// if the neighbors are on the same
 	if (!strncmp(hostname, &hostname_recv_buf[128*neighbor_rank], 128)) {
 	  int neighbor_gpuid = gpuid_recv_buf[neighbor_rank];
-	  int canAccessPeer[2];
-	  cudaDeviceCanAccessPeer(&canAccessPeer[0], gpuid, neighbor_gpuid);
-	  cudaDeviceCanAccessPeer(&canAccessPeer[1], neighbor_gpuid, gpuid);
-
-	  int accessRank[2] = { };
-	  if (canAccessPeer[0]*canAccessPeer[1] != 0) {
-	    cudaDeviceGetP2PAttribute(&accessRank[0], cudaDevP2PAttrPerformanceRank, gpuid, neighbor_gpuid);
-	    cudaDeviceGetP2PAttribute(&accessRank[1], cudaDevP2PAttrPerformanceRank, neighbor_gpuid, gpuid);
-	  }
-
-	  // enable P2P if we can access the peer or if peer is self
-          if ((canAccessPeer[0] * canAccessPeer[1] != 0 && accessRank[0] <= enable_p2p_max_access_rank
-               && accessRank[1] <= enable_p2p_max_access_rank)
-              || gpuid == neighbor_gpuid) {
+
+          bool can_access_peer = comm_peer2peer_possible(gpuid, neighbor_gpuid);
+          int access_rank = comm_peer2peer_performance(gpuid, neighbor_gpuid);
+
+	  // enable P2P if we can access the peer
+          if ((can_access_peer && access_rank <= enable_p2p_max_access_rank) || gpuid == neighbor_gpuid) {
             peer2peer_enabled[dir][dim] = true;
             if (getVerbosity() > QUDA_SILENT) {
               printf("Peer-to-peer enabled for rank %d (gpu=%d) with neighbor %d (gpu=%d) dir=%d, dim=%d, access rank "
-                     "= (%d, %d)\n",
-                     comm_rank(), gpuid, neighbor_rank, neighbor_gpuid, dir, dim, accessRank[0], accessRank[1]);
+                     "= %d\n",
+                     comm_rank(), gpuid, neighbor_rank, neighbor_gpuid, dir, dim, access_rank);
             }
           } else {
             intranode_enabled[dir][dim] = true;
@@ -313,8 +302,6 @@ void comm_peer2peer_init(const char* hostname_recv_buf)
   comm_barrier();
 
   peer2peer_present = comm_peer2peer_enabled_global();
-
-  checkCudaErrorNoSync();
 }
 
 bool comm_peer2peer_present() { return peer2peer_present; }
@@ -468,32 +455,6 @@ int comm_coord(int dim)
   return comm_coords(topo)[dim];
 }
 
-inline bool isHost(const void *buffer)
-{
-  CUmemorytype memType;
-  void *attrdata[] = {(void *)&memType};
-  CUpointer_attribute attributes[2] = {CU_POINTER_ATTRIBUTE_MEMORY_TYPE};
-  CUresult err = cuPointerGetAttributes(1, attributes, attrdata, (CUdeviceptr)buffer);
-  if (err != CUDA_SUCCESS) {
-    const char *str;
-    cuGetErrorName(err, &str);
-    errorQuda("cuPointerGetAttributes returned error %s", str);
-  }
-
-  switch (memType) {
-  case CU_MEMORYTYPE_DEVICE: return false;
-  case CU_MEMORYTYPE_ARRAY:
-    errorQuda("Using array memory for communications buffer is not supported");
-    return false;
-  case CU_MEMORYTYPE_UNIFIED:
-    errorQuda("Using unified memory for communications buffer is not supported");
-    return false;
-  case CU_MEMORYTYPE_HOST:
-  default: // memory not allocated by CUDA allocaters will default to being host memory
-    return true;
-  }
-}
-
 /**
  * Send to the "dir" direction in the "dim" dimension
  */
@@ -501,26 +462,6 @@ MsgHandle *comm_declare_send_relative_(const char *func, const char *file, int l
 				       void *buffer, int dim, int dir, size_t nbytes)
 {
   if (getVerbosity() >= QUDA_DEBUG_VERBOSE) printfQuda("%s called (%s:%d in %s())\n", __func__, file, line, func);
-#ifdef HOST_DEBUG
-  checkCudaError(); // check and clear error state first
-
-  if (isHost(buffer)) {
-    // test this memory allocation is ok by doing a memcpy from it
-    void *tmp = safe_malloc(nbytes);
-    try {
-      std::copy(static_cast<char*>(buffer), static_cast<char*>(buffer)+nbytes, static_cast<char*>(tmp));
-    } catch(std::exception &e) {
-      printfQuda("ERROR: buffer failed (%s:%d in %s(), dim=%d, dir=%d, nbytes=%zu)\n", file, line, func, dim, dir, nbytes);
-      errorQuda("aborting");
-    }
-    host_free(tmp);
-  } else {
-    // test this memory allocation is ok by doing a memcpy from it
-    void *tmp = device_malloc(nbytes);
-    qudaMemcpy(tmp, buffer, nbytes, qudaMemcpyDeviceToDevice);
-    device_free(tmp);
-  }
-#endif
 
   int disp[QUDA_MAX_DIM] = {0};
   disp[dim] = dir;
@@ -535,22 +476,6 @@ MsgHandle *comm_declare_receive_relative_(const char *func, const char *file, in
 					  void *buffer, int dim, int dir, size_t nbytes)
 {
   if (getVerbosity() >= QUDA_DEBUG_VERBOSE) printfQuda("%s called (%s:%d in %s())\n", __func__, file, line, func);
-#ifdef HOST_DEBUG
-  checkCudaError(); // check and clear error state first
-
-  if (isHost(buffer)) {
-    // test this memory allocation is ok by filling it
-    try {
-      std::fill(static_cast<char*>(buffer), static_cast<char*>(buffer)+nbytes, 0);
-    } catch(std::exception &e) {
-      printfQuda("ERROR: buffer failed (%s:%d in %s(), dim=%d, dir=%d, nbytes=%zu)\n", file, line, func, dim, dir, nbytes);
-      errorQuda("aborting");
-    }
-  } else {
-    // test this memory allocation is ok by doing a memset
-    qudaMemset(buffer, 0, nbytes);
-  }
-#endif
 
   int disp[QUDA_MAX_DIM] = {0};
   disp[dim] = dir;
@@ -565,28 +490,6 @@ MsgHandle *comm_declare_strided_send_relative_(const char *func, const char *fil
 					       void *buffer, int dim, int dir, size_t blksize, int nblocks, size_t stride)
 {
   if (getVerbosity() >= QUDA_DEBUG_VERBOSE) printfQuda("%s called (%s:%d in %s())\n", __func__, file, line, func);
-#ifdef HOST_DEBUG
-  checkCudaError(); // check and clear error state first
-
-  if (isHost(buffer)) {
-    // test this memory allocation is ok by doing a memcpy from it
-    void *tmp = safe_malloc(blksize*nblocks);
-    try {
-      for (int i=0; i<nblocks; i++)
-	std::copy(static_cast<char*>(buffer)+i*stride, static_cast<char*>(buffer)+i*stride+blksize, static_cast<char*>(tmp));
-    } catch(std::exception &e) {
-      printfQuda("ERROR: buffer failed (%s:%d in %s(), dim=%d, dir=%d, blksize=%zu nblocks=%d stride=%zu)\n",
-		 file, line, func, dim, dir, blksize, nblocks, stride);
-      errorQuda("aborting");
-    }
-    host_free(tmp);
-  } else {
-    // test this memory allocation is ok by doing a memcpy from it
-    void *tmp = device_malloc(blksize*nblocks);
-    qudaMemcpy2D(tmp, blksize, buffer, stride, blksize, nblocks, qudaMemcpyDeviceToDevice);
-    device_free(tmp);
-  }
-#endif
 
   int disp[QUDA_MAX_DIM] = {0};
   disp[dim] = dir;
@@ -602,24 +505,6 @@ MsgHandle *comm_declare_strided_receive_relative_(const char *func, const char *
 						  void *buffer, int dim, int dir, size_t blksize, int nblocks, size_t stride)
 {
   if (getVerbosity() >= QUDA_DEBUG_VERBOSE) printfQuda("%s called (%s:%d in %s())\n", __func__, file, line, func);
-#ifdef HOST_DEBUG
-  checkCudaError(); // check and clear error state first
-
-  if (isHost(buffer)) {
-    // test this memory allocation is ok by filling it
-    try {
-      for (int i=0; i<nblocks; i++)
-	std::fill(static_cast<char*>(buffer)+i*stride, static_cast<char*>(buffer)+i*stride+blksize, 0);
-    } catch(std::exception &e) {
-      printfQuda("ERROR: buffer failed (%s:%d in %s(), dim=%d, dir=%d, blksize=%zu nblocks=%d stride=%zu)\n",
-		 file, line, func, dim, dir, blksize, nblocks, stride);
-      errorQuda("aborting");
-    }
-  } else {
-    // test this memory allocation is ok by doing a memset
-    qudaMemset2D(buffer, stride, 0, blksize, nblocks);
-  }
-#endif
 
   int disp[QUDA_MAX_DIM] = {0};
   disp[dim] = dir;
@@ -704,13 +589,10 @@ bool comm_gdr_blacklist() {
     if (blacklist_env) { // set the policies to tune for explicitly
       std::stringstream blacklist_list(blacklist_env);
 
-      int device_count;
-      cudaGetDeviceCount(&device_count);
-
       int excluded_device;
       while (blacklist_list >> excluded_device) {
 	// check this is a valid device
-	if ( excluded_device < 0 || excluded_device >= device_count ) {
+	if ( excluded_device < 0 || excluded_device >= quda::device::get_device_count() ) {
 	  errorQuda("Cannot blacklist invalid GPU device ordinal %d", excluded_device);
 	}
 
@@ -743,9 +625,8 @@ void comm_init_common(int ndim, const int *dims, QudaCommsMap rank_from_coords,
     if (!strncmp(comm_hostname(), &hostname_recv_buf[128 * i], 128)) { gpuid++; }
   }
 
-  int device_count;
-  cudaGetDeviceCount(&device_count);
-  if (device_count == 0) { warningQuda("No CUDA devices found"); }
+  int device_count = quda::device::get_device_count();
+  if (device_count == 0) { warningQuda("No devices found"); }
   if (gpuid >= device_count) {
     char *enable_mps_env = getenv("QUDA_ENABLE_MPS");
     if (enable_mps_env && strcmp(enable_mps_env, "1") == 0) {
@@ -776,8 +657,6 @@ void comm_init_common(int ndim, const int *dims, QudaCommsMap rank_from_coords,
       std::stringstream device_list;                       // formatted (no commas)
 
       int device;
-      int deviceCount;
-      cudaGetDeviceCount(&deviceCount);
       while (device_list_raw >> device) {
         // check this is a valid policy choice
         if (device < 0) { errorQuda("Invalid CUDA_VISIBLE_DEVICE ordinal %d", device); }
diff --git a/lib/comm_mpi.cpp b/lib/comm_mpi.cpp
index b34c9ad86d..4f10be7ca5 100644
--- a/lib/comm_mpi.cpp
+++ b/lib/comm_mpi.cpp
@@ -83,9 +83,9 @@ int comm_rank(void)
 }
 
 
-int comm_size(void)
+size_t comm_size(void)
 {
-  return size;
+  return static_cast<size_t>(size);
 }
 
 
diff --git a/lib/comm_qmp.cpp b/lib/comm_qmp.cpp
index 6b97cdc7e9..1e83206c6d 100644
--- a/lib/comm_qmp.cpp
+++ b/lib/comm_qmp.cpp
@@ -107,9 +107,9 @@ int comm_rank(void)
 }
 
 
-int comm_size(void)
+size_t comm_size(void)
 {
-  return QMP_get_number_of_nodes();
+  return static_cast<size_t>(QMP_get_number_of_nodes());
 }
 
 
diff --git a/lib/comm_single.cpp b/lib/comm_single.cpp
index 2726a3793a..26ce5d5a34 100644
--- a/lib/comm_single.cpp
+++ b/lib/comm_single.cpp
@@ -13,7 +13,7 @@ void comm_init(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *m
 
 int comm_rank(void) { return 0; }
 
-int comm_size(void) { return 1; }
+size_t comm_size(void) { return 1; }
 
 void comm_gather_hostname(char *hostname_recv_buf) {
   strncpy(hostname_recv_buf, comm_hostname(), 128);
diff --git a/lib/copy_gauge_extended.cu b/lib/copy_gauge_extended.cu
index 3252f9687c..f3f9025201 100644
--- a/lib/copy_gauge_extended.cu
+++ b/lib/copy_gauge_extended.cu
@@ -12,7 +12,7 @@ namespace quda {
     FloatOut *Out;
     FloatIn *In;
 
-    unsigned int minThreads() const { return in.LocalVolumeCB(); }
+    unsigned int minThreads() const { return in.VolumeCB() == out.VolumeCB() ? in.VolumeCB() : in.LocalVolumeCB(); }
 
   public:
     CopyGaugeEx(GaugeField &out, const GaugeField &in, QudaFieldLocation location,
diff --git a/lib/covDev.cu b/lib/covDev.cu
index 521403e3f2..591ec31316 100644
--- a/lib/covDev.cu
+++ b/lib/covDev.cu
@@ -155,7 +155,9 @@ namespace quda
   void ApplyCovDev(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int mu, int parity,
                    bool dagger, const int *comm_override, TimeProfile &profile)
   {
+    pushKernelPackT(true); // non-spin projection requires kernel packing
     instantiate<CovDevApply>(out, in, U, mu, parity, dagger, comm_override, profile);
+    popKernelPackT();
   }
 #else
   void ApplyCovDev(ColorSpinorField &, const ColorSpinorField &, const GaugeField &, int, int,
diff --git a/lib/cpu_gauge_field.cpp b/lib/cpu_gauge_field.cpp
index 4e01b509f2..85e02b4e9e 100644
--- a/lib/cpu_gauge_field.cpp
+++ b/lib/cpu_gauge_field.cpp
@@ -1,4 +1,5 @@
 #include <quda_internal.h>
+#include <timer.h>
 #include <gauge_field.h>
 #include <assert.h>
 #include <string.h>
diff --git a/lib/cuda_color_spinor_field.cpp b/lib/cuda_color_spinor_field.cpp
index 8287e35a9b..93b54ad239 100644
--- a/lib/cuda_color_spinor_field.cpp
+++ b/lib/cuda_color_spinor_field.cpp
@@ -892,7 +892,7 @@ namespace quda {
       // first wait on send to backwards
       if (comm_peer2peer_enabled(0,dim)) {
 	comm_wait(mh_send_p2p_back[bufferIndex][dim]);
-	cudaEventSynchronize(ipcCopyEvent[bufferIndex][0][dim]);
+	qudaEventSynchronize(ipcCopyEvent[bufferIndex][0][dim]);
       } else if (gdr_send) {
 	comm_wait(mh_send_rdma_back[bufferIndex][dim]);
       } else {
@@ -902,7 +902,7 @@ namespace quda {
       // second wait on receive from forwards
       if (comm_peer2peer_enabled(1,dim)) {
 	comm_wait(mh_recv_p2p_fwd[bufferIndex][dim]);
-	cudaEventSynchronize(ipcRemoteCopyEvent[bufferIndex][1][dim]);
+	qudaEventSynchronize(ipcRemoteCopyEvent[bufferIndex][1][dim]);
       } else if (gdr_recv) {
 	comm_wait(mh_recv_rdma_fwd[bufferIndex][dim]);
       } else {
@@ -914,7 +914,7 @@ namespace quda {
       // first wait on send to forwards
       if (comm_peer2peer_enabled(1,dim)) {
 	comm_wait(mh_send_p2p_fwd[bufferIndex][dim]);
-	cudaEventSynchronize(ipcCopyEvent[bufferIndex][1][dim]);
+	qudaEventSynchronize(ipcCopyEvent[bufferIndex][1][dim]);
       } else if (gdr_send) {
 	comm_wait(mh_send_rdma_fwd[bufferIndex][dim]);
       } else {
@@ -924,7 +924,7 @@ namespace quda {
       // second wait on receive from backwards
       if (comm_peer2peer_enabled(0,dim)) {
 	comm_wait(mh_recv_p2p_back[bufferIndex][dim]);
-	cudaEventSynchronize(ipcRemoteCopyEvent[bufferIndex][0][dim]);
+	qudaEventSynchronize(ipcRemoteCopyEvent[bufferIndex][0][dim]);
       } else if (gdr_recv) {
 	comm_wait(mh_recv_rdma_back[bufferIndex][dim]);
       } else {
diff --git a/lib/cuda_gauge_field.cpp b/lib/cuda_gauge_field.cpp
index d9bd3f03c6..a9d38526b3 100644
--- a/lib/cuda_gauge_field.cpp
+++ b/lib/cuda_gauge_field.cpp
@@ -1,6 +1,7 @@
-#include <string.h>
-#include <gauge_field.h>
+#include <cstring>
 #include <typeinfo>
+#include <gauge_field.h>
+#include <timer.h>
 #include <blas_quda.h>
 #include <device.h>
 
diff --git a/lib/dslash_domain_wall_5d.cu b/lib/dslash_domain_wall_5d.cu
index 645eca7ac0..36315d3756 100644
--- a/lib/dslash_domain_wall_5d.cu
+++ b/lib/dslash_domain_wall_5d.cu
@@ -82,7 +82,9 @@ namespace quda
   void ApplyDomainWall5D(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double m_f,
       const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile)
   {
+    pushKernelPackT(true); // with 5-d checkerboarding we must use kernel packing
     instantiate<DomainWall5DApply>(out, in, U, a, m_f, x, parity, dagger, comm_override, profile);
+    popKernelPackT();
   }
 #else
   void ApplyDomainWall5D(ColorSpinorField &, const ColorSpinorField &, const GaugeField &, double, double,
diff --git a/lib/dslash_pack2.cu b/lib/dslash_pack2.cu
index d3fb9c735f..c27d89abbf 100644
--- a/lib/dslash_pack2.cu
+++ b/lib/dslash_pack2.cu
@@ -6,6 +6,7 @@
 // if not STRIPED then this means we assign one thread block per direction / dimension
 
 #include <dslash_quda.h>
+#include <tune_quda.h>
 #include <kernels/dslash_pack.cuh>
 #include <instantiate.h>
 
diff --git a/lib/dslash_policy.cuh b/lib/dslash_policy.cuh
index e39bea6fbe..d231de9e1b 100644
--- a/lib/dslash_policy.cuh
+++ b/lib/dslash_policy.cuh
@@ -1,6 +1,7 @@
 #include <array>
 #include <tune_quda.h>
 #include <index_helper.cuh>
+#include <timer.h>
 
 namespace quda
 {
@@ -10,12 +11,10 @@ namespace quda
 
     extern int it;
 
-    extern cudaEvent_t packEnd[]; // double buffered
-    extern cudaEvent_t gatherStart[];
-    extern cudaEvent_t gatherEnd[];
-    extern cudaEvent_t scatterStart[];
-    extern cudaEvent_t scatterEnd[];
-    extern cudaEvent_t dslashStart[]; // double buffered
+    extern qudaEvent_t packEnd[]; // double buffered
+    extern qudaEvent_t gatherEnd[];
+    extern qudaEvent_t scatterEnd[];
+    extern qudaEvent_t dslashStart[]; // double buffered
 
     // FIX this is a hack from hell
     // Auxiliary work that can be done while waiting on comms to finish
@@ -185,7 +184,7 @@ namespace quda
       if (!dslash.dslashParam.commDim[i]) continue;
 
       for (int dir=1; dir>=0; dir--) { // forwards gather
-        cudaEvent_t &event = (i!=3 || getKernelPackT()) ? packEnd[in.bufferIndex] : dslashStart[in.bufferIndex];
+        auto &event = (i!=3 || getKernelPackT()) ? packEnd[in.bufferIndex] : dslashStart[in.bufferIndex];
 
         PROFILE(qudaStreamWaitEvent(device::get_stream(2*i+dir), event, 0), profile, QUDA_PROFILE_STREAM_WAIT_EVENT);
 
@@ -567,7 +566,7 @@ namespace quda
           if (!dslashParam.commDim[i]) continue;
 
           if (!pack_event) {
-            cudaEventSynchronize(packEnd[in->bufferIndex]);
+            qudaEventSynchronize(packEnd[in->bufferIndex]);
             pack_event = true;
           }
 
@@ -650,7 +649,7 @@ namespace quda
           if (!dslashParam.commDim[i]) continue;
 
           if (!pack_event) {
-            cudaEventSynchronize(packEnd[in->bufferIndex]);
+            qudaEventSynchronize(packEnd[in->bufferIndex]);
             pack_event = true;
           }
 
diff --git a/lib/dslash_quda.cu b/lib/dslash_quda.cu
index c7e20d6fa1..87debe4dab 100644
--- a/lib/dslash_quda.cu
+++ b/lib/dslash_quda.cu
@@ -50,12 +50,10 @@ namespace quda {
     static constexpr int nDir = 2;
     static constexpr int nStream = nDim * nDir + 1;
 
-    cudaEvent_t packEnd[2];
-    cudaEvent_t gatherStart[nStream];
-    cudaEvent_t gatherEnd[nStream];
-    cudaEvent_t scatterStart[nStream];
-    cudaEvent_t scatterEnd[nStream];
-    cudaEvent_t dslashStart[2];
+    qudaEvent_t packEnd[2];
+    qudaEvent_t gatherEnd[nStream];
+    qudaEvent_t scatterEnd[nStream];
+    qudaEvent_t dslashStart[2];
 
     // these variables are used for benchmarking the dslash components in isolation
     bool dslash_pack_compute;
@@ -88,22 +86,17 @@ namespace quda {
   void createDslashEvents()
   {
     using namespace dslash;
-    // add cudaEventDisableTiming for lower sync overhead
     for (int i=0; i<nStream; i++) {
-      cudaEventCreateWithFlags(&gatherStart[i], cudaEventDisableTiming);
-      cudaEventCreateWithFlags(&gatherEnd[i], cudaEventDisableTiming);
-      cudaEventCreateWithFlags(&scatterStart[i], cudaEventDisableTiming);
-      cudaEventCreateWithFlags(&scatterEnd[i], cudaEventDisableTiming);
+      gatherEnd[i] = qudaEventCreate();
+      scatterEnd[i] = qudaEventCreate();
     }
     for (int i=0; i<2; i++) {
-      cudaEventCreateWithFlags(&packEnd[i], cudaEventDisableTiming);
-      cudaEventCreateWithFlags(&dslashStart[i], cudaEventDisableTiming);
+      packEnd[i] = qudaEventCreate();
+      dslashStart[i] = qudaEventCreate();
     }
 
     aux_worker = NULL;
 
-    checkCudaError();
-
     dslash_pack_compute = true;
     dslash_interior_compute = true;
     dslash_exterior_compute = true;
@@ -125,24 +118,19 @@ namespace quda {
     strcat(policy_string, ",pol=");
   }
 
-
   void destroyDslashEvents()
   {
     using namespace dslash;
 
     for (int i=0; i<nStream; i++) {
-      cudaEventDestroy(gatherStart[i]);
-      cudaEventDestroy(gatherEnd[i]);
-      cudaEventDestroy(scatterStart[i]);
-      cudaEventDestroy(scatterEnd[i]);
+      qudaEventDestroy(gatherEnd[i]);
+      qudaEventDestroy(scatterEnd[i]);
     }
 
     for (int i=0; i<2; i++) {
-      cudaEventDestroy(packEnd[i]);
-      cudaEventDestroy(dslashStart[i]);
+      qudaEventDestroy(packEnd[i]);
+      qudaEventDestroy(dslashStart[i]);
     }
-
-    checkCudaError();
   }
 
   template <typename Float, int nColor> class GammaApply : public TunableKernel2D {
diff --git a/lib/eigensolve_quda.cpp b/lib/eigensolve_quda.cpp
index f0404c33c5..6f3e18f4e1 100644
--- a/lib/eigensolve_quda.cpp
+++ b/lib/eigensolve_quda.cpp
@@ -8,6 +8,7 @@
 
 #include <quda_internal.h>
 #include <eigensolve_quda.h>
+#include <random_quda.h>
 #include <qio_field.h>
 #include <color_spinor_field.h>
 #include <blas_quda.h>
@@ -135,11 +136,9 @@ namespace quda
       }
     } else {
       RNG *rng = new RNG(*kSpace[0], 1234);
-      rng->Init();
       for (int b = 0; b < block_size; b++) {
         if (sqrt(blas::norm2(*kSpace[b])) == 0.0) { spinorNoise(*kSpace[b], *rng, QUDA_NOISE_UNIFORM); }
       }
-      rng->Release();
       delete rng;
     }
     bool orthed = false;
@@ -382,9 +381,7 @@ namespace quda
       in.Source(QUDA_RANDOM_SOURCE);
     } else {
       RNG *rng = new RNG(in, 1234);
-      rng->Init();
       spinorNoise(in, *rng, QUDA_NOISE_UNIFORM);
-      rng->Release();
       delete rng;
     }
 
diff --git a/lib/extract_gauge_ghost_mg.cu b/lib/extract_gauge_ghost_mg.cu
index 75823e320a..d2dbcbac44 100644
--- a/lib/extract_gauge_ghost_mg.cu
+++ b/lib/extract_gauge_ghost_mg.cu
@@ -1,6 +1,3 @@
-// trove cannot deal with the large matrices that MG uses so we need
-// to disable it (regardless we're using fine-grained access)
-#define DISABLE_TROVE
 #define FINE_GRAINED_ACCESS
 
 #include <gauge_field_order.h>
diff --git a/lib/gauge_field.cpp b/lib/gauge_field.cpp
index c0a2bdab9b..96d8343556 100644
--- a/lib/gauge_field.cpp
+++ b/lib/gauge_field.cpp
@@ -1,6 +1,7 @@
-#include <gauge_field.h>
 #include <typeinfo>
+#include <gauge_field.h>
 #include <blas_quda.h>
+#include <timer.h>
 
 namespace quda {
 
diff --git a/lib/gauge_fix_fft.cu b/lib/gauge_fix_fft.cu
index b3c740b73d..a7d955334a 100644
--- a/lib/gauge_fix_fft.cu
+++ b/lib/gauge_fix_fft.cu
@@ -4,8 +4,7 @@
 #include <gauge_field.h>
 #include <gauge_field_order.h>
 #include <unitarization_links.h>
-#include <atomic.cuh>
-#include <index_helper.cuh>
+#include <gauge_tools.h>
 
 #include <FFT_Plans.h>
 #include <instantiate.h>
@@ -87,14 +86,16 @@ namespace quda {
 
   template <typename Arg> class GaugeFixerFFT : TunableKernel2D {
     Arg &arg;
+    const GaugeField &field;
     GaugeFixFFTKernel type;
     char aux_tmp[TuneKey::aux_n];
     unsigned int minThreads() const { return arg.threads.x; }
 
   public:
-    GaugeFixerFFT(Arg &arg, const GaugeField &meta) :
-      TunableKernel2D(meta, 2),
-      arg(arg)
+    GaugeFixerFFT(Arg &arg, const GaugeField &field) :
+      TunableKernel2D(field, 2),
+      arg(arg),
+      field(field)
     {
       strcpy(aux_tmp, aux);
     }
diff --git a/lib/gauge_fix_ovr.cu b/lib/gauge_fix_ovr.cu
index fda9a93f62..1594d284e0 100644
--- a/lib/gauge_fix_ovr.cu
+++ b/lib/gauge_fix_ovr.cu
@@ -1,5 +1,6 @@
 #include <quda_internal.h>
 #include <gauge_field.h>
+#include <gauge_tools.h>
 #include <unitarization_links.h>
 #include <comm_quda.h>
 #include <gauge_fix_ovr_hit_devf.cuh>
@@ -219,7 +220,7 @@ public:
     {
       unsigned int blockx = param.block.x / 8;
       if (param.aux.x > 2) blockx = param.block.x / 4;
-      unsigned int gx  = (arg.threads + blockx - 1) / blockx;
+      unsigned int gx  = std::max((arg.threads + blockx - 1) / blockx, 1u);
       return dim3(gx, 1, 1);
     }
 
@@ -705,7 +706,7 @@ public:
             qudaMemcpyAsync(sendg[d], sendg_d[d], bytes[d], qudaMemcpyDeviceToHost, device::get_stream(4 + d));
           }
           //compute interior points
-          gfixIntPoints.apply(device::get_stream(8));
+          gfixIntPoints.apply(device::get_default_stream());
 
           for ( int d = 0; d < 4; d++ ) {
             if ( !commDimPartitioned(d)) continue;
diff --git a/lib/gauge_random.cu b/lib/gauge_random.cu
index 3d9fe4f80b..6bc07409b5 100644
--- a/lib/gauge_random.cu
+++ b/lib/gauge_random.cu
@@ -1,5 +1,6 @@
 #include <quda_internal.h>
 #include <gauge_field.h>
+#include <random_quda.h>
 #include <instantiate.h>
 #include <tunable_nd.h>
 #include <kernels/gauge_random.cuh>
@@ -35,9 +36,9 @@ namespace quda {
     {
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       if (group) {
-        launch<GaussGauge>(tp, stream, GaugeGaussArg<Float, nColor, recon, true>(U, rng, sigma));
+        launch<GaussGauge>(tp, stream, GaugeGaussArg<Float, nColor, recon, true>(U, rng.State(), sigma));
       } else {
-        launch<GaussGauge>(tp, stream, GaugeGaussArg<Float, nColor, recon, false>(U, rng, sigma));
+        launch<GaussGauge>(tp, stream, GaugeGaussArg<Float, nColor, recon, false>(U, rng.State(), sigma));
       }
     }
 
@@ -67,9 +68,7 @@ namespace quda {
   void gaugeGauss(GaugeField &U, unsigned long long seed, double sigma)
   {
     RNG *randstates = new RNG(U, seed);
-    randstates->Init();
     quda::gaugeGauss(U, *randstates, sigma);
-    randstates->Release();
     delete randstates;
   }
 }
diff --git a/lib/interface/CMakeLists.txt b/lib/interface/CMakeLists.txt
index f357a9e31c..6cc2fcb358 100644
--- a/lib/interface/CMakeLists.txt
+++ b/lib/interface/CMakeLists.txt
@@ -1,3 +1,6 @@
 # add interface files / options
 target_sources(quda_cpp PRIVATE blas_interface.cpp)
 
+if(BUILD_FORTRAN_INTERFACE)
+  target_sources(quda_cpp PRIVATE fortran_interface.cpp)
+endif()
diff --git a/lib/interface/blas_interface.cpp b/lib/interface/blas_interface.cpp
index c2f0883c46..f7e0c7ba8d 100644
--- a/lib/interface/blas_interface.cpp
+++ b/lib/interface/blas_interface.cpp
@@ -1,4 +1,5 @@
 #include <quda.h>
+#include <timer.h>
 #include <blas_lapack.h>
 #include <tune_quda.h>
 
diff --git a/lib/interface_quda.cpp b/lib/interface_quda.cpp
index 79baa355eb..07d03c9667 100644
--- a/lib/interface_quda.cpp
+++ b/lib/interface_quda.cpp
@@ -7,9 +7,9 @@
 #include <complex.h>
 
 #include <quda.h>
-#include <quda_fortran.h>
 #include <quda_internal.h>
 #include <device.h>
+#include <timer.h>
 #include <comm_quda.h>
 #include <tune_quda.h>
 #include <blas_quda.h>
@@ -524,6 +524,8 @@ void initQudaMemory()
 
   if (!comms_initialized) init_default_comms();
 
+  loadTuneCache();
+
   device::create_context();
   createDslashEvents();
 
@@ -536,8 +538,6 @@ void initQudaMemory()
   num_failures_h = static_cast<int*>(mapped_malloc(sizeof(int)));
   num_failures_d = static_cast<int*>(get_mapped_device_pointer(num_failures_h));
 
-  loadTuneCache();
-
   for (int d=0; d<4; d++) R[d] = 2 * (redundant_comms || commDimPartitioned(d));
 
   profileInit.TPSTOP(QUDA_PROFILE_INIT);
@@ -2188,6 +2188,9 @@ namespace quda
       return (gaugeFatPrecise != nullptr) and param->cuda_prec == gaugeFatPrecise->Precision();
     }
   }
+
+  GaugeField* getResidentGauge() { return gaugePrecise; }
+
 } // namespace quda
 
 void checkClover(QudaInvertParam *param) {
@@ -5200,246 +5203,6 @@ double momActionQuda(void* momentum, QudaGaugeParam* param)
   return action;
 }
 
-/*
-  The following functions are for the Fortran interface.
-*/
-
-void init_quda_(int *dev) { initQuda(*dev); }
-void init_quda_device_(int *dev) { initQudaDevice(*dev); }
-void init_quda_memory_() { initQudaMemory(); }
-void end_quda_() { endQuda(); }
-void load_gauge_quda_(void *h_gauge, QudaGaugeParam *param) { loadGaugeQuda(h_gauge, param); }
-void free_gauge_quda_() { freeGaugeQuda(); }
-void free_sloppy_gauge_quda_() { freeSloppyGaugeQuda(); }
-void load_clover_quda_(void *h_clover, void *h_clovinv, QudaInvertParam *inv_param)
-{ loadCloverQuda(h_clover, h_clovinv, inv_param); }
-void free_clover_quda_(void) { freeCloverQuda(); }
-void dslash_quda_(void *h_out, void *h_in, QudaInvertParam *inv_param,
-    QudaParity *parity) { dslashQuda(h_out, h_in, inv_param, *parity); }
-void clover_quda_(void *h_out, void *h_in, QudaInvertParam *inv_param,
-    QudaParity *parity, int *inverse) { cloverQuda(h_out, h_in, inv_param, *parity, *inverse); }
-void mat_quda_(void *h_out, void *h_in, QudaInvertParam *inv_param)
-{ MatQuda(h_out, h_in, inv_param); }
-void mat_dag_mat_quda_(void *h_out, void *h_in, QudaInvertParam *inv_param)
-{ MatDagMatQuda(h_out, h_in, inv_param); }
-void invert_quda_(void *hp_x, void *hp_b, QudaInvertParam *param) {
-  fflush(stdout);
-  // ensure that fifth dimension is set to 1
-  if (param->dslash_type == QUDA_ASQTAD_DSLASH || param->dslash_type == QUDA_STAGGERED_DSLASH) param->Ls = 1;
-  invertQuda(hp_x, hp_b, param);
-  fflush(stdout);
-}
-
-void invert_multishift_quda_(void *h_x, void *hp_b, QudaInvertParam *param) {
-  // ensure that fifth dimension is set to 1
-  if (param->dslash_type == QUDA_ASQTAD_DSLASH || param->dslash_type == QUDA_STAGGERED_DSLASH) param->Ls = 1;
-
-  if (!gaugePrecise) errorQuda("Resident gauge field not allocated");
-
-  // get data into array of pointers
-  int nSpin = (param->dslash_type == QUDA_STAGGERED_DSLASH || param->dslash_type == QUDA_ASQTAD_DSLASH) ? 1 : 4;
-
-  // compute offset assuming TIFR padded ordering (FIXME)
-  if (param->dirac_order != QUDA_TIFR_PADDED_DIRAC_ORDER)
-    errorQuda("Fortran multi-shift solver presently only supports QUDA_TIFR_PADDED_DIRAC_ORDER");
-
-  const int *X = gaugePrecise->X();
-  size_t cb_offset = (X[0]/2) * X[1] * (X[2] + 4) * X[3] * gaugePrecise->Ncolor() * nSpin * 2 * param->cpu_prec;
-  void *hp_x[QUDA_MAX_MULTI_SHIFT];
-  for (int i=0; i<param->num_offset; i++) hp_x[i] = static_cast<char*>(h_x) + i*cb_offset;
-
-  invertMultiShiftQuda(hp_x, hp_b, param);
-}
-
-void flush_chrono_quda_(int *index) { flushChronoQuda(*index); }
-
-void register_pinned_quda_(void *ptr, size_t *bytes) {
-  cudaHostRegister(ptr, *bytes, cudaHostRegisterDefault);
-  checkCudaError();
-}
-
-void unregister_pinned_quda_(void *ptr) {
-  cudaHostUnregister(ptr);
-  checkCudaError();
-}
-
-void new_quda_gauge_param_(QudaGaugeParam *param) {
-  *param = newQudaGaugeParam();
-}
-void new_quda_invert_param_(QudaInvertParam *param) {
-  *param = newQudaInvertParam();
-}
-
-void update_gauge_field_quda_(void *gauge, void *momentum, double *dt,
-    bool *conj_mom, bool *exact,
-    QudaGaugeParam *param) {
-  updateGaugeFieldQuda(gauge, momentum, *dt, (int)*conj_mom, (int)*exact, param);
-}
-
-static inline int opp(int dir) { return 7-dir; }
-
-static void createGaugeForcePaths(int **paths, int dir, int num_loop_types){
-
-  int index=0;
-  // Plaquette paths
-  if (num_loop_types >= 1)
-    for(int i=0; i<4; ++i){
-      if(i==dir) continue;
-      paths[index][0] = i;        paths[index][1] = opp(dir);   paths[index++][2] = opp(i);
-      paths[index][0] = opp(i);   paths[index][1] = opp(dir);   paths[index++][2] = i;
-    }
-
-  // Rectangle Paths
-  if (num_loop_types >= 2)
-    for(int i=0; i<4; ++i){
-      if(i==dir) continue;
-      paths[index][0] = paths[index][1] = i;       paths[index][2] = opp(dir); paths[index][3] = paths[index][4] = opp(i);
-      index++;
-      paths[index][0] = paths[index][1] = opp(i);  paths[index][2] = opp(dir); paths[index][3] = paths[index][4] = i;
-      index++;
-      paths[index][0] = dir; paths[index][1] = i; paths[index][2] = paths[index][3] = opp(dir); paths[index][4] = opp(i);
-      index++;
-      paths[index][0] = dir; paths[index][1] = opp(i); paths[index][2] = paths[index][3] = opp(dir); paths[index][4] = i;
-      index++;
-      paths[index][0] = i;  paths[index][1] = paths[index][2] = opp(dir); paths[index][3] = opp(i); paths[index][4] = dir;
-      index++;
-      paths[index][0] = opp(i);  paths[index][1] = paths[index][2] = opp(dir); paths[index][3] = i; paths[index][4] = dir;
-      index++;
-    }
-
-  if (num_loop_types >= 3) {
-    // Staple paths
-    for(int i=0; i<4; ++i){
-      for(int j=0; j<4; ++j){
-	if(i==dir || j==dir || i==j) continue;
-	paths[index][0] = i; paths[index][1] = j; paths[index][2] = opp(dir); paths[index][3] = opp(i), paths[index][4] = opp(j);
-	index++;
-	paths[index][0] = i; paths[index][1] = opp(j); paths[index][2] = opp(dir); paths[index][3] = opp(i), paths[index][4] = j;
-	index++;
-	paths[index][0] = opp(i); paths[index][1] = j; paths[index][2] = opp(dir); paths[index][3] = i, paths[index][4] = opp(j);
-	index++;
-	paths[index][0] = opp(i); paths[index][1] = opp(j); paths[index][2] = opp(dir); paths[index][3] = i, paths[index][4] = j;
-	index++;
-     }
-    }
-  }
-
-}
-
-void compute_gauge_force_quda_(void *mom, void *gauge, int *num_loop_types, double *coeff, double *dt,
-			       QudaGaugeParam *param) {
-
-  int numPaths = 0;
-  switch (*num_loop_types) {
-  case 1:
-    numPaths = 6;
-    break;
-  case 2:
-    numPaths = 24;
-    break;
-  case 3:
-    numPaths = 48;
-    break;
-  default:
-    errorQuda("Invalid num_loop_types = %d\n", *num_loop_types);
-  }
-
-  auto *loop_coeff = static_cast<double*>(safe_malloc(numPaths*sizeof(double)));
-  int *path_length = static_cast<int*>(safe_malloc(numPaths*sizeof(int)));
-
-  if (*num_loop_types >= 1) for(int i= 0; i< 6; ++i) {
-      loop_coeff[i] = coeff[0];
-      path_length[i] = 3;
-    }
-  if (*num_loop_types >= 2) for(int i= 6; i<24; ++i) {
-      loop_coeff[i] = coeff[1];
-      path_length[i] = 5;
-    }
-  if (*num_loop_types >= 3) for(int i=24; i<48; ++i) {
-      loop_coeff[i] = coeff[2];
-      path_length[i] = 5;
-    }
-
-  int** input_path_buf[4];
-  for(int dir=0; dir<4; ++dir){
-    input_path_buf[dir] = static_cast<int**>(safe_malloc(numPaths*sizeof(int*)));
-    for(int i=0; i<numPaths; ++i){
-      input_path_buf[dir][i] = static_cast<int*>(safe_malloc(path_length[i]*sizeof(int)));
-    }
-    createGaugeForcePaths(input_path_buf[dir], dir, *num_loop_types);
-  }
-
-  int max_length = 6;
-
-  computeGaugeForceQuda(mom, gauge, input_path_buf, path_length, loop_coeff, numPaths, max_length, *dt, param);
-
-  for(auto & dir : input_path_buf){
-    for(int i=0; i<numPaths; ++i) host_free(dir[i]);
-    host_free(dir);
-  }
-
-  host_free(path_length);
-  host_free(loop_coeff);
-}
-
-void compute_staggered_force_quda_(void* h_mom, double *dt, double *delta, void *gauge, void *x, QudaGaugeParam *gauge_param, QudaInvertParam *inv_param) {
-  computeStaggeredForceQuda(h_mom, *dt, *delta, gauge, (void**)x, gauge_param, inv_param);
-}
-
-// apply the staggered phases
-void apply_staggered_phase_quda_() {
-  if (getVerbosity() >= QUDA_VERBOSE) printfQuda("applying staggered phase\n");
-  if (gaugePrecise) {
-    gaugePrecise->applyStaggeredPhase();
-  } else {
-    errorQuda("No persistent gauge field");
-  }
-}
-
-// remove the staggered phases
-void remove_staggered_phase_quda_() {
-  if (getVerbosity() >= QUDA_VERBOSE) printfQuda("removing staggered phase\n");
-  if (gaugePrecise) {
-    gaugePrecise->removeStaggeredPhase();
-  } else {
-    errorQuda("No persistent gauge field");
-  }
-  qudaDeviceSynchronize();
-}
-
-// evaluate the kinetic term
-void kinetic_quda_(double *kin, void* momentum, QudaGaugeParam* param) {
-  *kin = momActionQuda(momentum, param);
-}
-
-
-/**
- * BQCD wants a node mapping with x varying fastest.
- */
-static int bqcd_rank_from_coords(const int *coords, void *fdata)
-{
-  int *dims = static_cast<int *>(fdata);
-
-  int rank = coords[3];
-  for (int i = 2; i >= 0; i--) {
-    rank = dims[i] * rank + coords[i];
-  }
-  return rank;
-}
-
-void comm_set_gridsize_(int *grid)
-{
-  initCommsGridQuda(4, grid, bqcd_rank_from_coords, static_cast<void *>(grid));
-}
-
-/**
- * Exposed due to poor derived MPI datatype performance with GPUDirect RDMA
- */
-void set_kernel_pack_t_(int* pack)
-{
-  bool pack_ = *pack ? true : false;
-  setKernelPackT(pack_);
-}
 
 void gaussGaugeQuda(unsigned long long seed, double sigma)
 {
@@ -5470,10 +5233,6 @@ void gaussGaugeQuda(unsigned long long seed, double sigma)
 /*
  * Computes the total, spatial and temporal plaquette averages of the loaded gauge configuration.
  */
-void plaq_quda_(double plaq[3]) {
-  plaqQuda(plaq);
-}
-
 void plaqQuda(double plaq[3])
 {
   profilePlaq.TPSTART(QUDA_PROFILE_TOTAL);
@@ -5496,20 +5255,11 @@ void plaqQuda(double plaq[3])
 /*
  * Performs a deep copy from the internal extendedGaugeResident field.
  */
-void copyExtendedResidentGaugeQuda(void* resident_gauge, QudaFieldLocation loc)
+void copyExtendedResidentGaugeQuda(void* resident_gauge)
 {
-  //profilePlaq.TPSTART(QUDA_PROFILE_TOTAL);
-
   if (!gaugePrecise) errorQuda("Cannot perform deep copy of resident gauge field as there is no resident gauge field");
-
-  cudaGaugeField *data = extendedGaugeResident ? extendedGaugeResident : createExtendedGauge(*gaugePrecise, R, profilePlaq);
-  extendedGaugeResident = data;
-
-  auto* io_gauge = (cudaGaugeField*)resident_gauge;
-
-  copyExtendedGauge(*io_gauge, *extendedGaugeResident, loc);
-
-  //profilePlaq.TPSTOP(QUDA_PROFILE_TOTAL);
+  extendedGaugeResident = extendedGaugeResident ? extendedGaugeResident : createExtendedGauge(*gaugePrecise, R, profilePlaq);
+  static_cast<GaugeField*>(resident_gauge)->copy(*extendedGaugeResident);
 }
 
 void performWuppertalnStep(void *h_out, void *h_in, QudaInvertParam *inv_param, unsigned int n_steps, double alpha)
diff --git a/lib/inv_eigcg_quda.cpp b/lib/inv_eigcg_quda.cpp
index 5ef8a14f61..a05062d877 100644
--- a/lib/inv_eigcg_quda.cpp
+++ b/lib/inv_eigcg_quda.cpp
@@ -174,51 +174,6 @@ namespace quda {
      return;
    }
 
-#ifdef MAGMA_LIB
-   //(supposed to be a pure) magma version: 
-   template <> void ComputeRitz<libtype::magma_lib>(EigCGArgs &args)
-   {
-     const int m = args.m;
-     const int k = args.k;
-     //Solve m dim eigenproblem:
-     args.ritzVecs = args.Tm;
-     Complex *evecm = static_cast<Complex*>( args.ritzVecs.data());
-     double  *evalm = static_cast<double *>( args.Tmvals.data());
-
-     cudaHostRegister(static_cast<void *>(evecm), m*m*sizeof(Complex),  cudaHostRegisterDefault);
-     magma_Xheev(evecm, m, m, evalm, sizeof(Complex));
-     //Solve m-1 dim eigenproblem:
-     DenseMatrix ritzVecsm1(args.Tm);
-     Complex *evecm1 = static_cast<Complex*>( ritzVecsm1.data());
-
-     cudaHostRegister(static_cast<void *>(evecm1), m*m*sizeof(Complex),  cudaHostRegisterDefault);
-     magma_Xheev(evecm1, (m-1), m, evalm, sizeof(Complex));
-     // fill 0s in mth element of old evecs:
-     for(int l = 1; l <= m ; l++) evecm1[l*m-1] = 0.0 ;
-     // Attach the first n_ev old evecs at the end of the n_ev latest ones:
-     memcpy(&evecm[k*m], evecm1, k*m*sizeof(Complex));
-     //?
-     // Orthogonalize the 2*n_ev (new+old) vectors evecm=QR:
-
-     MatrixXcd Q2k(MatrixXcd::Identity(m, 2*k));
-     HouseholderQR<MatrixXcd> ritzVecs2k_qr( Map<MatrixXcd, Unaligned >(args.ritzVecs.data(), m, 2*k) );
-     Q2k.applyOnTheLeft( ritzVecs2k_qr.householderQ() );
-
-     //2. Construct H = QH*Tm*Q :
-     args.H2k = Q2k.adjoint()*args.Tm*Q2k;
-
-     /* solve the small evecm1 2n_ev x 2n_ev eigenproblem */
-     SelfAdjointEigenSolver<MatrixXcd> es_h2k(args.H2k);
-     Block<MatrixXcd>(args.ritzVecs.derived(), 0, 0, m, 2*k) = Q2k * es_h2k.eigenvectors();
-     args.Tmvals.segment(0,2*k) = es_h2k.eigenvalues();//this is ok
-//?
-     cudaHostUnregister(evecm);
-     cudaHostUnregister(evecm1);
-   }
-#else
-  template <> void ComputeRitz<libtype::magma_lib>(EigCGArgs &) { errorQuda("Magma library was not built."); }
-#endif
-
   // set the required parameters for the inner solver
   static void fillEigCGInnerSolverParam(SolverParam &inner, const SolverParam &outer, bool use_sloppy_partial_accumulator = true)
   {
diff --git a/lib/inv_gmresdr_quda.cpp b/lib/inv_gmresdr_quda.cpp
index f9d8aa7705..72af57a159 100644
--- a/lib/inv_gmresdr_quda.cpp
+++ b/lib/inv_gmresdr_quda.cpp
@@ -94,43 +94,6 @@ namespace quda {
 
   template <libtype which_lib> void ComputeHarmonicRitz(GMResDRArgs &) { errorQuda("\nUnknown library type.\n"); }
 
-#ifdef MAGMA_LIB
-  template <> void ComputeHarmonicRitz<libtype::magma_lib>(GMResDRArgs &args)
-  {
-    DenseMatrix cH = args.H.block(0, 0, args.m, args.m).adjoint();
-    DenseMatrix Gk = args.H.block(0, 0, args.m, args.m);
-
-    VectorSet harVecs = MatrixXcd::Zero(args.m, args.m);
-    Vector harVals = VectorXcd::Zero(args.m);
-
-    Vector em = VectorXcd::Zero(args.m);
-
-    em(args.m - 1) = norm(args.H(args.m, args.m - 1));
-
-    cudaHostRegister(static_cast<void *>(cH.data()), args.m * args.m * sizeof(Complex), cudaHostRegisterDefault);
-    magma_Xgesv(static_cast<void *>(em.data()), args.m, args.m, static_cast<void *>(cH.data()), args.m, sizeof(Complex));
-    cudaHostUnregister(cH.data());
-
-    Gk.col(args.m - 1) += em;
-
-    cudaHostRegister(static_cast<void *>(Gk.data()), args.m * args.m * sizeof(Complex), cudaHostRegisterDefault);
-    magma_Xgeev(static_cast<void *>(Gk.data()), args.m, args.m, static_cast<void *>(harVecs.data()),
-                static_cast<void *>(harVals.data()), args.m, sizeof(Complex));
-    cudaHostUnregister(Gk.data());
-
-    std::vector<SortedEvals> sorted_evals;
-    sorted_evals.reserve(args.m);
-
-    for (int e = 0; e < args.m; e++) sorted_evals.push_back(SortedEvals(abs(harVals.data()[e]), e));
-    std::stable_sort(sorted_evals.begin(), sorted_evals.end(), SortedEvals::SelectSmall);
-
-    for (int e = 0; e < args.k; e++)
-      memcpy(args.ritzVecs.col(e).data(), harVecs.col(sorted_evals[e]._idx).data(), (args.m) * sizeof(Complex));
-  }
-#else
-  template <> void ComputeHarmonicRitz<libtype::magma_lib>(GMResDRArgs &) { errorQuda("Magma library was not built"); }
-#endif
-
   template <> void ComputeHarmonicRitz<libtype::eigen_lib>(GMResDRArgs &args)
   {
 
@@ -163,26 +126,6 @@ namespace quda {
 
   template <libtype which_lib> void ComputeEta(GMResDRArgs &) { errorQuda("\nUnknown library type.\n"); }
 
-#ifdef MAGMA_LIB
-  template <> void ComputeEta<libtype::magma_lib>(GMResDRArgs &args)
-  {
-    DenseMatrix Htemp(DenseMatrix::Zero(args.m + 1, args.m));
-    Htemp = args.H;
-
-    Complex *ctemp = static_cast<Complex *>(args.ritzVecs.col(0).data());
-    memcpy(ctemp, args.c, (args.m + 1) * sizeof(Complex));
-
-    cudaHostRegister(static_cast<void *>(Htemp.data()), (args.m + 1) * args.m * sizeof(Complex), cudaHostRegisterDefault);
-    magma_Xgels(static_cast<void *>(Htemp.data()), ctemp, args.m + 1, args.m, args.m + 1, sizeof(Complex));
-    cudaHostUnregister(Htemp.data());
-
-    memcpy(args.eta.data(), ctemp, args.m * sizeof(Complex));
-    memset(ctemp, 0, (args.m + 1) * sizeof(Complex));
-  }
-#else
-  template <> void ComputeEta<libtype::magma_lib>(GMResDRArgs &) { errorQuda("MAGMA library was not built"); }
-#endif
-
   template <> void ComputeEta<libtype::eigen_lib>(GMResDRArgs &args)
   {
 
diff --git a/lib/lattice_field.cpp b/lib/lattice_field.cpp
index a57f28030b..2cdba53689 100644
--- a/lib/lattice_field.cpp
+++ b/lib/lattice_field.cpp
@@ -19,8 +19,8 @@ namespace quda {
   MsgHandle* LatticeField::mh_recv_p2p_fwd[2][QUDA_MAX_DIM] { };
   MsgHandle* LatticeField::mh_recv_p2p_back[2][QUDA_MAX_DIM] { };
 
-  cudaEvent_t LatticeField::ipcCopyEvent[2][2][QUDA_MAX_DIM];
-  cudaEvent_t LatticeField::ipcRemoteCopyEvent[2][2][QUDA_MAX_DIM];
+  qudaEvent_t LatticeField::ipcCopyEvent[2][2][QUDA_MAX_DIM];
+  qudaEvent_t LatticeField::ipcRemoteCopyEvent[2][2][QUDA_MAX_DIM];
 
   void *LatticeField::ghost_pinned_send_buffer_h[2] = {nullptr, nullptr};
   void *LatticeField::ghost_pinned_send_buffer_hd[2] = {nullptr, nullptr};
@@ -210,9 +210,9 @@ namespace quda {
     volumeCB = (siteSubset == QUDA_FULL_SITE_SUBSET) ? volume / 2 : volume;
     localVolumeCB = (siteSubset == QUDA_FULL_SITE_SUBSET) ? localVolume / 2 : localVolume;
     stride = volumeCB + pad;
-  
+
     // for parity fields the factor of half is present for all surfaces dimensions except x, so add it manually
-    for (int i=0; i<nDim; i++) 
+    for (int i=0; i<nDim; i++)
       surfaceCB[i] = (siteSubset == QUDA_FULL_SITE_SUBSET || i==0) ? surface[i] / 2 : surface[i];
 
     // for 5-dimensional fields, we only communicate in the space-time dimensions
@@ -374,7 +374,6 @@ namespace quda {
     } // loop over dimension
 
     initComms = true;
-    checkCudaError();
   }
 
   void LatticeField::destroyComms()
@@ -409,121 +408,19 @@ namespace quda {
 
   }
 
-  void LatticeField::createIPCComms() {
+  void LatticeField::createIPCComms()
+  {
     if ( initIPCComms && !ghost_field_reset ) return;
 
     if (!initComms) errorQuda("Can only be called after create comms");
     if ((!ghost_recv_buffer_d[0] || !ghost_recv_buffer_d[1]) && comm_size() > 1)
       errorQuda("ghost_field appears not to be allocated");
 
-    // handles for obtained ghost pointers
-    cudaIpcMemHandle_t ipcRemoteGhostDestHandle[2][2][QUDA_MAX_DIM];
-
     for (int b=0; b<2; b++) {
-      for (int dim=0; dim<4; ++dim) {
-	if (comm_dim(dim)==1) continue;
-	for (int dir=0; dir<2; ++dir) {
-	  MsgHandle* sendHandle = nullptr;
-	  MsgHandle* receiveHandle = nullptr;
-	  int disp = (dir == 1) ? +1 : -1;
-
-          // first set up receive
-          if (comm_peer2peer_enabled(1-dir,dim)) {
-	    receiveHandle = comm_declare_receive_relative(&ipcRemoteGhostDestHandle[b][1-dir][dim],
-							  dim, -disp,
-							  sizeof(ipcRemoteGhostDestHandle[b][1-dir][dim]));
-	  }
-	  // now send
-          cudaIpcMemHandle_t ipcLocalGhostDestHandle;
-          if (comm_peer2peer_enabled(dir,dim)) {
-	    cudaIpcGetMemHandle(&ipcLocalGhostDestHandle, ghost_recv_buffer_d[b]);
-	    sendHandle = comm_declare_send_relative(&ipcLocalGhostDestHandle,
-						    dim, disp,
-						    sizeof(ipcLocalGhostDestHandle));
-	  }
-	  if (receiveHandle) comm_start(receiveHandle);
-	  if (sendHandle) comm_start(sendHandle);
-
-	  if (receiveHandle) comm_wait(receiveHandle);
-	  if (sendHandle) comm_wait(sendHandle);
-
-	  if (sendHandle) comm_free(sendHandle);
-	  if (receiveHandle) comm_free(receiveHandle);
-	}
-      }
-
-      checkCudaError();
-
-      // open the remote memory handles and set the send ghost pointers
-      for (int dim=0; dim<4; ++dim) {
-	if (comm_dim(dim)==1) continue;
-        // even if comm_dim(2) == 2, we might not have p2p enabled in both directions, so check this
-        const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
-	for (int dir=0; dir<num_dir; ++dir) {
-	  if (!comm_peer2peer_enabled(dir,dim)) continue;
-	  void **ghostDest = &(ghost_remote_send_buffer_d[b][dim][dir]);
-	  cudaIpcOpenMemHandle(ghostDest, ipcRemoteGhostDestHandle[b][dir][dim],
-			       cudaIpcMemLazyEnablePeerAccess);
-	}
-	if (num_dir == 1) ghost_remote_send_buffer_d[b][dim][1] = ghost_remote_send_buffer_d[b][dim][0];
-      }
-    } // buffer index
-
-    checkCudaError();
-
-    // handles for obtained events
-    cudaIpcEventHandle_t ipcRemoteEventHandle[2][2][QUDA_MAX_DIM];
-
-    // Note that no b index is necessary here
-    // Now communicate the event handles
-    for (int dim=0; dim<4; ++dim) {
-      if (comm_dim(dim)==1) continue;
-      for (int dir=0; dir<2; ++dir) {
-	for (int b=0; b<2; b++) {
-
-	  MsgHandle* sendHandle = NULL;
-	  MsgHandle* receiveHandle = NULL;
-	  int disp = (dir == 1) ? +1 : -1;
-
-	  // first set up receive
-	  if (comm_peer2peer_enabled(1-dir,dim)) {
-	    receiveHandle = comm_declare_receive_relative(&ipcRemoteEventHandle[b][1-dir][dim], dim, -disp,
-							  sizeof(ipcRemoteEventHandle[b][1-dir][dim]));
-	  }
-
-	  // now send
-          cudaIpcEventHandle_t ipcLocalEventHandle;
-          if (comm_peer2peer_enabled(dir,dim)) {
-	    cudaEventCreate(&ipcCopyEvent[b][dir][dim], cudaEventDisableTiming | cudaEventInterprocess);
-	    cudaIpcGetEventHandle(&ipcLocalEventHandle, ipcCopyEvent[b][dir][dim]);
-
-	    sendHandle = comm_declare_send_relative(&ipcLocalEventHandle, dim, disp,
-						    sizeof(ipcLocalEventHandle));
-	  }
-
-	  if (receiveHandle) comm_start(receiveHandle);
-	  if (sendHandle) comm_start(sendHandle);
-
-	  if (receiveHandle) comm_wait(receiveHandle);
-	  if (sendHandle) comm_wait(sendHandle);
-
-	  if (sendHandle) comm_free(sendHandle);
-	  if (receiveHandle) comm_free(receiveHandle);
-
-	} // buffer index
-      }
-    }
-
-    checkCudaError();
-
-    for (int dim=0; dim<4; ++dim) {
-      if (comm_dim(dim)==1) continue;
-      for (int dir=0; dir<2; ++dir) {
-	if (!comm_peer2peer_enabled(dir,dim)) continue;
-	for (int b=0; b<2; b++) {
-	  cudaIpcOpenEventHandle(&(ipcRemoteCopyEvent[b][dir][dim]), ipcRemoteEventHandle[b][dir][dim]);
-	}
-      }
+      // set remote send buffer to ghost receive buffers on neighboring processes
+      comm_create_neighbor_memory(ghost_remote_send_buffer_d[b], ghost_recv_buffer_d[b]);
+      // get remote events
+      comm_create_neighbor_event(ipcRemoteCopyEvent[b], ipcCopyEvent[b]);
     }
 
     // Create message handles for IPC synchronization
@@ -547,14 +444,13 @@ namespace quda {
 	}
       }
     }
-    checkCudaError();
 
     initIPCComms = true;
     ghost_field_reset = false;
   }
 
-  void LatticeField::destroyIPCComms() {
-
+  void LatticeField::destroyIPCComms()
+  {
     if (!initIPCComms) return;
 
     // ensure that all processes bring down their communicators
@@ -562,35 +458,28 @@ namespace quda {
     qudaDeviceSynchronize();
     comm_barrier();
 
+    for (int b = 0; b < 2; b++) {
+      comm_destroy_neighbor_memory(ghost_remote_send_buffer_d[b]);
+      comm_destroy_neighbor_event(ipcRemoteCopyEvent[b], ipcCopyEvent[b]);
+    }
+
     for (int dim=0; dim<4; ++dim) {
 
       if (comm_dim(dim)==1) continue;
-      const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
 
       for (int b=0; b<2; b++) {
 	if (comm_peer2peer_enabled(1,dim)) {
-	  if (mh_send_p2p_fwd[b][dim] || mh_recv_p2p_fwd[b][dim]) {
-	    cudaEventDestroy(ipcCopyEvent[b][1][dim]);
-	    // only close this handle if it doesn't alias the back ghost
-	    if (num_dir == 2) cudaIpcCloseMemHandle(ghost_remote_send_buffer_d[b][dim][1]);
-	  }
           if (mh_send_p2p_fwd[b][dim]) comm_free(mh_send_p2p_fwd[b][dim]);
           if (mh_recv_p2p_fwd[b][dim]) comm_free(mh_recv_p2p_fwd[b][dim]);
         }
 
 	if (comm_peer2peer_enabled(0,dim)) {
-	  if (mh_send_p2p_back[b][dim] || mh_recv_p2p_back[b][dim]) {
-	    cudaEventDestroy(ipcCopyEvent[b][0][dim]);
-	    cudaIpcCloseMemHandle(ghost_remote_send_buffer_d[b][dim][0]);
-	  }
           if (mh_send_p2p_back[b][dim]) comm_free(mh_send_p2p_back[b][dim]);
           if (mh_recv_p2p_back[b][dim]) comm_free(mh_recv_p2p_back[b][dim]);
         }
       } // buffer
     } // iterate over dim
 
-    checkCudaError();
-
     // local take down complete - now synchronize to ensure globally complete
     qudaDeviceSynchronize();
     comm_barrier();
@@ -600,19 +489,19 @@ namespace quda {
 
   bool LatticeField::ipcCopyComplete(int dir, int dim)
   {
-    return (cudaSuccess == cudaEventQuery(ipcCopyEvent[bufferIndex][dir][dim]) ? true : false);
+    return qudaEventQuery(ipcCopyEvent[bufferIndex][dir][dim]);
   }
 
   bool LatticeField::ipcRemoteCopyComplete(int dir, int dim)
   {
-    return (cudaSuccess == cudaEventQuery(ipcRemoteCopyEvent[bufferIndex][dir][dim]) ? true : false);
+    return qudaEventQuery(ipcRemoteCopyEvent[bufferIndex][dir][dim]);
   }
 
-  const cudaEvent_t& LatticeField::getIPCCopyEvent(int dir, int dim) const {
+  const qudaEvent_t& LatticeField::getIPCCopyEvent(int dir, int dim) const {
     return ipcCopyEvent[bufferIndex][dir][dim];
   }
 
-  const cudaEvent_t& LatticeField::getIPCRemoteCopyEvent(int dir, int dim) const {
+  const qudaEvent_t& LatticeField::getIPCRemoteCopyEvent(int dir, int dim) const {
     return ipcRemoteCopyEvent[bufferIndex][dir][dim];
   }
 
@@ -677,13 +566,13 @@ namespace quda {
     }
   }
 
-  QudaFieldLocation LatticeField::Location() const { 
+  QudaFieldLocation LatticeField::Location() const {
     QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION;
-    if (typeid(*this)==typeid(cudaCloverField) || 
+    if (typeid(*this)==typeid(cudaCloverField) ||
 	typeid(*this)==typeid(cudaColorSpinorField) ||
 	typeid(*this)==typeid(cudaGaugeField)) {
-      location = QUDA_CUDA_FIELD_LOCATION; 
-    } else if (typeid(*this)==typeid(cpuCloverField) || 
+      location = QUDA_CUDA_FIELD_LOCATION;
+    } else if (typeid(*this)==typeid(cpuCloverField) ||
 	       typeid(*this)==typeid(cpuColorSpinorField) ||
 	       typeid(*this)==typeid(cpuGaugeField)) {
       location = QUDA_CPU_FIELD_LOCATION;
@@ -696,7 +585,7 @@ namespace quda {
   void LatticeField::read(char *) {
     errorQuda("Not implemented");
   }
-  
+
   void LatticeField::write(char *) {
     errorQuda("Not implemented");
   }
@@ -710,7 +599,7 @@ namespace quda {
       const GaugeField &gField = static_cast<const GaugeField&>(*this);
       if (gField.Order() == 2 || gField.Order() == 4)
 	return static_cast<int>(gField.Order());
-    } else if (typeid(*this) == typeid(const cudaCloverField)) { 
+    } else if (typeid(*this) == typeid(const cudaCloverField)) {
       const CloverField &cField = static_cast<const CloverField&>(*this);
       if (cField.Order() == 2 || cField.Order() == 4)
 	return static_cast<int>(cField.Order());
@@ -725,7 +614,7 @@ namespace quda {
   {
     output << "nDim = " << param.nDim << std::endl;
     for (int i=0; i<param.nDim; i++) {
-      output << "x[" << i << "] = " << param.x[i] << std::endl;    
+      output << "x[" << i << "] = " << param.x[i] << std::endl;
     }
     output << "pad = " << param.pad << std::endl;
     output << "precision = " << param.Precision() << std::endl;
diff --git a/lib/momentum.cu b/lib/momentum.cu
index 80aee15b5c..3d6325b52d 100644
--- a/lib/momentum.cu
+++ b/lib/momentum.cu
@@ -125,7 +125,7 @@ namespace quda {
     void apply(const qudaStream_t &stream)
     {
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      launch<MomUpdate, max_reducer2>(force_max, tp, stream, arg);
+      launch<MomUpdate>(force_max, tp, stream, arg);
     }
 
     void preTune() { arg.mom.save();}
diff --git a/lib/multigrid.cpp b/lib/multigrid.cpp
index 87cde62492..99cd3527e8 100644
--- a/lib/multigrid.cpp
+++ b/lib/multigrid.cpp
@@ -2,6 +2,7 @@
 
 #include <multigrid.h>
 #include <tune_quda.h>
+#include <random_quda.h>
 #include <vector_io.h>
 
 // for building the KD inverse op
@@ -79,7 +80,6 @@ namespace quda
     }
 
     rng = new RNG(*param.B[0], 1234);
-    rng->Init();
 
     if (param.transfer_type == QUDA_TRANSFER_AGGREGATE) {
       if (param.level < param.Nlevel - 1) {
@@ -699,7 +699,6 @@ namespace quda
     }
 
     if (rng) {
-      rng->Release();
       delete rng;
     }
 
diff --git a/lib/pgauge_heatbath.cu b/lib/pgauge_heatbath.cu
index 9ce7a3d367..cfae69874d 100644
--- a/lib/pgauge_heatbath.cu
+++ b/lib/pgauge_heatbath.cu
@@ -1,4 +1,5 @@
 #include <quda_internal.h>
+#include <timer.h>
 #include <tune_quda.h>
 #include <gauge_field.h>
 #include <comm_quda.h>
@@ -14,7 +15,7 @@ namespace quda {
   class GaugeHB : TunableKernel1D {
     GaugeField &U;
     Float beta;
-    RNG &rngstate;
+    RNG &rng;
     int mu;
     int parity;
     bool heatbath; // true = heatbath, false = over relaxation
@@ -22,11 +23,11 @@ namespace quda {
     unsigned int minThreads() const { return U.LocalVolumeCB(); }
 
   public:
-    GaugeHB(GaugeField &U, double beta, RNG &rngstate) :
+    GaugeHB(GaugeField &U, double beta, RNG &rng) :
       TunableKernel1D(U),
       U(U),
       beta(static_cast<Float>(beta)),
-      rngstate(rngstate),
+      rng(rng),
       mu(0),
       parity(0),
       heatbath(false)
@@ -47,22 +48,22 @@ namespace quda {
     {
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       if (heatbath) {
-        MonteArg<Float, nColor, recon, true> arg(U, beta, rngstate, mu, parity);
+        MonteArg<Float, nColor, recon, true> arg(U, beta, rng.State(), mu, parity);
         launch<HB>(tp, stream, arg);
       } else {
-        MonteArg<Float, nColor, recon, false> arg(U, beta, rngstate, mu, parity);
+        MonteArg<Float, nColor, recon, false> arg(U, beta, rng.State(), mu, parity);
         launch<HB>(tp, stream, arg);
       }
     }
 
     void preTune() {
       U.backup();
-      if (heatbath) rngstate.backup();
+      if (heatbath) rng.backup();
     }
 
     void postTune() {
       U.restore();
-      if (heatbath) rngstate.restore();
+      if (heatbath) rng.restore();
     }
 
     long long flops() const
diff --git a/lib/pgauge_init.cu b/lib/pgauge_init.cu
index 8c5a4fd125..ceeed78494 100644
--- a/lib/pgauge_init.cu
+++ b/lib/pgauge_init.cu
@@ -52,7 +52,7 @@ namespace quda {
     void apply(const qudaStream_t &stream)
     {
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
-      launch<HotStart>(tp, stream, InitGaugeHotArg<Float, nColors, recon>(U, rng));
+      launch<HotStart>(tp, stream, InitGaugeHotArg<Float, nColors, recon>(U, rng.State()));
     }
 
     void preTune() { rng.backup(); }
diff --git a/lib/random.cu b/lib/random.cu
index 2c532eec36..774f60fae7 100644
--- a/lib/random.cu
+++ b/lib/random.cu
@@ -1,79 +1,44 @@
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-
-#include <quda_internal.h>
-#include <tune_quda.h>
-#include <random_helper.h>
-#include <comm_quda.h>
-#include <index_helper.cuh>
+#include <util_quda.h>
+#include <random_quda.h>
+#include <malloc_quda.h>
+#include <tunable_nd.h>
+#include <kernels/random_init.cuh>
 
 namespace quda {
 
-  dim3 GetGridDim(size_t threads, size_t size) { return dim3((size + threads - 1) / threads, 1, 1); }
+  class RNGInit : public TunableKernel2D {
+
+    RNG &rng;
+    const LatticeField &meta;
+    unsigned long long seed;
+    unsigned int minThreads() const { return meta.VolumeCB(); }
+    bool tuneSharedBytes() const { return false; }
 
-  struct rngArg {
-    int commDim[QUDA_MAX_DIM];
-    int commCoord[QUDA_MAX_DIM];
-    int X[QUDA_MAX_DIM];
-    rngArg(const int X_[]) {
-      for (int i=0; i<4; i++) {
-        commDim[i] = comm_dim(i);
-        commCoord[i] = comm_coord(i);
-        X[i] = X_[i];
-      }
+  public:
+    RNGInit(RNG &rng, const LatticeField &meta, unsigned long long seed) :
+      TunableKernel2D(meta, meta.SiteSubset()),
+      rng(rng),
+      meta(meta),
+      seed(seed)
+    {
+      apply(device::get_default_stream());
     }
-  };
 
-  /**
-     @brief CUDA kernel to initialize CURAND RNG states
-     @param state CURAND RNG state array
-     @param seed initial seed for RNG
-     @param size size of the CURAND RNG state array
-     @param arg Metadata needed for computing multi-gpu offsets
-  */
-  __global__ void kernel_random(RNGState *state, unsigned long long seed, int size_cb, rngArg arg)
-  {
-    QUDA_RT_CONSTS;
-    int id = blockIdx.x * blockDim.x + threadIdx.x;
-    int parity = blockIdx.y * blockDim.y + threadIdx.y;
-    if (id < size_cb) {
-      // Each thread gets same seed, a different sequence number, no offset
-      int x[4];
-      getCoords(x, id, arg.X, parity);
-      for (int i = 0; i < 4; i++) x[i] += arg.commCoord[i] * arg.X[i];
-      int idd
-        = (((x[3] * arg.commDim[2] * arg.X[2] + x[2]) * arg.commDim[1] * arg.X[1]) + x[1]) * arg.commDim[0] * arg.X[0]
-        + x[0];
-      random_init(seed, idd, 0, state[parity * size_cb + id]);
+    void apply(const qudaStream_t &stream)
+    {
+      TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
+      launch_device<init_random>(tp, stream, rngArg(rng.State(), seed, meta));
     }
-  }
 
-  /**
-     @brief Call CUDA kernel to initialize CURAND RNG states
-     @param state CURAND RNG state array
-     @param seed initial seed for RNG
-     @param size_cb Checkerboarded size of the CURAND RNG state array
-     @param n_parity Number of parities (1 or 2)
-     @param X array of lattice dimensions
-  */
-  void launch_kernel_random(RNGState *state, unsigned long long seed, int size_cb, int n_parity, int X[4])
-  {
-    TuneParam tp;
-    tp.block = dim3(128, 1, 1);
-    tp.grid = GetGridDim(tp.block.x, size_cb);
-    rngArg arg(X);
-    tp.block.y = n_parity;
-    qudaLaunchKernel(kernel_random, tp, device::get_default_stream(), state, seed, size_cb, arg);
-  }
+    long long flops() const { return 0; }
+    long long bytes() const { return 0; }
+  };
 
   RNG::RNG(const LatticeField &meta, unsigned long long seedin) :
-    seed(seedin),
     size(meta.Volume()),
-    size_cb(meta.VolumeCB())
+    state((RNGState *)device_malloc(size * sizeof(RNGState)), [](RNGState *ptr){ device_free(ptr); } ),
+    seed(seedin)
   {
-    state = nullptr;
-    for (int i = 0; i < 4; i++) X[i] = meta.X()[i];
 #if defined(XORWOW)
     if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Using randStateXORWOW\n");
 #elif defined(RG32k3a)
@@ -81,73 +46,25 @@ namespace quda {
 #else
     if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Using randStateMRG32k3a\n");
 #endif
-  }
 
-  RNG::RNG(const LatticeFieldParam &param, unsigned long long seedin) : seed(seedin), size(1), size_cb(1)
-  {
-    state = nullptr;
-    for (int i = 0; i < 4; i++) {
-      X[i] = param.x[i];
-      size *= X[i];
-    }
-    size_cb = size / param.siteSubset;
+    if (getVerbosity() >= QUDA_DEBUG_VERBOSE)
+      printfQuda("Allocated array of random numbers with size: %.2f MB\n",
+                 size * sizeof(RNGState) / (float)(1048576));
 
-#if defined(XORWOW)
-    if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Using curandStateXORWOW\n");
-#elif defined(RG32k3a)
-    if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Using curandStateMRG32k3a\n");
-#else
-    if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Using curandStateMRG32k3a\n");
-#endif
-  }
-
-  /**
-     @brief Initialize CURAND RNG states
-  */
-  void RNG::Init() {
-    AllocateRNG();
-    launch_kernel_random(state, seed, size_cb, size / size_cb, X);
-  }
-
-  /**
-     @brief Allocate Device memory for CURAND RNG states
-  */
-  void RNG::AllocateRNG() {
-    if (size > 0 && state == nullptr) {
-      state = (RNGState *)device_malloc(size * sizeof(RNGState));
-      qudaMemset(state, 0, size * sizeof(RNGState));
-      if (getVerbosity() >= QUDA_DEBUG_VERBOSE)
-        printfQuda("Allocated array of random numbers with size: %.2f MB\n",
-                   size * sizeof(RNGState) / (float)(1048576));
-    } else {
-      errorQuda("Array of random numbers not allocated, array size: %d !\nExiting...\n", size);
-    }
-  }
-
-  /**
-     @brief Release Device memory for CURAND RNG states
-  */
-  void RNG::Release() {
-    if (size > 0 && state != nullptr) {
-      device_free(state);
-      if (getVerbosity() >= QUDA_DEBUG_VERBOSE)
-        printfQuda("Free array of random numbers with size: %.2f MB\n", size * sizeof(RNGState) / (float)(1048576));
-      size = 0;
-      state = NULL;
-    }
+    RNGInit(*this, meta, seed);
   }
 
   /*! @brief Backup CURAND array states initialization */
   void RNG::backup()
   {
     backup_state = (RNGState *)safe_malloc(size * sizeof(RNGState));
-    qudaMemcpy(backup_state, state, size * sizeof(RNGState), qudaMemcpyDeviceToHost);
+    qudaMemcpy(backup_state, state.get(), size * sizeof(RNGState), qudaMemcpyDeviceToHost);
   }
 
   /*! @brief Restore CURAND array states initialization */
   void RNG::restore()
   {
-    qudaMemcpy(state, backup_state, size * sizeof(RNGState), qudaMemcpyHostToDevice);
+    qudaMemcpy(state.get(), backup_state, size * sizeof(RNGState), qudaMemcpyHostToDevice);
     host_free(backup_state);
   }
 
diff --git a/lib/reduce_helper.cu b/lib/reduce_helper.cu
index b89b278070..a978042953 100644
--- a/lib/reduce_helper.cu
+++ b/lib/reduce_helper.cu
@@ -1,7 +1,8 @@
 #include <quda_internal.h>
 #include <malloc_quda.h>
 #include <reduce_helper.h>
-#include <tune_quda.h>
+#include <tunable_nd.h>
+#include <kernels/reduce_init.cuh>
 
 // These are used for reduction kernels
 static device_reduce_t *d_reduce = nullptr;
@@ -9,7 +10,7 @@ static device_reduce_t *h_reduce = nullptr;
 static device_reduce_t *hd_reduce = nullptr;
 
 static count_t *reduce_count = nullptr;
-static cudaEvent_t reduceEnd;
+static qudaEvent_t reduceEnd;
 
 namespace quda
 {
@@ -22,7 +23,7 @@ namespace quda
     void *get_mapped_buffer() { return hd_reduce; }
     void *get_host_buffer() { return h_reduce; }
     count_t *get_count() { return reduce_count; }
-    cudaEvent_t &get_event() { return reduceEnd; }
+    qudaEvent_t &get_event() { return reduceEnd; }
 
     size_t buffer_size()
     {
@@ -48,11 +49,23 @@ namespace quda
       return bytes;
     }
 
-    // need to use placement new constructor to initialize the atomic counters
-    template <typename T> __global__ void init_count(T *counter)
-    {
-      for (int i = 0; i < max_n_reduce(); i++) new (counter + i) T {0};
-    }
+    template <typename T>
+    struct init_reduce : public TunableKernel1D {
+      T *reduce_count;
+      long long bytes() const { return max_n_reduce() * sizeof(T); }
+      unsigned int minThreads() const { return max_n_reduce(); }
+
+      init_reduce(T *reduce_count) :
+        TunableKernel1D(max_n_reduce()),
+        reduce_count(reduce_count)
+      { apply(device::get_default_stream()); }
+
+      void apply(const qudaStream_t &stream)
+      {
+        auto tp = tuneLaunch(*this, getTuning(), getVerbosity());
+        launch_device<init_count>(tp, stream, init_arg<T>(reduce_count));
+      }
+    };
 
     void init()
     {
@@ -78,21 +91,15 @@ namespace quda
 
       if (!reduce_count) {
         reduce_count = static_cast<count_t *>(device_malloc(max_n_reduce() * sizeof(decltype(*reduce_count))));
-        TuneParam tp;
-        tp.grid = dim3(1, 1, 1);
-        tp.block = dim3(1, 1, 1);
-
-        qudaLaunchKernel(init_count<count_t>, tp, device::get_default_stream(), reduce_count);
+        init_reduce<count_t> init(reduce_count);
       }
 
-      cudaEventCreateWithFlags(&reduceEnd, cudaEventDisableTiming);
-
-      checkCudaError();
+      reduceEnd = qudaEventCreate();
     }
 
     void destroy()
     {
-      cudaEventDestroy(reduceEnd);
+      qudaEventDestroy(reduceEnd);
 
       if (reduce_count) {
         device_free(reduce_count);
diff --git a/lib/restrictor.cu b/lib/restrictor.cu
index 1da207a8a8..82e4b9269e 100644
--- a/lib/restrictor.cu
+++ b/lib/restrictor.cu
@@ -11,7 +11,11 @@ namespace quda {
     // size is equal to number of fine points per aggregate, rounded
     // up to a whole power of two.  So for example, 2x2x2x2 and
     // 3x3x3x1 aggregation would both use the same block size 32
+#ifndef QUDA_FAST_COMPILE_REDUCE
     static constexpr std::array<unsigned int, 6> block = {32, 64, 128, 256, 512, 1024};
+#else
+    static constexpr std::array<unsigned int, 1> block = {1024};
+#endif
 
     /**
        @brief Return the first power of two block that is larger than the required size
@@ -24,7 +28,11 @@ namespace quda {
     }
   };
 
+#ifndef QUDA_FAST_COMPILE_REDUCE
   constexpr std::array<unsigned int, 6> Aggregates::block;
+#else
+  constexpr std::array<unsigned int, 1> Aggregates::block;
+#endif
 
   template <typename Float, typename vFloat, int fineSpin, int fineColor, int coarseSpin, int coarseColor>
   class RestrictLaunch : public TunableBlock2D {
diff --git a/lib/spinor_noise.cu b/lib/spinor_noise.cu
index fdf8569a63..37729db098 100644
--- a/lib/spinor_noise.cu
+++ b/lib/spinor_noise.cu
@@ -1,4 +1,5 @@
 #include <color_spinor_field.h>
+#include <random_quda.h>
 #include <tunable_nd.h>
 #include <kernels/spinor_noise.cuh>
 
@@ -26,10 +27,10 @@ namespace quda {
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       switch (type) {
       case QUDA_NOISE_GAUSS:
-        launch<NoiseSpinor>(tp, stream, SpinorNoiseArg<real, Ns, Nc, order, QUDA_NOISE_GAUSS>(v, rng));
+        launch<NoiseSpinor>(tp, stream, SpinorNoiseArg<real, Ns, Nc, order, QUDA_NOISE_GAUSS>(v, rng.State()));
         break;
       case QUDA_NOISE_UNIFORM:
-        launch<NoiseSpinor>(tp, stream, SpinorNoiseArg<real, Ns, Nc, order, QUDA_NOISE_UNIFORM>(v, rng));
+        launch<NoiseSpinor>(tp, stream, SpinorNoiseArg<real, Ns, Nc, order, QUDA_NOISE_UNIFORM>(v, rng.State()));
         break;
       default: errorQuda("Noise type %d not implemented", type);
       }
@@ -132,9 +133,7 @@ namespace quda {
   void spinorNoise(ColorSpinorField &src, unsigned long long seed, QudaNoiseType type)
   {
     RNG *randstates = new RNG(src, seed);
-    randstates->Init();
     spinorNoise(src, *randstates, type);
-    randstates->Release();
     delete randstates;
   }
 
diff --git a/lib/targets/cuda/CMakeLists.txt b/lib/targets/cuda/CMakeLists.txt
index b7b7b0e6b9..2c1c3c5b21 100644
--- a/lib/targets/cuda/CMakeLists.txt
+++ b/lib/targets/cuda/CMakeLists.txt
@@ -1,5 +1,5 @@
 # add target specific files / options
-target_sources(quda_cpp PRIVATE quda_api.cpp device.cpp malloc.cpp tune.cpp blas_lapack_cublas.cpp)
+target_sources(quda_cpp PRIVATE quda_api.cpp device.cpp malloc.cpp blas_lapack_cublas.cpp comm_target.cpp)
 
 if(QUDA_JITIFY)
   target_sources(quda_cpp PRIVATE jitify_helper.cpp)
diff --git a/lib/targets/cuda/blas_lapack_cublas.cpp b/lib/targets/cuda/blas_lapack_cublas.cpp
index 36b2672c70..ead59bb383 100644
--- a/lib/targets/cuda/blas_lapack_cublas.cpp
+++ b/lib/targets/cuda/blas_lapack_cublas.cpp
@@ -1,5 +1,6 @@
 #include <complex.h>
 #include <blas_lapack.h>
+#include <timer.h>
 #ifdef NATIVE_LAPACK_LIB
 #include <cublas_v2.h>
 #include <malloc_quda.h>
diff --git a/lib/targets/cuda/device.cpp b/lib/targets/cuda/device.cpp
index 4b7d2081df..81ee666fc3 100644
--- a/lib/targets/cuda/device.cpp
+++ b/lib/targets/cuda/device.cpp
@@ -2,6 +2,7 @@
 #include <cuda_profiler_api.h>
 #include <util_quda.h>
 #include <quda_internal.h>
+#include <quda_cuda_api.h>
 
 #ifdef QUDA_NVML
 #include <nvml.h>
@@ -15,6 +16,9 @@ static cudaDeviceProp deviceProp;
 static cudaStream_t *streams;
 static const int Nstream = 9;
 
+#define CHECK_CUDA_ERROR(func)                                          \
+  cuda::set_runtime_error(func, #func, __func__, __FILE__, __STRINGIFY__(__LINE__));
+
 namespace quda
 {
 
@@ -29,11 +33,11 @@ namespace quda
       initialized = true;
 
       int driver_version;
-      cudaDriverGetVersion(&driver_version);
+      CHECK_CUDA_ERROR(cudaDriverGetVersion(&driver_version));
       printfQuda("CUDA Driver version = %d\n", driver_version);
 
       int runtime_version;
-      cudaRuntimeGetVersion(&runtime_version);
+      CHECK_CUDA_ERROR(cudaRuntimeGetVersion(&runtime_version));
       printfQuda("CUDA Runtime version = %d\n", runtime_version);
 
 #ifdef QUDA_NVML
@@ -48,25 +52,15 @@ namespace quda
       if (NVML_SUCCESS != result) errorQuda("NVML Shutdown failed with error %d", result);
 #endif
 
-      int deviceCount;
-      cudaGetDeviceCount(&deviceCount);
-      if (deviceCount == 0) {
-        errorQuda("No CUDA devices found");
-      }
-
-      for (int i = 0; i < deviceCount; i++) {
-        cudaGetDeviceProperties(&deviceProp, i);
-        checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
+      for (int i = 0; i < get_device_count(); i++) {
+        CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, i));
         if (getVerbosity() >= QUDA_SUMMARIZE) {
           printfQuda("Found device %d: %s\n", i, deviceProp.name);
         }
       }
 
-      cudaGetDeviceProperties(&deviceProp, dev);
-      checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
-      if (deviceProp.major < 1) {
-        errorQuda("Device %d does not support CUDA", dev);
-      }
+      CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, dev));
+      if (deviceProp.major < 1) errorQuda("Device %d does not support CUDA", dev);
 
       // Check GPU and QUDA build compatibiliy
       // 4 cases:
@@ -77,7 +71,7 @@ namespace quda
 
       const int my_major = __COMPUTE_CAPABILITY__ / 100;
       const int my_minor = (__COMPUTE_CAPABILITY__  - my_major * 100) / 10;
-      // b) UDA was compiled for a higher compute capability
+      // b) QUDA was compiled for a higher compute capability
       if (deviceProp.major * 100 + deviceProp.minor * 10 < __COMPUTE_CAPABILITY__)
         errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. ** \n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
 
@@ -95,12 +89,13 @@ namespace quda
         warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- This might result in a lower performance. Please consider adjusting QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
       }
 
+      if (!deviceProp.unifiedAddressing) errorQuda("Device %d does not support unified addressing", dev);
+
       if (getVerbosity() >= QUDA_SUMMARIZE) {
         printfQuda("Using device %d: %s\n", dev, deviceProp.name);
       }
 #ifndef USE_QDPJIT
-      cudaSetDevice(dev);
-      checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
+      CHECK_CUDA_ERROR(cudaSetDevice(dev));
 #endif
 
 #ifdef NUMA_NVML
@@ -112,21 +107,27 @@ namespace quda
       }
 #endif
 
-      cudaDeviceSetCacheConfig(cudaFuncCachePreferL1);
+      CHECK_CUDA_ERROR(cudaDeviceSetCacheConfig(cudaFuncCachePreferL1));
       //cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte);
       // cudaGetDeviceProperties(&deviceProp, dev);
     }
 
-    void print_device_properties()
+    int get_device_count()
     {
+      static int device_count = 0;
+      if (device_count == 0) {
+        CHECK_CUDA_ERROR(cudaGetDeviceCount(&device_count));
+        if (device_count == 0) errorQuda("No CUDA devices found");
+      }
+      return device_count;
+    }
 
-      int dev_count;
-      cudaGetDeviceCount(&dev_count);
-      int device;
-      for (device = 0; device < dev_count; device++) {
+    void print_device_properties()
+    {
+      for (int device = 0; device < get_device_count(); device++) {
 
         // cudaDeviceProp deviceProp;
-        cudaGetDeviceProperties(&deviceProp, device);
+        CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, device));
         printfQuda("%d - name:                    %s\n", device, deviceProp.name);
         printfQuda("%d - totalGlobalMem:          %lu bytes ( %.2f Gbytes)\n", device, deviceProp.totalGlobalMem,
                    deviceProp.totalGlobalMem / (float)(1024 * 1024 * 1024));
@@ -186,20 +187,17 @@ namespace quda
 
       int greatestPriority;
       int leastPriority;
-      cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority);
+      CHECK_CUDA_ERROR(cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority));
       for (int i=0; i<Nstream-1; i++) {
-        cudaStreamCreateWithPriority(&streams[i], cudaStreamDefault, greatestPriority);
+        CHECK_CUDA_ERROR(cudaStreamCreateWithPriority(&streams[i], cudaStreamDefault, greatestPriority));
       }
-      cudaStreamCreateWithPriority(&streams[Nstream-1], cudaStreamDefault, leastPriority);
-
-      checkCudaError();
-
+      CHECK_CUDA_ERROR(cudaStreamCreateWithPriority(&streams[Nstream-1], cudaStreamDefault, leastPriority));
     }
 
     void destroy()
     {
       if (streams) {
-        for (int i=0; i<Nstream; i++) cudaStreamDestroy(streams[i]);
+        for (int i=0; i<Nstream; i++) CHECK_CUDA_ERROR(cudaStreamDestroy(streams[i]));
         delete []streams;
         streams = nullptr;
       }
@@ -207,7 +205,7 @@ namespace quda
       char *device_reset_env = getenv("QUDA_DEVICE_RESET");
       if (device_reset_env && strcmp(device_reset_env,"1") == 0) {
         // end this CUDA context
-        cudaDeviceReset();
+        CHECK_CUDA_ERROR(cudaDeviceReset());
       }
     }
 
@@ -258,7 +256,7 @@ namespace quda
     {
       static int max_shared_bytes = 0;
       if (!max_shared_bytes)
-        cudaDeviceGetAttribute(&max_shared_bytes, cudaDevAttrMaxSharedMemoryPerBlockOptin, comm_gpuid());
+        CHECK_CUDA_ERROR(cudaDeviceGetAttribute(&max_shared_bytes, cudaDevAttrMaxSharedMemoryPerBlockOptin, comm_gpuid()));
       return max_shared_bytes;
     }
 
@@ -276,7 +274,8 @@ namespace quda
     {
 #if CUDA_VERSION >= 11000
       static int max_blocks_per_sm = 0;
-      if (!max_blocks_per_sm) cudaDeviceGetAttribute(&max_blocks_per_sm, cudaDevAttrMaxBlocksPerMultiprocessor, comm_gpuid());
+      if (!max_blocks_per_sm)
+        CHECK_CUDA_ERROR(cudaDeviceGetAttribute(&max_blocks_per_sm, cudaDevAttrMaxBlocksPerMultiprocessor, comm_gpuid()));
       return max_blocks_per_sm;
 #else
       // these variables are taken from Table 14 of the CUDA 10.2 prgramming guide
diff --git a/lib/targets/cuda/jitify_helper.cpp b/lib/targets/cuda/jitify_helper.cpp
index 4a2c0629d4..f956c34faa 100644
--- a/lib/targets/cuda/jitify_helper.cpp
+++ b/lib/targets/cuda/jitify_helper.cpp
@@ -1,5 +1,6 @@
 #include <tune_quda.h>
 #include <quda_api.h>
+#include <quda_cuda_api.h>
 
 #ifdef HOST_DEBUG
 
@@ -69,8 +70,6 @@ namespace quda {
     }
   }
 
-  void qudaSetErrorString(const std::string &);
-
   qudaError_t launch_jitify(const std::string &file, const std::string &kernel,
                             const std::vector<std::string> &template_args,
                             const TuneParam &tp, const qudaStream_t &stream,
@@ -80,7 +79,8 @@ namespace quda {
     std::string kernel_file(std::string("kernels/") + file);
     create_jitify_program_v2(kernel_file);
 
-    auto instance = program_map[kernel_file]->kernel(kernel).instantiate(template_args);
+    std::string kernel_name(std::string("quda::") + kernel);
+    auto instance = program_map[kernel_file]->kernel(kernel_name).instantiate(template_args);
 
     if (tp.set_max_shared_bytes && device::max_dynamic_shared_memory() > device::max_default_shared_memory()) {
       instance.set_func_attribute(CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, 100);
@@ -97,14 +97,8 @@ namespace quda {
     auto configured_instance = instance.configure(tp.grid, tp.block, tp.shared_bytes, device::get_cuda_stream(stream));
     auto error = configured_instance.launch(arg_ptrs, arg_types);
 
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorString(error, &str);
-      qudaSetErrorString(std::string(str));
-      if (!activeTuning()) { // error only if not tuning
-        errorQuda("jitify returned error %s", str);
-      }
-    }
+    if (error != CUDA_SUCCESS)
+      cuda::set_driver_error(error, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
 
     return error == CUDA_SUCCESS ? QUDA_SUCCESS : QUDA_ERROR;
   }
diff --git a/lib/targets/cuda/malloc.cpp b/lib/targets/cuda/malloc.cpp
index b2a1d86fda..d4ac8e62ba 100644
--- a/lib/targets/cuda/malloc.cpp
+++ b/lib/targets/cuda/malloc.cpp
@@ -477,7 +477,6 @@ namespace quda
 
   QudaFieldLocation get_pointer_location(const void *ptr)
   {
-
     CUpointer_attribute attribute[] = {CU_POINTER_ATTRIBUTE_MEMORY_TYPE};
     CUmemorytype mem_type;
     void *data[] = {&mem_type};
@@ -510,6 +509,23 @@ namespace quda
     return device;
   }
 
+  void register_pinned_(const char *func, const char *file, int line, void *ptr, size_t bytes)
+  {
+    auto error = cudaHostRegister(ptr, bytes, cudaHostRegisterDefault);
+    if (error != cudaSuccess) {
+      errorQuda("cudaHostRegister failed with error %s (%s:%d in %s()",
+                cudaGetErrorString(error), file, line, func);
+    }
+  }
+
+  void unregister_pinned_(const char *func, const char *file, int line, void *ptr)
+  {
+    auto error = cudaHostUnregister(ptr);
+    if (error != cudaSuccess) {
+      errorQuda("cudaHostUnregister failed with error %s (%s:%d in %s()",
+                cudaGetErrorString(error), file, line, func);
+    }
+  }
 
   namespace pool
   {
diff --git a/lib/targets/cuda/quda_api.cpp b/lib/targets/cuda/quda_api.cpp
index a53053891a..49caefeaba 100644
--- a/lib/targets/cuda/quda_api.cpp
+++ b/lib/targets/cuda/quda_api.cpp
@@ -2,6 +2,7 @@
 #include <tune_quda.h>
 #include <uint_to_char.h>
 #include <quda_internal.h>
+#include <timer.h>
 #include <device.h>
 
 // if this macro is defined then we use the driver API, else use the
@@ -22,6 +23,49 @@
 
 namespace quda {
 
+  static qudaError_t last_error = QUDA_SUCCESS;
+  static std::string last_error_str("CUDA_SUCCESS");
+
+  qudaError_t qudaGetLastError()
+  {
+    auto rtn = last_error;
+    last_error = QUDA_SUCCESS;
+    return rtn;
+  }
+
+  std::string qudaGetLastErrorString()
+  {
+    auto rtn = last_error_str;
+    last_error_str = QUDA_SUCCESS;
+    return rtn;
+  }
+
+  namespace cuda {
+
+    void set_runtime_error(cudaError_t error, const char *api_func, const char *func, const char *file, const char *line,
+                           bool allow_error = false)
+    {
+      if (error == cudaSuccess) return;
+      last_error = error == cudaSuccess ? QUDA_SUCCESS : QUDA_ERROR;
+      last_error_str = cudaGetErrorString(error);
+      if (!allow_error) errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, cudaGetErrorString(error), file, line, func);
+    }
+
+    void set_driver_error(CUresult error, const char *api_func, const char *func, const char *file, const char *line,
+                          bool allow_error = false)
+    {
+      if (error == CUDA_SUCCESS) return;
+      last_error = error == CUDA_SUCCESS ? QUDA_SUCCESS : QUDA_ERROR;
+      const char *str;
+      cuGetErrorName(error, &str);
+      last_error_str = str;
+      if (!allow_error) errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, str, file, line, func);
+    }
+
+  }
+
+  using namespace cuda;
+
   // Agnostic way to return a cuda API flag
   namespace {
     inline
@@ -39,7 +83,7 @@ namespace quda {
       }
     }
   }
-  
+
   // No need to abstract these across the library so keep these definitions local to CUDA target
 
   /**
@@ -90,7 +134,7 @@ namespace quda {
     // no driver API variant here since we have C++ functions
     PROFILE(cudaError_t error = cudaLaunchKernel(func, tp.grid, tp.block, args, tp.shared_bytes, device::get_cuda_stream(stream)),
             QUDA_PROFILE_LAUNCH_KERNEL);
-    if (error != cudaSuccess && !activeTuning()) errorQuda("(CUDA) %s", cudaGetErrorString(error));
+    set_runtime_error(error, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
     return error == cudaSuccess ? QUDA_SUCCESS : QUDA_ERROR;
   }
 
@@ -105,6 +149,9 @@ namespace quda {
     const bool async;
     const char *name;
     const bool active_tuning;
+    const char *func;
+    const char *file;
+    const char *line;
 
     unsigned int sharedBytesPerThread() const { return 0; }
     unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
@@ -112,7 +159,8 @@ namespace quda {
   public:
     inline QudaMem(void *dst, const void *src, size_t count, cudaMemcpyKind kind, const qudaStream_t &stream,
                    bool async, const char *func, const char *file, const char *line) :
-      dst(dst), src(src), count(count), value(0), copy(true), kind(kind), async(async), active_tuning(activeTuning())
+      dst(dst), src(src), count(count), value(0), copy(true), kind(kind), async(async), active_tuning(activeTuning()),
+      func(func), file(file), line(line)
     {
       if (!async) {
         switch (kind) {
@@ -170,21 +218,23 @@ namespace quda {
       if (copy) {
         if (async) {
 #ifdef USE_DRIVER_API
+          CUresult error = CUDA_SUCCESS;
           switch (kind) {
           case cudaMemcpyDeviceToHost:
-            PROFILE(cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
+            PROFILE(error = cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
             break;
           case cudaMemcpyHostToDevice:
-            PROFILE(cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
+            PROFILE(error = cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
             break;
           case cudaMemcpyDeviceToDevice:
-            PROFILE(cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
+            PROFILE(error = cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
             break;
           case cudaMemcpyDefault:
-            PROFILE(cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
+            PROFILE(error = cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
             break;
           default: errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
           }
+          set_driver_error(error, "cuMemcpyAsync", func, file, line, active_tuning);
 #else
 #ifdef API_PROFILE
           QudaProfileType type = QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC;
@@ -196,33 +246,38 @@ namespace quda {
           default: errorQuda("Unsupported cudaMemcpyTypeAsync %d", kind);
           }
 #endif
-          PROFILE(cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)), type);
+          cudaError_t error;
+          PROFILE(error = cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)), type);
+          set_runtime_error(error, "cudaMemcpyAsync", func, file, line, active_tuning);
 #endif
         } else {
 #ifdef USE_DRIVER_API
+          CUresult error = CUDA_SUCCESS;
           switch (kind) {
-          case cudaMemcpyDeviceToHost: cuMemcpyDtoH(dst, (CUdeviceptr)src, count); break;
-          case cudaMemcpyHostToDevice: cuMemcpyHtoD((CUdeviceptr)dst, src, count); break;
+          case cudaMemcpyDeviceToHost: error = cuMemcpyDtoH(dst, (CUdeviceptr)src, count); break;
+          case cudaMemcpyHostToDevice: error = cuMemcpyHtoD((CUdeviceptr)dst, src, count); break;
           case cudaMemcpyHostToHost: memcpy(dst, src, count); break;
-          case cudaMemcpyDeviceToDevice: cuMemcpyDtoD((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
-          case cudaMemcpyDefault: cuMemcpy((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
+          case cudaMemcpyDeviceToDevice: error = cuMemcpyDtoD((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
+          case cudaMemcpyDefault: error = cuMemcpy((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
           default: errorQuda("Unsupported cudaMemcpyType %d", kind);
           }
+          set_driver_error(error, "cuMemcpy", func, file, line, active_tuning);
 #else
-          cudaMemcpy(dst, src, count, kind);
+          cudaError_t error = cudaMemcpy(dst, src, count, kind);
+          set_runtime_error(error, "cudaMemcpy", func, file, line, active_tuning);
 #endif
         }
       } else {
 #ifdef USE_DRIVER_API
-        if (async)
-          cuMemsetD32Async((CUdeviceptr)dst, value, count / 4, device::get_cuda_stream(stream));
-        else
+        CUresult error = async ?
+          cuMemsetD32Async((CUdeviceptr)dst, value, count / 4, device::get_cuda_stream(stream)) :
           cuMemsetD32((CUdeviceptr)dst, value, count / 4);
+        set_driver_error(error, "cuMemset", func, file, line, active_tuning);
 #else
-        if (async)
-          cudaMemsetAsync(dst, value, count, device::get_cuda_stream(stream));
-        else
+        cudaError_t error = async ?
+          cudaMemsetAsync(dst, value, count, device::get_cuda_stream(stream)) :
           cudaMemset(dst, value, count);
+        set_runtime_error(error, "cudaMemset", func, file, line, active_tuning);
 #endif
       }
     }
@@ -244,9 +299,6 @@ namespace quda {
                    const char *func, const char *file, const char *line) {
     if (count == 0) return;
     QudaMem copy(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_default_stream(), false, func, file, line);
-    cudaError_t error = cudaGetLastError();
-    if (error != cudaSuccess)
-      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
   void qudaMemcpyAsync_(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const qudaStream_t &stream,
@@ -352,15 +404,10 @@ namespace quda {
       errorQuda("Unsupported cuMemcpyType2DAsync %d", qudaMemcpyKindToAPI(kind));
     }
     PROFILE(auto error = cuMemcpy2DAsync(&param, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuMemcpy2DAsync returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    set_driver_error(error, "cuMemcpy2DAsync", func, file, line);
 #else
     PROFILE(auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
-    if (error != cudaSuccess)
-      errorQuda("cudaMemcpy2DAsync returned error %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, "cudaMemcpy2DAsync", func, file, line);
 #endif
   }
 
@@ -368,17 +415,13 @@ namespace quda {
                              const qudaStream_t &stream, const char *func, const char *file, const char *line)
   {
     auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, device::get_cuda_stream(stream));
-    if (error != cudaSuccess)
-      errorQuda("cudaMemcpy2DAsync returned %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, "cudaMemcpy2DAsync", func, file, line);
   }
 
   void qudaMemset_(void *ptr, int value, size_t count, const char *func, const char *file, const char *line)
   {
     if (count == 0) return;
     QudaMem set(ptr, value, count, device::get_default_stream(), false, func, file, line);
-    cudaError_t error = cudaGetLastError();
-    if (error != cudaSuccess && !activeTuning())
-      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
   void qudaMemsetAsync_(void *ptr, int value, size_t count, const qudaStream_t &stream, const char *func,
@@ -386,22 +429,20 @@ namespace quda {
   {
     if (count == 0) return;
     QudaMem copy(ptr, value, count, stream, true, func, file, line);
-    cudaError_t error = cudaGetLastError();
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
   void qudaMemset2D_(void *ptr, size_t pitch, int value, size_t width, size_t height, const char *func,
                      const char *file, const char *line)
   {
     cudaError_t error = cudaMemset2D(ptr, pitch, value, width, height);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
   }
 
   void qudaMemset2DAsync_(void *ptr, size_t pitch, int value, size_t width, size_t height, const qudaStream_t &stream,
                           const char *func, const char *file, const char *line)
   {
     cudaError_t error = cudaMemset2DAsync(ptr, pitch, value, width, height, device::get_cuda_stream(stream));
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
   }
 
   void qudaMemPrefetchAsync_(void *ptr, size_t count, QudaFieldLocation mem_space, const qudaStream_t &stream,
@@ -416,76 +457,100 @@ namespace quda {
       errorQuda("Invalid QudaFieldLocation.");
 
     cudaError_t error = cudaMemPrefetchAsync(ptr, count, dev_id, device::get_cuda_stream(stream));
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
   }
 
-  bool qudaEventQuery_(cudaEvent_t &event, const char *func, const char *file, const char *line)
+  bool qudaEventQuery_(qudaEvent_t &quda_event, const char *func, const char *file, const char *line)
   {
+    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuEventQuery(event), QUDA_PROFILE_EVENT_QUERY);
     switch (error) {
     case CUDA_SUCCESS: return true;
     case CUDA_ERROR_NOT_READY: return false;
-    default: {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuEventQuery returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    default: set_driver_error(error, __func__, func, file, line);
     }
 #else
     PROFILE(cudaError_t error = cudaEventQuery(event), QUDA_PROFILE_EVENT_QUERY);
     switch (error) {
     case cudaSuccess: return true;
     case cudaErrorNotReady: return false;
-    default: errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    default: set_runtime_error(error, __func__, func, file, line);
     }
 #endif
     return false;
   }
 
-  void qudaEventRecord_(cudaEvent_t &event, qudaStream_t stream, const char *func, const char *file, const char *line)
+  void qudaEventRecord_(qudaEvent_t &quda_event, qudaStream_t stream, const char *func, const char *file, const char *line)
   {
+    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuEventRecord(event, device::get_cuda_stream(stream)), QUDA_PROFILE_EVENT_RECORD);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuEventRecord returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    set_driver_error(error, __func__, func, file, line);
 #else
     PROFILE(cudaError_t error = cudaEventRecord(event, device::get_cuda_stream(stream)), QUDA_PROFILE_EVENT_RECORD);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
 #endif
   }
 
-  void qudaStreamWaitEvent_(qudaStream_t stream, cudaEvent_t event, unsigned int flags, const char *func,
+  void qudaStreamWaitEvent_(qudaStream_t stream, qudaEvent_t quda_event, unsigned int flags, const char *func,
                             const char *file, const char *line)
   {
+    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuStreamWaitEvent(device::get_cuda_stream(stream), event, flags), QUDA_PROFILE_STREAM_WAIT_EVENT);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuStreamWaitEvent returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    set_driver_error(error, __func__, func, file, line);
 #else
     PROFILE(cudaError_t error = cudaStreamWaitEvent(device::get_cuda_stream(stream), event, flags), QUDA_PROFILE_STREAM_WAIT_EVENT);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
 #endif
   }
 
-  void qudaEventSynchronize_(const cudaEvent_t &event, const char *func, const char *file, const char *line)
+  qudaEvent_t qudaEventCreate_(const char *func, const char *file, const char *line)
+  {
+    cudaEvent_t cuda_event;
+    cudaError_t error = cudaEventCreateWithFlags(&cuda_event, cudaEventDisableTiming);
+    set_runtime_error(error, __func__, func, file, line);
+    qudaEvent_t quda_event;
+    quda_event.event = reinterpret_cast<void*>(cuda_event);
+    return quda_event;
+  }
+
+  qudaEvent_t qudaChronoEventCreate_(const char *func, const char *file, const char *line)
+  {
+    cudaEvent_t cuda_event;
+    cudaError_t error = cudaEventCreate(&cuda_event);
+    set_runtime_error(error, __func__, func, file, line);
+    qudaEvent_t quda_event;
+    quda_event.event = reinterpret_cast<void*>(cuda_event);
+    return quda_event;
+  }
+
+  float qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
+                              const char *func, const char *file, const char *line)
+  {
+    float elapsed_time;
+    cudaError_t error = cudaEventElapsedTime(&elapsed_time, reinterpret_cast<cudaEvent_t>(start.event),
+                                             reinterpret_cast<cudaEvent_t>(stop.event));
+    set_runtime_error(error, __func__, func, file, line);
+    return elapsed_time / 1000;
+  }
+
+  void qudaEventDestroy_(qudaEvent_t &event, const char *func, const char *file, const char *line)
+  {
+    cudaError_t error = cudaEventDestroy(reinterpret_cast<cudaEvent_t&>(event.event));
+    set_runtime_error(error, __func__, func, file, line);
+  }
+
+  void qudaEventSynchronize_(const qudaEvent_t &quda_event, const char *func, const char *file, const char *line)
   {
+    const cudaEvent_t &event = reinterpret_cast<const cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuEventSynchronize(event), QUDA_PROFILE_EVENT_SYNCHRONIZE);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuEventSynchronize returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    set_driver_error(error, __func__, func, file, line);
 #else
     PROFILE(cudaError_t error = cudaEventSynchronize(event), QUDA_PROFILE_EVENT_SYNCHRONIZE);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
 #endif
   }
 
@@ -493,15 +558,10 @@ namespace quda {
   {
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuStreamSynchronize(device::get_cuda_stream(stream)), QUDA_PROFILE_STREAM_SYNCHRONIZE);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("(CUDA) cuStreamSynchronize returned error %s\n (%s:%s in %s())\n", str, file, line, func);
-    }
+    set_driver_error(error, __func__, file, line, func, activeTuning());
 #else
     PROFILE(cudaError_t error = cudaStreamSynchronize(device::get_cuda_stream(stream)), QUDA_PROFILE_STREAM_SYNCHRONIZE);
-    if (error != cudaSuccess && !activeTuning())
-      errorQuda("(CUDA) %s\n (%s:%s in %s())", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, file, line, func, activeTuning());
 #endif
   }
 
@@ -509,15 +569,10 @@ namespace quda {
   {
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuCtxSynchronize(), QUDA_PROFILE_DEVICE_SYNCHRONIZE);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuCtxSynchronize returned error %s (%s:%s in %s())\n", str, file, line, func);
-    }
+    set_driver_error(error, __func__, func, file, line, activeTuning());
 #else
     PROFILE(cudaError_t error = cudaDeviceSynchronize(), QUDA_PROFILE_DEVICE_SYNCHRONIZE);
-    if (error != cudaSuccess)
-      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line, activeTuning());
 #endif
   }
 
@@ -525,8 +580,7 @@ namespace quda {
   {
     void *ptr;
     cudaError_t error = cudaGetSymbolAddress(&ptr, symbol);
-    if (error != cudaSuccess)
-      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
     return ptr;
   }
 
@@ -535,7 +589,7 @@ namespace quda {
   {
     // no driver API variant here since we have C++ functions
     PROFILE(cudaError_t error = cudaFuncSetAttribute(kernel, attr, value), QUDA_PROFILE_FUNC_SET_ATTRIBUTE);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
   }
 
   void qudaFuncGetAttributes_(cudaFuncAttributes &attr, const void *kernel, const char *func, const char *file,
@@ -543,19 +597,7 @@ namespace quda {
   {
     // no driver API variant here since we have C++ functions
     PROFILE(cudaError_t error = cudaFuncGetAttributes(&attr, kernel), QUDA_PROFILE_FUNC_SET_ATTRIBUTE);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
-  }
-
-  static std::string error_str("CUDA_SUCCESS");
-
-  void qudaSetErrorString(const std::string &error_str_)
-  {
-    error_str = error_str_;
-  }
-
-  std::string qudaGetLastErrorString()
-  {
-    return error_str;
+    set_runtime_error(error, __func__, func, file, line);
   }
 
   void printAPIProfile() {
diff --git a/lib/targets/cuda/tune.cpp b/lib/targets/cuda/tune.cpp
deleted file mode 100644
index 5b6fced824..0000000000
--- a/lib/targets/cuda/tune.cpp
+++ /dev/null
@@ -1,849 +0,0 @@
-#include <tune_quda.h>
-#include <comm_quda.h>
-#include <quda.h>     // for QUDA_VERSION_STRING
-#include <sys/stat.h> // for stat()
-#include <fcntl.h>
-#include <cfloat> // for FLT_MAX
-#include <ctime>
-#include <fstream>
-#include <typeinfo>
-#include <map>
-#include <list>
-#include <unistd.h>
-#include <uint_to_char.h>
-
-#include <deque>
-#include <queue>
-#include <functional>
-
-//#define LAUNCH_TIMER
-extern char *gitversion;
-
-namespace quda
-{
-  static TuneKey last_key;
-
-  TuneKey getLastTuneKey() { return quda::last_key; }
-
-  typedef std::map<TuneKey, TuneParam> map;
-
-  struct TraceKey {
-
-    TuneKey key;
-    float time;
-
-    long device_bytes;
-    long pinned_bytes;
-    long mapped_bytes;
-    long host_bytes;
-
-    TraceKey() {}
-
-    TraceKey(const TuneKey &key, float time) :
-      key(key),
-      time(time),
-      device_bytes(device_allocated_peak()),
-      pinned_bytes(pinned_allocated_peak()),
-      mapped_bytes(mapped_allocated_peak()),
-      host_bytes(host_allocated_peak())
-    {
-    }
-
-    TraceKey(const TraceKey &) = default;
-    TraceKey(TraceKey &&) = default;
-    TraceKey &operator=(const TraceKey &) = default;
-    TraceKey &operator=(TraceKey &&) = default;
-  };
-
-  // linked list that is augmented each time we call a kernel
-  static std::list<TraceKey> trace_list;
-  static int enable_trace = 0;
-
-  int traceEnabled()
-  {
-    static bool init = false;
-
-    if (!init) {
-      char *enable_trace_env = getenv("QUDA_ENABLE_TRACE");
-      if (enable_trace_env) {
-        if (strcmp(enable_trace_env, "1") == 0) {
-          // only explicitly posted trace events are included
-          enable_trace = 1;
-        } else if (strcmp(enable_trace_env, "2") == 0) {
-          // enable full kernel trace and posted trace events
-          enable_trace = 2;
-        }
-      }
-      init = true;
-    }
-    return enable_trace;
-  }
-
-  void postTrace_(const char *func, const char *file, int line)
-  {
-    if (traceEnabled() >= 1) {
-      char aux[TuneKey::aux_n];
-      strcpy(aux, file);
-      strcat(aux, ":");
-      char tmp[TuneKey::aux_n];
-      i32toa(tmp, line);
-      strcat(aux, tmp);
-      TuneKey key("", func, aux);
-      TraceKey trace_entry(key, 0.0);
-      trace_list.push_back(trace_entry);
-    }
-  }
-
-  static const std::string quda_hash = QUDA_HASH; // defined in lib/Makefile
-  static std::string resource_path;
-  static map tunecache;
-  static map::iterator it;
-  static size_t initial_cache_size = 0;
-
-#define STR_(x) #x
-#define STR(x) STR_(x)
-  static const std::string quda_version
-    = STR(QUDA_VERSION_MAJOR) "." STR(QUDA_VERSION_MINOR) "." STR(QUDA_VERSION_SUBMINOR);
-#undef STR
-#undef STR_
-
-  /** tuning in progress? */
-  static bool tuning = false;
-
-  bool activeTuning() { return tuning; }
-
-  static bool profile_count = true;
-
-  void disableProfileCount() { profile_count = false; }
-  void enableProfileCount() { profile_count = true; }
-
-  const map &getTuneCache() { return tunecache; }
-
-  /**
-   * Deserialize tunecache from an istream, useful for reading a file or receiving from other nodes.
-   */
-  static void deserializeTuneCache(std::istream &in)
-  {
-    std::string line;
-    std::stringstream ls;
-
-    TuneKey key;
-    TuneParam param;
-
-    std::string v;
-    std::string n;
-    std::string a;
-
-    int check;
-
-    while (in.good()) {
-      getline(in, line);
-      if (!line.length()) continue; // skip blank lines (e.g., at end of file)
-      ls.clear();
-      ls.str(line);
-      ls >> v >> n >> a >> param.block.x >> param.block.y >> param.block.z;
-      check = snprintf(key.volume, key.volume_n, "%s", v.c_str());
-      if (check < 0 || check >= key.volume_n) errorQuda("Error writing volume string (check = %d)", check);
-      check = snprintf(key.name, key.name_n, "%s", n.c_str());
-      if (check < 0 || check >= key.name_n) errorQuda("Error writing name string (check=%d)", check);
-      check = snprintf(key.aux, key.aux_n, "%s", a.c_str());
-      if (check < 0 || check >= key.aux_n) errorQuda("Error writing aux string (check=%d)", check);
-      ls >> param.grid.x >> param.grid.y >> param.grid.z >> param.shared_bytes >> param.aux.x >> param.aux.y
-        >> param.aux.z >> param.aux.w >> param.time;
-      ls.ignore(1);               // throw away tab before comment
-      getline(ls, param.comment); // assume anything remaining on the line is a comment
-      param.comment += "\n";      // our convention is to include the newline, since ctime() likes to do this
-      tunecache[key] = param;
-    }
-  }
-
-  /**
-   * Serialize tunecache to an ostream, useful for writing to a file or sending to other nodes.
-   */
-  static void serializeTuneCache(std::ostream &out)
-  {
-    map::iterator entry;
-
-    for (entry = tunecache.begin(); entry != tunecache.end(); entry++) {
-      TuneKey key = entry->first;
-      TuneParam param = entry->second;
-
-      out << std::setw(16) << key.volume << "\t" << key.name << "\t" << key.aux << "\t";
-      out << param.block.x << "\t" << param.block.y << "\t" << param.block.z << "\t";
-      out << param.grid.x << "\t" << param.grid.y << "\t" << param.grid.z << "\t";
-      out << param.shared_bytes << "\t" << param.aux.x << "\t" << param.aux.y << "\t" << param.aux.z << "\t"
-          << param.aux.w << "\t";
-      out << param.time << "\t" << param.comment; // param.comment ends with a newline
-    }
-  }
-
-  template <class T> struct less_significant : std::binary_function<T, T, bool> {
-    inline bool operator()(const T &lhs, const T &rhs)
-    {
-      return lhs.second.time * lhs.second.n_calls < rhs.second.time * rhs.second.n_calls;
-    }
-  };
-
-  /**
-   * Serialize tunecache to an ostream, useful for writing to a file or sending to other nodes.
-   */
-  static void serializeProfile(std::ostream &out, std::ostream &async_out)
-  {
-    map::iterator entry;
-    double total_time = 0.0;
-    double async_total_time = 0.0;
-
-    // first let's sort the entries in decreasing order of significance
-    typedef std::pair<TuneKey, TuneParam> profile_t;
-    typedef std::priority_queue<profile_t, std::deque<profile_t>, less_significant<profile_t>> queue_t;
-    queue_t q(tunecache.begin(), tunecache.end());
-
-    // now compute total time spent in kernels so we can give each kernel a significance
-    for (entry = tunecache.begin(); entry != tunecache.end(); entry++) {
-      TuneKey key = entry->first;
-      TuneParam param = entry->second;
-
-      char tmp[TuneKey::aux_n] = {};
-      strncpy(tmp, key.aux, TuneKey::aux_n);
-      bool is_policy_kernel = strncmp(tmp, "policy_kernel", 13) == 0 ? true : false;
-      bool is_policy = (strncmp(tmp, "policy", 6) == 0 && !is_policy_kernel) ? true : false;
-      if (param.n_calls > 0 && !is_policy) total_time += param.n_calls * param.time;
-      if (param.n_calls > 0 && is_policy) async_total_time += param.n_calls * param.time;
-    }
-
-    while (!q.empty()) {
-      TuneKey key = q.top().first;
-      TuneParam param = q.top().second;
-
-      char tmp[TuneKey::aux_n] = {};
-      strncpy(tmp, key.aux, TuneKey::aux_n);
-      bool is_policy_kernel = strncmp(tmp, "policy_kernel", 13) == 0 ? true : false;
-      bool is_policy = (strncmp(tmp, "policy", 6) == 0 && !is_policy_kernel) ? true : false;
-      bool is_nested_policy = (strncmp(tmp, "nested_policy", 6) == 0) ? true : false; // nested policies not included
-
-      // synchronous profile
-      if (param.n_calls > 0 && !is_policy && !is_nested_policy) {
-        double time = param.n_calls * param.time;
-
-        out << std::setw(12) << param.n_calls * param.time << "\t" << std::setw(12) << (time / total_time) * 100 << "\t";
-        out << std::setw(12) << param.n_calls << "\t" << std::setw(12) << param.time << "\t" << std::setw(16)
-            << key.volume << "\t";
-        out << key.name << "\t" << key.aux << "\t" << param.comment; // param.comment ends with a newline
-      }
-
-      // async policy profile
-      if (param.n_calls > 0 && is_policy) {
-        double time = param.n_calls * param.time;
-
-        async_out << std::setw(12) << param.n_calls * param.time << "\t" << std::setw(12)
-                  << (time / async_total_time) * 100 << "\t";
-        async_out << std::setw(12) << param.n_calls << "\t" << std::setw(12) << param.time << "\t" << std::setw(16)
-                  << key.volume << "\t";
-        async_out << key.name << "\t" << key.aux << "\t" << param.comment; // param.comment ends with a newline
-      }
-
-      q.pop();
-    }
-
-    out << std::endl << "# Total time spent in kernels = " << total_time << " seconds" << std::endl;
-    async_out << std::endl
-              << "# Total time spent in asynchronous execution = " << async_total_time << " seconds" << std::endl;
-  }
-
-  /**
-   * Serialize trace to an ostream, useful for writing to a file or sending to other nodes.
-   */
-  static void serializeTrace(std::ostream &out)
-  {
-    for (auto it = trace_list.begin(); it != trace_list.end(); it++) {
-
-      TuneKey &key = it->key;
-
-      // special case kernel members of a policy
-      char tmp[TuneKey::aux_n] = {};
-      strncpy(tmp, key.aux, TuneKey::aux_n);
-      bool is_policy_kernel = strcmp(tmp, "policy_kernel") == 0 ? true : false;
-
-      out << std::setw(12) << it->time << "\t";
-      out << std::setw(12) << it->device_bytes << "\t";
-      out << std::setw(12) << it->pinned_bytes << "\t";
-      out << std::setw(12) << it->mapped_bytes << "\t";
-      out << std::setw(12) << it->host_bytes << "\t";
-      out << std::setw(16) << key.volume << "\t";
-      if (is_policy_kernel) out << "\t";
-      out << key.name << "\t";
-      if (!is_policy_kernel) out << "\t";
-      out << key.aux << std::endl;
-    }
-  }
-
-  /**
-   * Distribute the tunecache from node 0 to all other nodes.
-   */
-  static void broadcastTuneCache()
-  {
-#ifdef MULTI_GPU
-    std::stringstream serialized;
-    size_t size;
-
-    if (comm_rank() == 0) {
-      serializeTuneCache(serialized);
-      size = serialized.str().length();
-    }
-    comm_broadcast(&size, sizeof(size_t));
-
-    if (size > 0) {
-      if (comm_rank() == 0) {
-        comm_broadcast(const_cast<char *>(serialized.str().c_str()), size);
-      } else {
-        char *serstr = new char[size + 1];
-        comm_broadcast(serstr, size);
-        serstr[size] = '\0'; // null-terminate
-        serialized.str(serstr);
-        deserializeTuneCache(serialized);
-        delete[] serstr;
-      }
-    }
-#endif
-  }
-
-  /*
-   * Read tunecache from disk.
-   */
-  void loadTuneCache()
-  {
-    if (getTuning() == QUDA_TUNE_NO) {
-      warningQuda("Autotuning disabled");
-      return;
-    }
-
-    char *path;
-    struct stat pstat;
-    std::string cache_path, line, token;
-    std::ifstream cache_file;
-    std::stringstream ls;
-
-    path = getenv("QUDA_RESOURCE_PATH");
-
-    if (!path) {
-      warningQuda("Environment variable QUDA_RESOURCE_PATH is not set.");
-      warningQuda("Caching of tuned parameters will be disabled.");
-      return;
-    } else if (stat(path, &pstat) || !S_ISDIR(pstat.st_mode)) {
-      warningQuda("The path \"%s\" specified by QUDA_RESOURCE_PATH does not exist or is not a directory.", path);
-      warningQuda("Caching of tuned parameters will be disabled.");
-      return;
-    } else {
-      resource_path = path;
-    }
-
-    bool version_check = true;
-    char *override_version_env = getenv("QUDA_TUNE_VERSION_CHECK");
-    if (override_version_env && strcmp(override_version_env, "0") == 0) {
-      version_check = false;
-      warningQuda("Disabling QUDA tunecache version check");
-    }
-
-#ifdef MULTI_GPU
-    if (comm_rank() == 0) {
-#endif
-
-      cache_path = resource_path;
-      cache_path += "/tunecache.tsv";
-      cache_file.open(cache_path.c_str());
-
-      if (cache_file) {
-
-        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
-        getline(cache_file, line);
-        ls.str(line);
-        ls >> token;
-        if (token.compare("tunecache")) errorQuda("Bad format in %s", cache_path.c_str());
-        ls >> token;
-        if (version_check && token.compare(quda_version))
-          errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
-                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
-                    cache_path.c_str());
-        ls >> token;
-#ifdef GITVERSION
-        if (version_check && token.compare(gitversion))
-          errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
-                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
-                    cache_path.c_str());
-#else
-      if (version_check && token.compare(quda_version))
-        errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
-                  "QUDA_RESOURCE_PATH environment variable to point to a new path.",
-                  cache_path.c_str());
-#endif
-        ls >> token;
-        if (version_check && token.compare(quda_hash))
-          errorQuda("Cache file %s does not match current QUDA build. \nPlease delete this file or set the "
-                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
-                    cache_path.c_str());
-
-        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
-        getline(cache_file, line); // eat the blank line
-
-        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
-        getline(cache_file, line); // eat the description line
-
-        deserializeTuneCache(cache_file);
-
-        cache_file.close();
-        initial_cache_size = tunecache.size();
-
-        if (getVerbosity() >= QUDA_SUMMARIZE) {
-          printfQuda("Loaded %d sets of cached parameters from %s\n", static_cast<int>(initial_cache_size),
-                     cache_path.c_str());
-        }
-
-      } else {
-        warningQuda("Cache file not found.  All kernels will be re-tuned (if tuning is enabled).");
-      }
-
-#ifdef MULTI_GPU
-    }
-#endif
-
-    broadcastTuneCache();
-  }
-
-  /**
-   * Write tunecache to disk.
-   */
-  void saveTuneCache(bool error)
-  {
-    time_t now;
-    int lock_handle;
-    std::string lock_path, cache_path;
-    std::ofstream cache_file;
-
-    if (resource_path.empty()) return;
-
-      // FIXME: We should really check to see if any nodes have tuned a kernel that was not also tuned on node 0, since as things
-      //       stand, the corresponding launch parameters would never get cached to disk in this situation.  This will come up if we
-      //       ever support different subvolumes per GPU (as might be convenient for lattice volumes that don't divide evenly).
-
-#ifdef MULTI_GPU
-    if (comm_rank() == 0) {
-#endif
-
-      if (tunecache.size() == initial_cache_size && !error) return;
-
-      // Acquire lock.  Note that this is only robust if the filesystem supports flock() semantics, which is true for
-      // NFS on recent versions of linux but not Lustre by default (unless the filesystem was mounted with "-o flock").
-      lock_path = resource_path + "/tunecache.lock";
-      lock_handle = open(lock_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
-      if (lock_handle == -1) {
-        warningQuda("Unable to lock cache file.  Tuned launch parameters will not be cached to disk.  "
-                    "If you are certain that no other instances of QUDA are accessing this filesystem, "
-                    "please manually remove %s",
-                    lock_path.c_str());
-        return;
-      }
-      char msg[] = "If no instances of applications using QUDA are running,\n"
-                   "this lock file shouldn't be here and is safe to delete.";
-      int stat = write(lock_handle, msg, sizeof(msg)); // check status to avoid compiler warning
-      if (stat == -1) warningQuda("Unable to write to lock file for some bizarre reason");
-
-      cache_path = resource_path + (error ? "/tunecache_error.tsv" : "/tunecache.tsv");
-      cache_file.open(cache_path.c_str());
-
-      if (getVerbosity() >= QUDA_SUMMARIZE) {
-        printfQuda("Saving %d sets of cached parameters to %s\n", static_cast<int>(tunecache.size()), cache_path.c_str());
-      }
-
-      time(&now);
-      cache_file << "tunecache\t" << quda_version;
-#ifdef GITVERSION
-      cache_file << "\t" << gitversion;
-#else
-    cache_file << "\t" << quda_version;
-#endif
-      cache_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
-      cache_file << std::setw(16) << "volume"
-                 << "\tname\taux\tblock.x\tblock.y\tblock.z\tgrid.x\tgrid.y\tgrid.z\tshared_bytes\taux.x\taux.y\taux."
-                    "z\taux.w\ttime\tcomment"
-                 << std::endl;
-      serializeTuneCache(cache_file);
-      cache_file.close();
-
-      // Release lock.
-      close(lock_handle);
-      remove(lock_path.c_str());
-
-      initial_cache_size = tunecache.size();
-
-#ifdef MULTI_GPU
-    } else {
-      // give process 0 time to write out its tunecache if needed, but
-      // doesn't cause a hang if error is not triggered on process 0
-      if (error) sleep(10);
-    }
-#endif
-  }
-
-  static bool policy_tuning = false;
-  bool policyTuning() { return policy_tuning; }
-
-  void setPolicyTuning(bool policy_tuning_) { policy_tuning = policy_tuning_; }
-
-  // flush profile, setting counts to zero
-  void flushProfile()
-  {
-    for (map::iterator entry = tunecache.begin(); entry != tunecache.end(); entry++) {
-      // set all n_calls = 0
-      TuneParam &param = entry->second;
-      param.n_calls = 0;
-    }
-  }
-
-  // save profile
-  void saveProfile(const std::string label)
-  {
-    time_t now;
-    int lock_handle;
-    std::string lock_path, profile_path, async_profile_path, trace_path;
-    std::ofstream profile_file, async_profile_file, trace_file;
-
-    if (resource_path.empty()) return;
-
-#ifdef MULTI_GPU
-    if (comm_rank() == 0) {
-#endif
-
-      // Acquire lock.  Note that this is only robust if the filesystem supports flock() semantics, which is true for
-      // NFS on recent versions of linux but not Lustre by default (unless the filesystem was mounted with "-o flock").
-      lock_path = resource_path + "/profile.lock";
-      lock_handle = open(lock_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
-      if (lock_handle == -1) {
-        warningQuda("Unable to lock profile file.  Profile will not be saved to disk.  "
-                    "If you are certain that no other instances of QUDA are accessing this filesystem, "
-                    "please manually remove %s",
-                    lock_path.c_str());
-        return;
-      }
-      char msg[] = "If no instances of applications using QUDA are running,\n"
-                   "this lock file shouldn't be here and is safe to delete.";
-      int stat = write(lock_handle, msg, sizeof(msg)); // check status to avoid compiler warning
-      if (stat == -1) warningQuda("Unable to write to lock file for some bizarre reason");
-
-      // profile counter for writing out unique profiles
-      static int count = 0;
-
-      char *profile_fname = getenv("QUDA_PROFILE_OUTPUT_BASE");
-
-      if (!profile_fname) {
-        warningQuda(
-          "Environment variable QUDA_PROFILE_OUTPUT_BASE not set; writing to profile.tsv and profile_async.tsv");
-        profile_path = resource_path + "/profile_" + std::to_string(count) + ".tsv";
-        async_profile_path = resource_path + "/profile_async_" + std::to_string(count) + ".tsv";
-        if (traceEnabled()) trace_path = resource_path + "/trace_" + std::to_string(count) + ".tsv";
-      } else {
-        profile_path = resource_path + "/" + profile_fname + "_" + std::to_string(count) + ".tsv";
-        async_profile_path = resource_path + "/" + profile_fname + "_" + std::to_string(count) + "_async.tsv";
-        if (traceEnabled())
-          trace_path = resource_path + "/" + profile_fname + "_trace_" + std::to_string(count) + ".tsv";
-      }
-
-      count++;
-
-      profile_file.open(profile_path.c_str());
-      async_profile_file.open(async_profile_path.c_str());
-      if (traceEnabled()) trace_file.open(trace_path.c_str());
-
-      if (getVerbosity() >= QUDA_SUMMARIZE) {
-        // compute number of non-zero entries that will be output in the profile
-        int n_entry = 0;
-        int n_policy = 0;
-        for (map::iterator entry = tunecache.begin(); entry != tunecache.end(); entry++) {
-          // if a policy entry, then we can ignore
-          char tmp[TuneKey::aux_n] = {};
-          strncpy(tmp, entry->first.aux, TuneKey::aux_n);
-          TuneParam param = entry->second;
-          bool is_policy = strcmp(tmp, "policy") == 0 ? true : false;
-          if (param.n_calls > 0 && !is_policy) n_entry++;
-          if (param.n_calls > 0 && is_policy) n_policy++;
-        }
-
-        printfQuda("Saving %d sets of cached parameters to %s\n", n_entry, profile_path.c_str());
-        printfQuda("Saving %d sets of cached profiles to %s\n", n_policy, async_profile_path.c_str());
-        if (traceEnabled())
-          printfQuda("Saving trace list with %lu entries to %s\n", trace_list.size(), trace_path.c_str());
-      }
-
-      time(&now);
-
-      std::string Label = label.empty() ? "profile" : label;
-
-      profile_file << Label << "\t" << quda_version;
-#ifdef GITVERSION
-      profile_file << "\t" << gitversion;
-#else
-    profile_file << "\t" << quda_version;
-#endif
-      profile_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
-      profile_file << std::setw(12) << "total time"
-                   << "\t" << std::setw(12) << "percentage"
-                   << "\t" << std::setw(12) << "calls"
-                   << "\t" << std::setw(12) << "time / call"
-                   << "\t" << std::setw(16) << "volume"
-                   << "\tname\taux\tcomment" << std::endl;
-
-      async_profile_file << Label << "\t" << quda_version;
-#ifdef GITVERSION
-      async_profile_file << "\t" << gitversion;
-#else
-    async_profile_file << "\t" << quda_version;
-#endif
-      async_profile_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
-      async_profile_file << std::setw(12) << "total time"
-                         << "\t" << std::setw(12) << "percentage"
-                         << "\t" << std::setw(12) << "calls"
-                         << "\t" << std::setw(12) << "time / call"
-                         << "\t" << std::setw(16) << "volume"
-                         << "\tname\taux\tcomment" << std::endl;
-
-      serializeProfile(profile_file, async_profile_file);
-
-      profile_file.close();
-      async_profile_file.close();
-
-      if (traceEnabled()) {
-        trace_file << "trace"
-                   << "\t" << quda_version;
-#ifdef GITVERSION
-        trace_file << "\t" << gitversion;
-#else
-      trace_file << "\t" << quda_version;
-#endif
-        trace_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
-
-        trace_file << std::setw(12) << "time\t" << std::setw(12) << "device-mem\t" << std::setw(12) << "pinned-mem\t";
-        trace_file << std::setw(12) << "mapped-mem\t" << std::setw(12) << "host-mem\t";
-        trace_file << std::setw(16) << "volume"
-                   << "\tname\taux" << std::endl;
-
-        serializeTrace(trace_file);
-
-        trace_file.close();
-      }
-
-      // Release lock.
-      close(lock_handle);
-      remove(lock_path.c_str());
-
-#ifdef MULTI_GPU
-    }
-#endif
-  }
-
-  static TimeProfile launchTimer("tuneLaunch");
-
-  /**
-   * Return the optimal launch parameters for a given kernel, either
-   * by retrieving them from tunecache or autotuning on the spot.
-   */
-  TuneParam &tuneLaunch(Tunable &tunable, QudaTune enabled, QudaVerbosity verbosity)
-  {
-#ifdef LAUNCH_TIMER
-    launchTimer.TPSTART(QUDA_PROFILE_TOTAL);
-    launchTimer.TPSTART(QUDA_PROFILE_INIT);
-#endif
-
-    TuneKey key = tunable.tuneKey();
-    if (use_managed_memory()) strcat(key.aux, ",managed");
-    last_key = key;
-    static TuneParam param;
-
-#ifdef LAUNCH_TIMER
-    launchTimer.TPSTOP(QUDA_PROFILE_INIT);
-    launchTimer.TPSTART(QUDA_PROFILE_PREAMBLE);
-#endif
-
-    static const Tunable *active_tunable; // for error checking
-    it = tunecache.find(key);
-
-    // first check if we have the tuned value and return if we have it
-    if (enabled == QUDA_TUNE_YES && it != tunecache.end()) {
-
-#ifdef LAUNCH_TIMER
-      launchTimer.TPSTOP(QUDA_PROFILE_PREAMBLE);
-      launchTimer.TPSTART(QUDA_PROFILE_COMPUTE);
-#endif
-
-      TuneParam &param = it->second;
-
-      if (verbosity >= QUDA_DEBUG_VERBOSE) {
-        printfQuda("Launching %s with %s at vol=%s with %s\n", key.name, key.aux, key.volume,
-                   tunable.paramString(param).c_str());
-      }
-
-#ifdef LAUNCH_TIMER
-      launchTimer.TPSTOP(QUDA_PROFILE_COMPUTE);
-      launchTimer.TPSTART(QUDA_PROFILE_EPILOGUE);
-#endif
-
-      tunable.checkLaunchParam(param);
-
-      // we could be tuning outside of the current scope
-      if (!tuning && profile_count) param.n_calls++;
-
-#ifdef LAUNCH_TIMER
-      launchTimer.TPSTOP(QUDA_PROFILE_EPILOGUE);
-      launchTimer.TPSTOP(QUDA_PROFILE_TOTAL);
-#endif
-
-      if (traceEnabled() >= 2) {
-        TraceKey trace_entry(key, param.time);
-        trace_list.push_back(trace_entry);
-      }
-
-      return param;
-    }
-
-#ifdef LAUNCH_TIMER
-    launchTimer.TPSTOP(QUDA_PROFILE_PREAMBLE);
-    launchTimer.TPSTOP(QUDA_PROFILE_TOTAL);
-#endif
-
-    if (enabled == QUDA_TUNE_NO) {
-      tunable.defaultTuneParam(param);
-      tunable.checkLaunchParam(param);
-      if (verbosity >= QUDA_DEBUG_VERBOSE) {
-        printfQuda("Launching %s with %s at vol=%s with %s (untuned)\n", key.name, key.aux, key.volume,
-                   tunable.paramString(param).c_str());
-      }
-    } else if (!tuning) {
-
-      /* As long as global reductions are not disabled, only do the
-         tuning on node 0, else do the tuning on all nodes since we
-         can't guarantee that all nodes are partaking */
-      if (comm_rank() == 0 || !commGlobalReduction() || policyTuning()) {
-        TuneParam best_param;
-        cudaError_t error = cudaSuccess;
-        cudaEvent_t start, end;
-        float elapsed_time, best_time;
-        time_t now;
-
-        tuning = true;
-        active_tunable = &tunable;
-        best_time = FLT_MAX;
-
-        if (verbosity >= QUDA_DEBUG_VERBOSE) printfQuda("PreTune %s\n", key.name);
-        tunable.preTune();
-
-        cudaEventCreate(&start);
-        cudaEventCreate(&end);
-
-        if (verbosity >= QUDA_DEBUG_VERBOSE) {
-          printfQuda("Tuning %s with %s at vol=%s\n", key.name, key.aux, key.volume);
-        }
-
-        Timer tune_timer;
-        tune_timer.Start(__func__, __FILE__, __LINE__);
-
-        tunable.initTuneParam(param);
-        while (tuning) {
-          cudaDeviceSynchronize();
-          cudaGetLastError(); // clear error counter
-          tunable.checkLaunchParam(param);
-          if (verbosity >= QUDA_DEBUG_VERBOSE) {
-            printfQuda("About to call tunable.apply block=(%d,%d,%d) grid=(%d,%d,%d) shared_bytes=%d aux=(%d,%d,%d)\n",
-                       param.block.x, param.block.y, param.block.z, param.grid.x, param.grid.y, param.grid.z,
-                       param.shared_bytes, param.aux.x, param.aux.y, param.aux.z);
-          }
-          tunable.apply(); // do initial call in case we need to jit compile for these parameters or if policy tuning
-
-          cudaEventRecord(start, 0);
-          for (int i = 0; i < tunable.tuningIter(); i++) {
-            tunable.apply(); // calls tuneLaunch() again, which simply returns the currently active param
-          }
-          cudaEventRecord(end, 0);
-          cudaEventSynchronize(end);
-          cudaEventElapsedTime(&elapsed_time, start, end);
-          cudaDeviceSynchronize();
-          error = cudaGetLastError();
-
-          { // check that error state is cleared
-            cudaDeviceSynchronize();
-            cudaError_t error = cudaGetLastError();
-            if (error != cudaSuccess) errorQuda("Failed to clear error state %s\n", cudaGetErrorString(error));
-          }
-
-          elapsed_time /= (1e3 * tunable.tuningIter());
-          if ((elapsed_time < best_time) && (error == cudaSuccess) && (tunable.launchError() == QUDA_SUCCESS)) {
-            best_time = elapsed_time;
-            best_param = param;
-          }
-          if ((verbosity >= QUDA_DEBUG_VERBOSE)) {
-            if (error == cudaSuccess && tunable.launchError() == QUDA_SUCCESS) {
-              printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(),
-                         tunable.perfString(elapsed_time).c_str());
-            } else {
-              if (tunable.launchError() == QUDA_SUCCESS) { // must be regular error
-                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), cudaGetErrorString(error));
-              } else { // else must be a manually thrown error
-                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), qudaGetLastErrorString().c_str());
-              }
-            }
-          }
-          tuning = tunable.advanceTuneParam(param);
-          tunable.launchError() = QUDA_SUCCESS;
-        }
-
-        tune_timer.Stop(__func__, __FILE__, __LINE__);
-
-        if (best_time == FLT_MAX) {
-          errorQuda("Auto-tuning failed for %s with %s at vol=%s", key.name, key.aux, key.volume);
-        }
-        if (verbosity >= QUDA_VERBOSE) {
-          printfQuda("Tuned %s giving %s for %s with %s\n", tunable.paramString(best_param).c_str(),
-                     tunable.perfString(best_time).c_str(), key.name, key.aux);
-        }
-        time(&now);
-        best_param.comment = "# " + tunable.perfString(best_time);
-        best_param.comment += ", tuning took " + std::to_string(tune_timer.Last()) + " seconds at ";
-        best_param.comment += ctime(&now); // includes a newline
-        best_param.time = best_time;
-
-        cudaEventDestroy(start);
-        cudaEventDestroy(end);
-
-        if (verbosity >= QUDA_DEBUG_VERBOSE) printfQuda("PostTune %s\n", key.name);
-        tuning = true;
-        tunable.postTune();
-        tuning = false;
-        param = best_param;
-        tunecache[key] = best_param;
-      }
-      if (commGlobalReduction() || policyTuning()) broadcastTuneCache();
-
-      // check this process is getting the key that is expected
-      if (tunecache.find(key) == tunecache.end()) {
-        errorQuda("Failed to find key entry (%s:%s:%s)", key.name, key.volume, key.aux);
-      }
-      param = tunecache[key]; // read this now for all processes
-
-      if (traceEnabled() >= 2) {
-        TraceKey trace_entry(key, param.time);
-        trace_list.push_back(trace_entry);
-      }
-
-    } else if (&tunable != active_tunable) {
-      errorQuda("Unexpected call to tuneLaunch() in %s::apply()", typeid(tunable).name());
-    }
-
-    param.n_calls = profile_count ? 1 : 0;
-
-    return param;
-  }
-
-  void printLaunchTimer()
-  {
-#ifdef LAUNCH_TIMER
-    launchTimer.Print();
-#endif
-  }
-} // namespace quda
diff --git a/lib/targets/generic/blas_lapack_eigen.cpp b/lib/targets/generic/blas_lapack_eigen.cpp
index 5702e52a94..53c4d0046e 100644
--- a/lib/targets/generic/blas_lapack_eigen.cpp
+++ b/lib/targets/generic/blas_lapack_eigen.cpp
@@ -1,4 +1,5 @@
 #include <complex>
+#include <timer.h>
 #include <blas_lapack.h>
 #include <eigen_helper.h>
 
diff --git a/lib/targets/hip/tune.cpp b/lib/targets/hip/tune.cpp
deleted file mode 100644
index 5b6fced824..0000000000
--- a/lib/targets/hip/tune.cpp
+++ /dev/null
@@ -1,849 +0,0 @@
-#include <tune_quda.h>
-#include <comm_quda.h>
-#include <quda.h>     // for QUDA_VERSION_STRING
-#include <sys/stat.h> // for stat()
-#include <fcntl.h>
-#include <cfloat> // for FLT_MAX
-#include <ctime>
-#include <fstream>
-#include <typeinfo>
-#include <map>
-#include <list>
-#include <unistd.h>
-#include <uint_to_char.h>
-
-#include <deque>
-#include <queue>
-#include <functional>
-
-//#define LAUNCH_TIMER
-extern char *gitversion;
-
-namespace quda
-{
-  static TuneKey last_key;
-
-  TuneKey getLastTuneKey() { return quda::last_key; }
-
-  typedef std::map<TuneKey, TuneParam> map;
-
-  struct TraceKey {
-
-    TuneKey key;
-    float time;
-
-    long device_bytes;
-    long pinned_bytes;
-    long mapped_bytes;
-    long host_bytes;
-
-    TraceKey() {}
-
-    TraceKey(const TuneKey &key, float time) :
-      key(key),
-      time(time),
-      device_bytes(device_allocated_peak()),
-      pinned_bytes(pinned_allocated_peak()),
-      mapped_bytes(mapped_allocated_peak()),
-      host_bytes(host_allocated_peak())
-    {
-    }
-
-    TraceKey(const TraceKey &) = default;
-    TraceKey(TraceKey &&) = default;
-    TraceKey &operator=(const TraceKey &) = default;
-    TraceKey &operator=(TraceKey &&) = default;
-  };
-
-  // linked list that is augmented each time we call a kernel
-  static std::list<TraceKey> trace_list;
-  static int enable_trace = 0;
-
-  int traceEnabled()
-  {
-    static bool init = false;
-
-    if (!init) {
-      char *enable_trace_env = getenv("QUDA_ENABLE_TRACE");
-      if (enable_trace_env) {
-        if (strcmp(enable_trace_env, "1") == 0) {
-          // only explicitly posted trace events are included
-          enable_trace = 1;
-        } else if (strcmp(enable_trace_env, "2") == 0) {
-          // enable full kernel trace and posted trace events
-          enable_trace = 2;
-        }
-      }
-      init = true;
-    }
-    return enable_trace;
-  }
-
-  void postTrace_(const char *func, const char *file, int line)
-  {
-    if (traceEnabled() >= 1) {
-      char aux[TuneKey::aux_n];
-      strcpy(aux, file);
-      strcat(aux, ":");
-      char tmp[TuneKey::aux_n];
-      i32toa(tmp, line);
-      strcat(aux, tmp);
-      TuneKey key("", func, aux);
-      TraceKey trace_entry(key, 0.0);
-      trace_list.push_back(trace_entry);
-    }
-  }
-
-  static const std::string quda_hash = QUDA_HASH; // defined in lib/Makefile
-  static std::string resource_path;
-  static map tunecache;
-  static map::iterator it;
-  static size_t initial_cache_size = 0;
-
-#define STR_(x) #x
-#define STR(x) STR_(x)
-  static const std::string quda_version
-    = STR(QUDA_VERSION_MAJOR) "." STR(QUDA_VERSION_MINOR) "." STR(QUDA_VERSION_SUBMINOR);
-#undef STR
-#undef STR_
-
-  /** tuning in progress? */
-  static bool tuning = false;
-
-  bool activeTuning() { return tuning; }
-
-  static bool profile_count = true;
-
-  void disableProfileCount() { profile_count = false; }
-  void enableProfileCount() { profile_count = true; }
-
-  const map &getTuneCache() { return tunecache; }
-
-  /**
-   * Deserialize tunecache from an istream, useful for reading a file or receiving from other nodes.
-   */
-  static void deserializeTuneCache(std::istream &in)
-  {
-    std::string line;
-    std::stringstream ls;
-
-    TuneKey key;
-    TuneParam param;
-
-    std::string v;
-    std::string n;
-    std::string a;
-
-    int check;
-
-    while (in.good()) {
-      getline(in, line);
-      if (!line.length()) continue; // skip blank lines (e.g., at end of file)
-      ls.clear();
-      ls.str(line);
-      ls >> v >> n >> a >> param.block.x >> param.block.y >> param.block.z;
-      check = snprintf(key.volume, key.volume_n, "%s", v.c_str());
-      if (check < 0 || check >= key.volume_n) errorQuda("Error writing volume string (check = %d)", check);
-      check = snprintf(key.name, key.name_n, "%s", n.c_str());
-      if (check < 0 || check >= key.name_n) errorQuda("Error writing name string (check=%d)", check);
-      check = snprintf(key.aux, key.aux_n, "%s", a.c_str());
-      if (check < 0 || check >= key.aux_n) errorQuda("Error writing aux string (check=%d)", check);
-      ls >> param.grid.x >> param.grid.y >> param.grid.z >> param.shared_bytes >> param.aux.x >> param.aux.y
-        >> param.aux.z >> param.aux.w >> param.time;
-      ls.ignore(1);               // throw away tab before comment
-      getline(ls, param.comment); // assume anything remaining on the line is a comment
-      param.comment += "\n";      // our convention is to include the newline, since ctime() likes to do this
-      tunecache[key] = param;
-    }
-  }
-
-  /**
-   * Serialize tunecache to an ostream, useful for writing to a file or sending to other nodes.
-   */
-  static void serializeTuneCache(std::ostream &out)
-  {
-    map::iterator entry;
-
-    for (entry = tunecache.begin(); entry != tunecache.end(); entry++) {
-      TuneKey key = entry->first;
-      TuneParam param = entry->second;
-
-      out << std::setw(16) << key.volume << "\t" << key.name << "\t" << key.aux << "\t";
-      out << param.block.x << "\t" << param.block.y << "\t" << param.block.z << "\t";
-      out << param.grid.x << "\t" << param.grid.y << "\t" << param.grid.z << "\t";
-      out << param.shared_bytes << "\t" << param.aux.x << "\t" << param.aux.y << "\t" << param.aux.z << "\t"
-          << param.aux.w << "\t";
-      out << param.time << "\t" << param.comment; // param.comment ends with a newline
-    }
-  }
-
-  template <class T> struct less_significant : std::binary_function<T, T, bool> {
-    inline bool operator()(const T &lhs, const T &rhs)
-    {
-      return lhs.second.time * lhs.second.n_calls < rhs.second.time * rhs.second.n_calls;
-    }
-  };
-
-  /**
-   * Serialize tunecache to an ostream, useful for writing to a file or sending to other nodes.
-   */
-  static void serializeProfile(std::ostream &out, std::ostream &async_out)
-  {
-    map::iterator entry;
-    double total_time = 0.0;
-    double async_total_time = 0.0;
-
-    // first let's sort the entries in decreasing order of significance
-    typedef std::pair<TuneKey, TuneParam> profile_t;
-    typedef std::priority_queue<profile_t, std::deque<profile_t>, less_significant<profile_t>> queue_t;
-    queue_t q(tunecache.begin(), tunecache.end());
-
-    // now compute total time spent in kernels so we can give each kernel a significance
-    for (entry = tunecache.begin(); entry != tunecache.end(); entry++) {
-      TuneKey key = entry->first;
-      TuneParam param = entry->second;
-
-      char tmp[TuneKey::aux_n] = {};
-      strncpy(tmp, key.aux, TuneKey::aux_n);
-      bool is_policy_kernel = strncmp(tmp, "policy_kernel", 13) == 0 ? true : false;
-      bool is_policy = (strncmp(tmp, "policy", 6) == 0 && !is_policy_kernel) ? true : false;
-      if (param.n_calls > 0 && !is_policy) total_time += param.n_calls * param.time;
-      if (param.n_calls > 0 && is_policy) async_total_time += param.n_calls * param.time;
-    }
-
-    while (!q.empty()) {
-      TuneKey key = q.top().first;
-      TuneParam param = q.top().second;
-
-      char tmp[TuneKey::aux_n] = {};
-      strncpy(tmp, key.aux, TuneKey::aux_n);
-      bool is_policy_kernel = strncmp(tmp, "policy_kernel", 13) == 0 ? true : false;
-      bool is_policy = (strncmp(tmp, "policy", 6) == 0 && !is_policy_kernel) ? true : false;
-      bool is_nested_policy = (strncmp(tmp, "nested_policy", 6) == 0) ? true : false; // nested policies not included
-
-      // synchronous profile
-      if (param.n_calls > 0 && !is_policy && !is_nested_policy) {
-        double time = param.n_calls * param.time;
-
-        out << std::setw(12) << param.n_calls * param.time << "\t" << std::setw(12) << (time / total_time) * 100 << "\t";
-        out << std::setw(12) << param.n_calls << "\t" << std::setw(12) << param.time << "\t" << std::setw(16)
-            << key.volume << "\t";
-        out << key.name << "\t" << key.aux << "\t" << param.comment; // param.comment ends with a newline
-      }
-
-      // async policy profile
-      if (param.n_calls > 0 && is_policy) {
-        double time = param.n_calls * param.time;
-
-        async_out << std::setw(12) << param.n_calls * param.time << "\t" << std::setw(12)
-                  << (time / async_total_time) * 100 << "\t";
-        async_out << std::setw(12) << param.n_calls << "\t" << std::setw(12) << param.time << "\t" << std::setw(16)
-                  << key.volume << "\t";
-        async_out << key.name << "\t" << key.aux << "\t" << param.comment; // param.comment ends with a newline
-      }
-
-      q.pop();
-    }
-
-    out << std::endl << "# Total time spent in kernels = " << total_time << " seconds" << std::endl;
-    async_out << std::endl
-              << "# Total time spent in asynchronous execution = " << async_total_time << " seconds" << std::endl;
-  }
-
-  /**
-   * Serialize trace to an ostream, useful for writing to a file or sending to other nodes.
-   */
-  static void serializeTrace(std::ostream &out)
-  {
-    for (auto it = trace_list.begin(); it != trace_list.end(); it++) {
-
-      TuneKey &key = it->key;
-
-      // special case kernel members of a policy
-      char tmp[TuneKey::aux_n] = {};
-      strncpy(tmp, key.aux, TuneKey::aux_n);
-      bool is_policy_kernel = strcmp(tmp, "policy_kernel") == 0 ? true : false;
-
-      out << std::setw(12) << it->time << "\t";
-      out << std::setw(12) << it->device_bytes << "\t";
-      out << std::setw(12) << it->pinned_bytes << "\t";
-      out << std::setw(12) << it->mapped_bytes << "\t";
-      out << std::setw(12) << it->host_bytes << "\t";
-      out << std::setw(16) << key.volume << "\t";
-      if (is_policy_kernel) out << "\t";
-      out << key.name << "\t";
-      if (!is_policy_kernel) out << "\t";
-      out << key.aux << std::endl;
-    }
-  }
-
-  /**
-   * Distribute the tunecache from node 0 to all other nodes.
-   */
-  static void broadcastTuneCache()
-  {
-#ifdef MULTI_GPU
-    std::stringstream serialized;
-    size_t size;
-
-    if (comm_rank() == 0) {
-      serializeTuneCache(serialized);
-      size = serialized.str().length();
-    }
-    comm_broadcast(&size, sizeof(size_t));
-
-    if (size > 0) {
-      if (comm_rank() == 0) {
-        comm_broadcast(const_cast<char *>(serialized.str().c_str()), size);
-      } else {
-        char *serstr = new char[size + 1];
-        comm_broadcast(serstr, size);
-        serstr[size] = '\0'; // null-terminate
-        serialized.str(serstr);
-        deserializeTuneCache(serialized);
-        delete[] serstr;
-      }
-    }
-#endif
-  }
-
-  /*
-   * Read tunecache from disk.
-   */
-  void loadTuneCache()
-  {
-    if (getTuning() == QUDA_TUNE_NO) {
-      warningQuda("Autotuning disabled");
-      return;
-    }
-
-    char *path;
-    struct stat pstat;
-    std::string cache_path, line, token;
-    std::ifstream cache_file;
-    std::stringstream ls;
-
-    path = getenv("QUDA_RESOURCE_PATH");
-
-    if (!path) {
-      warningQuda("Environment variable QUDA_RESOURCE_PATH is not set.");
-      warningQuda("Caching of tuned parameters will be disabled.");
-      return;
-    } else if (stat(path, &pstat) || !S_ISDIR(pstat.st_mode)) {
-      warningQuda("The path \"%s\" specified by QUDA_RESOURCE_PATH does not exist or is not a directory.", path);
-      warningQuda("Caching of tuned parameters will be disabled.");
-      return;
-    } else {
-      resource_path = path;
-    }
-
-    bool version_check = true;
-    char *override_version_env = getenv("QUDA_TUNE_VERSION_CHECK");
-    if (override_version_env && strcmp(override_version_env, "0") == 0) {
-      version_check = false;
-      warningQuda("Disabling QUDA tunecache version check");
-    }
-
-#ifdef MULTI_GPU
-    if (comm_rank() == 0) {
-#endif
-
-      cache_path = resource_path;
-      cache_path += "/tunecache.tsv";
-      cache_file.open(cache_path.c_str());
-
-      if (cache_file) {
-
-        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
-        getline(cache_file, line);
-        ls.str(line);
-        ls >> token;
-        if (token.compare("tunecache")) errorQuda("Bad format in %s", cache_path.c_str());
-        ls >> token;
-        if (version_check && token.compare(quda_version))
-          errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
-                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
-                    cache_path.c_str());
-        ls >> token;
-#ifdef GITVERSION
-        if (version_check && token.compare(gitversion))
-          errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
-                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
-                    cache_path.c_str());
-#else
-      if (version_check && token.compare(quda_version))
-        errorQuda("Cache file %s does not match current QUDA version. \nPlease delete this file or set the "
-                  "QUDA_RESOURCE_PATH environment variable to point to a new path.",
-                  cache_path.c_str());
-#endif
-        ls >> token;
-        if (version_check && token.compare(quda_hash))
-          errorQuda("Cache file %s does not match current QUDA build. \nPlease delete this file or set the "
-                    "QUDA_RESOURCE_PATH environment variable to point to a new path.",
-                    cache_path.c_str());
-
-        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
-        getline(cache_file, line); // eat the blank line
-
-        if (!cache_file.good()) errorQuda("Bad format in %s", cache_path.c_str());
-        getline(cache_file, line); // eat the description line
-
-        deserializeTuneCache(cache_file);
-
-        cache_file.close();
-        initial_cache_size = tunecache.size();
-
-        if (getVerbosity() >= QUDA_SUMMARIZE) {
-          printfQuda("Loaded %d sets of cached parameters from %s\n", static_cast<int>(initial_cache_size),
-                     cache_path.c_str());
-        }
-
-      } else {
-        warningQuda("Cache file not found.  All kernels will be re-tuned (if tuning is enabled).");
-      }
-
-#ifdef MULTI_GPU
-    }
-#endif
-
-    broadcastTuneCache();
-  }
-
-  /**
-   * Write tunecache to disk.
-   */
-  void saveTuneCache(bool error)
-  {
-    time_t now;
-    int lock_handle;
-    std::string lock_path, cache_path;
-    std::ofstream cache_file;
-
-    if (resource_path.empty()) return;
-
-      // FIXME: We should really check to see if any nodes have tuned a kernel that was not also tuned on node 0, since as things
-      //       stand, the corresponding launch parameters would never get cached to disk in this situation.  This will come up if we
-      //       ever support different subvolumes per GPU (as might be convenient for lattice volumes that don't divide evenly).
-
-#ifdef MULTI_GPU
-    if (comm_rank() == 0) {
-#endif
-
-      if (tunecache.size() == initial_cache_size && !error) return;
-
-      // Acquire lock.  Note that this is only robust if the filesystem supports flock() semantics, which is true for
-      // NFS on recent versions of linux but not Lustre by default (unless the filesystem was mounted with "-o flock").
-      lock_path = resource_path + "/tunecache.lock";
-      lock_handle = open(lock_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
-      if (lock_handle == -1) {
-        warningQuda("Unable to lock cache file.  Tuned launch parameters will not be cached to disk.  "
-                    "If you are certain that no other instances of QUDA are accessing this filesystem, "
-                    "please manually remove %s",
-                    lock_path.c_str());
-        return;
-      }
-      char msg[] = "If no instances of applications using QUDA are running,\n"
-                   "this lock file shouldn't be here and is safe to delete.";
-      int stat = write(lock_handle, msg, sizeof(msg)); // check status to avoid compiler warning
-      if (stat == -1) warningQuda("Unable to write to lock file for some bizarre reason");
-
-      cache_path = resource_path + (error ? "/tunecache_error.tsv" : "/tunecache.tsv");
-      cache_file.open(cache_path.c_str());
-
-      if (getVerbosity() >= QUDA_SUMMARIZE) {
-        printfQuda("Saving %d sets of cached parameters to %s\n", static_cast<int>(tunecache.size()), cache_path.c_str());
-      }
-
-      time(&now);
-      cache_file << "tunecache\t" << quda_version;
-#ifdef GITVERSION
-      cache_file << "\t" << gitversion;
-#else
-    cache_file << "\t" << quda_version;
-#endif
-      cache_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
-      cache_file << std::setw(16) << "volume"
-                 << "\tname\taux\tblock.x\tblock.y\tblock.z\tgrid.x\tgrid.y\tgrid.z\tshared_bytes\taux.x\taux.y\taux."
-                    "z\taux.w\ttime\tcomment"
-                 << std::endl;
-      serializeTuneCache(cache_file);
-      cache_file.close();
-
-      // Release lock.
-      close(lock_handle);
-      remove(lock_path.c_str());
-
-      initial_cache_size = tunecache.size();
-
-#ifdef MULTI_GPU
-    } else {
-      // give process 0 time to write out its tunecache if needed, but
-      // doesn't cause a hang if error is not triggered on process 0
-      if (error) sleep(10);
-    }
-#endif
-  }
-
-  static bool policy_tuning = false;
-  bool policyTuning() { return policy_tuning; }
-
-  void setPolicyTuning(bool policy_tuning_) { policy_tuning = policy_tuning_; }
-
-  // flush profile, setting counts to zero
-  void flushProfile()
-  {
-    for (map::iterator entry = tunecache.begin(); entry != tunecache.end(); entry++) {
-      // set all n_calls = 0
-      TuneParam &param = entry->second;
-      param.n_calls = 0;
-    }
-  }
-
-  // save profile
-  void saveProfile(const std::string label)
-  {
-    time_t now;
-    int lock_handle;
-    std::string lock_path, profile_path, async_profile_path, trace_path;
-    std::ofstream profile_file, async_profile_file, trace_file;
-
-    if (resource_path.empty()) return;
-
-#ifdef MULTI_GPU
-    if (comm_rank() == 0) {
-#endif
-
-      // Acquire lock.  Note that this is only robust if the filesystem supports flock() semantics, which is true for
-      // NFS on recent versions of linux but not Lustre by default (unless the filesystem was mounted with "-o flock").
-      lock_path = resource_path + "/profile.lock";
-      lock_handle = open(lock_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
-      if (lock_handle == -1) {
-        warningQuda("Unable to lock profile file.  Profile will not be saved to disk.  "
-                    "If you are certain that no other instances of QUDA are accessing this filesystem, "
-                    "please manually remove %s",
-                    lock_path.c_str());
-        return;
-      }
-      char msg[] = "If no instances of applications using QUDA are running,\n"
-                   "this lock file shouldn't be here and is safe to delete.";
-      int stat = write(lock_handle, msg, sizeof(msg)); // check status to avoid compiler warning
-      if (stat == -1) warningQuda("Unable to write to lock file for some bizarre reason");
-
-      // profile counter for writing out unique profiles
-      static int count = 0;
-
-      char *profile_fname = getenv("QUDA_PROFILE_OUTPUT_BASE");
-
-      if (!profile_fname) {
-        warningQuda(
-          "Environment variable QUDA_PROFILE_OUTPUT_BASE not set; writing to profile.tsv and profile_async.tsv");
-        profile_path = resource_path + "/profile_" + std::to_string(count) + ".tsv";
-        async_profile_path = resource_path + "/profile_async_" + std::to_string(count) + ".tsv";
-        if (traceEnabled()) trace_path = resource_path + "/trace_" + std::to_string(count) + ".tsv";
-      } else {
-        profile_path = resource_path + "/" + profile_fname + "_" + std::to_string(count) + ".tsv";
-        async_profile_path = resource_path + "/" + profile_fname + "_" + std::to_string(count) + "_async.tsv";
-        if (traceEnabled())
-          trace_path = resource_path + "/" + profile_fname + "_trace_" + std::to_string(count) + ".tsv";
-      }
-
-      count++;
-
-      profile_file.open(profile_path.c_str());
-      async_profile_file.open(async_profile_path.c_str());
-      if (traceEnabled()) trace_file.open(trace_path.c_str());
-
-      if (getVerbosity() >= QUDA_SUMMARIZE) {
-        // compute number of non-zero entries that will be output in the profile
-        int n_entry = 0;
-        int n_policy = 0;
-        for (map::iterator entry = tunecache.begin(); entry != tunecache.end(); entry++) {
-          // if a policy entry, then we can ignore
-          char tmp[TuneKey::aux_n] = {};
-          strncpy(tmp, entry->first.aux, TuneKey::aux_n);
-          TuneParam param = entry->second;
-          bool is_policy = strcmp(tmp, "policy") == 0 ? true : false;
-          if (param.n_calls > 0 && !is_policy) n_entry++;
-          if (param.n_calls > 0 && is_policy) n_policy++;
-        }
-
-        printfQuda("Saving %d sets of cached parameters to %s\n", n_entry, profile_path.c_str());
-        printfQuda("Saving %d sets of cached profiles to %s\n", n_policy, async_profile_path.c_str());
-        if (traceEnabled())
-          printfQuda("Saving trace list with %lu entries to %s\n", trace_list.size(), trace_path.c_str());
-      }
-
-      time(&now);
-
-      std::string Label = label.empty() ? "profile" : label;
-
-      profile_file << Label << "\t" << quda_version;
-#ifdef GITVERSION
-      profile_file << "\t" << gitversion;
-#else
-    profile_file << "\t" << quda_version;
-#endif
-      profile_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
-      profile_file << std::setw(12) << "total time"
-                   << "\t" << std::setw(12) << "percentage"
-                   << "\t" << std::setw(12) << "calls"
-                   << "\t" << std::setw(12) << "time / call"
-                   << "\t" << std::setw(16) << "volume"
-                   << "\tname\taux\tcomment" << std::endl;
-
-      async_profile_file << Label << "\t" << quda_version;
-#ifdef GITVERSION
-      async_profile_file << "\t" << gitversion;
-#else
-    async_profile_file << "\t" << quda_version;
-#endif
-      async_profile_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
-      async_profile_file << std::setw(12) << "total time"
-                         << "\t" << std::setw(12) << "percentage"
-                         << "\t" << std::setw(12) << "calls"
-                         << "\t" << std::setw(12) << "time / call"
-                         << "\t" << std::setw(16) << "volume"
-                         << "\tname\taux\tcomment" << std::endl;
-
-      serializeProfile(profile_file, async_profile_file);
-
-      profile_file.close();
-      async_profile_file.close();
-
-      if (traceEnabled()) {
-        trace_file << "trace"
-                   << "\t" << quda_version;
-#ifdef GITVERSION
-        trace_file << "\t" << gitversion;
-#else
-      trace_file << "\t" << quda_version;
-#endif
-        trace_file << "\t" << quda_hash << "\t# Last updated " << ctime(&now) << std::endl;
-
-        trace_file << std::setw(12) << "time\t" << std::setw(12) << "device-mem\t" << std::setw(12) << "pinned-mem\t";
-        trace_file << std::setw(12) << "mapped-mem\t" << std::setw(12) << "host-mem\t";
-        trace_file << std::setw(16) << "volume"
-                   << "\tname\taux" << std::endl;
-
-        serializeTrace(trace_file);
-
-        trace_file.close();
-      }
-
-      // Release lock.
-      close(lock_handle);
-      remove(lock_path.c_str());
-
-#ifdef MULTI_GPU
-    }
-#endif
-  }
-
-  static TimeProfile launchTimer("tuneLaunch");
-
-  /**
-   * Return the optimal launch parameters for a given kernel, either
-   * by retrieving them from tunecache or autotuning on the spot.
-   */
-  TuneParam &tuneLaunch(Tunable &tunable, QudaTune enabled, QudaVerbosity verbosity)
-  {
-#ifdef LAUNCH_TIMER
-    launchTimer.TPSTART(QUDA_PROFILE_TOTAL);
-    launchTimer.TPSTART(QUDA_PROFILE_INIT);
-#endif
-
-    TuneKey key = tunable.tuneKey();
-    if (use_managed_memory()) strcat(key.aux, ",managed");
-    last_key = key;
-    static TuneParam param;
-
-#ifdef LAUNCH_TIMER
-    launchTimer.TPSTOP(QUDA_PROFILE_INIT);
-    launchTimer.TPSTART(QUDA_PROFILE_PREAMBLE);
-#endif
-
-    static const Tunable *active_tunable; // for error checking
-    it = tunecache.find(key);
-
-    // first check if we have the tuned value and return if we have it
-    if (enabled == QUDA_TUNE_YES && it != tunecache.end()) {
-
-#ifdef LAUNCH_TIMER
-      launchTimer.TPSTOP(QUDA_PROFILE_PREAMBLE);
-      launchTimer.TPSTART(QUDA_PROFILE_COMPUTE);
-#endif
-
-      TuneParam &param = it->second;
-
-      if (verbosity >= QUDA_DEBUG_VERBOSE) {
-        printfQuda("Launching %s with %s at vol=%s with %s\n", key.name, key.aux, key.volume,
-                   tunable.paramString(param).c_str());
-      }
-
-#ifdef LAUNCH_TIMER
-      launchTimer.TPSTOP(QUDA_PROFILE_COMPUTE);
-      launchTimer.TPSTART(QUDA_PROFILE_EPILOGUE);
-#endif
-
-      tunable.checkLaunchParam(param);
-
-      // we could be tuning outside of the current scope
-      if (!tuning && profile_count) param.n_calls++;
-
-#ifdef LAUNCH_TIMER
-      launchTimer.TPSTOP(QUDA_PROFILE_EPILOGUE);
-      launchTimer.TPSTOP(QUDA_PROFILE_TOTAL);
-#endif
-
-      if (traceEnabled() >= 2) {
-        TraceKey trace_entry(key, param.time);
-        trace_list.push_back(trace_entry);
-      }
-
-      return param;
-    }
-
-#ifdef LAUNCH_TIMER
-    launchTimer.TPSTOP(QUDA_PROFILE_PREAMBLE);
-    launchTimer.TPSTOP(QUDA_PROFILE_TOTAL);
-#endif
-
-    if (enabled == QUDA_TUNE_NO) {
-      tunable.defaultTuneParam(param);
-      tunable.checkLaunchParam(param);
-      if (verbosity >= QUDA_DEBUG_VERBOSE) {
-        printfQuda("Launching %s with %s at vol=%s with %s (untuned)\n", key.name, key.aux, key.volume,
-                   tunable.paramString(param).c_str());
-      }
-    } else if (!tuning) {
-
-      /* As long as global reductions are not disabled, only do the
-         tuning on node 0, else do the tuning on all nodes since we
-         can't guarantee that all nodes are partaking */
-      if (comm_rank() == 0 || !commGlobalReduction() || policyTuning()) {
-        TuneParam best_param;
-        cudaError_t error = cudaSuccess;
-        cudaEvent_t start, end;
-        float elapsed_time, best_time;
-        time_t now;
-
-        tuning = true;
-        active_tunable = &tunable;
-        best_time = FLT_MAX;
-
-        if (verbosity >= QUDA_DEBUG_VERBOSE) printfQuda("PreTune %s\n", key.name);
-        tunable.preTune();
-
-        cudaEventCreate(&start);
-        cudaEventCreate(&end);
-
-        if (verbosity >= QUDA_DEBUG_VERBOSE) {
-          printfQuda("Tuning %s with %s at vol=%s\n", key.name, key.aux, key.volume);
-        }
-
-        Timer tune_timer;
-        tune_timer.Start(__func__, __FILE__, __LINE__);
-
-        tunable.initTuneParam(param);
-        while (tuning) {
-          cudaDeviceSynchronize();
-          cudaGetLastError(); // clear error counter
-          tunable.checkLaunchParam(param);
-          if (verbosity >= QUDA_DEBUG_VERBOSE) {
-            printfQuda("About to call tunable.apply block=(%d,%d,%d) grid=(%d,%d,%d) shared_bytes=%d aux=(%d,%d,%d)\n",
-                       param.block.x, param.block.y, param.block.z, param.grid.x, param.grid.y, param.grid.z,
-                       param.shared_bytes, param.aux.x, param.aux.y, param.aux.z);
-          }
-          tunable.apply(); // do initial call in case we need to jit compile for these parameters or if policy tuning
-
-          cudaEventRecord(start, 0);
-          for (int i = 0; i < tunable.tuningIter(); i++) {
-            tunable.apply(); // calls tuneLaunch() again, which simply returns the currently active param
-          }
-          cudaEventRecord(end, 0);
-          cudaEventSynchronize(end);
-          cudaEventElapsedTime(&elapsed_time, start, end);
-          cudaDeviceSynchronize();
-          error = cudaGetLastError();
-
-          { // check that error state is cleared
-            cudaDeviceSynchronize();
-            cudaError_t error = cudaGetLastError();
-            if (error != cudaSuccess) errorQuda("Failed to clear error state %s\n", cudaGetErrorString(error));
-          }
-
-          elapsed_time /= (1e3 * tunable.tuningIter());
-          if ((elapsed_time < best_time) && (error == cudaSuccess) && (tunable.launchError() == QUDA_SUCCESS)) {
-            best_time = elapsed_time;
-            best_param = param;
-          }
-          if ((verbosity >= QUDA_DEBUG_VERBOSE)) {
-            if (error == cudaSuccess && tunable.launchError() == QUDA_SUCCESS) {
-              printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(),
-                         tunable.perfString(elapsed_time).c_str());
-            } else {
-              if (tunable.launchError() == QUDA_SUCCESS) { // must be regular error
-                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), cudaGetErrorString(error));
-              } else { // else must be a manually thrown error
-                printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), qudaGetLastErrorString().c_str());
-              }
-            }
-          }
-          tuning = tunable.advanceTuneParam(param);
-          tunable.launchError() = QUDA_SUCCESS;
-        }
-
-        tune_timer.Stop(__func__, __FILE__, __LINE__);
-
-        if (best_time == FLT_MAX) {
-          errorQuda("Auto-tuning failed for %s with %s at vol=%s", key.name, key.aux, key.volume);
-        }
-        if (verbosity >= QUDA_VERBOSE) {
-          printfQuda("Tuned %s giving %s for %s with %s\n", tunable.paramString(best_param).c_str(),
-                     tunable.perfString(best_time).c_str(), key.name, key.aux);
-        }
-        time(&now);
-        best_param.comment = "# " + tunable.perfString(best_time);
-        best_param.comment += ", tuning took " + std::to_string(tune_timer.Last()) + " seconds at ";
-        best_param.comment += ctime(&now); // includes a newline
-        best_param.time = best_time;
-
-        cudaEventDestroy(start);
-        cudaEventDestroy(end);
-
-        if (verbosity >= QUDA_DEBUG_VERBOSE) printfQuda("PostTune %s\n", key.name);
-        tuning = true;
-        tunable.postTune();
-        tuning = false;
-        param = best_param;
-        tunecache[key] = best_param;
-      }
-      if (commGlobalReduction() || policyTuning()) broadcastTuneCache();
-
-      // check this process is getting the key that is expected
-      if (tunecache.find(key) == tunecache.end()) {
-        errorQuda("Failed to find key entry (%s:%s:%s)", key.name, key.volume, key.aux);
-      }
-      param = tunecache[key]; // read this now for all processes
-
-      if (traceEnabled() >= 2) {
-        TraceKey trace_entry(key, param.time);
-        trace_list.push_back(trace_entry);
-      }
-
-    } else if (&tunable != active_tunable) {
-      errorQuda("Unexpected call to tuneLaunch() in %s::apply()", typeid(tunable).name());
-    }
-
-    param.n_calls = profile_count ? 1 : 0;
-
-    return param;
-  }
-
-  void printLaunchTimer()
-  {
-#ifdef LAUNCH_TIMER
-    launchTimer.Print();
-#endif
-  }
-} // namespace quda
diff --git a/lib/timer.cpp b/lib/timer.cpp
index da1472353e..d8c481a287 100644
--- a/lib/timer.cpp
+++ b/lib/timer.cpp
@@ -78,7 +78,7 @@ namespace quda {
   const int TimeProfile::nvtx_num_colors = sizeof(nvtx_colors)/sizeof(uint32_t);
 #endif
 
-  Timer TimeProfile::global_profile[QUDA_PROFILE_COUNT];
+  Timer<> TimeProfile::global_profile[QUDA_PROFILE_COUNT];
   bool TimeProfile::global_switchOff[QUDA_PROFILE_COUNT] = {};
   int TimeProfile::global_total_level[QUDA_PROFILE_COUNT] = {};
 
diff --git a/tests/blas_test.cpp b/tests/blas_test.cpp
index 5c48610efe..dbdaed64f5 100644
--- a/tests/blas_test.cpp
+++ b/tests/blas_test.cpp
@@ -2,6 +2,7 @@
 #include <stdlib.h>
 
 #include <quda_internal.h>
+#include <timer.h>
 #include <color_spinor_field.h>
 #include <blas_quda.h>
 
@@ -362,10 +363,8 @@ double benchmark(Kernel kernel, const int niter)
   quda::Complex * A2 = new quda::Complex[Nsrc*Nsrc]; // for the block cDotProductNorm test
   double *Ar = new double[Nsrc * Msrc];
 
-  cudaEvent_t start, end;
-  cudaEventCreate(&start);
-  cudaEventCreate(&end);
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
+  device_timer_t timer;
+  timer.start();
 
   {
     switch (kernel) {
@@ -529,19 +528,15 @@ double benchmark(Kernel kernel, const int niter)
     }
   }
 
-  cudaEventRecord(end, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(end);
-  float runTime;
-  cudaEventElapsedTime(&runTime, start, end);
-  cudaEventDestroy(start);
-  cudaEventDestroy(end);
+  timer.stop();
+
   delete[] A;
   delete[] B;
   delete[] C;
   delete[] A2;
   delete[] Ar;
-  double secs = runTime / 1000;
-  return secs;
+
+  return timer.last();
 }
 
 #define ERROR(a) fabs(blas::norm2(*a##D) - blas::norm2(*a##H)) / blas::norm2(*a##H)
diff --git a/tests/contract_test.cpp b/tests/contract_test.cpp
index 4dad33d67b..19267957fc 100644
--- a/tests/contract_test.cpp
+++ b/tests/contract_test.cpp
@@ -104,16 +104,8 @@ int main(int argc, char **argv)
 //-----------------------------------------------------------------------------
 
 // Performs the CPU GPU comparison with the given parameters
-int test(int contractionType, int Prec)
+int test(int contractionType, QudaPrecision test_prec)
 {
-  QudaPrecision test_prec = QUDA_INVALID_PRECISION;
-  switch (Prec) {
-  case 0: test_prec = QUDA_SINGLE_PRECISION; break;
-  case 1: test_prec = QUDA_DOUBLE_PRECISION; break;
-  default: errorQuda("Undefined QUDA precision type %d\n", Prec);
-  }
-  if ((test_prec & QUDA_PRECISION) == 0) GTEST_SKIP();
-
   int X[4] = {xdim, ydim, zdim, tdim};
 
   QudaInvertParam inv_param = newQudaInvertParam();
@@ -195,8 +187,9 @@ class ContractionTest : public ::testing::TestWithParam<::testing::tuple<int, in
 // Sets up the Google test
 TEST_P(ContractionTest, verify)
 {
-  int prec = ::testing::get<0>(GetParam());
+  QudaPrecision prec = getPrecision(::testing::get<0>(GetParam()));
   int contractionType = ::testing::get<1>(GetParam());
+  if ((QUDA_PRECISION & prec) == 0) GTEST_SKIP();
   auto faults = test(contractionType, prec);
   EXPECT_EQ(faults, 0) << "CPU and GPU implementations do not agree";
 }
@@ -213,4 +206,4 @@ std::string getContractName(testing::TestParamInfo<::testing::tuple<int, int>> p
 }
 
 // Instantiate all test cases
-INSTANTIATE_TEST_SUITE_P(QUDA, ContractionTest, Combine(Range(0, 2), Range(0, NcontractType)), getContractName);
+INSTANTIATE_TEST_SUITE_P(QUDA, ContractionTest, Combine(Range(2, 4), Range(0, NcontractType)), getContractName);
diff --git a/tests/covdev_test.cpp b/tests/covdev_test.cpp
index 6617bc7d1f..d293cfbf64 100644
--- a/tests/covdev_test.cpp
+++ b/tests/covdev_test.cpp
@@ -161,24 +161,13 @@ void end(void)
 
 double dslashCUDA(int niter, int mu)
 {
-  cudaEvent_t start, end;
-  cudaEventCreate(&start);
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(start);
+  device_timer_t timer;
+  timer.start();
 
   for (int i = 0; i < niter; i++) dirac->MCD(*cudaSpinorOut, *cudaSpinor, mu);
 
-  cudaEventCreate(&end);
-  cudaEventRecord(end, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(end);
-  float runTime;
-  cudaEventElapsedTime(&runTime, start, end);
-  cudaEventDestroy(start);
-  cudaEventDestroy(end);
-
-  double secs = runTime / 1000; //stopwatchReadSeconds();
-
-  return secs;
+  timer.stop();
+  return timer.last();
 }
 
 void covdevRef(int mu)
diff --git a/tests/dslash_ctest.cpp b/tests/dslash_ctest.cpp
index ce96836938..1fb527b39c 100644
--- a/tests/dslash_ctest.cpp
+++ b/tests/dslash_ctest.cpp
@@ -318,20 +318,17 @@ struct DslashTime {
 // execute kernel
 DslashTime dslashCUDA(int niter)
 {
-
   DslashTime dslash_time;
-  timeval tstart, tstop;
 
-  cudaEvent_t start, end;
-  cudaEventCreate(&start);
-  cudaEventCreate(&end);
+  host_timer_t host_timer;
+  device_timer_t device_timer;
 
   comm_barrier();
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
+  device_timer.start();
 
   for (int i = 0; i < niter; i++) {
 
-    gettimeofday(&tstart, NULL);
+    host_timer.start();
 
     if (dslash_type == QUDA_DOMAIN_WALL_4D_DSLASH) {
       switch (dtest_type) {
@@ -523,27 +520,18 @@ DslashTime dslashCUDA(int niter)
       }
     }
 
-    gettimeofday(&tstop, NULL);
-    long ds = tstop.tv_sec - tstart.tv_sec;
-    long dus = tstop.tv_usec - tstart.tv_usec;
-    double elapsed = ds + 0.000001*dus;
+    host_timer.stop();
 
-    dslash_time.cpu_time += elapsed;
+    dslash_time.cpu_time += host_timer.last();
     // skip first and last iterations since they may skew these metrics if comms are not synchronous
     if (i>0 && i<niter) {
-      if (elapsed < dslash_time.cpu_min) dslash_time.cpu_min = elapsed;
-      if (elapsed > dslash_time.cpu_max) dslash_time.cpu_max = elapsed;
+      dslash_time.cpu_min = std::min(dslash_time.cpu_min, host_timer.last());
+      dslash_time.cpu_max = std::max(dslash_time.cpu_max, host_timer.last());
     }
   }
 
-  cudaEventRecord(end, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(end);
-  float runTime;
-  cudaEventElapsedTime(&runTime, start, end);
-  cudaEventDestroy(start);
-  cudaEventDestroy(end);
-
-  dslash_time.event_time = runTime / 1000;
+  device_timer.stop();
+  dslash_time.event_time = device_timer.last();
 
   return dslash_time;
 }
diff --git a/tests/dslash_test.cpp b/tests/dslash_test.cpp
index 0c46d3573d..8d335c5c76 100644
--- a/tests/dslash_test.cpp
+++ b/tests/dslash_test.cpp
@@ -290,18 +290,16 @@ struct DslashTime {
 DslashTime dslashCUDA(int niter) {
 
   DslashTime dslash_time;
-  timeval tstart, tstop;
 
-  cudaEvent_t start, end;
-  cudaEventCreate(&start);
-  cudaEventCreate(&end);
+  host_timer_t host_timer;
+  device_timer_t device_timer;
 
   comm_barrier();
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
+  device_timer.start();
 
   for (int i = 0; i < niter; i++) {
 
-    gettimeofday(&tstart, nullptr);
+    host_timer.start();
 
     if (dslash_type == QUDA_DOMAIN_WALL_4D_DSLASH){
       switch (dtest_type) {
@@ -493,27 +491,18 @@ DslashTime dslashCUDA(int niter) {
       }
     }
 
-    gettimeofday(&tstop, nullptr);
-    long ds = tstop.tv_sec - tstart.tv_sec;
-    long dus = tstop.tv_usec - tstart.tv_usec;
-    double elapsed = ds + 0.000001*dus;
+    host_timer.stop();
 
-    dslash_time.cpu_time += elapsed;
+    dslash_time.cpu_time += host_timer.last();
     // skip first and last iterations since they may skew these metrics if comms are not synchronous
     if (i>0 && i<niter) {
-      if (elapsed < dslash_time.cpu_min) dslash_time.cpu_min = elapsed;
-      if (elapsed > dslash_time.cpu_max) dslash_time.cpu_max = elapsed;
+      dslash_time.cpu_min = std::min(dslash_time.cpu_min, host_timer.last());
+      dslash_time.cpu_max = std::max(dslash_time.cpu_max, host_timer.last());
     }
   }
-    
-  cudaEventRecord(end, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(end);
-  float runTime;
-  cudaEventElapsedTime(&runTime, start, end);
-  cudaEventDestroy(start);
-  cudaEventDestroy(end);
-
-  dslash_time.event_time = runTime / 1000;
+
+  device_timer.stop();
+  dslash_time.event_time = device_timer.last();
 
   return dslash_time;
 }
diff --git a/tests/eigensolve_test.cpp b/tests/eigensolve_test.cpp
index 0dd675907b..88f5c375ec 100644
--- a/tests/eigensolve_test.cpp
+++ b/tests/eigensolve_test.cpp
@@ -1,10 +1,10 @@
 #include <stdlib.h>
 #include <stdio.h>
-#include <time.h>
 #include <math.h>
 #include <string.h>
 #include <algorithm>
 
+#include <timer.h>
 #include <host_utils.h>
 #include <command_line_params.h>
 #include "misc.h"
@@ -154,16 +154,15 @@ int main(int argc, char **argv)
   // requested data, at the requested prec. All the information needed to perfom the
   // solve is in the eig_param container. If eig_param.arpack_check == true and
   // precision is double, the routine will use ARPACK rather than the GPU.
-  struct timeval start, end;
-  gettimeofday(&start, NULL);
+  host_timer_t host_timer;
+  host_timer.start();
   if (eig_param.arpack_check && !(eig_inv_param.cpu_prec == QUDA_DOUBLE_PRECISION)) {
     errorQuda("ARPACK check only available in double precision");
   }
 
   eigensolveQuda(host_evecs, host_evals, &eig_param);
-  gettimeofday(&end, NULL);
-  double time = ((end.tv_sec - start.tv_sec) * 1000000u + end.tv_usec - start.tv_usec) / 1.e6;
-  printfQuda("Time for %s solution = %f\n", eig_param.arpack_check ? "ARPACK" : "QUDA", time);
+  host_timer.stop();
+  printfQuda("Time for %s solution = %f\n", eig_param.arpack_check ? "ARPACK" : "QUDA", host_timer.last());
   // QUDA eigensolver test COMPLETE
   //----------------------------------------------------------------------------
 
diff --git a/tests/gauge_alg_test.cpp b/tests/gauge_alg_test.cpp
index d72f947dbc..f4a69b0478 100644
--- a/tests/gauge_alg_test.cpp
+++ b/tests/gauge_alg_test.cpp
@@ -4,6 +4,7 @@
 
 #include <quda.h>
 #include <quda_internal.h>
+#include <timer.h>
 #include <gauge_field.h>
 
 #include <comm_quda.h>
@@ -22,7 +23,16 @@
 using namespace quda;
 
 class GaugeAlgTest : public ::testing::Test {
- protected:
+
+  QudaGaugeParam param;
+  host_timer_t a0;
+  host_timer_t a1;
+  RNG * randstates;
+
+protected:
+  cudaGaugeField *U;
+  double3 plaq;
+
   void SetReunitarizationConsts(){
     const double unitarize_eps = 1e-14;
     const double max_error = 1e-10;
@@ -115,17 +125,16 @@ class GaugeAlgTest : public ::testing::Test {
     U = new cudaGaugeField(gParam);
 #endif
     // CURAND random generator initialization
-    randstates = new RNG(gParam, 1234);
-    randstates->Init();
+    randstates = new RNG(*U, 1234);
 
-    nsteps = 10;
-    nhbsteps = 4;
-    novrsteps = 4;
-    coldstart = false;
-    beta_value = 6.2;
+    int nsteps = 10;
+    int nhbsteps = 4;
+    int novrsteps = 4;
+    bool coldstart = false;
+    double beta_value = 6.2;
 
-    a0.Start(__func__, __FILE__, __LINE__);
-    a1.Start(__func__, __FILE__, __LINE__);
+    a0.start();
+    a1.start();
 
     int *num_failures_h = (int *)mapped_malloc(sizeof(int));
     int *num_failures_d = (int *)get_mapped_device_pointer(num_failures_h);
@@ -151,9 +160,9 @@ class GaugeAlgTest : public ::testing::Test {
 
       plaquette(*U);
     }
-    a1.Stop(__func__, __FILE__, __LINE__);
+    a1.stop();
 
-    printfQuda("Time Monte -> %.6f s\n", a1.Last());
+    printfQuda("Time Monte -> %.6f s\n", a1.last());
     plaq = plaquette(*U);
     printfQuda("Plaq: %.16e, %.16e, %.16e\n", plaq.x, plaq.y, plaq.z);
 
@@ -161,7 +170,7 @@ class GaugeAlgTest : public ::testing::Test {
   }
 
   virtual void TearDown() {
-    detu = getLinkDeterminant(*U);
+    auto detu = getLinkDeterminant(*U);
     double2 tru = getLinkTrace(*U);
     printfQuda("Det: %.16e:%.16e\n", detu.x, detu.y);
     printfQuda("Tr: %.16e:%.16e\n", tru.x/3.0, tru.y/3.0);
@@ -170,30 +179,16 @@ class GaugeAlgTest : public ::testing::Test {
     //Release all temporary memory used for data exchange between GPUs in multi-GPU mode
     PGaugeExchangeFree();
 
-    a0.Stop(__func__, __FILE__, __LINE__);
-    printfQuda("Time -> %.6f s\n", a0.Last());
-    randstates->Release();
+    a0.stop();
+    printfQuda("Time -> %.6f s\n", a0.last());
     delete randstates;
   }
 
-  QudaGaugeParam param;
-
-  Timer a0,a1;
-  double2 detu;
-  double3 plaq;
-  cudaGaugeField *U;
-  int nsteps;
-  int nhbsteps;
-  int novrsteps;
-  bool coldstart;
-  double beta_value;
-  RNG * randstates;
-
 };
 
 TEST_F(GaugeAlgTest, Generation)
 {
-  detu = getLinkDeterminant(*U);
+  auto detu = getLinkDeterminant(*U);
   plaq = plaquette(*U);
   bool testgen = false;
   //check plaquette value for beta = 6.2
diff --git a/tests/heatbath_test.cpp b/tests/heatbath_test.cpp
index 461af3d764..f1206e513a 100644
--- a/tests/heatbath_test.cpp
+++ b/tests/heatbath_test.cpp
@@ -126,7 +126,6 @@ int main(int argc, char **argv)
     cudaGaugeField *gaugeEx = new cudaGaugeField(gParamEx);
     // CURAND random generator initialization
     RNG *randstates = new RNG(*gauge, 1234);
-    randstates->Init();
 
     int nsteps = heatbath_num_steps;
     int nwarm = heatbath_warmup_steps;
@@ -140,32 +139,22 @@ int main(int argc, char **argv)
     printfQuda("  %d Warmup steps\n", nwarm);
     printfQuda("  %d Measurement steps\n", nsteps);
 
-    if (strcmp(latfile, "")) { // Copy in loaded gauge field
-
-      printfQuda("Loading the gauge field in %s\n", latfile);
-
-      loadGaugeQuda(load_gauge, &gauge_param);
-      // Get pointer to internal resident gauge field
-      cudaGaugeField* extendedGaugeResident = new cudaGaugeField(gParamEx);
-      copyExtendedResidentGaugeQuda((void*)extendedGaugeResident, QUDA_CUDA_FIELD_LOCATION);
-      InitGaugeField(*gaugeEx);
-      copyExtendedGauge(*gaugeEx, *extendedGaugeResident, QUDA_CUDA_FIELD_LOCATION);
-      delete extendedGaugeResident;
-
-    } else if (link_recon != QUDA_RECONSTRUCT_8 && coldstart) {
-      InitGaugeField( *gaugeEx);
+    if (strcmp(latfile, "")) { // We loaded in a gauge field
+      // copy internal extended field to gaugeEx
+      copyExtendedResidentGaugeQuda((void*)gaugeEx);
     } else {
-      InitGaugeField( *gaugeEx, *randstates );
-    }
+      if (coldstart) InitGaugeField(*gaugeEx);
+      else InitGaugeField(*gaugeEx, *randstates);
 
-    // copy into regular field
-    copyExtendedGauge(*gauge, *gaugeEx, QUDA_CUDA_FIELD_LOCATION);
+      // copy into regular field
+      copyExtendedGauge(*gauge, *gaugeEx, QUDA_CUDA_FIELD_LOCATION);
 
-    // load the gauge field from gauge
-    gauge_param.gauge_order = gauge->Order();
-    gauge_param.location = QUDA_CUDA_FIELD_LOCATION;
+      // load the gauge field from gauge
+      gauge_param.gauge_order = gauge->Order();
+      gauge_param.location = QUDA_CUDA_FIELD_LOCATION;
 
-    loadGaugeQuda(gauge->Gauge_p(), &gauge_param);
+      loadGaugeQuda(gauge->Gauge_p(), &gauge_param);
+    }
 
     QudaGaugeObservableParam param = newQudaGaugeObservableParam();
     param.compute_plaquette = QUDA_BOOLEAN_TRUE;
@@ -245,7 +234,6 @@ int main(int argc, char **argv)
     //Release all temporary memory used for data exchange between GPUs in multi-GPU mode
     PGaugeExchangeFree();
 
-    randstates->Release();
     delete randstates;
   }
 
diff --git a/tests/host_reference/gauge_force_reference.cpp b/tests/host_reference/gauge_force_reference.cpp
index 341391a550..50f7f7aeb1 100644
--- a/tests/host_reference/gauge_force_reference.cpp
+++ b/tests/host_reference/gauge_force_reference.cpp
@@ -88,6 +88,30 @@ typedef struct {
   double space;
 } danti_hermitmat;
 
+// convenience struct for passing around lattice meta data
+struct lattice_t {
+  int n_color;
+  size_t volume;
+  size_t volume_ex;
+  int x[4];
+  int r[4];
+  int e[4];
+
+  lattice_t(const GaugeField &lat) :
+    n_color(lat.Ncolor()),
+    volume(1),
+    volume_ex(lat.Volume())
+  {
+    for (int d = 0; d < 4; d++) {
+      x[d] = lat.X()[d] - 2 * lat.R()[d];
+      r[d] = lat.R()[d];
+      e[d] = lat.X()[d];
+      volume *= x[d];
+    }
+  };
+
+};
+
 extern int neighborIndexFullLattice(int i, int dx4, int dx3, int dx2, int dx1);
 
 template <typename su3_matrix> void su3_adjoint(su3_matrix *a, su3_matrix *b)
@@ -213,67 +237,46 @@ template <typename su3_matrix> void print_su3_matrix(su3_matrix *a)
   }
 }
 
-int gf_neighborIndexFullLattice(int i, int dx4, int dx3, int dx2, int dx1)
+int gf_neighborIndexFullLattice(size_t i, int dx[], const lattice_t &lat)
 {
   int oddBit = 0;
-  int half_idx = i;
-  if (i >= Vh) {
+  int x[4];
+  auto half_idx = i;
+  if (i >= lat.volume/2) {
     oddBit = 1;
-    half_idx = i - Vh;
+    half_idx = i - lat.volume/2;
+  }
+
+  auto za = half_idx / (lat.x[0] / 2) ;
+  auto x0h = half_idx - za * (lat.x[0] / 2);
+  auto zb = za / lat.x[1];
+  x[1] = za - zb * lat.x[1];
+  x[3] = zb / lat.x[2];
+  x[2] = zb - x[3] * lat.x[2];
+  auto x1odd = (x[1] + x[2] + x[3] + oddBit) & 1;
+  x[0] = 2 * x0h + x1odd;
+
+  for (int d = 0; d < 4; d++) {
+    x[d] = comm_dim_partitioned(d) ? x[d] + dx[d] : (x[d] + dx[d] + lat.x[d]) % lat.x[d];
   }
-  int X1 = Z[0];
-  int X2 = Z[1];
-  int X3 = Z[2];
-  // int X4 = Z[3];
-  int X1h = X1 / 2;
-
-  int za = half_idx / X1h;
-  int x1h = half_idx - za * X1h;
-  int zb = za / X2;
-  int x2 = za - zb * X2;
-  int x4 = zb / X3;
-  int x3 = zb - x4 * X3;
-  int x1odd = (x2 + x3 + x4 + oddBit) & 1;
-  int x1 = 2 * x1h + x1odd;
-
-#ifdef MULTI_GPU
-  x4 = x4 + dx4;
-  x3 = x3 + dx3;
-  x2 = x2 + dx2;
-  x1 = x1 + dx1;
-
-  int nbr_half_idx = ((x4 + 2) * (E[2] * E[1] * E[0]) + (x3 + 2) * (E[1] * E[0]) + (x2 + 2) * (E[0]) + (x1 + 2)) / 2;
-#else
-  x4 = (x4 + dx4 + Z[3]) % Z[3];
-  x3 = (x3 + dx3 + Z[2]) % Z[2];
-  x2 = (x2 + dx2 + Z[1]) % Z[1];
-  x1 = (x1 + dx1 + Z[0]) % Z[0];
-
-  int nbr_half_idx = (x4 * (Z[2] * Z[1] * Z[0]) + x3 * (Z[1] * Z[0]) + x2 * (Z[0]) + x1) / 2;
-#endif
-
-  int oddBitChanged = (dx4 + dx3 + dx2 + dx1) % 2;
+  size_t nbr_half_idx = ((x[3] + lat.r[3]) * (lat.e[2] * lat.e[1] * lat.e[0]) + (x[2] + lat.r[2]) * (lat.e[1] * lat.e[0]) + (x[1] + lat.r[1]) * (lat.e[0]) + (x[0] + lat.r[0])) / 2;
+
+  int oddBitChanged = (dx[3] + dx[2] + dx[1] + dx[0]) % 2;
   if (oddBitChanged) { oddBit = 1 - oddBit; }
   int ret = nbr_half_idx;
-  if (oddBit) {
-#ifdef MULTI_GPU
-    ret = Vh_ex + nbr_half_idx;
-#else
-    ret = Vh + nbr_half_idx;
-#endif
-  }
+  if (oddBit) ret += lat.volume_ex / 2;
 
   return ret;
 }
 
-// this functon compute one path for all lattice sites
+// this function compute one path for all lattice sites
 template <typename su3_matrix, typename Float>
-static void compute_path_product(su3_matrix *staple, su3_matrix **sitelink, int *path, int len, Float loop_coeff, int dir)
+static void compute_path_product(su3_matrix *staple, su3_matrix **sitelink, int *path, int len, Float loop_coeff, int dir, const lattice_t &lat)
 {
   su3_matrix prev_matrix, curr_matrix, tmat;
   int dx[4];
 
-  for (int i = 0; i < V; i++) {
+  for (size_t i = 0; i < lat.volume; i++) {
     memset(dx, 0, sizeof(dx));
     memset(&curr_matrix, 0, sizeof(curr_matrix));
 
@@ -294,7 +297,7 @@ static void compute_path_product(su3_matrix *staple, su3_matrix **sitelink, int
         lnkdir = OPP_DIR(path[j]);
       }
 
-      int nbr_idx = gf_neighborIndexFullLattice(i, dx[3], dx[2], dx[1], dx[0]);
+      int nbr_idx = gf_neighborIndexFullLattice(i, dx, lat);
       su3_matrix *lnk = sitelink[lnkdir] + nbr_idx;
 
       if (GOES_FORWARDS(path[j])) {
@@ -317,9 +320,9 @@ static void compute_path_product(su3_matrix *staple, su3_matrix **sitelink, int
 }
 
 template <typename su3_matrix, typename anti_hermitmat, typename Float>
-static void update_mom(anti_hermitmat *momentum, int dir, su3_matrix **sitelink, su3_matrix *staple, Float eb3)
+static void update_mom(anti_hermitmat *momentum, int dir, su3_matrix **sitelink, su3_matrix *staple, Float eb3, const lattice_t &lat)
 {
-  for (int i = 0; i < V; i++) {
+  for (size_t i = 0; i < lat.volume; i++) {
     su3_matrix tmat1;
     su3_matrix tmat2;
     su3_matrix tmat3;
@@ -340,52 +343,47 @@ static void update_mom(anti_hermitmat *momentum, int dir, su3_matrix **sitelink,
  *
  */
 void gauge_force_reference_dir(void *refMom, int dir, double eb3, void **sitelink, void **sitelink_ex,
-                               QudaPrecision prec, int **path_dir, int *length, void *loop_coeff, int num_paths)
+                               QudaPrecision prec, int **path_dir, int *length, void *loop_coeff, int num_paths, const lattice_t &lat)
 {
-  void *staple;
-  int gSize = prec;
-
-  staple = malloc(V * gauge_site_size * gSize);
-  if (staple == NULL) {
-    fprintf(stderr, "ERROR: malloc failed for staple in functon %s\n", __FUNCTION__);
-    exit(1);
-  }
-
-  memset(staple, 0, V * gauge_site_size * gSize);
+  size_t size = V * 2 * lat.n_color * lat.n_color * prec;
+  void *staple = safe_malloc(size);
+  memset(staple, 0, size);
 
   for (int i = 0; i < num_paths; i++) {
     if (prec == QUDA_DOUBLE_PRECISION) {
       double *my_loop_coeff = (double *)loop_coeff;
-      compute_path_product((dsu3_matrix *)staple, (dsu3_matrix **)sitelink_ex, path_dir[i], length[i], my_loop_coeff[i], dir);
+      compute_path_product((dsu3_matrix *)staple, (dsu3_matrix **)sitelink_ex, path_dir[i], length[i], my_loop_coeff[i], dir, lat);
     } else {
       float *my_loop_coeff = (float *)loop_coeff;
-      compute_path_product((fsu3_matrix *)staple, (fsu3_matrix **)sitelink_ex, path_dir[i], length[i], my_loop_coeff[i], dir);
+      compute_path_product((fsu3_matrix *)staple, (fsu3_matrix **)sitelink_ex, path_dir[i], length[i], my_loop_coeff[i], dir, lat);
     }
   }
 
   if (prec == QUDA_DOUBLE_PRECISION) {
-    update_mom((danti_hermitmat *)refMom, dir, (dsu3_matrix **)sitelink, (dsu3_matrix *)staple, (double)eb3);
+    update_mom((danti_hermitmat *)refMom, dir, (dsu3_matrix **)sitelink, (dsu3_matrix *)staple, (double)eb3, lat);
   } else {
-    update_mom((fanti_hermitmat *)refMom, dir, (fsu3_matrix **)sitelink, (fsu3_matrix *)staple, (float)eb3);
+    update_mom((fanti_hermitmat *)refMom, dir, (fsu3_matrix **)sitelink, (fsu3_matrix *)staple, (float)eb3, lat);
   }
 
-  free(staple);
+  host_free(staple);
 }
 
 void gauge_force_reference(void *refMom, double eb3, void **sitelink, QudaPrecision prec, int ***path_dir, int *length,
                            void *loop_coeff, int num_paths)
 {
   // created extended field
-  int R[] = {2, 2, 2, 2};
+  int R[4];
+  for (int d = 0; d < 4; d++) R[d] = 2 * comm_dim_partitioned(d);
   QudaGaugeParam param = newQudaGaugeParam();
   setGaugeParam(param);
   param.gauge_order = QUDA_QDP_GAUGE_ORDER;
   param.t_boundary = QUDA_PERIODIC_T;
 
   auto qdp_ex = quda::createExtendedGauge((void **)sitelink, param, R);
+  lattice_t lat(*qdp_ex);
 
   for (int dir = 0; dir < 4; dir++) {
-    gauge_force_reference_dir(refMom, dir, eb3, sitelink, (void **)qdp_ex->Gauge_p(), prec, path_dir[dir], length, loop_coeff, num_paths);
+    gauge_force_reference_dir(refMom, dir, eb3, sitelink, (void **)qdp_ex->Gauge_p(), prec, path_dir[dir], length, loop_coeff, num_paths, lat);
   }
 
   delete qdp_ex;
diff --git a/tests/host_reference/hisq_force_reference2.cpp b/tests/host_reference/hisq_force_reference2.cpp
index b466983f36..f6876ae212 100644
--- a/tests/host_reference/hisq_force_reference2.cpp
+++ b/tests/host_reference/hisq_force_reference2.cpp
@@ -135,8 +135,10 @@ extern int Vh_ex;
 	
     public: 
       Matrix(); // default constructor
-      Matrix(const Matrix<N,T>& mat) = default;
-      Matrix& operator=(const Matrix<N,T>& mat) = default;
+      Matrix(const Matrix<N,T> &) = default;
+      Matrix(Matrix<N,T> &&) = default;
+      Matrix& operator=(const Matrix<N,T> &) = default;
+      Matrix& operator=(Matrix<N,T> &&) = default;
       Matrix & operator += (const Matrix<N,T>& mat);
       Matrix & operator -= (const Matrix<N,T>& mat);
       const T& operator()(int i, int j) const;
diff --git a/tests/invert_test.cpp b/tests/invert_test.cpp
index d61c298d30..5662693951 100644
--- a/tests/invert_test.cpp
+++ b/tests/invert_test.cpp
@@ -290,8 +290,7 @@ int main(int argc, char **argv)
   std::vector<double> gflops(Nsrc);
   std::vector<int> iter(Nsrc);
 
-  auto *rng = new quda::RNG(quda::LatticeFieldParam(gauge_param), 1234);
-  rng->Init();
+  auto *rng = new quda::RNG(*check, 1234);
 
   for (int i = 0; i < Nsrc; i++) {
 
@@ -315,7 +314,6 @@ int main(int argc, char **argv)
   // QUDA invert test COMPLETE
   //----------------------------------------------------------------------------
 
-  rng->Release();
   delete rng;
 
   // free the multigrid solver
diff --git a/tests/multigrid_benchmark_test.cpp b/tests/multigrid_benchmark_test.cpp
index 052d02495e..a03700f01b 100644
--- a/tests/multigrid_benchmark_test.cpp
+++ b/tests/multigrid_benchmark_test.cpp
@@ -164,12 +164,10 @@ void freeFields()
 
 DiracCoarse *dirac;
 
-double benchmark(int test, const int niter) {
-
-  cudaEvent_t start, end;
-  cudaEventCreate(&start);
-  cudaEventCreate(&end);
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
+double benchmark(int test, const int niter)
+{
+  device_timer_t device_timer;
+  device_timer.start();
 
   switch(test) {
   case 0:
@@ -185,18 +183,10 @@ double benchmark(int test, const int niter) {
     errorQuda("Undefined test %d", test);
   }
 
-  cudaEventRecord(end, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(end);
-  float runTime;
-  cudaEventElapsedTime(&runTime, start, end);
-  cudaEventDestroy(start);
-  cudaEventDestroy(end);
-
-  double secs = runTime / 1000;
-  return secs;
+  device_timer.stop();
+  return device_timer.last();
 }
 
-
 const char *names[] = {
   "Dslash",
   "Mat",
diff --git a/tests/multigrid_evolve_test.cpp b/tests/multigrid_evolve_test.cpp
index 420ac6e4e2..afc6f262a2 100644
--- a/tests/multigrid_evolve_test.cpp
+++ b/tests/multigrid_evolve_test.cpp
@@ -42,7 +42,7 @@ void CallUnitarizeLinks(quda::cudaGaugeField *cudaInGauge)
   qudaMemset(num_failures_dev, 0, sizeof(int));
   unitarizeLinks(*cudaInGauge, num_failures_dev);
 
-  qudaMemcpy(&num_failures, num_failures_dev, sizeof(int), cudaMemcpyDeviceToHost);
+  qudaMemcpy(&num_failures, num_failures_dev, sizeof(int), qudaMemcpyDeviceToHost);
   if (num_failures > 0) errorQuda("Error in the unitarization\n");
   device_free(num_failures_dev);
 }
@@ -248,7 +248,6 @@ int main(int argc, char **argv)
 
     // CURAND random generator initialization
     RNG *randstates = new RNG(*gauge, 1234);
-    randstates->Init();
     int nsteps = 10;
     int nhbsteps = 1;
     int novrsteps = 1;
@@ -450,7 +449,6 @@ int main(int argc, char **argv)
     //Release all temporary memory used for data exchange between GPUs in multi-GPU mode
     PGaugeExchangeFree();
 
-    randstates->Release();
     delete randstates;
   }
 
diff --git a/tests/pack_test.cpp b/tests/pack_test.cpp
index 06541b83ff..f422d62198 100644
--- a/tests/pack_test.cpp
+++ b/tests/pack_test.cpp
@@ -9,6 +9,7 @@
 
 #include <color_spinor_field.h>
 #include <blas_quda.h>
+#include <timer.h>
 
 using namespace quda;
 
@@ -93,7 +94,9 @@ void end() {
   endQuda();
 }
 
-void packTest() {
+void packTest()
+{
+  host_timer_t host_timer;
 
   printfQuda("Sending fields to GPU...\n");
 
@@ -109,15 +112,15 @@ void packTest() {
     cpsParam.pad = param.ga_pad;
     cudaGaugeField cudaCpsGauge(cpsParam);
 
-    stopwatchStart();
+    host_timer.start();
     cudaCpsGauge.loadCPUField(cpsCpuGauge);
-    double cpsGtime = stopwatchReadSeconds();
-    printfQuda("CPS Gauge send time = %e seconds\n", cpsGtime);
+    host_timer.stop();
+    printfQuda("CPS Gauge send time = %e seconds\n", host_timer.last());
 
-    stopwatchStart();
+    host_timer.start();
     cudaCpsGauge.saveCPUField(cpsCpuGauge);
-    double cpsGRtime = stopwatchReadSeconds();
-    printfQuda("CPS Gauge restore time = %e seconds\n", cpsGRtime);
+    host_timer.stop();
+    printfQuda("CPS Gauge restore time = %e seconds\n", host_timer.last());
   }
 #endif
 
@@ -133,28 +136,27 @@ void packTest() {
     qdpParam.pad = param.ga_pad;
     cudaGaugeField cudaQdpGauge(qdpParam);
 
-    stopwatchStart();
+    host_timer.start();
     cudaQdpGauge.loadCPUField(qdpCpuGauge);
-    double qdpGtime = stopwatchReadSeconds();
-    printfQuda("QDP Gauge send time = %e seconds\n", qdpGtime);
+    host_timer.stop();
+    printfQuda("QDP Gauge send time = %e seconds\n", host_timer.last());
 
-    stopwatchStart();
+    host_timer.start();
     cudaQdpGauge.saveCPUField(qdpCpuGauge);
-    double qdpGRtime = stopwatchReadSeconds();
-    printfQuda("QDP Gauge restore time = %e seconds\n", qdpGRtime);
+    host_timer.stop();
+    printfQuda("QDP Gauge restore time = %e seconds\n", host_timer.last());
   }
 #endif
 
-  stopwatchStart();
-
+  host_timer.start();
   *cudaSpinor = *spinor;
-  double sSendTime = stopwatchReadSeconds();
-  printfQuda("Spinor send time = %e seconds\n", sSendTime);
+  host_timer.stop();
+  printfQuda("Spinor send time = %e seconds\n", host_timer.last());
 
-  stopwatchStart();
+  host_timer.start();
   *spinor2 = *cudaSpinor;
-  double sRecTime = stopwatchReadSeconds();
-  printfQuda("Spinor receive time = %e seconds\n", sRecTime);
+  host_timer.stop();
+  printfQuda("Spinor receive time = %e seconds\n", host_timer.last());
 
   double spinor_norm = blas::norm2(*spinor);
   double cuda_spinor_norm = blas::norm2(*cudaSpinor);
diff --git a/tests/staggered_dslash_ctest.cpp b/tests/staggered_dslash_ctest.cpp
index 0e4c6e3db2..05be0c6937 100644
--- a/tests/staggered_dslash_ctest.cpp
+++ b/tests/staggered_dslash_ctest.cpp
@@ -300,19 +300,16 @@ struct DslashTime {
 DslashTime dslashCUDA(int niter) {
 
   DslashTime dslash_time;
-  timeval tstart, tstop;
 
-  cudaEvent_t start, end;
-  cudaEventCreate(&start);
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(start);
+  host_timer_t host_timer;
+  device_timer_t device_timer;
 
   comm_barrier();
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
+  device_timer.start();
 
   for (int i = 0; i < niter; i++) {
 
-    gettimeofday(&tstart, NULL);
+    host_timer.start();
 
     switch (dtest_type) {
     case dslash_test_type::Dslash: dirac->Dslash(*cudaSpinorOut, *cudaSpinor, parity); break;
@@ -321,28 +318,18 @@ DslashTime dslashCUDA(int niter) {
     default: errorQuda("Test type %d not defined on staggered dslash.\n", static_cast<int>(dtest_type));
     }
 
-    gettimeofday(&tstop, NULL);
-    long ds = tstop.tv_sec - tstart.tv_sec;
-    long dus = tstop.tv_usec - tstart.tv_usec;
-    double elapsed = ds + 0.000001*dus;
+    host_timer.stop();
 
-    dslash_time.cpu_time += elapsed;
+    dslash_time.cpu_time += host_timer.last();
     // skip first and last iterations since they may skew these metrics if comms are not synchronous
     if (i>0 && i<niter) {
-      if (elapsed < dslash_time.cpu_min) dslash_time.cpu_min = elapsed;
-      if (elapsed > dslash_time.cpu_max) dslash_time.cpu_max = elapsed;
+      dslash_time.cpu_min = std::min(dslash_time.cpu_min, host_timer.last());
+      dslash_time.cpu_max = std::max(dslash_time.cpu_max, host_timer.last());
     }
   }
 
-  cudaEventCreate(&end);
-  cudaEventRecord(end, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(end);
-  float runTime;
-  cudaEventElapsedTime(&runTime, start, end);
-  cudaEventDestroy(start);
-  cudaEventDestroy(end);
-
-  dslash_time.event_time = runTime / 1000;
+  device_timer.stop();
+  dslash_time.event_time = device_timer.last();
 
   return dslash_time;
 }
diff --git a/tests/staggered_dslash_test.cpp b/tests/staggered_dslash_test.cpp
index 82dedc3b14..0bf285fa23 100644
--- a/tests/staggered_dslash_test.cpp
+++ b/tests/staggered_dslash_test.cpp
@@ -277,19 +277,16 @@ struct DslashTime {
 DslashTime dslashCUDA(int niter) {
 
   DslashTime dslash_time;
-  timeval tstart, tstop;
 
-  cudaEvent_t start, end;
-  cudaEventCreate(&start);
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(start);
+  host_timer_t host_timer;
+  device_timer_t device_timer;
 
   comm_barrier();
-  cudaEventRecord(start, device::get_cuda_stream(device::get_default_stream()));
+  device_timer.start();
 
   for (int i = 0; i < niter; i++) {
 
-    gettimeofday(&tstart, NULL);
+    host_timer.start();
 
     switch (dtest_type) {
     case dslash_test_type::Dslash: dirac->Dslash(*cudaSpinorOut, *cudaSpinor, parity); break;
@@ -298,28 +295,18 @@ DslashTime dslashCUDA(int niter) {
     default: errorQuda("Test type %d not defined on staggered dslash.\n", static_cast<int>(dtest_type));
     }
 
-    gettimeofday(&tstop, NULL);
-    long ds = tstop.tv_sec - tstart.tv_sec;
-    long dus = tstop.tv_usec - tstart.tv_usec;
-    double elapsed = ds + 0.000001*dus;
+    host_timer.stop();
 
-    dslash_time.cpu_time += elapsed;
+    dslash_time.cpu_time += host_timer.last();
     // skip first and last iterations since they may skew these metrics if comms are not synchronous
     if (i>0 && i<niter) {
-      if (elapsed < dslash_time.cpu_min) dslash_time.cpu_min = elapsed;
-      if (elapsed > dslash_time.cpu_max) dslash_time.cpu_max = elapsed;
+      dslash_time.cpu_min = std::min(dslash_time.cpu_min, host_timer.last());
+      dslash_time.cpu_max = std::max(dslash_time.cpu_max, host_timer.last());
     }
   }
 
-  cudaEventCreate(&end);
-  cudaEventRecord(end, device::get_cuda_stream(device::get_default_stream()));
-  cudaEventSynchronize(end);
-  float runTime;
-  cudaEventElapsedTime(&runTime, start, end);
-  cudaEventDestroy(start);
-  cudaEventDestroy(end);
-
-  dslash_time.event_time = runTime / 1000;
+  device_timer.stop();
+  dslash_time.event_time = device_timer.last();
 
   return dslash_time;
 }
diff --git a/tests/staggered_invert_test.cpp b/tests/staggered_invert_test.cpp
index 3640ddfc4c..bb06be9f3a 100644
--- a/tests/staggered_invert_test.cpp
+++ b/tests/staggered_invert_test.cpp
@@ -286,8 +286,7 @@ int main(int argc, char **argv)
   //-----------------------------------------------------------------------------------
 
   // Prepare rng
-  auto *rng = new quda::RNG(quda::LatticeFieldParam(gauge_param), 1234);
-  rng->Init();
+  auto *rng = new quda::RNG(*ref, 1234);
 
   // Performance measuring
   std::vector<double> time(Nsrc);
@@ -383,7 +382,6 @@ int main(int argc, char **argv)
   if (Nsrc > 1) performanceStats(time, gflops, iter);
 
   // Free RNG
-  rng->Release();
   delete rng;
 
   // Free the multigrid solver
diff --git a/tests/unitarize_link_test.cpp b/tests/unitarize_link_test.cpp
index 370b849e66..0893725a1e 100644
--- a/tests/unitarize_link_test.cpp
+++ b/tests/unitarize_link_test.cpp
@@ -4,6 +4,7 @@
 #include <sys/time.h>
 
 #include "quda.h"
+#include "timer.h"
 #include "gauge_field.h"
 #include "host_utils.h"
 #include <command_line_params.h>
diff --git a/tests/utils/face_gauge.cpp b/tests/utils/face_gauge.cpp
index 1f87a4417e..8dcc2323b7 100644
--- a/tests/utils/face_gauge.cpp
+++ b/tests/utils/face_gauge.cpp
@@ -4,7 +4,6 @@
 #include <sys/time.h>
 #include <assert.h>
 
-#include <quda_internal.h>
 #include <comm_quda.h>
 
 #include <host_utils.h>
diff --git a/tests/utils/host_utils.cpp b/tests/utils/host_utils.cpp
index 5aa64ab893..2bed9b57da 100644
--- a/tests/utils/host_utils.cpp
+++ b/tests/utils/host_utils.cpp
@@ -174,6 +174,7 @@ void constructHostGaugeField(void **gauge, QudaGaugeParam &gauge_param, int argc
   int construct_type = 0;
   if (strcmp(latfile, "")) {
     // load in the command line supplied gauge field using QIO and LIME
+    if (getVerbosity() >= QUDA_VERBOSE) printfQuda("Loading the gauge field in %s\n", latfile);
     read_gauge_field(latfile, gauge, gauge_param.cpu_prec, gauge_param.X, argc, argv);
     construct_type = 2;
   } else {
@@ -336,7 +337,6 @@ void setDims(int *X) {
   Vsh_z = Vs_z/2;
   Vsh_t = Vs_t/2;
 
-
   E1=X[0]+4; E2=X[1]+4; E3=X[2]+4; E4=X[3]+4;
   E1h=E1/2;
   E[0] = E1;
@@ -345,7 +345,6 @@ void setDims(int *X) {
   E[3] = E4;
   V_ex = E1*E2*E3*E4;
   Vh_ex = V_ex/2;
-
 }
 
 void dw_setDims(int *X, const int L5)
@@ -1607,20 +1606,6 @@ double mom_action(void *mom, QudaPrecision prec, int len)
   return action;
 }
 
-static struct timeval startTime;
-
-void stopwatchStart() { gettimeofday(&startTime, NULL); }
-
-double stopwatchReadSeconds()
-{
-  struct timeval endTime;
-  gettimeofday(&endTime, NULL);
-
-  long ds = endTime.tv_sec - startTime.tv_sec;
-  long dus = endTime.tv_usec - startTime.tv_usec;
-  return ds + 0.000001*dus;
-}
-
 void performanceStats(std::vector<double> &time, std::vector<double> &gflops, std::vector<int> &iter)
 {
   auto mean_time = 0.0;
diff --git a/tests/utils/host_utils.h b/tests/utils/host_utils.h
index 1c7a5a6698..0bfe48158b 100644
--- a/tests/utils/host_utils.h
+++ b/tests/utils/host_utils.h
@@ -216,8 +216,6 @@ void cpu_xpy(QudaPrecision prec, void *x, void *y, int size);
 #endif
 
 // Use for profiling
-void stopwatchStart();
-double stopwatchReadSeconds();
 void performanceStats(double *time, double *gflops);
 
 inline QudaPrecision getPrecision(int i)

From a7e70ac3ebca177f3869a4e7827b587afc5c0703 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 17 Feb 2021 21:58:49 +0000
Subject: [PATCH 020/198] omptarget: borrow hip/aos.h

---
 include/targets/omptarget/aos.h | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 include/targets/omptarget/aos.h

diff --git a/include/targets/omptarget/aos.h b/include/targets/omptarget/aos.h
new file mode 100644
index 0000000000..262c46e51d
--- /dev/null
+++ b/include/targets/omptarget/aos.h
@@ -0,0 +1,27 @@
+#pragma once
+
+namespace quda {
+
+  template <typename T, int n> __host__ __device__ void block_load(T out[n], const T *in)
+  {
+#pragma unroll
+    for (int i = 0; i < n; i++) out[i] = in[i];
+  }
+
+  template <typename T, int n> __host__ __device__ void block_store(T *out, const T in[n])
+  {
+#pragma unroll
+    for (int i = 0; i < n; i++) out[i] = in[i];
+  }
+
+  template <typename T> __host__ __device__ void block_load(T &out, const T *in)
+  {
+    out = *in;
+  }
+
+  template <typename T> __host__ __device__ void block_store(T *out, const T &in)
+  {
+    *out = in;
+  }
+
+}

From 0eb109aa4b74806576725e366ee1079d7b109bc2 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 3 Mar 2021 20:15:39 +0000
Subject: [PATCH 021/198] update omptarget

---
 include/targets/omptarget/aos.h               |  52 +++-
 .../omptarget/block_reduction_kernel.h        |  14 +-
 include/targets/omptarget/kernel.h            |  17 +-
 include/targets/omptarget/random_helper.h     |   2 -
 include/targets/omptarget/reduction_kernel.h  |  40 ++-
 include/targets/omptarget/warp_collective.h   |  23 ++
 lib/targets/omptarget/blas_lapack_cublas.cpp  |   1 +
 lib/targets/omptarget/device.cpp              |  82 +++---
 lib/targets/omptarget/malloc.cpp              |  20 +-
 lib/targets/omptarget/quda_api.cpp            | 271 ++++++++++--------
 10 files changed, 313 insertions(+), 209 deletions(-)
 create mode 100644 include/targets/omptarget/warp_collective.h

diff --git a/include/targets/omptarget/aos.h b/include/targets/omptarget/aos.h
index 262c46e51d..8724bfbeed 100644
--- a/include/targets/omptarget/aos.h
+++ b/include/targets/omptarget/aos.h
@@ -1,27 +1,73 @@
 #pragma once
 
+// trove requires CUDA and has issues with device debug
+/*
+#if !defined(DEVICE_DEBUG)
+#include <trove/ptr.h>
+#else
+*/
+#define DISABLE_TROVE
+/*
+#endif
+*/
+
 namespace quda {
 
+  /**
+     @brief This is just a dummy structure we use for trove to define the
+     required structure size
+     @param real Real number type
+     @param length Number of elements in the structure
+  */
+  template <typename T, int n> struct S {
+    T v[n];
+    __host__ __device__ const T &operator[](int i) const { return v[i]; }
+    __host__ __device__ T &operator[](int i) { return v[i]; }
+  };
+
   template <typename T, int n> __host__ __device__ void block_load(T out[n], const T *in)
   {
+    using struct_t = S<T, n>;
+#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
+    struct_t v = *(trove::coalesced_ptr<const struct_t>(reinterpret_cast<const struct_t*>(in)));
+#else
+    struct_t v = *(reinterpret_cast<const struct_t*>(in));
+#endif
+
 #pragma unroll
-    for (int i = 0; i < n; i++) out[i] = in[i];
+    for (int i = 0; i < n; i++) out[i] = v[i];
   }
 
   template <typename T, int n> __host__ __device__ void block_store(T *out, const T in[n])
   {
+    using struct_t = S<T, n>;
+    struct_t v;
 #pragma unroll
-    for (int i = 0; i < n; i++) out[i] = in[i];
+    for (int i = 0; i < n; i++) v[i] = in[i];
+
+#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
+    *(trove::coalesced_ptr<struct_t>(reinterpret_cast<struct_t *>(out))) = v;
+#else
+    *(reinterpret_cast<struct_t*>(out)) = v;
+#endif
   }
 
   template <typename T> __host__ __device__ void block_load(T &out, const T *in)
   {
-    out = *in;
+#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
+    out = *(trove::coalesced_ptr<const T>(in));
+#else
+    out = *(in);
+#endif
   }
 
   template <typename T> __host__ __device__ void block_store(T *out, const T &in)
   {
+#if defined( __CUDA_ARCH__) && !defined(DISABLE_TROVE)
+    *(trove::coalesced_ptr<T>(out)) = in;
+#else
     *out = in;
+#endif
   }
 
 }
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 83018deb30..da3452e3bf 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -13,16 +13,16 @@ namespace quda {
      which is the effective matrix dimension that we are tranposing in
      this mapping.
    */
-  template <typename Arg> __device__ int virtual_block_idx(const Arg &arg)
+  template <typename Arg> __device__ constexpr int virtual_block_idx(const Arg &arg)
   {
     QUDA_RT_CONSTS;
-    int block_idx = blockIdx.x;
+    auto block_idx = blockIdx.x;
     if (arg.swizzle) {
       // the portion of the grid that is exactly divisible by the number of SMs
-      const int gridp = gridDim.x - gridDim.x % arg.swizzle_factor;
+      const auto gridp = gridDim.x - gridDim.x % arg.swizzle_factor;
 
       block_idx = blockIdx.x;
-      if (blockIdx.x < gridp) {
+      if (block_idx < gridp) {
         // this is the portion of the block that we are going to transpose
         const int i = blockIdx.x % arg.swizzle_factor;
         const int j = blockIdx.x / arg.swizzle_factor;
@@ -41,13 +41,13 @@ namespace quda {
      templated, e.g., for efficient reductions, and typically the y
      thread dimension is a trivial vectorizable dimension.
   */
-  template <int block_size, template <int, typename> class Transformer, typename Arg>
-  __launch_bounds__(Arg::launch_bounds ? block_size : 0) __global__ void BlockKernel2D(Arg arg)
+  template <unsigned int block_size, template <int, typename> class Transformer, typename Arg>
+  __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0) __global__ void BlockKernel2D(Arg arg)
   {
     QUDA_RT_CONSTS;
     const dim3 block_idx(virtual_block_idx(arg), blockIdx.y, 0);
     const dim3 thread_idx(threadIdx.x, threadIdx.y, 0);
-    const int j = blockDim.y*blockIdx.y + threadIdx.y;
+    auto j = blockDim.y*blockIdx.y + threadIdx.y;
     if (j >= arg.threads.y) return;
 
     Transformer<block_size, Arg> t(arg);
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 4c7ff3e240..42c966e8dc 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -8,7 +8,7 @@ namespace quda {
     QUDA_RT_CONSTS;
     Functor<Arg> f(arg);
 
-    int i = threadIdx.x + blockIdx.x * blockDim.x;
+    auto i = threadIdx.x + blockIdx.x * blockDim.x;
 
     while (i < arg.threads.x) {
       f(i);
@@ -22,8 +22,8 @@ namespace quda {
     QUDA_RT_CONSTS;
     Functor<Arg> f(arg);
 
-    int i = threadIdx.x + blockIdx.x * blockDim.x;
-    int j = threadIdx.y + blockIdx.y * blockDim.y;
+    auto i = threadIdx.x + blockIdx.x * blockDim.x;
+    auto j = threadIdx.y + blockIdx.y * blockDim.y;
     if (j >= arg.threads.y) return;
 
     while (i < arg.threads.x) {
@@ -38,9 +38,9 @@ namespace quda {
     QUDA_RT_CONSTS;
     Functor<Arg> f(arg);
 
-    int i = threadIdx.x + blockIdx.x * blockDim.x;
-    int j = threadIdx.y + blockIdx.y * blockDim.y;
-    int k = threadIdx.z + blockIdx.z * blockDim.z;
+    auto i = threadIdx.x + blockIdx.x * blockDim.x;
+    auto j = threadIdx.y + blockIdx.y * blockDim.y;
+    auto k = threadIdx.z + blockIdx.z * blockDim.z;
     if (j >= arg.threads.y) return;
     if (k >= arg.threads.z) return;
 
@@ -50,9 +50,8 @@ namespace quda {
     }
   }
 
-  template <template <typename> class Functor, typename Arg>
-  // __launch_bounds__(Arg::block_dim, Arg::min_blocks)
-  __global__ void raw_kernel(Arg arg)
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
     Functor<Arg> f(arg);
     f();
diff --git a/include/targets/omptarget/random_helper.h b/include/targets/omptarget/random_helper.h
index d7449cf6a2..b9d06882ac 100644
--- a/include/targets/omptarget/random_helper.h
+++ b/include/targets/omptarget/random_helper.h
@@ -1,8 +1,6 @@
 #pragma once
 
 #include <random>
-
-#include <random_quda.h>
 // #include <curand_kernel.h>
 
 namespace quda {
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 2fc92e3cbd..15d367b86f 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -4,51 +4,49 @@
 
 namespace quda {
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer,
-            template <typename> class Reducer, typename Arg>
+  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __global__ void Reduction2D(Arg arg)
-  {/*
+  {
+    QUDA_RT_CONSTS;
     using reduce_t = typename Transformer<Arg>::reduce_t;
     Transformer<Arg> t(arg);
-    Reducer<reduce_t> r;
 
-    int idx = threadIdx.x + blockIdx.x * blockDim.x;
-    int j = threadIdx.y;
+    auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+    auto j = threadIdx.y;
 
     reduce_t value = arg.init();
 
     while (idx < arg.threads.x) {
-      value = t(value, r, idx, j);
-      idx += blockDim.x * gridDim.x;
+      value = t(value, idx, j);
+      if (grid_stride) idx += blockDim.x * gridDim.x; else break;
     }
 
     // perform final inter-block reduction and write out result
-    reduce<block_size_x, block_size_y, decltype(r)>(arg, value);
-  */}
+    reduce<block_size_x, block_size_y>(arg, t, value);
+  }
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer,
-            template <typename> class Reducer, typename Arg>
+  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __global__ void MultiReduction(Arg arg)
-  {/*
+  {
+    QUDA_RT_CONSTS;
     using reduce_t = typename Transformer<Arg>::reduce_t;
     Transformer<Arg> t(arg);
-    Reducer<reduce_t> r;
 
-    int idx = threadIdx.x + blockIdx.x * blockDim.x;
-    int j = threadIdx.y + blockIdx.y * blockDim.y;
-    int k = threadIdx.z;
+    auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+    auto j = threadIdx.y + blockIdx.y * blockDim.y;
+    auto k = threadIdx.z;
 
     if (j >= arg.threads.y) return;
 
     reduce_t value = arg.init();
 
     while (idx < arg.threads.x) {
-      value = t(value, r, idx, j, k);
-      idx += blockDim.x * gridDim.x;
+      value = t(value, idx, j, k);
+      if (grid_stride) idx += blockDim.x * gridDim.x; else break;
     }
 
     // perform final inter-block reduction and write out result
-    reduce<block_size_x, block_size_y, decltype(r)>(arg, value, j);
-  */}
+    reduce<block_size_x, block_size_y>(arg, t, value, j);
+  }
 
 }
diff --git a/include/targets/omptarget/warp_collective.h b/include/targets/omptarget/warp_collective.h
new file mode 100644
index 0000000000..cf3f592b74
--- /dev/null
+++ b/include/targets/omptarget/warp_collective.h
@@ -0,0 +1,23 @@
+namespace quda {
+
+  template <int warp_split, typename T> __device__ __host__ inline T warp_combine(T &x)
+  {
+#ifdef __CUDA_ARCH__
+    constexpr int warp_size = device::warp_size();
+    if (warp_split > 1) {
+#pragma unroll
+      for (int i = 0; i < x.size(); i++) {
+        // reduce down to the first group of column-split threads
+#pragma unroll
+        for (int offset = warp_size / 2; offset >= warp_size / warp_split; offset /= 2) {
+          // TODO - add support for non-converged warps
+          x[i].real(x[i].real() + __shfl_down_sync(device::warp_converged_mask(), x[i].real(), offset));
+          x[i].imag(x[i].imag() + __shfl_down_sync(device::warp_converged_mask(), x[i].imag(), offset));
+        }
+      }
+    }
+#endif
+    return x;
+  }
+
+}
diff --git a/lib/targets/omptarget/blas_lapack_cublas.cpp b/lib/targets/omptarget/blas_lapack_cublas.cpp
index 36b2672c70..ead59bb383 100644
--- a/lib/targets/omptarget/blas_lapack_cublas.cpp
+++ b/lib/targets/omptarget/blas_lapack_cublas.cpp
@@ -1,5 +1,6 @@
 #include <complex.h>
 #include <blas_lapack.h>
+#include <timer.h>
 #ifdef NATIVE_LAPACK_LIB
 #include <cublas_v2.h>
 #include <malloc_quda.h>
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 1587d7911d..425b09be68 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -104,25 +104,16 @@ namespace quda
       if (NVML_SUCCESS != result) errorQuda("NVML Shutdown failed with error %d", result);
 #endif
 
-      int deviceCount;
-      cudaGetDeviceCount(&deviceCount);
-      if (deviceCount == 0) {
-        warningQuda("No CUDA devices found");
-      }
-
-      for (int i = 0; i < deviceCount; i++) {
+      for (int i = 0; i < get_device_count(); i++) {
         cudaGetDeviceProperties(&deviceProp, i);
-        checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
         if (getVerbosity() >= QUDA_SUMMARIZE) {
           printfQuda("Found device %d: %s\n", i, deviceProp.name);
         }
       }
 
-      cudaGetDeviceProperties(&deviceProp, dev);
-      checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
-      if (deviceProp.major < 1) {
-        errorQuda("Device %d does not support CUDA", dev);
-      }
+/*
+      CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, dev));
+      if (deviceProp.major < 1) errorQuda("Device %d does not support CUDA", dev);
 
       // Check GPU and QUDA build compatibiliy
       // 4 cases:
@@ -133,9 +124,9 @@ namespace quda
 
       const int my_major = __COMPUTE_CAPABILITY__ / 100;
       const int my_minor = (__COMPUTE_CAPABILITY__  - my_major * 100) / 10;
-      // b) UDA was compiled for a higher compute capability
+      // b) QUDA was compiled for a higher compute capability
       if (deviceProp.major * 100 + deviceProp.minor * 10 < __COMPUTE_CAPABILITY__)
-        warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. ** \n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
+        errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. ** \n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
 
       // c) QUDA was compiled for a lower compute capability
       if (deviceProp.major < my_major) {
@@ -143,20 +134,22 @@ namespace quda
         if (allow_jit_env && strcmp(allow_jit_env, "1") == 0) {
           if (getVerbosity() > QUDA_SILENT) warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- Jitting the PTX since QUDA_ALLOW_JIT=1 was set. Note that this will take some time.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
         } else {
-          warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n If you want the PTX to be jitted for your current GPU arch please set the enviroment variable QUDA_ALLOW_JIT=1.", deviceProp.major, deviceProp.minor, my_major, my_minor);
+          errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n If you want the PTX to be jitted for your current GPU arch please set the enviroment variable QUDA_ALLOW_JIT=1.", deviceProp.major, deviceProp.minor, my_major, my_minor);
         }
       }
       // d) QUDA built for same major compute capability but lower minor
       if (deviceProp.major == my_major and deviceProp.minor > my_minor) {
         warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- This might result in a lower performance. Please consider adjusting QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
       }
+*/
+
+      if (!deviceProp.unifiedAddressing) errorQuda("Device %d does not support unified addressing", dev);
 
       if (getVerbosity() >= QUDA_SUMMARIZE) {
         printfQuda("Using device %d: %s\n", dev, deviceProp.name);
       }
 #ifndef USE_QDPJIT
-      cudaSetDevice(dev);
-      checkCudaErrorNoSync(); // "NoSync" for correctness in HOST_DEBUG mode
+      CHECK_CUDA_ERROR(cudaSetDevice(dev));
 #endif
 
 #ifdef NUMA_NVML
@@ -168,29 +161,36 @@ namespace quda
       }
 #endif
 
-      cudaDeviceSetCacheConfig(cudaFuncCachePreferL1);
+      CHECK_CUDA_ERROR(cudaDeviceSetCacheConfig(cudaFuncCachePreferL1));
       //cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte);
       // cudaGetDeviceProperties(&deviceProp, dev);
     }
 
+    int get_device_count()
+    {
+      static int device_count = 0;
+      if (device_count == 0) {
+        CHECK_CUDA_ERROR(cudaGetDeviceCount(&device_count));
+        if (device_count == 0) warningQuda("No CUDA devices found");
+      }
+      return device_count;
+    }
+
     void print_device_properties()
     {
 
-      int dev_count;
-      cudaGetDeviceCount(&dev_count);
-      int device;
-      for (device = 0; device < dev_count; device++) {
+      for (int device = 0; device < get_device_count(); device++) {
 
         // cudaDeviceProp deviceProp;
-        cudaGetDeviceProperties(&deviceProp, device);
+        CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, device));
         printfQuda("%d - name:                    %s\n", device, deviceProp.name);
-//        printfQuda("%d - totalGlobalMem:          %lu bytes ( %.2f Gbytes)\n", device, deviceProp.totalGlobalMem,
-//                   deviceProp.totalGlobalMem / (float)(1024 * 1024 * 1024));
+        printfQuda("%d - totalGlobalMem:          %lu bytes ( %.2f Gbytes)\n", device, deviceProp.totalGlobalMem,
+                   deviceProp.totalGlobalMem / (float)(1024 * 1024 * 1024));
         printfQuda("%d - sharedMemPerBlock:       %lu bytes ( %.2f Kbytes)\n", device, deviceProp.sharedMemPerBlock,
                    deviceProp.sharedMemPerBlock / (float)1024);
-//        printfQuda("%d - regsPerBlock:            %d\n", device, deviceProp.regsPerBlock);
-//        printfQuda("%d - warpSize:                %d\n", device, deviceProp.warpSize);
-//        printfQuda("%d - memPitch:                %lu\n", device, deviceProp.memPitch);
+        printfQuda("%d - regsPerBlock:            %d\n", device, deviceProp.regsPerBlock);
+        printfQuda("%d - warpSize:                %d\n", device, deviceProp.warpSize);
+        printfQuda("%d - memPitch:                %lu\n", device, deviceProp.memPitch);
         printfQuda("%d - maxThreadsPerBlock:      %d\n", device, deviceProp.maxThreadsPerBlock);
         printfQuda("%d - maxThreadsDim[0]:        %d\n", device, deviceProp.maxThreadsDim[0]);
         printfQuda("%d - maxThreadsDim[1]:        %d\n", device, deviceProp.maxThreadsDim[1]);
@@ -198,10 +198,10 @@ namespace quda
         printfQuda("%d - maxGridSize[0]:          %d\n", device, deviceProp.maxGridSize[0]);
         printfQuda("%d - maxGridSize[1]:          %d\n", device, deviceProp.maxGridSize[1]);
         printfQuda("%d - maxGridSize[2]:          %d\n", device, deviceProp.maxGridSize[2]);
-//        printfQuda("%d - totalConstMem:           %lu bytes ( %.2f Kbytes)\n", device, deviceProp.totalConstMem,
-//                   deviceProp.totalConstMem / (float)1024);
+        printfQuda("%d - totalConstMem:           %lu bytes ( %.2f Kbytes)\n", device, deviceProp.totalConstMem,
+                   deviceProp.totalConstMem / (float)1024);
         printfQuda("%d - compute capability:      %d.%d\n", device, deviceProp.major, deviceProp.minor);
-//        printfQuda("%d - deviceOverlap            %s\n", device, (deviceProp.deviceOverlap ? "true" : "false"));
+        printfQuda("%d - deviceOverlap            %s\n", device, (deviceProp.deviceOverlap ? "true" : "false"));
         printfQuda("%d - multiProcessorCount      %d\n", device, deviceProp.multiProcessorCount);
 /*
         printfQuda("%d - kernelExecTimeoutEnabled %s\n", device,
@@ -243,20 +243,17 @@ namespace quda
 
       int greatestPriority;
       int leastPriority;
-      cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority);
+      CHECK_CUDA_ERROR(cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority));
       for (int i=0; i<Nstream-1; i++) {
-        cudaStreamCreateWithPriority(&streams[i], cudaStreamDefault, greatestPriority);
+        CHECK_CUDA_ERROR(cudaStreamCreateWithPriority(&streams[i], cudaStreamDefault, greatestPriority));
       }
-      cudaStreamCreateWithPriority(&streams[Nstream-1], cudaStreamDefault, leastPriority);
-
-      checkCudaError();
-
+      CHECK_CUDA_ERROR(cudaStreamCreateWithPriority(&streams[Nstream-1], cudaStreamDefault, leastPriority));
     }
 
     void destroy()
     {
       if (streams) {
-        for (int i=0; i<Nstream; i++) cudaStreamDestroy(streams[i]);
+        for (int i=0; i<Nstream; i++) CHECK_CUDA_ERROR(cudaStreamDestroy(streams[i]));
         delete []streams;
         streams = nullptr;
       }
@@ -264,7 +261,7 @@ namespace quda
       char *device_reset_env = getenv("QUDA_DEVICE_RESET");
       if (device_reset_env && strcmp(device_reset_env,"1") == 0) {
         // end this CUDA context
-        cudaDeviceReset();
+        CHECK_CUDA_ERROR(cudaDeviceReset());
       }
     }
 
@@ -315,7 +312,7 @@ namespace quda
     {
       static int max_shared_bytes = 0;
       if (!max_shared_bytes)
-        cudaDeviceGetAttribute(&max_shared_bytes, cudaDevAttrMaxSharedMemoryPerBlockOptin, comm_gpuid());
+        CHECK_CUDA_ERROR(cudaDeviceGetAttribute(&max_shared_bytes, cudaDevAttrMaxSharedMemoryPerBlockOptin, comm_gpuid()));
       return max_shared_bytes;
     }
 
@@ -333,7 +330,8 @@ namespace quda
     {
 #if CUDA_VERSION >= 11000
       static int max_blocks_per_sm = 0;
-      if (!max_blocks_per_sm) cudaDeviceGetAttribute(&max_blocks_per_sm, cudaDevAttrMaxBlocksPerMultiprocessor, comm_gpuid());
+      if (!max_blocks_per_sm)
+        CHECK_CUDA_ERROR(cudaDeviceGetAttribute(&max_blocks_per_sm, cudaDevAttrMaxBlocksPerMultiprocessor, comm_gpuid()));
       return max_blocks_per_sm;
 #else
       // these variables are taken from Table 14 of the CUDA 10.2 prgramming guide
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index 60d4cd5b2a..f02fea8933 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -323,6 +323,7 @@ namespace quda
       errorQuda("cudaHostAlloc failed of size %zu (%s:%d in %s())\n", size, file, line, func); }
     }
 #else
+    /* FIXME: this likely won't work. */
     void *ptr = aligned_malloc(a, size);
     ompwip([=](){printfQuda("mapped_malloc_ host: %p\n",ptr);});
     if(0<omp_get_num_devices()){
@@ -538,8 +539,25 @@ namespace quda
                 cudaGetErrorString(error), file, line, func);
     }
     return device;
-  */}
+  }
 
+  void register_pinned_(const char *func, const char *file, int line, void *ptr, size_t bytes)
+  {ompwip();
+    auto error = cudaHostRegister(ptr, bytes, cudaHostRegisterDefault);
+    if (error != cudaSuccess) {
+      errorQuda("cudaHostRegister failed with error %s (%s:%d in %s()",
+                cudaGetErrorString(error), file, line, func);
+    }
+  }
+
+  void unregister_pinned_(const char *func, const char *file, int line, void *ptr)
+  {ompwip();
+    auto error = cudaHostUnregister(ptr);
+    if (error != cudaSuccess) {
+      errorQuda("cudaHostUnregister failed with error %s (%s:%d in %s()",
+                cudaGetErrorString(error), file, line, func);
+    }
+  }
 
   namespace pool
   {
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 6b902944d1..59c2436009 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -2,10 +2,9 @@
 #include <tune_quda.h>
 #include <uint_to_char.h>
 #include <quda_internal.h>
+#include <timer.h>
 #include <device.h>
 
-#include <quda_api.h>
-
 enum cudaFuncAttribute{cudaFuncAttributePreferredSharedMemoryCarveout};
 using cudaFuncAttributes = int;
 using CUdeviceptr = void*;
@@ -28,24 +27,67 @@ using CUdeviceptr = void*;
 
 namespace quda {
 
+  static qudaError_t last_error = QUDA_SUCCESS;
+  static std::string last_error_str("CUDA_SUCCESS");
+
+  qudaError_t qudaGetLastError()
+  {
+    auto rtn = last_error;
+    last_error = QUDA_SUCCESS;
+    return rtn;
+  }
+
+  std::string qudaGetLastErrorString()
+  {
+    auto rtn = last_error_str;
+    last_error_str = QUDA_SUCCESS;
+    return rtn;
+  }
+
+  namespace cuda {
+
+    void set_runtime_error(cudaError_t error, const char *api_func, const char *func, const char *file, const char *line,
+                           bool allow_error = false)
+    {
+      if (error == cudaSuccess) return;
+      last_error = error == cudaSuccess ? QUDA_SUCCESS : QUDA_ERROR;
+      last_error_str = cudaGetErrorString(error);
+      if (!allow_error) errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, cudaGetErrorString(error), file, line, func);
+    }
+
+    void set_driver_error(CUresult error, const char *api_func, const char *func, const char *file, const char *line,
+                          bool allow_error = false)
+    {
+      if (error == CUDA_SUCCESS) return;
+      last_error = error == CUDA_SUCCESS ? QUDA_SUCCESS : QUDA_ERROR;
+      const char *str;
+      cuGetErrorName(error, &str);
+      last_error_str = str;
+      if (!allow_error) errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, str, file, line, func);
+    }
+
+  }
+
+  using namespace cuda;
+
   // Agnostic way to return a cuda API flag
   namespace {
     inline
     cudaMemcpyKind qudaMemcpyKindToAPI( const qudaMemcpyKind& k)
     {
       switch(k) {
-      case qudaMemcpyHostToHost : return cudaMemcpyHostToHost; break;
-      case qudaMemcpyHostToDevice : return cudaMemcpyHostToDevice; break;
-      case qudaMemcpyDeviceToHost : return cudaMemcpyDeviceToHost; break;
-      case qudaMemcpyDeviceToDevice : return cudaMemcpyDeviceToDevice; break;
-      case qudaMemcpyDefault : return cudaMemcpyDefault; break;
+      case qudaMemcpyHostToHost : return cudaMemcpyHostToHost;
+      case qudaMemcpyHostToDevice : return cudaMemcpyHostToDevice;
+      case qudaMemcpyDeviceToHost : return cudaMemcpyDeviceToHost;
+      case qudaMemcpyDeviceToDevice : return cudaMemcpyDeviceToDevice;
+      case qudaMemcpyDefault : return cudaMemcpyDefault;
       default:
 	errorQuda(" unknown value for qudaMemcpyKind %d", static_cast<int>(k));
 	return cudaMemcpyDefault; // keep warnings away
       }
     }
   }
-  
+
   // No need to abstract these across the library so keep these definitions local to CUDA target
 
   /**
@@ -96,11 +138,12 @@ namespace quda {
       }
 */
     }
+
 /*
     // no driver API variant here since we have C++ functions
     PROFILE(cudaError_t error = cudaLaunchKernel(func, tp.grid, tp.block, args, tp.shared_bytes, device::get_cuda_stream(stream)),
             QUDA_PROFILE_LAUNCH_KERNEL);
-    if (error != cudaSuccess && !activeTuning()) errorQuda("(CUDA) %s", cudaGetErrorString(error));
+    set_runtime_error(error, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
     return error == cudaSuccess ? QUDA_SUCCESS : QUDA_ERROR;
 */
     return QUDA_SUCCESS;
@@ -117,6 +160,9 @@ namespace quda {
     const bool async;
     const char *name;
     const bool active_tuning;
+    const char *func;
+    const char *file;
+    const char *line;
 
     unsigned int sharedBytesPerThread() const { return 0; }
     unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
@@ -124,7 +170,8 @@ namespace quda {
   public:
     inline QudaMem(void *dst, const void *src, size_t count, cudaMemcpyKind kind, const qudaStream_t &stream,
                    bool async, const char *func, const char *file, const char *line) :
-      dst(dst), src(src), count(count), value(0), copy(true), kind(kind), async(async), active_tuning(activeTuning())
+      dst(dst), src(src), count(count), value(0), copy(true), kind(kind), async(async), active_tuning(activeTuning()),
+      func(func), file(file), line(line)
     {
       if (!async) {
         switch (kind) {
@@ -182,21 +229,23 @@ namespace quda {
       if (copy) {
         if (async) {
 #ifdef USE_DRIVER_API
+          CUresult error = CUDA_SUCCESS;
           switch (kind) {
           case cudaMemcpyDeviceToHost:
-            PROFILE(cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
+            PROFILE(error = cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
             break;
           case cudaMemcpyHostToDevice:
-            PROFILE(cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
+            PROFILE(error = cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
             break;
           case cudaMemcpyDeviceToDevice:
-            PROFILE(cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
+            PROFILE(error = cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
             break;
           case cudaMemcpyDefault:
-            PROFILE(cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
+            PROFILE(error = cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
             break;
           default: errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
           }
+          set_driver_error(error, "cuMemcpyAsync", func, file, line, active_tuning);
 #else
 #ifdef API_PROFILE
           QudaProfileType type = QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC;
@@ -208,54 +257,38 @@ namespace quda {
           default: errorQuda("Unsupported cudaMemcpyTypeAsync %d", kind);
           }
 #endif
-          PROFILE(cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)), type);
+          cudaError_t error;
+          PROFILE(error = cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)), type);
+          set_runtime_error(error, "cudaMemcpyAsync", func, file, line, active_tuning);
 #endif
         } else {
 #ifdef USE_DRIVER_API
+          CUresult error = CUDA_SUCCESS;
           switch (kind) {
-          case cudaMemcpyDeviceToHost: cuMemcpyDtoH(dst, (CUdeviceptr)src, count); break;
-          case cudaMemcpyHostToDevice: cuMemcpyHtoD((CUdeviceptr)dst, src, count); break;
+          case cudaMemcpyDeviceToHost: error = cuMemcpyDtoH(dst, (CUdeviceptr)src, count); break;
+          case cudaMemcpyHostToDevice: error = cuMemcpyHtoD((CUdeviceptr)dst, src, count); break;
           case cudaMemcpyHostToHost: memcpy(dst, src, count); break;
-          case cudaMemcpyDeviceToDevice: cuMemcpyDtoD((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
-          case cudaMemcpyDefault: cuMemcpy((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
+          case cudaMemcpyDeviceToDevice: error = cuMemcpyDtoD((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
+          case cudaMemcpyDefault: error = cuMemcpy((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
           default: errorQuda("Unsupported cudaMemcpyType %d", kind);
           }
+          set_driver_error(error, "cuMemcpy", func, file, line, active_tuning);
 #else
-          printfQuda("memcpy %p <- %p\n",dst,src);
-          switch (kind) {
-          case cudaMemcpyHostToHost: memcpy(dst,(void*)src,count); break;
-          case cudaMemcpyHostToDevice:
-            if(0<omp_get_num_devices()) omp_target_memcpy(dst,(void*)src,count,0,0,omp_get_default_device(),omp_get_initial_device());
-            else{warningQuda("cudaMemcpyHostToDevice without a device"); memcpy(dst,(void*)src,count);}
-            break;
-          case cudaMemcpyDeviceToHost:
-            if(0<omp_get_num_devices()) omp_target_memcpy(dst,(void*)src,count,0,0,omp_get_initial_device(),omp_get_default_device());
-            else{warningQuda("cudaMemcpyDeviceToHost without a device"); memcpy(dst,(void*)src,count);}
-            break;
-          case cudaMemcpyDeviceToDevice:
-            warningQuda("unimplemented for cudaMemcpyDeviceToDevice");
-            break;
-          case cudaMemcpyDefault:
-            warningQuda("OMP treating cudaMemcpyDefault as cudaMemcpyHostToDevice");
-            if(0<omp_get_num_devices()) omp_target_memcpy(dst,(void*)src,count,0,0,omp_get_default_device(),omp_get_initial_device());
-            else{warningQuda("cudaMemcpyHostToDevice without a device"); memcpy(dst,(void*)src,count);}
-            break;
-          default: errorQuda("Unsupported cudaMemcpyType %d", kind);
-          }
-          // cudaMemcpy(dst, src, count, kind);
+          cudaError_t error = cudaMemcpy(dst, src, count, kind);
+          set_runtime_error(error, "cudaMemcpy", func, file, line, active_tuning);
 #endif
         }
       } else {
 #ifdef USE_DRIVER_API
-        if (async)
-          cuMemsetD32Async((CUdeviceptr)dst, value, count / 4, device::get_cuda_stream(stream));
-        else
+        CUresult error = async ?
+          cuMemsetD32Async((CUdeviceptr)dst, value, count / 4, device::get_cuda_stream(stream)) :
           cuMemsetD32((CUdeviceptr)dst, value, count / 4);
+        set_driver_error(error, "cuMemset", func, file, line, active_tuning);
 #else
-        if (async)
-          cudaMemsetAsync(dst, value, count, device::get_cuda_stream(stream));
-        else
+        cudaError_t error = async ?
+          cudaMemsetAsync(dst, value, count, device::get_cuda_stream(stream)) :
           cudaMemset(dst, value, count);
+        set_runtime_error(error, "cudaMemset", func, file, line, active_tuning);
 #endif
       }
     }
@@ -277,9 +310,6 @@ namespace quda {
                    const char *func, const char *file, const char *line) {
     if (count == 0) return;
     QudaMem copy(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_default_stream(), false, func, file, line);
-    cudaError_t error = cudaGetLastError();
-    if (error != cudaSuccess)
-      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
   void qudaMemcpyAsync_(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const qudaStream_t &stream,
@@ -385,15 +415,10 @@ namespace quda {
       errorQuda("Unsupported cuMemcpyType2DAsync %d", qudaMemcpyKindToAPI(kind));
     }
     PROFILE(auto error = cuMemcpy2DAsync(&param, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuMemcpy2DAsync returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    set_driver_error(error, "cuMemcpy2DAsync", func, file, line);
 #else
     PROFILE(auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
-    if (error != cudaSuccess)
-      errorQuda("cudaMemcpy2DAsync returned error %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, "cudaMemcpy2DAsync", func, file, line);
 #endif
   }
 
@@ -401,17 +426,13 @@ namespace quda {
                              const qudaStream_t &stream, const char *func, const char *file, const char *line)
   {
     auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, device::get_cuda_stream(stream));
-    if (error != cudaSuccess)
-      errorQuda("cudaMemcpy2DAsync returned %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, "cudaMemcpy2DAsync", func, file, line);
   }
 
   void qudaMemset_(void *ptr, int value, size_t count, const char *func, const char *file, const char *line)
   {
     if (count == 0) return;
     QudaMem set(ptr, value, count, device::get_default_stream(), false, func, file, line);
-    cudaError_t error = cudaGetLastError();
-    if (error != cudaSuccess && !activeTuning())
-      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
   void qudaMemsetAsync_(void *ptr, int value, size_t count, const qudaStream_t &stream, const char *func,
@@ -419,27 +440,27 @@ namespace quda {
   {
     if (count == 0) return;
     QudaMem copy(ptr, value, count, stream, true, func, file, line);
-    cudaError_t error = cudaGetLastError();
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
   void qudaMemset2D_(void *ptr, size_t pitch, int value, size_t width, size_t height, const char *func,
                      const char *file, const char *line)
   {
     cudaError_t error = cudaMemset2D(ptr, pitch, value, width, height);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
   }
 
   void qudaMemset2DAsync_(void *ptr, size_t pitch, int value, size_t width, size_t height, const qudaStream_t &stream,
                           const char *func, const char *file, const char *line)
   {
     cudaError_t error = cudaMemset2DAsync(ptr, pitch, value, width, height, device::get_cuda_stream(stream));
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
   }
 
   void qudaMemPrefetchAsync_(void *ptr, size_t count, QudaFieldLocation mem_space, const qudaStream_t &stream,
                              const char *func, const char *file, const char *line)
-  {ompwip("doing nothing");/*
+  {
+    ompwip("doing nothing");
+/*
     int dev_id = 0;
     if (mem_space == QUDA_CUDA_FIELD_LOCATION)
       dev_id = comm_gpuid();
@@ -449,28 +470,26 @@ namespace quda {
       errorQuda("Invalid QudaFieldLocation.");
 
     cudaError_t error = cudaMemPrefetchAsync(ptr, count, dev_id, device::get_cuda_stream(stream));
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
-  */}
+    set_runtime_error(error, __func__, func, file, line);
+*/
+  }
 
-  bool qudaEventQuery_(cudaEvent_t &event, const char *func, const char *file, const char *line)
+  bool qudaEventQuery_(qudaEvent_t &quda_event, const char *func, const char *file, const char *line)
   {
+    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuEventQuery(event), QUDA_PROFILE_EVENT_QUERY);
     switch (error) {
     case CUDA_SUCCESS: return true;
     case CUDA_ERROR_NOT_READY: return false;
-    default: {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuEventQuery returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    default: set_driver_error(error, __func__, func, file, line);
     }
 #else
     PROFILE(cudaError_t error = cudaEventQuery(event), QUDA_PROFILE_EVENT_QUERY);
     switch (error) {
     case cudaSuccess: return true;
     case cudaErrorNotReady: return false;
-    default: errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    default: set_runtime_error(error, __func__, func, file, line);
     }
 #endif
     return false;
@@ -478,47 +497,74 @@ namespace quda {
 
   void qudaEventRecord_(cudaEvent_t &event, qudaStream_t stream, const char *func, const char *file, const char *line)
   {
+    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuEventRecord(event, device::get_cuda_stream(stream)), QUDA_PROFILE_EVENT_RECORD);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuEventRecord returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    set_driver_error(error, __func__, func, file, line);
 #else
     PROFILE(cudaError_t error = cudaEventRecord(event, device::get_cuda_stream(stream)), QUDA_PROFILE_EVENT_RECORD);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
 #endif
   }
 
   void qudaStreamWaitEvent_(qudaStream_t stream, cudaEvent_t event, unsigned int flags, const char *func,
                             const char *file, const char *line)
   {
+    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuStreamWaitEvent(device::get_cuda_stream(stream), event, flags), QUDA_PROFILE_STREAM_WAIT_EVENT);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuStreamWaitEvent returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    set_driver_error(error, __func__, func, file, line);
 #else
     PROFILE(cudaError_t error = cudaStreamWaitEvent(device::get_cuda_stream(stream), event, flags), QUDA_PROFILE_STREAM_WAIT_EVENT);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
 #endif
   }
 
+  qudaEvent_t qudaEventCreate_(const char *func, const char *file, const char *line)
+  {
+    cudaEvent_t cuda_event;
+    cudaError_t error = cudaEventCreateWithFlags(&cuda_event, cudaEventDisableTiming);
+    set_runtime_error(error, __func__, func, file, line);
+    qudaEvent_t quda_event;
+    quda_event.event = reinterpret_cast<void*>(cuda_event);
+    return quda_event;
+  }
+
+  qudaEvent_t qudaChronoEventCreate_(const char *func, const char *file, const char *line)
+  {
+    cudaEvent_t cuda_event;
+    cudaError_t error = cudaEventCreate(&cuda_event);
+    set_runtime_error(error, __func__, func, file, line);
+    qudaEvent_t quda_event;
+    quda_event.event = reinterpret_cast<void*>(cuda_event);
+    return quda_event;
+  }
+
+  float qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
+                              const char *func, const char *file, const char *line)
+  {
+    float elapsed_time;
+    cudaError_t error = cudaEventElapsedTime(&elapsed_time, reinterpret_cast<cudaEvent_t>(start.event),
+                                             reinterpret_cast<cudaEvent_t>(stop.event));
+    set_runtime_error(error, __func__, func, file, line);
+    return elapsed_time / 1000;
+  }
+
+  void qudaEventDestroy_(qudaEvent_t &event, const char *func, const char *file, const char *line)
+  {
+    cudaError_t error = cudaEventDestroy(reinterpret_cast<cudaEvent_t&>(event.event));
+    set_runtime_error(error, __func__, func, file, line);
+  }
+
   void qudaEventSynchronize_(const cudaEvent_t &event, const char *func, const char *file, const char *line)
   {
+    const cudaEvent_t &event = reinterpret_cast<const cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuEventSynchronize(event), QUDA_PROFILE_EVENT_SYNCHRONIZE);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuEventSynchronize returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
+    set_driver_error(error, __func__, func, file, line);
 #else
     PROFILE(cudaError_t error = cudaEventSynchronize(event), QUDA_PROFILE_EVENT_SYNCHRONIZE);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
 #endif
   }
 
@@ -526,15 +572,10 @@ namespace quda {
   {
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuStreamSynchronize(device::get_cuda_stream(stream)), QUDA_PROFILE_STREAM_SYNCHRONIZE);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("(CUDA) cuStreamSynchronize returned error %s\n (%s:%s in %s())\n", str, file, line, func);
-    }
+    set_driver_error(error, __func__, file, line, func, activeTuning());
 #else
     PROFILE(cudaError_t error = cudaStreamSynchronize(device::get_cuda_stream(stream)), QUDA_PROFILE_STREAM_SYNCHRONIZE);
-    if (error != cudaSuccess && !activeTuning())
-      errorQuda("(CUDA) %s\n (%s:%s in %s())", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, file, line, func, activeTuning());
 #endif
   }
 
@@ -542,15 +583,10 @@ namespace quda {
   {
 #ifdef USE_DRIVER_API
     PROFILE(CUresult error = cuCtxSynchronize(), QUDA_PROFILE_DEVICE_SYNCHRONIZE);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuCtxSynchronize returned error %s (%s:%s in %s())\n", str, file, line, func);
-    }
+    set_driver_error(error, __func__, func, file, line, activeTuning());
 #else
     PROFILE(cudaError_t error = cudaDeviceSynchronize(), QUDA_PROFILE_DEVICE_SYNCHRONIZE);
-    if (error != cudaSuccess)
-      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line, activeTuning());
 #endif
   }
 
@@ -558,8 +594,7 @@ namespace quda {
   {
     void *ptr;
     cudaError_t error = cudaGetSymbolAddress(&ptr, symbol);
-    if (error != cudaSuccess)
-      errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
     return ptr;
   }
 
@@ -568,7 +603,7 @@ namespace quda {
   {
     // no driver API variant here since we have C++ functions
     PROFILE(cudaError_t error = cudaFuncSetAttribute(kernel, attr, value), QUDA_PROFILE_FUNC_SET_ATTRIBUTE);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    set_runtime_error(error, __func__, func, file, line);
   }
 
   void qudaFuncGetAttributes_(cudaFuncAttributes &attr, const void *kernel, const char *func, const char *file,
@@ -576,19 +611,7 @@ namespace quda {
   {
     // no driver API variant here since we have C++ functions
     PROFILE(cudaError_t error = cudaFuncGetAttributes(&attr, kernel), QUDA_PROFILE_FUNC_SET_ATTRIBUTE);
-    if (error != cudaSuccess) errorQuda("(CUDA) %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
-  }
-
-  static std::string error_str("CUDA_SUCCESS");
-
-  void qudaSetErrorString(const std::string &error_str_)
-  {
-    error_str = error_str_;
-  }
-
-  std::string qudaGetLastErrorString()
-  {
-    return error_str;
+    set_runtime_error(error, __func__, func, file, line);
   }
 
   void printAPIProfile() {

From bf6c6ba6eb73a2b5ad65a98d4fc73aa405a591d0 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 3 Mar 2021 22:23:46 +0000
Subject: [PATCH 022/198] fix build

---
 include/blas_helper.cuh                   |   1 +
 include/quda_api.h                        | 194 +++-------------------
 include/targets/omptarget/math_helper.cuh |   4 +
 include/targets/omptarget/quda_cuda_api.h |  21 +++
 lib/targets/generic/blas_lapack_eigen.cpp |   2 +-
 lib/targets/omptarget/device.cpp          |  39 ++++-
 lib/targets/omptarget/malloc.cpp          |  26 ++-
 lib/targets/omptarget/quda_api.cpp        |  69 +++++++-
 8 files changed, 165 insertions(+), 191 deletions(-)
 create mode 100644 include/targets/omptarget/quda_cuda_api.h

diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index c6d722c17b..6d4c970748 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -2,6 +2,7 @@
 
 #include <color_spinor_field.h>
 #include <reduce_helper.h>
+#include <math_helper.cuh>
 
 //#define QUAD_SUM
 #ifdef QUAD_SUM
diff --git a/include/quda_api.h b/include/quda_api.h
index 6823922881..da023d0ccf 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -1,179 +1,23 @@
 #pragma once
 
 #ifdef QUDA_BACKEND_OMPTARGET
-
-#include <omp.h>
-#include <cstdlib>
-#include <cstring>
-#include <iostream>
-#include <functional>
-#include <string_view>
-
-#include <util_quda.h>
-
-template <typename T>
-constexpr auto type_name() noexcept {
-  std::string_view name = "Error: unsupported compiler", prefix, suffix;
-#ifdef __clang__
-  name = __PRETTY_FUNCTION__;
-  prefix = "auto type_name() [T = ";
-  suffix = "]";
-#elif defined(__GNUC__)
-  name = __PRETTY_FUNCTION__;
-  prefix = "constexpr auto type_name() [with T = ";
-  suffix = "]";
-#endif
-  name.remove_prefix(prefix.size());
-  name.remove_suffix(suffix.size());
-  return name;
-}
-
-template <typename...Arg>
-struct seq_args_call{
-  static constexpr size_t n = sizeof...(Arg);
-  template <typename F, std::size_t...IX>
-  void operator()(F *func, void *args[n], std::index_sequence<IX...>)
-  {(*func)(*(Arg*)args[IX]...);}
-};
-
-template <typename T>
-T * to_device(const T * x, size_t s) {
-  if(0<omp_get_num_devices()){
-    const int d = omp_get_default_device();
-    const int h = omp_get_initial_device();
-    void *p = omp_target_alloc(s, d);
-    printfQuda("# to_device: host_ptr@%d = %p  device_ptr@%d = %p  size = %zu\n", h, x, d, p, s);
-    omp_target_memcpy(p, (void *)x, s, 0, 0, d, h);
-    return (T *)p;
-  }else{
-    return x;
-  }
-}
-
-template <typename T>
-T * to_device(const T& x) {
-  constexpr size_t s = sizeof(T);
-  return to_device(&x, s);
-}
-
+// OMPTARGET SPECIFIC workarounds
 #define __host__
 #define __device__
 #define __shared__
 #define __global__
 #define __constant__ static
-#define __launch_bounds__(x)
+#define __launch_bounds__(...)
 #define __syncthreads() _Pragma("omp barrier")
-
-#define CUDA_SUCCESS QUDA_SUCCESS
-
-using size_t = std::size_t;
-
 #define __forceinline__ inline __attribute__((always_inline))
 
-enum cudaMemcpyKind{cudaMemcpyHostToHost, cudaMemcpyHostToDevice, cudaMemcpyDeviceToHost, cudaMemcpyDeviceToDevice, cudaMemcpyDefault};
-enum cudaError_t{cudaSuccess,cudaErrorNotReady};
-enum {cudaEventDisableTiming,cudaEventInterprocess};
-enum {cudaHostRegisterDefault};
-enum {cudaIpcMemLazyEnablePeerAccess};
-enum {cudaStreamDefault};
-
-static inline cudaError_t
-ompwip_(const char * const file, const size_t line, const char * const func, const char * const msg, std::function<void(void)>f = [](){})
-{
-  if(0==omp_get_team_num()&&0==omp_get_thread_num()) std::cerr<<"OMP WIP:"<<msg<<": "<<file<<':'<<line<<' '<<func<<std::endl;
-  f();
-  return cudaSuccess;
-}
-static inline cudaError_t
-ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f = [](){})
-{return ompwip_(file,line,func,"",f);}
-#define ompwip(...) ompwip_(__FILE__,__LINE__,__PRETTY_FUNCTION__,##__VA_ARGS__)
-
-#define __shfl_down_sync(a,b,c) ompwip("__shfl_down_sync")
-
-#define cuMemcpy(a,b,c) ompwip()
-#define cuMemcpyAsync(a,b,c,d) ompwip()
-#define cuMemcpyDtoD(a,b,c) ompwip()
-#define cuMemcpyDtoDAsync(a,b,c,d) ompwip()
-#define cuMemcpyDtoH(a,b,c) ompwip()
-#define cuMemcpyDtoHAsync(a,b,c,d) ompwip()
-#define cuMemcpyHtoD(a,b,c) ompwip()
-#define cuMemcpyHtoDAsync(a,b,c,d) ompwip()
-#define cudaMemcpy(a,b,c,d) ompwip([&](){printfQuda("memcpy %p <- %p\n",a,b);ompwipMemcpy(a,(void*)b,c,d);})
-#define cudaMemcpy2D(a,b,c,d,e,f,g) ompwip()
-#define cudaMemcpy2DAsync(a,b,c,d,e,f,g,h) ompwip()
-#define cudaMemcpyAsync(a,b,c,d,e) ompwip()
-#define cudaMemcpyToSymbolAsync(a,b,c,d,e,f) ompwip()
-#define cudaMemset(a,b,c) ompwip([&](){printfQuda("memset %p\n",a);ompwipMemset(a,b,c);})
-#define cudaMemset2D(a,b,c,d,e) ompwip()
-#define cudaMemset2DAsync(a,b,c,d,e,f) ompwip()
-#define cudaMemsetAsync(a,b,c,d) ompwip()
-
-static inline void
-ompwipMemset(void *p, unsigned char b, std::size_t s)
-{
-#pragma omp target teams distribute parallel for simd is_device_ptr(p)
-  for(std::size_t i=0;i<s;++i) *(unsigned char *)p = b;
-}
+#define CUDA_SUCCESS QUDA_SUCCESS
 
-static inline void
-ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
-{
-  switch(k){
-  case cudaMemcpyHostToHost: memcpy(d,s,c); break;
-  case cudaMemcpyHostToDevice:
-    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
-    else warningQuda("cudaMemcpyHostToDevice without a device");
-    break;
-  case cudaMemcpyDeviceToHost:
-    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
-    else warningQuda("cudaMemcpyDeviceToHost without a device");
-    break;
-  case cudaMemcpyDeviceToDevice:
-    warningQuda("unimplemented for cudaMemcpyDeviceToDevice");
-    break;
-  case cudaMemcpyDefault:
-    warningQuda("unimplemented for cudaMemcpyDefault");
-    break;
-  default: errorQuda("Unsupported cudaMemcpyType %d", k);
-  }
-}
+#define QUDA_RT_CONSTS ompwip();const dim3 threadIdx(omp_get_thread_num(),0,0),blockDim(omp_get_num_threads()),blockIdx(omp_get_team_num(),0,0),gridDim(omp_get_num_teams())
 
-#define cudaDeviceSynchronize() ompwip()
-#define cudaEventCreate(a,...) ompwip([&](){(*(a))=0;})
-#define cudaEventCreateWithFlags(a,b) ompwip()
-#define cudaEventDestroy(a) ompwip()
-#define cudaEventElapsedTime(a,b,c) ompwip([&](){(*(a))=0;})
-#define cudaEventQuery(a) ompwip()
-#define cudaEventRecord(a,b) ompwip()
-#define cudaEventSynchronize(a) ompwip()
-#define cudaFuncGetAttributes(a,b) ompwip()
-#define cudaFuncSetAttribute(a,b,c) ompwip()
-#define cudaGetErrorString(a) "FIXME"
-#define cudaGetLastError() ompwip()
-#define cudaGetSymbolAddress(a,b) ompwip()
-#define cudaHostRegister(a,b,c) ompwip()
-#define cudaHostUnregister(a) ompwip()
-#define cudaIpcCloseMemHandle(a) ompwip()
-#define cudaIpcGetEventHandle(a,b) ompwip([&](){(*(a))=0;})
-#define cudaIpcGetMemHandle(a,b) ompwip([&](){(*(a))=0;})
-#define cudaIpcOpenEventHandle(a,b) ompwip([&](){(*(a))=0;})
-#define cudaIpcOpenMemHandle(a,b,c) ompwip()
-#define cudaStreamCreate(a) ompwip([&](){(*(a))=0;})
-#define cudaStreamDestroy(a) ompwip()
-#define cudaStreamSynchronize(a) ompwip()
-#define cudaStreamWaitEvent(a,b,c) ompwip()
-
-static char FIXME[]="FIXME";
-#define cuGetErrorString(a,b) ompwip([&](){(*(b))=FIXME;})
-#define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
-
-using qudaStream_t = int;
-using cudaStream_t = int;
-using CUresult = int;
-using cudaIpcMemHandle_t = int;
-using cudaIpcEventHandle_t = int;
-using cudaEvent_t = int;
+#include <functional>
+#include <iostream>
+#include <omp.h>
 
 struct dim3 {unsigned int x,y,z;
   constexpr dim3():x(1u),y(1u),z(1u){}
@@ -209,18 +53,24 @@ static inline double3 make_double3(double x,double y,double z){return double3{x,
 struct double4 {double x,y,z,w;};
 static inline double4 make_double4(double x,double y,double z,double w){return double4{x,y,z,w};}
 
-#define QUDA_RT_CONSTS ompwip();const dim3 threadIdx(omp_get_thread_num(),0,0),blockDim(omp_get_num_threads()),blockIdx(omp_get_team_num(),0,0),gridDim(omp_get_num_teams())
-
-#else  // QUDA_BACKEND_OMPTARGET
+static inline int
+ompwip_(const char * const file, const size_t line, const char * const func, const char * const msg, std::function<void(void)>f = [](){})
+{
+  if(0==omp_get_team_num()&&0==omp_get_thread_num()) std::cerr<<"OMP WIP:"<<msg<<": "<<file<<':'<<line<<' '<<func<<std::endl;
+  f();
+  return 0;
+}
+static inline int
+ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f = [](){})
+{return ompwip_(file,line,func,"",f);}
+#define ompwip(...) ompwip_(__FILE__,__LINE__,__PRETTY_FUNCTION__,##__VA_ARGS__)
 
-#define QUDA_RT_CONSTS
+using cudaStream_t = int;  // device.h:/cudaStream_t
+using CUresult = int;  // ../lib/coarse_op.cuh:/CUresult
 
-#ifndef __CUDACC_RTC__
-#include <cuda.h>
-#include <cuda_runtime.h>
-#endif
+#else // QUDA_BACKEND_OMPTARGET
 
-#endif  // QUDA_BACKEND_OMPTARGET
+#endif // QUDA_BACKEND_OMPTARGET
 
 #include <quda_define.h>
 #include <string>
diff --git a/include/targets/omptarget/math_helper.cuh b/include/targets/omptarget/math_helper.cuh
index 285fdc1284..f50e133536 100644
--- a/include/targets/omptarget/math_helper.cuh
+++ b/include/targets/omptarget/math_helper.cuh
@@ -119,5 +119,9 @@ namespace quda {
     return std::pow(a, b);
   }
 
+  template <typename real> __device__ __host__ inline real fdivide(real a, real b)
+  {
+    return a / b;
+  }
 
 }
diff --git a/include/targets/omptarget/quda_cuda_api.h b/include/targets/omptarget/quda_cuda_api.h
new file mode 100644
index 0000000000..b84d0ba949
--- /dev/null
+++ b/include/targets/omptarget/quda_cuda_api.h
@@ -0,0 +1,21 @@
+using cudaError_t = int;
+
+namespace quda {
+
+  /**
+     @file quda_cuda_api.h
+     @brief Header file that declares some functions that will be called from within the CUDA target
+  */
+
+  namespace cuda {
+
+    void set_runtime_error(cudaError_t error, const char *api_func, const char *func, const char *file, const char *line,
+                           bool allow_error = false);
+
+    // defined in quda_api.cpp
+    void set_driver_error(CUresult error, const char *api_func, const char *func, const char *file, const char *line,
+                          bool allow_error = false);
+
+
+  }
+}
diff --git a/lib/targets/generic/blas_lapack_eigen.cpp b/lib/targets/generic/blas_lapack_eigen.cpp
index 53c4d0046e..881b8e4b63 100644
--- a/lib/targets/generic/blas_lapack_eigen.cpp
+++ b/lib/targets/generic/blas_lapack_eigen.cpp
@@ -55,7 +55,7 @@ namespace quda
                                   QudaFieldLocation location)
       {
         if (getVerbosity() >= QUDA_VERBOSE)
-          printfQuda("BatchInvertMatrix (generic - Eigen): Nc = %d, batch = %llu\n", n, batch);
+          printfQuda("BatchInvertMatrix (generic - Eigen): Nc = %d, batch = %lu\n", n, batch);
 
         size_t size = 2 * n * n * batch * prec;
         void *A_h = (location == QUDA_CUDA_FIELD_LOCATION ? pool_pinned_malloc(size) : A);
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 425b09be68..e4ea3d5aa2 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -1,6 +1,7 @@
 #include <omp.h>
 #include <util_quda.h>
 #include <quda_internal.h>
+#include <quda_cuda_api.h>
 
 #ifdef QUDA_NVML
 #include <nvml.h>
@@ -21,13 +22,21 @@ struct cudaDeviceProp{
   unsigned int maxThreadsDim[3];
   unsigned int maxGridSize[3];
   unsigned int multiProcessorCount;
+  unsigned int warpSize;
+  unsigned long totalGlobalMem;
+  unsigned long totalConstMem;
+  unsigned long memPitch;
+  unsigned int regsPerBlock;
+  int unifiedAddressing;
+  int deviceOverlap;
 };
 
 static void cudaDriverGetVersion(int*v){*v=0;}
 static void cudaRuntimeGetVersion(int*v){*v=0;}
-static void cudaGetDeviceCount(int*c){*c=omp_get_num_devices();}
-static void cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
+static int cudaGetDeviceCount(int*c){*c=omp_get_num_devices();return 0;}
+static int cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
 {
+  /* FIXME totally fake numbers */
   if(0<omp_get_num_devices()){
     p->name = ompdevname;
     p->major = 7;
@@ -57,20 +66,34 @@ static void cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
     p->maxGridSize[2] = 8;
     p->multiProcessorCount = 64;
   }
+  p->warpSize = 32;
+  p->totalGlobalMem = 1u<34;
+  p->totalConstMem = 1u<34;
+  p->memPitch = 1u<34;
+  p->regsPerBlock = 1024;
+  p->unifiedAddressing = 1;
+  p->deviceOverlap = 0;
+  return 0;
 }
 
-#define cudaSetDevice omp_set_default_device
-#define cudaDeviceSetCacheConfig(a)
-static void cudaDeviceGetStreamPriorityRange(int*lo,int*hi){lo=0;hi=0;}
-static void cudaStreamCreateWithPriority(qudaStream_t*s,int lo,int hi){*s=0;}
-#define cudaDeviceReset()
+enum {cudaFuncCachePreferL1};
+enum {cudaStreamDefault};
+static int cudaSetDevice(int d){omp_set_default_device(d); return 0;}
+static int cudaDeviceSetCacheConfig(int a){ompwip();return 0;}
+static int cudaDeviceGetStreamPriorityRange(int*lo,int*hi){ompwip();lo=0;hi=0;return 0;}
+static int cudaStreamCreateWithPriority(cudaStream_t*s,int lo,int hi){ompwip();*s=0;return 0;}
+static int cudaStreamDestroy(cudaStream_t s){ompwip();return 0;}
+static int cudaDeviceReset(){ompwip();return 0;}
 enum {cudaDevAttrMaxSharedMemoryPerBlockOptin};
-static void cudaDeviceGetAttribute(int*b,int o,int i){*b=98304;}
+static int cudaDeviceGetAttribute(int*b,int o,int i){ompwip();*b=98304;return 0;}
 
 static cudaDeviceProp deviceProp;
 static cudaStream_t *streams;
 static const int Nstream = 9;
 
+#define CHECK_CUDA_ERROR(func)                                          \
+  cuda::set_runtime_error(func, #func, __func__, __FILE__, __STRINGIFY__(__LINE__));
+
 namespace quda
 {
 
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index f02fea8933..b08eb9b4a5 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -453,17 +453,20 @@ namespace quda
     if (alloc[HOST].count(ptr)) {
       track_free(HOST, ptr);
       free(ptr);
-    } else if (alloc[PINNED].count(ptr)) {ompwip("alloc[PINNED].count(ptr)");/*
+    } else if (alloc[PINNED].count(ptr)) {
+      ompwip("alloc[PINNED].count(ptr)");
+/*
       cudaError_t err = cudaHostUnregister(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to unregister pinned memory (%s:%d in %s())\n", file, line, func); }
       track_free(PINNED, ptr);
       free(ptr);
-    */} else if (alloc[MAPPED].count(ptr)) {
+*/
+    } else if (alloc[MAPPED].count(ptr)) {
 #ifdef HOST_ALLOC
       ompwip("HOST_ALLOC alloc[MAPPED].count(ptr)");/*
       cudaError_t err = cudaFreeHost(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to free host memory (%s:%d in %s())\n", file, line, func); }
-      */
+*/
 #else
       ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr)");/*
       cudaError_t err = cudaHostUnregister(ptr);
@@ -471,7 +474,7 @@ namespace quda
         errorQuda("Failed to unregister host-mapped memory (%s:%d in %s())\n", file, line, func);
       }
       free(ptr);
-      */
+*/
 #endif
       // track_free(MAPPED, ptr);
     } else {
@@ -531,7 +534,9 @@ namespace quda
   */}
 
   void *get_mapped_device_pointer_(const char *func, const char *file, int line, const void *host)
-  {ompwip("return nullptr");return nullptr;/*
+  {
+    ompwip("return nullptr");return nullptr;
+/*
     void *device;
     auto error = cudaHostGetDevicePointer(&device, const_cast<void *>(host), 0);
     if (error != cudaSuccess) {
@@ -539,24 +544,31 @@ namespace quda
                 cudaGetErrorString(error), file, line, func);
     }
     return device;
+*/
   }
 
   void register_pinned_(const char *func, const char *file, int line, void *ptr, size_t bytes)
-  {ompwip();
+  {
+    ompwip();
+/*
     auto error = cudaHostRegister(ptr, bytes, cudaHostRegisterDefault);
     if (error != cudaSuccess) {
       errorQuda("cudaHostRegister failed with error %s (%s:%d in %s()",
                 cudaGetErrorString(error), file, line, func);
     }
+*/
   }
 
   void unregister_pinned_(const char *func, const char *file, int line, void *ptr)
-  {ompwip();
+  {
+    ompwip();
+/*
     auto error = cudaHostUnregister(ptr);
     if (error != cudaSuccess) {
       errorQuda("cudaHostUnregister failed with error %s (%s:%d in %s()",
                 cudaGetErrorString(error), file, line, func);
     }
+*/
   }
 
   namespace pool
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 59c2436009..a63d2d53c0 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -8,6 +8,69 @@
 enum cudaFuncAttribute{cudaFuncAttributePreferredSharedMemoryCarveout};
 using cudaFuncAttributes = int;
 using CUdeviceptr = void*;
+using cudaError_t = int;
+enum {cudaSuccess,cudaErrorNotReady};
+enum cudaMemcpyKind{cudaMemcpyHostToHost, cudaMemcpyHostToDevice, cudaMemcpyDeviceToHost, cudaMemcpyDeviceToDevice, cudaMemcpyDefault};
+
+static char FIXME[]="OMP FIXME";
+
+#define cudaGetErrorString(a) "OMP FIXME"
+#define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
+
+#define cudaMemcpy(a,b,c,d) ompwip([&](){printfQuda("memcpy %p <- %p\n",a,b);ompwipMemcpy(a,(void*)b,c,d);})
+#define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){printfQuda("memcpy %p <- %p\n",a,b);ompwipMemcpy(a,(void*)b,c,d);})
+#define cudaMemcpy2D(a,b,c,d,e,f,g) ompwip("unimplemented")
+#define cudaMemcpy2DAsync(a,b,c,d,e,f,g,h) ompwip("unimplemented")
+#define cudaMemset(a,b,c) ompwip([&](){printfQuda("memset %p\n",a);ompwipMemset(a,b,c);})
+#define cudaMemsetAsync(a,b,c,d) ompwip([&](){printfQuda("memset %p\n",a);ompwipMemset(a,b,c);})
+#define cudaMemset2D(a,b,c,d,e) ompwip("unimplemented")
+#define cudaMemset2DAsync(a,b,c,d,e,f) ompwip("unimplemented")
+
+static inline void
+ompwipMemset(void *p, unsigned char b, std::size_t s)
+{
+#pragma omp target teams distribute parallel for simd is_device_ptr(p)
+  for(std::size_t i=0;i<s;++i) *(unsigned char *)p = b;
+}
+
+static inline void
+ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
+{
+  switch(k){
+  case cudaMemcpyHostToHost: memcpy(d,s,c); break;
+  case cudaMemcpyHostToDevice:
+    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+    else warningQuda("cudaMemcpyHostToDevice without a device");
+    break;
+  case cudaMemcpyDeviceToHost:
+    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
+    else warningQuda("cudaMemcpyDeviceToHost without a device");
+    break;
+  case cudaMemcpyDeviceToDevice:
+    warningQuda("unimplemented for cudaMemcpyDeviceToDevice");
+    break;
+  case cudaMemcpyDefault:
+    warningQuda("unimplemented for cudaMemcpyDefault");
+    break;
+  default: errorQuda("Unsupported cudaMemcpyType %d", k);
+  }
+}
+
+using cudaEvent_t = int*;
+enum {cudaEventDisableTiming};
+#define cudaEventCreate(a) ompwip("cudaEventCreate")
+#define cudaEventCreateWithFlags(a,b) ompwip("cudaEventCreateWithFlags")
+#define cudaEventElapsedTime(a,b,c) ompwip([&](){(*(a))=0;})
+#define cudaEventQuery(a) ompwip("cudaEventQuery")
+#define cudaEventRecord(a,b) ompwip("cudaEventRecord")
+#define cudaEventSynchronize(a) ompwip("cudaEventSynchronize")
+#define cudaEventDestroy(a) ompwip("cudaEventDestroy")
+#define cudaStreamWaitEvent(a,b,c) ompwip("cudaStreamWaitEvent")
+#define cudaStreamSynchronize(a) ompwip("cudaStreamSynchronize")
+#define cudaDeviceSynchronize() ompwip("cudaDeviceSynchronize")
+#define cudaGetSymbolAddress(a,b) ompwip("cudaGetSymbolAddress")
+#define cudaFuncGetAttributes(a,b) ompwip("cudaFuncGetAttributes")
+#define cudaFuncSetAttribute(a,b,c) ompwip("cudaFuncSetAttribute")
 
 // if this macro is defined then we use the driver API, else use the
 // runtime API.  Typically the driver API has 10-20% less overhead
@@ -495,7 +558,7 @@ namespace quda {
     return false;
   }
 
-  void qudaEventRecord_(cudaEvent_t &event, qudaStream_t stream, const char *func, const char *file, const char *line)
+  void qudaEventRecord_(qudaEvent_t &quda_event, qudaStream_t stream, const char *func, const char *file, const char *line)
   {
     cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API
@@ -507,7 +570,7 @@ namespace quda {
 #endif
   }
 
-  void qudaStreamWaitEvent_(qudaStream_t stream, cudaEvent_t event, unsigned int flags, const char *func,
+  void qudaStreamWaitEvent_(qudaStream_t stream, qudaEvent_t quda_event, unsigned int flags, const char *func,
                             const char *file, const char *line)
   {
     cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
@@ -556,7 +619,7 @@ namespace quda {
     set_runtime_error(error, __func__, func, file, line);
   }
 
-  void qudaEventSynchronize_(const cudaEvent_t &event, const char *func, const char *file, const char *line)
+  void qudaEventSynchronize_(const qudaEvent_t &quda_event, const char *func, const char *file, const char *line)
   {
     const cudaEvent_t &event = reinterpret_cast<const cudaEvent_t&>(quda_event.event);
 #ifdef USE_DRIVER_API

From 679ef7c7edfee98c58186bf0c11d45a9449593ca Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 4 Mar 2021 02:01:40 +0000
Subject: [PATCH 023/198] omp: add missing file

---
 include/complex_quda.h                |   2 +
 lib/targets/omptarget/comm_target.cpp | 176 ++++++++++++++++++++++++++
 2 files changed, 178 insertions(+)
 create mode 100644 lib/targets/omptarget/comm_target.cpp

diff --git a/include/complex_quda.h b/include/complex_quda.h
index ee428d412c..8cc381c628 100644
--- a/include/complex_quda.h
+++ b/include/complex_quda.h
@@ -25,6 +25,8 @@
 #include <sstream>
 #include <cstdint>
 
+#include <quda_api.h>
+
 namespace quda {
   namespace gauge {
     template<typename Float, typename storeFloat> struct fieldorder_wrapper;
diff --git a/lib/targets/omptarget/comm_target.cpp b/lib/targets/omptarget/comm_target.cpp
new file mode 100644
index 0000000000..7d95c583f6
--- /dev/null
+++ b/lib/targets/omptarget/comm_target.cpp
@@ -0,0 +1,176 @@
+#include <comm_quda.h>
+#include <quda_api.h>
+#include <quda_cuda_api.h>
+#include <algorithm>
+
+#define CHECK_CUDA_ERROR(func)                                          \
+  quda::cuda::set_runtime_error(func, #func, __func__, __FILE__, __STRINGIFY__(__LINE__));
+
+bool comm_peer2peer_possible(int local_gpuid, int neighbor_gpuid)
+{
+  ompwip("comm_peer2peer_possible -> false");
+  return false;
+/*
+  int canAccessPeer[2];
+  CHECK_CUDA_ERROR(cudaDeviceCanAccessPeer(&canAccessPeer[0], local_gpuid, neighbor_gpuid));
+  CHECK_CUDA_ERROR(cudaDeviceCanAccessPeer(&canAccessPeer[1], neighbor_gpuid, local_gpuid));
+
+  // require symmetric peer-to-peer access to enable peer-to-peer
+  return canAccessPeer[0] && canAccessPeer[1];
+*/
+}
+
+int comm_peer2peer_performance(int local_gpuid, int neighbor_gpuid)
+{
+  ompwip("comm_peer2peer_performance -> 0");
+  return 0;
+/*
+  int accessRank[2] = { };
+  if (comm_peer2peer_possible(local_gpuid, neighbor_gpuid)) {
+    CHECK_CUDA_ERROR(cudaDeviceGetP2PAttribute(&accessRank[0], cudaDevP2PAttrPerformanceRank, local_gpuid, neighbor_gpuid));
+    CHECK_CUDA_ERROR(cudaDeviceGetP2PAttribute(&accessRank[1], cudaDevP2PAttrPerformanceRank, neighbor_gpuid, local_gpuid));
+  }
+
+  // return the slowest direction of access (lower is faster)
+  return std::max(accessRank[0], accessRank[1]);
+*/
+}
+
+void comm_create_neighbor_memory(void *remote[QUDA_MAX_DIM][2], void *local)
+{
+  ompwip("unimplemented");
+/*
+  // handles for obtained ghost pointers
+  cudaIpcMemHandle_t remote_handle[2][QUDA_MAX_DIM];
+
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    for (int dir=0; dir<2; ++dir) {
+      MsgHandle* sendHandle = nullptr;
+      MsgHandle* receiveHandle = nullptr;
+      int disp = (dir == 1) ? +1 : -1;
+
+      // first set up receive
+      if (comm_peer2peer_enabled(1-dir,dim)) {
+        receiveHandle = comm_declare_receive_relative(&remote_handle[1-dir][dim],
+                                                      dim, -disp, sizeof(remote_handle));
+      }
+      // now send
+      cudaIpcMemHandle_t local_handle;
+      CHECK_CUDA_ERROR(cudaIpcGetMemHandle(&local_handle, local));
+      if (comm_peer2peer_enabled(dir,dim)) {
+        sendHandle = comm_declare_send_relative(&local_handle, dim, disp, sizeof(local_handle));
+      }
+      if (receiveHandle) comm_start(receiveHandle);
+      if (sendHandle) comm_start(sendHandle);
+
+      if (receiveHandle) comm_wait(receiveHandle);
+      if (sendHandle) comm_wait(sendHandle);
+
+      if (sendHandle) comm_free(sendHandle);
+      if (receiveHandle) comm_free(receiveHandle);
+    }
+  }
+
+  // open the remote memory handles and set the send ghost pointers
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    // even if comm_dim(2) == 2, we might not have p2p enabled in both directions, so check this
+    const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
+    for (int dir=0; dir<num_dir; ++dir) {
+      remote[dim][dir] = nullptr;
+      if (comm_peer2peer_enabled(dir,dim)) CHECK_CUDA_ERROR(cudaIpcOpenMemHandle(&remote[dim][dir], remote_handle[dir][dim], cudaIpcMemLazyEnablePeerAccess));
+    }
+    if (num_dir == 1) remote[dim][1] = remote[dim][0];
+  }
+*/
+}
+
+void comm_destroy_neighbor_memory(void *remote[QUDA_MAX_DIM][2])
+{
+  ompwip("unimplemented");
+/*
+  for (int dim=0; dim<4; ++dim) {
+
+    if (comm_dim(dim)==1) continue;
+    const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
+
+    if (comm_peer2peer_enabled(1,dim)) {
+      // only close this handle if it doesn't alias the back ghost
+      if (num_dir == 2 && remote[dim][1]) CHECK_CUDA_ERROR(cudaIpcCloseMemHandle(remote[dim][1]));
+    }
+
+    if (comm_peer2peer_enabled(0,dim)) {
+      if (remote[dim][0]) CHECK_CUDA_ERROR(cudaIpcCloseMemHandle(remote[dim][0]));
+    }
+  } // iterate over dim
+*/
+}
+  
+void comm_create_neighbor_event(qudaEvent_t remote[2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
+{
+  ompwip("unimplemented");
+/*
+  // handles for obtained events
+  cudaIpcEventHandle_t ipcRemoteEventHandle[2][QUDA_MAX_DIM];
+
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    for (int dir=0; dir<2; ++dir) {
+      MsgHandle* sendHandle = nullptr;
+      MsgHandle* receiveHandle = nullptr;
+      int disp = (dir == 1) ? +1 : -1;
+
+      // first set up receive
+      if (comm_peer2peer_enabled(1-dir,dim)) {
+        receiveHandle = comm_declare_receive_relative(&ipcRemoteEventHandle[1-dir][dim], dim, -disp,
+                                                      sizeof(ipcRemoteEventHandle[1-dir][dim]));
+      }
+
+      cudaIpcEventHandle_t handle;
+
+      // now send
+      if (comm_peer2peer_enabled(dir,dim)) {
+        cudaEvent_t event;
+        CHECK_CUDA_ERROR(cudaEventCreate(&event, cudaEventDisableTiming | cudaEventInterprocess));
+        local[dir][dim].event = reinterpret_cast<void*>(event);
+        CHECK_CUDA_ERROR(cudaIpcGetEventHandle(&handle, event));
+        sendHandle = comm_declare_send_relative(&handle, dim, disp, sizeof(handle));
+      }
+
+      if (receiveHandle) comm_start(receiveHandle);
+      if (sendHandle) comm_start(sendHandle);
+
+      if (receiveHandle) comm_wait(receiveHandle);
+      if (sendHandle) comm_wait(sendHandle);
+
+      if (sendHandle) comm_free(sendHandle);
+      if (receiveHandle) comm_free(receiveHandle);
+    }
+  }
+
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    for (int dir=0; dir<2; ++dir) {
+      if (!comm_peer2peer_enabled(dir,dim)) continue;
+      cudaEvent_t event;
+      CHECK_CUDA_ERROR(cudaIpcOpenEventHandle(&event, ipcRemoteEventHandle[dir][dim]));
+      remote[dir][dim].event = reinterpret_cast<void*>(event);
+    }
+  }
+*/
+}
+
+void comm_destroy_neighbor_event(qudaEvent_t [2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
+{
+  ompwip("unimplemented");
+/*
+  for (int dim=0; dim<4; ++dim) {
+    if (comm_dim(dim)==1) continue;
+    for (int dir = 0; dir <2; dir++) {
+      cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(local[dir][dim].event);
+      if (comm_peer2peer_enabled(dir, dim)) CHECK_CUDA_ERROR(cudaEventDestroy(event));
+    } 
+  } // iterate over dim
+*/
+}

From 4a31920d148ff54ea22ab5d335456ecf716c7831 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 15 Mar 2021 22:11:03 +0000
Subject: [PATCH 024/198] omp: workaround rng

---
 lib/gauge_random.cu | 3 +++
 lib/random.cu       | 3 +++
 lib/spinor_noise.cu | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/lib/gauge_random.cu b/lib/gauge_random.cu
index 6bc07409b5..89b597d15b 100644
--- a/lib/gauge_random.cu
+++ b/lib/gauge_random.cu
@@ -34,12 +34,15 @@ namespace quda {
 
     void apply(const qudaStream_t &stream)
     {
+      ompwip("skip nonfunctioning RNG.");
+/*
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       if (group) {
         launch<GaussGauge>(tp, stream, GaugeGaussArg<Float, nColor, recon, true>(U, rng.State(), sigma));
       } else {
         launch<GaussGauge>(tp, stream, GaugeGaussArg<Float, nColor, recon, false>(U, rng.State(), sigma));
       }
+*/
     }
 
     long long flops() const { return 0; }
diff --git a/lib/random.cu b/lib/random.cu
index 774f60fae7..71f348f5d0 100644
--- a/lib/random.cu
+++ b/lib/random.cu
@@ -26,8 +26,11 @@ namespace quda {
 
     void apply(const qudaStream_t &stream)
     {
+      ompwip("skip nonfunctioning RNG.");
+/*
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       launch_device<init_random>(tp, stream, rngArg(rng.State(), seed, meta));
+*/
     }
 
     long long flops() const { return 0; }
diff --git a/lib/spinor_noise.cu b/lib/spinor_noise.cu
index 37729db098..675bd8d48c 100644
--- a/lib/spinor_noise.cu
+++ b/lib/spinor_noise.cu
@@ -24,6 +24,8 @@ namespace quda {
     }
 
     void apply(const qudaStream_t &stream) {
+      ompwip("skip nonfunctioning RNG.");
+/*
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       switch (type) {
       case QUDA_NOISE_GAUSS:
@@ -34,6 +36,7 @@ namespace quda {
         break;
       default: errorQuda("Noise type %d not implemented", type);
       }
+*/
     }
 
     long long flops() const { return 0; }

From 3bbcd0813101f41bb3933ecc4234aceb7fb51afd Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 16 Mar 2021 03:16:13 +0000
Subject: [PATCH 025/198] omp: fix compilation after merge

---
 include/atomic.cuh                        | 34 ++++++++++++++++--
 include/blas_helper.cuh                   |  1 +
 include/cub_helper.cuh                    |  5 ++-
 include/targets/omptarget/math_helper.cuh | 11 +++---
 include/targets/omptarget/target_device.h | 44 +++++++++++++++--------
 lib/targets/omptarget/device.cpp          | 12 +++----
 6 files changed, 76 insertions(+), 31 deletions(-)

diff --git a/include/atomic.cuh b/include/atomic.cuh
index 9c52bc0df6..71a9662192 100644
--- a/include/atomic.cuh
+++ b/include/atomic.cuh
@@ -11,6 +11,17 @@
  */
 
 #ifdef QUDA_BACKEND_OMPTARGET
+static inline unsigned int
+__float_as_uint(float x)
+{
+  return reinterpret_cast<unsigned int &>(x);  // FIXME UB?
+}
+static inline unsigned int
+__uint_as_float(unsigned int x)
+{
+  return reinterpret_cast<float &>(x);  // FIXME UB?
+}
+
 template<typename T>
 static inline T atomicAdd(T*x,T v)
 {
@@ -19,6 +30,22 @@ static inline T atomicAdd(T*x,T v)
   {z=*x; *x+=v;}
   return z;
 }
+template<typename T>
+static inline T atomicCAS(T*x,T c,T v)
+{
+  T z;
+#pragma omp atomic capture
+  {z=*x; *x=z==c?v:z;}
+  return z;
+}
+template<typename T>
+static inline T atomicMax(T*x,T v)
+{
+  T z;
+#pragma omp atomic capture
+  {z=*x; *x=z<v?v:z;}
+  return z;
+}
 #endif
 
 /**
@@ -90,7 +117,7 @@ static inline __device__ short2 atomicAdd(short2 *addr, short2 val){
   return old.s;
 }
 
-union uint32_char2 { unsigned short i; char2 s; };
+union uint32_char2 { unsigned int i; char2 s; };
 
 /**
    @brief Implementation of char2 atomic addition using compare
@@ -179,8 +206,9 @@ template <typename T, int n> __device__ __host__ void atomic_fetch_add(vector_ty
 template <bool is_device> struct atomic_fetch_abs_max_impl {
   template <typename T> inline void operator()(T *addr, T val)
   {
-#pragma omp atomic update
-    *addr = std::max(*addr, val);
+    T z;
+#pragma omp atomic capture
+    {z=*addr; *addr=z<val?val:z;}
   }
 };
 
diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index e0845a8d5f..486df3e039 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -3,6 +3,7 @@
 #include <color_spinor_field.h>
 #include <reduce_helper.h>
 #include <load_store.h>
+#include <math_helper.cuh>
 
 //#define QUAD_SUM
 #ifdef QUAD_SUM
diff --git a/include/cub_helper.cuh b/include/cub_helper.cuh
index 3b10997349..fae6b4f660 100644
--- a/include/cub_helper.cuh
+++ b/include/cub_helper.cuh
@@ -75,9 +75,12 @@ namespace quda {
     template <typename dummy> struct sum<true, dummy> {
       __device__ inline T operator()(const T &value_, bool pipeline, int batch, bool all_sum)
       {
+        QUDA_RT_CONSTS;
         static __shared__ typename block_reduce_t::TempStorage storage[batch_size];
         block_reduce_t block_reduce(storage[batch]);
-        if (!pipeline) __syncthreads(); // only synchronize if we are not pipelining
+        if (!pipeline) {
+          __syncthreads(); // only synchronize if we are not pipelining
+        }
         T value = block_reduce.Sum(value_);
 
         if (all_sum) {
diff --git a/include/targets/omptarget/math_helper.cuh b/include/targets/omptarget/math_helper.cuh
index f50e133536..4d8034d4c5 100644
--- a/include/targets/omptarget/math_helper.cuh
+++ b/include/targets/omptarget/math_helper.cuh
@@ -1,16 +1,15 @@
 #pragma once
 
 #include <cmath>
+#include <target_device.h>
 
 namespace quda {
 
-  
   /**
-   * @brief Combined sin and cos colculation in QUDA NAMESPACE  
-   * @param a the angle 
+   * @brief Combined sin and cos colculation in QUDA NAMESPACE
+   * @param a the angle
    * @param s pointer to the storage for the result of the sin
    * @param c pointer to the storage for the result of the cos
-   *
    */
   template<typename T>
   inline __host__ __device__ void sincos(const T& a, T* s, T* c)
@@ -30,7 +29,7 @@ namespace quda {
    * Specialization to float arguments. Device function calls CUDA intrinsic
    *
   template<>
-  inline  __host__ __device__ void sincos(const float& a, float * s, float *c)
+  inline __host__ __device__ void sincos(const float& a, float * s, float *c)
   {
 #ifdef __CUDA_ARCH__
     __sincosf(a,s,c);
@@ -119,7 +118,7 @@ namespace quda {
     return std::pow(a, b);
   }
 
-  template <typename real> __device__ __host__ inline real fdivide(real a, real b)
+  template <typename real> __device__ __host__ inline real fdividef(real a, real b)
   {
     return a / b;
   }
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index f7b22a7667..33b8ad458f 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -2,31 +2,41 @@
 
 namespace quda {
 
-  namespace device {
+  namespace target {
 
-    /**
-       @brief Helper function that returns if the current execution
-       region is on the device
-    */
 #pragma omp begin declare variant match(device={kind(host)})
-    constexpr bool is_device() {return false;}
+    template <template <bool, typename ...> class f, typename ...Args>
+      __host__ __device__ auto dispatch(Args &&... args)
+    {
+      return f<false>()(args...);
+    }
 #pragma omp end declare variant
 #pragma omp begin declare variant match(device={kind(nohost)})
-    constexpr bool is_device() {return true;}
+    template <template <bool, typename ...> class f, typename ...Args>
+      __host__ __device__ auto dispatch(Args &&... args)
+    {
+      return f<true>()(args...);
+    }
 #pragma omp end declare variant
-    constexpr bool is_device() {return false;}
+
+    template <bool is_device> struct is_device_impl { constexpr bool operator()() { return false; } };
+    template <> struct is_device_impl<true> { constexpr bool operator()() { return true; } };
+
+    /**
+       @brief Helper function that returns if the current execution
+       region is on the device
+    */
+    __device__ __host__ inline bool is_device() { return dispatch<is_device_impl>(); }
+
+
+    template <bool is_device> struct is_host_impl { constexpr bool operator()() { return true; } };
+    template <> struct is_host_impl<true> { constexpr bool operator()() { return false; } };
 
     /**
        @brief Helper function that returns if the current execution
        region is on the host
     */
-#pragma omp begin declare variant match(device={kind(host)})
-    constexpr bool is_host() {return true;}
-#pragma omp end declare variant
-#pragma omp begin declare variant match(device={kind(nohost)})
-    constexpr bool is_host() {return false;}
-#pragma omp end declare variant
-    constexpr bool is_host() {return true;}
+    __device__ __host__ inline bool is_host() { return dispatch<is_host_impl>(); }
 
     /**
        @brief Helper function that returns the thread block
@@ -79,6 +89,10 @@ namespace quda {
 */
     }
 
+  }
+
+  namespace device {
+
     /**
        @brief Helper function that returns the warp-size of the
        architecture we are running on.
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index e4ea3d5aa2..8c9879ee15 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -31,8 +31,8 @@ struct cudaDeviceProp{
   int deviceOverlap;
 };
 
-static void cudaDriverGetVersion(int*v){*v=0;}
-static void cudaRuntimeGetVersion(int*v){*v=0;}
+static int cudaDriverGetVersion(int*v){*v=0;return 0;}
+static int cudaRuntimeGetVersion(int*v){*v=0;return 0;}
 static int cudaGetDeviceCount(int*c){*c=omp_get_num_devices();return 0;}
 static int cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
 {
@@ -108,11 +108,11 @@ namespace quda
       initialized = true;
 
       int driver_version;
-      cudaDriverGetVersion(&driver_version);
+      CHECK_CUDA_ERROR(cudaDriverGetVersion(&driver_version));
       printfQuda("CUDA Driver version = %d\n", driver_version);
 
       int runtime_version;
-      cudaRuntimeGetVersion(&runtime_version);
+      CHECK_CUDA_ERROR(cudaRuntimeGetVersion(&runtime_version));
       printfQuda("CUDA Runtime version = %d\n", runtime_version);
 
 #ifdef QUDA_NVML
@@ -128,7 +128,7 @@ namespace quda
 #endif
 
       for (int i = 0; i < get_device_count(); i++) {
-        cudaGetDeviceProperties(&deviceProp, i);
+        CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, i));
         if (getVerbosity() >= QUDA_SUMMARIZE) {
           printfQuda("Found device %d: %s\n", i, deviceProp.name);
         }
@@ -201,7 +201,6 @@ namespace quda
 
     void print_device_properties()
     {
-
       for (int device = 0; device < get_device_count(); device++) {
 
         // cudaDeviceProp deviceProp;
@@ -238,6 +237,7 @@ namespace quda
         case 3: printfQuda("%d - computeMode              3: cudaComputeModeExclusiveProcess\n", device); break;
         default: errorQuda("Unknown deviceProp.computeMode.");
         }
+
         printfQuda("%d - surfaceAlignment         %lu\n", device, deviceProp.surfaceAlignment);
         printfQuda("%d - concurrentKernels        %s\n", device, (deviceProp.concurrentKernels ? "true" : "false"));
         printfQuda("%d - ECCEnabled               %s\n", device, (deviceProp.ECCEnabled ? "true" : "false"));

From 681a2f01004509308d3aae86dc7b1b423dbb06e8 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 16 Mar 2021 05:19:08 +0000
Subject: [PATCH 026/198] omp: custom reduction compiles

---
 include/reducer.h           |  4 ++++
 include/tunable_nd.h        | 40 +++++++++++++++++++++++++++++++++++--
 include/tunable_reduction.h | 21 +++++++++++++++++++
 3 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/include/reducer.h b/include/reducer.h
index d59c33c1f9..ba6403b995 100644
--- a/include/reducer.h
+++ b/include/reducer.h
@@ -5,6 +5,10 @@ namespace quda {
   template <typename T> struct plus {
     static constexpr bool do_sum = true;
     __device__ __host__ T operator()(T a, T b) const { return a + b; }
+#ifdef QUDA_BACKEND_OMPTARGET
+    static T reduce_omp(T a, T b) { return a + b; }
+    static T init_omp() { return ::quda::zero<T>(); }
+#endif
   };
 
   template <typename T> struct maximum {
diff --git a/include/tunable_nd.h b/include/tunable_nd.h
index 9a28a8abee..16f4b007e5 100644
--- a/include/tunable_nd.h
+++ b/include/tunable_nd.h
@@ -67,7 +67,19 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip("OFFLOADING...");
+#ifdef QUDA_BACKEND_OMPTARGET
+      const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
+      const int ld = tp.block.x*tp.block.y*tp.grid.z;
+      const int tx = arg.threads.x;
+      Functor<Arg> f(const_cast<Arg &>(arg));
+#pragma omp target teams distribute parallel for simd num_teams(gd) thread_limit(ld) num_threads(ld)
+      for (int i = 0; i < tx; i++) {
+        f(i);
+      }
+#else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel1D), tp, stream, arg, param);
+#endif
     }
 
     template <template <typename> class Functor, typename Arg>
@@ -191,7 +203,6 @@ namespace quda {
 #else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel2D), tp, stream, arg, param);
 #endif
-      TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel2D), tp, stream, arg, param);
     }
 
     template <template <typename> class Functor, typename Arg>
@@ -356,8 +367,33 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg, const std::vector<constant_param_t> &param = dummy_param)
     {
-      ompwip();
+#if 1
+      ompwip("NOOOO...");
+#else
+/*
+InvalidArraySize: Array size must be at least 1:  [Src: /netbatch/donb884519_00/runDir/72/20210130_000000/llvm/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp:399 T->getArrayNumElements() >= 1 ]
+clang-12: error: llvm-spirv command failed with exit code 7 (use -v to see invocation)
+*/
+      ompwip("OFFLOADING...");
+#ifdef QUDA_BACKEND_OMPTARGET
+      const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
+      const int ld = tp.block.x*tp.block.y*tp.grid.z;
+      const int tx = arg.threads.x;
+      const int ty = arg.threads.y;
+      const int tz = arg.threads.z;
+      Functor<Arg> f(const_cast<Arg &>(arg));
+#pragma omp target teams distribute parallel for simd collapse(3) num_teams(gd) thread_limit(ld) num_threads(ld)
+      for (int i = 0; i < tx; i++) {
+        for (int j = 0; j < ty; j++) {
+          for (int k = 0; k < tz; k++) {
+            f(i, j, k);
+          }
+        }
+      }
+#else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel3D), tp, stream, arg, param);
+#endif
+#endif
     }
 
     template <template <typename> class Functor, typename Arg>
diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index 1d5d2bf8b2..4578b77a5b 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -78,7 +78,27 @@ namespace quda {
 #else
       for (unsigned int i = 0; i < param.size(); i++)
         qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
+      ompwip("OFFLOADING...");
+#ifdef QUDA_BACKEND_OMPTARGET
+      const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
+      const int ld = tp.block.x*tp.block.y*tp.grid.z;
+      const int tx = arg.threads.x;
+      const int ty = arg.threads.y;
+      using reduce_t = typename Transformer<Arg>::reduce_t;
+      Transformer<Arg> t(arg);
+      reduce_t value = arg.init();
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(Transformer<Arg>::init_omp())
+
+#pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld)
+      for (int j = 0; j < ty; j++) {
+        for (int i = 0; i < tx; i++) {
+          value = t(value, i, j);
+        }
+      }
+      arg.launch_error = QUDA_SUCCESS;
+#else
       arg.launch_error = launch<device::max_reduce_block_size<block_size_y>(), Transformer>(arg, tp, stream);
+#endif
 #endif
 
       if (!commAsyncReduction()) {
@@ -289,6 +309,7 @@ namespace quda {
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
                        const std::vector<constant_param_t> &param = dummy_param)
     {
+      ompwip("unimplemented multireduction");
 #ifdef JITIFY
       arg.launch_error = launch_jitify<Transformer, true, Arg, true>("MultiReduction", tp, stream, arg, param);
 #else

From 353b7ad6ad58fc1037d1cde915fd506686c63c66 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 16 Mar 2021 19:16:51 +0000
Subject: [PATCH 027/198] omp: actually doing reduction, more testing of memory
 functions

---
 include/float_vector.h             |  4 ++
 include/tunable_reduction.h        | 31 +++++++---
 lib/targets/omptarget/malloc.cpp   | 99 ++++++++++++++++++++++--------
 lib/targets/omptarget/quda_api.cpp | 16 ++---
 tests/blas_test.cpp                |  1 +
 5 files changed, 113 insertions(+), 38 deletions(-)

diff --git a/include/float_vector.h b/include/float_vector.h
index a6980c6679..a7d19df33b 100644
--- a/include/float_vector.h
+++ b/include/float_vector.h
@@ -146,4 +146,8 @@ namespace quda {
     for (int i = 0; i < n; i++) c[i] = a[i] + b[i];
     return c;
   }
+
+  template <int n> struct scalar<vector_type<double,n>> {
+    typedef double type;
+  };
 }
diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index 4578b77a5b..f35077a8bc 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -73,13 +73,10 @@ namespace quda {
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg,
                        const std::vector<constant_param_t> &param = dummy_param)
     {
-#ifdef JITIFY
-      arg.launch_error = launch_jitify<Transformer, true, Arg, true>("Reduction2D", tp, stream, arg, param);
-#else
+#ifdef QUDA_BACKEND_OMPTARGET
       for (unsigned int i = 0; i < param.size(); i++)
         qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
       ompwip("OFFLOADING...");
-#ifdef QUDA_BACKEND_OMPTARGET
       const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
       const int ld = tp.block.x*tp.block.y*tp.grid.z;
       const int tx = arg.threads.x;
@@ -89,16 +86,34 @@ namespace quda {
       reduce_t value = arg.init();
 #pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(Transformer<Arg>::init_omp())
 
-#pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld)
+#pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld) reduction(OMPReduce_:value)
       for (int j = 0; j < ty; j++) {
         for (int i = 0; i < tx; i++) {
           value = t(value, i, j);
         }
       }
-      arg.launch_error = QUDA_SUCCESS;
+      int input_size = vec_length<reduce_t>::value;
+      int output_size = result.size() * vec_length<T>::value;
+      if (output_size != input_size) errorQuda("Input %d and output %d length do not match", input_size, output_size);
+
+      // copy element by element to output vector
+      for (int i = 0; i < output_size; i++) {
+        std::cerr << "launch_device: reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[" << i << "]: " << reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i] << std::endl;
+        reinterpret_cast<typename scalar<T>::type*>(result.data())[i] =
+          reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i];
+      }
+
+      if (!activeTuning() && commGlobalReduction()) {
+        // FIXME - this will break when we have non-double reduction types, e.g., double-double on the host
+        comm_allreduce_array((double*)result.data(), result.size() * sizeof(T) / sizeof(double));
+      }
 #else
+#ifdef JITIFY
+      arg.launch_error = launch_jitify<Transformer, true, Arg, true>("Reduction2D", tp, stream, arg, param);
+#else
+      for (unsigned int i = 0; i < param.size(); i++)
+        qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
       arg.launch_error = launch<device::max_reduce_block_size<block_size_y>(), Transformer>(arg, tp, stream);
-#endif
 #endif
 
       if (!commAsyncReduction()) {
@@ -109,6 +124,7 @@ namespace quda {
           comm_allreduce_array((double*)result.data(), result.size() * sizeof(T) / sizeof(double));
         }
       }
+#endif
     }
 
     template <template <typename> class Transformer, typename Arg, typename T>
@@ -141,6 +157,7 @@ namespace quda {
 
       // copy element by element to output vector
       for (int i = 0; i < output_size; i++) {
+        std::cerr << "launch_host: reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[" << i << "]: " << reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i] << std::endl;
         reinterpret_cast<typename scalar<T>::type*>(result.data())[i] =
           reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i];
       }
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index b08eb9b4a5..0a3190d446 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -116,6 +116,10 @@ namespace quda
       if (total_pinned_bytes > max_total_pinned_bytes) { max_total_pinned_bytes = total_pinned_bytes; }
     }
     alloc[type][ptr] = a;
+{
+    const char *type_str[] = {"Device", "Device Pinned", "Host  ", "Pinned", "Mapped", "Managed"};
+    printfQuda("MALLOC: %s  %15p  %15lu  %s(), %s:%d\n", type_str[type], ptr, (unsigned long)a.base_size, a.func.c_str(), a.file.c_str(), a.line);
+}
   }
 
   static void track_free(const AllocType &type, void *ptr)
@@ -124,6 +128,11 @@ namespace quda
     total_bytes[type] -= size;
     if (type != DEVICE && type != DEVICE_PINNED) { total_host_bytes -= size; }
     if (type == PINNED || type == MAPPED) { total_pinned_bytes -= size; }
+{
+    auto a = alloc[type][ptr];
+    const char *type_str[] = {"Device", "Device Pinned", "Host  ", "Pinned", "Mapped", "Managed"};
+    printfQuda("FREE: %s  %15p  %15lu  %s(), %s:%d\n", type_str[type], ptr, (unsigned long)a.base_size, a.func.c_str(), a.file.c_str(), a.line);
+}
     alloc[type].erase(ptr);
   }
 
@@ -240,8 +249,11 @@ namespace quda
    * defined in malloc_quda.h.
    */
   void *device_pinned_malloc_(const char *func, const char *file, int line, size_t size)
-  {ompwip("return nullptr");return nullptr;/*
+  {
     if (!comm_peer2peer_present()) return device_malloc_(func, file, line, size);
+    ompwip("device_pinned_malloc_ with !comm_peer2peer_present() unimplemented");
+    return nullptr;
+/*
 
     MemAlloc a(func, file, line);
     void *ptr;
@@ -257,7 +269,8 @@ namespace quda
     cudaMemset(ptr, 0xff, size);
 #endif
     return ptr;
-  */}
+*/
+  }
 
   /**
    * Perform a standard malloc() with error-checking.  This function
@@ -289,21 +302,26 @@ namespace quda
    * shared with MPI via GPU Direct on some systems.
    */
   void *pinned_malloc_(const char *func, const char *file, int line, size_t size)
-  {ompwip("return nullptr");return nullptr;/*
+  {
     MemAlloc a(func, file, line);
     void *ptr = aligned_malloc(a, size);
-
+    ompwip("pinned_malloc_ uses aligned_malloc");
+    if(!ptr)
+      errorQuda("Failed to register pinned memory of size %zu (%s:%d in %s())\n", size, file, line, func);
+/*
     cudaError_t err = cudaHostRegister(ptr, a.base_size, cudaHostRegisterDefault);
     if (err != cudaSuccess) {
       errorQuda("Failed to register pinned memory of size %zu (%s:%d in %s())\n", size, file, line, func);
     }
+*/
     track_malloc(PINNED, a, ptr);
 #ifdef HOST_DEBUG
     memset(ptr, 0xff, a.base_size);
 #endif
     return ptr;
-  */}
+  }
 
+  static std::map<const void*,void*> omp_mapped_ptr;  // host -> device
   /**
    * Allocate page-locked ("pinned") host memory, and map it into the
    * GPU address space.  This function should only be called via the
@@ -323,9 +341,9 @@ namespace quda
       errorQuda("cudaHostAlloc failed of size %zu (%s:%d in %s())\n", size, file, line, func); }
     }
 #else
-    /* FIXME: this likely won't work. */
     void *ptr = aligned_malloc(a, size);
     ompwip([=](){printfQuda("mapped_malloc_ host: %p\n",ptr);});
+    print_trace();
     if(0<omp_get_num_devices()){
       int d = omp_get_default_device();
       void *dp = omp_target_alloc(a.base_size, d);  // FIXME keep it somewhere so we can free it
@@ -334,8 +352,11 @@ namespace quda
         errorQuda("%s:%d %s() Failed to allocate device memory of size %zu for mapped malloc\n", file, line, func, size);
       if(omp_target_associate_ptr(ptr, dp, a.base_size, 0, d))
         errorQuda("%s:%d %s() Failed to assocaite device memory to host pointer\n", file, line, func);
-    }else
+      omp_mapped_ptr[ptr] = dp;
+    }else{
       warningQuda("%s:%d %s() mapped malloc without a device", file, line, func);
+      omp_mapped_ptr[ptr] = ptr;
+    }
 #endif
     track_malloc(MAPPED, a, ptr);
 #ifdef HOST_DEBUG
@@ -350,7 +371,7 @@ namespace quda
    * defined in malloc_quda.h
    */
   void *managed_malloc_(const char *func, const char *file, int line, size_t size)
-  {ompwip("return nullptr");return nullptr;/*
+  {ompwip("managed_malloc_ unimplemented return nullptr");return nullptr;/*
     MemAlloc a(func, file, line);
     void *ptr;
 
@@ -406,6 +427,7 @@ namespace quda
    */
   void device_pinned_free_(const char *func, const char *file, int line, void *ptr)
   {
+    ompwip("device_pinned_free_ untested code path"); print_trace();
     if (!comm_peer2peer_present()) {
       device_free_(func, file, line, ptr);
       return;
@@ -432,7 +454,7 @@ namespace quda
    * malloc_quda.h
    */
   void managed_free_(const char *func, const char *file, int line, void *ptr)
-  {ompwip("doing nothing");/*
+  {ompwip("managed_free_ unimplemented doing nothing");/*
     if (!ptr) { errorQuda("Attempt to free NULL managed pointer (%s:%d in %s())\n", file, line, func); }
     if (!alloc[MANAGED].count(ptr)) {
       errorQuda("Attempt to free invalid managed pointer (%s:%d in %s())\n", file, line, func);
@@ -454,7 +476,7 @@ namespace quda
       track_free(HOST, ptr);
       free(ptr);
     } else if (alloc[PINNED].count(ptr)) {
-      ompwip("alloc[PINNED].count(ptr)");
+      ompwip("free PINNED ptr unimplemented");
 /*
       cudaError_t err = cudaHostUnregister(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to unregister pinned memory (%s:%d in %s())\n", file, line, func); }
@@ -463,12 +485,21 @@ namespace quda
 */
     } else if (alloc[MAPPED].count(ptr)) {
 #ifdef HOST_ALLOC
-      ompwip("HOST_ALLOC alloc[MAPPED].count(ptr)");/*
+      ompwip("HOST_ALLOC alloc[MAPPED].count(ptr) untested code path"); print_trace();
+      free(ptr);
+/*
       cudaError_t err = cudaFreeHost(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to free host memory (%s:%d in %s())\n", file, line, func); }
 */
 #else
-      ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr)");/*
+      ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr)");
+      void *dp = omp_mapped_ptr[ptr];
+      if(dp!=ptr){
+        omp_target_disassociate_ptr(ptr, omp_get_default_device());
+        omp_target_free(dp, omp_get_default_device());
+      }
+      free(ptr);
+/*
       cudaError_t err = cudaHostUnregister(ptr);
       if (err != cudaSuccess) {
         errorQuda("Failed to unregister host-mapped memory (%s:%d in %s())\n", file, line, func);
@@ -476,7 +507,7 @@ namespace quda
       free(ptr);
 */
 #endif
-      // track_free(MAPPED, ptr);
+      track_free(MAPPED, ptr);
     } else {
       printfQuda("ERROR: Attempt to free invalid host pointer (%s:%d in %s())\n", file, line, func);
       print_trace();
@@ -510,8 +541,19 @@ namespace quda
   }
 
   QudaFieldLocation get_pointer_location(const void *ptr)
-  {ompwip("return QUDA_INVALID_FIELD_LOCATION;");return QUDA_INVALID_FIELD_LOCATION;/*
-
+  {
+    ompwip("get_pointer_location");
+    QudaFieldLocation loc = QUDA_INVALID_FIELD_LOCATION;
+    if(alloc[DEVICE].count(const_cast<void*>(ptr)) || alloc[DEVICE_PINNED].count(const_cast<void*>(ptr))){
+      ompwip("get_pointer_location returns QUDA_CUDA_FIELD_LOCATION");
+      return QUDA_CUDA_FIELD_LOCATION;
+    }else if(alloc[HOST].count(const_cast<void*>(ptr)) || alloc[PINNED].count(const_cast<void*>(ptr)) || alloc[MAPPED].count(const_cast<void*>(ptr))){
+      ompwip("get_pointer_location returns QUDA_CPU_FIELD_LOCATION");
+      return QUDA_CPU_FIELD_LOCATION;
+    }
+    ompwip("WARNING: get_pointer_location returns QUDA_INVALID_FIELD_LOCATION");
+    return QUDA_INVALID_FIELD_LOCATION;
+/*
     CUpointer_attribute attribute[] = {CU_POINTER_ATTRIBUTE_MEMORY_TYPE};
     CUmemorytype mem_type;
     void *data[] = {&mem_type};
@@ -531,11 +573,14 @@ namespace quda
     case CU_MEMORYTYPE_HOST: return QUDA_CPU_FIELD_LOCATION;
     default: errorQuda("Unknown memory type %d", mem_type); return QUDA_INVALID_FIELD_LOCATION;
     }
-  */}
+*/
+  }
 
   void *get_mapped_device_pointer_(const char *func, const char *file, int line, const void *host)
   {
-    ompwip("return nullptr");return nullptr;
+    ompwip("get_mapped_device_pointer_");
+    print_trace();
+    return omp_mapped_ptr[host];
 /*
     void *device;
     auto error = cudaHostGetDevicePointer(&device, const_cast<void *>(host), 0);
@@ -549,7 +594,7 @@ namespace quda
 
   void register_pinned_(const char *func, const char *file, int line, void *ptr, size_t bytes)
   {
-    ompwip();
+    ompwip("register_pinned_ unimplemented");
 /*
     auto error = cudaHostRegister(ptr, bytes, cudaHostRegisterDefault);
     if (error != cudaSuccess) {
@@ -561,7 +606,7 @@ namespace quda
 
   void unregister_pinned_(const char *func, const char *file, int line, void *ptr)
   {
-    ompwip();
+    ompwip("unregister_pinned_ unimplemented");
 /*
     auto error = cudaHostUnregister(ptr);
     if (error != cudaSuccess) {
@@ -629,7 +674,8 @@ namespace quda
     }
 
     void *pinned_malloc_(const char *func, const char *file, int line, size_t nbytes)
-    {ompwip();
+    {
+      ompwip("pool::pinned_malloc_");
       void *ptr = nullptr;
       if (pinned_memory_pool) {
         std::multimap<size_t, void *>::iterator it;
@@ -658,7 +704,8 @@ namespace quda
     }
 
     void pinned_free_(const char *func, const char *file, int line, void *ptr)
-    {ompwip();
+    {
+      ompwip("pool::pinnd_free_");
       if (pinned_memory_pool) {
         if (!pinnedSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
         pinnedCache.insert(std::make_pair(pinnedSize[ptr], ptr));
@@ -670,6 +717,7 @@ namespace quda
 
     void *device_malloc_(const char *func, const char *file, int line, size_t nbytes)
     {
+      ompwip("pool::device_malloc_");
       void *ptr = nullptr;
       if (device_memory_pool) {
         std::multimap<size_t, void *>::iterator it;
@@ -699,7 +747,8 @@ namespace quda
     }
 
     void device_free_(const char *func, const char *file, int line, void *ptr)
-    {ompwip();
+    {
+      ompwip("pool::device_free_");
       if (device_memory_pool) {
         if (!deviceSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
         deviceCache.insert(std::make_pair(deviceSize[ptr], ptr));
@@ -710,7 +759,8 @@ namespace quda
     }
 
     void flush_pinned()
-    {ompwip();
+    {
+      ompwip("pool::flush_pinned");
       if (pinned_memory_pool) {
         std::multimap<size_t, void *>::iterator it;
         for (it = pinnedCache.begin(); it != pinnedCache.end(); it++) {
@@ -722,7 +772,8 @@ namespace quda
     }
 
     void flush_device()
-    {ompwip();
+    {
+      ompwip("pool:flush_device");
       if (device_memory_pool) {
         std::multimap<size_t, void *>::iterator it;
         for (it = deviceCache.begin(); it != deviceCache.end(); it++) {
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index a63d2d53c0..0167e8d615 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -17,12 +17,12 @@ static char FIXME[]="OMP FIXME";
 #define cudaGetErrorString(a) "OMP FIXME"
 #define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
 
-#define cudaMemcpy(a,b,c,d) ompwip([&](){printfQuda("memcpy %p <- %p\n",a,b);ompwipMemcpy(a,(void*)b,c,d);})
-#define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){printfQuda("memcpy %p <- %p\n",a,b);ompwipMemcpy(a,(void*)b,c,d);})
+#define cudaMemcpy(a,b,c,d) ompwip([&](){printfQuda("memcpy %p <- %p %ld\n",a,b,c);ompwipMemcpy(a,(void*)b,c,d);})
+#define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){printfQuda("memcpy %p <- %p %ld\n",a,b,c);ompwipMemcpy(a,(void*)b,c,d);})
 #define cudaMemcpy2D(a,b,c,d,e,f,g) ompwip("unimplemented")
 #define cudaMemcpy2DAsync(a,b,c,d,e,f,g,h) ompwip("unimplemented")
-#define cudaMemset(a,b,c) ompwip([&](){printfQuda("memset %p\n",a);ompwipMemset(a,b,c);})
-#define cudaMemsetAsync(a,b,c,d) ompwip([&](){printfQuda("memset %p\n",a);ompwipMemset(a,b,c);})
+#define cudaMemset(a,b,c) ompwip([&](){printfQuda("memset %p %d %ld\n",a,b,c);ompwipMemset(a,b,c);})
+#define cudaMemsetAsync(a,b,c,d) ompwip([&](){printfQuda("memset %p %d %ld\n",a,b,c);ompwipMemset(a,b,c);})
 #define cudaMemset2D(a,b,c,d,e) ompwip("unimplemented")
 #define cudaMemset2DAsync(a,b,c,d,e,f) ompwip("unimplemented")
 
@@ -40,17 +40,19 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
   case cudaMemcpyHostToHost: memcpy(d,s,c); break;
   case cudaMemcpyHostToDevice:
     if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
-    else warningQuda("cudaMemcpyHostToDevice without a device");
+    else{ warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy"); memcpy(d,s,c); }
     break;
   case cudaMemcpyDeviceToHost:
     if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
-    else warningQuda("cudaMemcpyDeviceToHost without a device");
+    else{ warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy"); memcpy(d,s,c); }
     break;
   case cudaMemcpyDeviceToDevice:
     warningQuda("unimplemented for cudaMemcpyDeviceToDevice");
     break;
   case cudaMemcpyDefault:
-    warningQuda("unimplemented for cudaMemcpyDefault");
+    warningQuda("cudaMemcpyDefault calling host to device");
+    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+    else{ warningQuda("cudaMemcpyDefault without a device, calling memcpy"); memcpy(d,s,c); }
     break;
   default: errorQuda("Unsupported cudaMemcpyType %d", k);
   }
diff --git a/tests/blas_test.cpp b/tests/blas_test.cpp
index dbdaed64f5..17ca4a6075 100644
--- a/tests/blas_test.cpp
+++ b/tests/blas_test.cpp
@@ -670,6 +670,7 @@ double test(Kernel kernel)
     break;
 
   case Kernel::norm2:
+    ompwip("*xD = *xH");
     *xD = *xH;
     error = fabs(blas::norm2(*xD) - blas::norm2(*xH)) / blas::norm2(*xH);
     break;

From c535838ae065856d29ba32fca9caaab4acd97832 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 31 Mar 2021 15:12:29 +0000
Subject: [PATCH 028/198] float_vector: add scalar and vec_length

---
 include/float_vector.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/include/float_vector.h b/include/float_vector.h
index a7d19df33b..ef179b67e1 100644
--- a/include/float_vector.h
+++ b/include/float_vector.h
@@ -147,7 +147,10 @@ namespace quda {
     return c;
   }
 
-  template <int n> struct scalar<vector_type<double,n>> {
-    typedef double type;
+  template <typename s, int n> struct scalar<vector_type<s,n>> {
+    typedef s type;
+  };
+  template <typename s, int n> struct vec_length<vector_type<s,n>> {
+    static const int value = n;
   };
 }

From a62543c8c8316e76c881ade34257e8849c973139 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 31 Mar 2021 15:13:35 +0000
Subject: [PATCH 029/198] omp: more debug output

---
 include/blas_helper.cuh            |  1 +
 include/kernels/reduce_core.cuh    |  5 ++
 lib/cuda_color_spinor_field.cpp    | 11 ++++
 lib/targets/omptarget/quda_api.cpp | 82 +++++++++++++++++++++++++++---
 4 files changed, 91 insertions(+), 8 deletions(-)

diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index 486df3e039..8d72c1e25e 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -187,6 +187,7 @@ namespace quda
         constexpr int len = 2 * n; // real-valued length
         float nrm = isFixed<store_t>::value ? SN::load_norm(x, parity) : 0.0;
 
+        // if(x>=0 && x<=4) printf("Spinor::load x %d parity %d spinor %p spinor[0] %g\n", x, parity, spinor, spinor[0]);
         vector_type<real, len> v_;
 
         constexpr int M = len / N;
diff --git a/include/kernels/reduce_core.cuh b/include/kernels/reduce_core.cuh
index bebb8bf60b..661aafcd3b 100644
--- a/include/kernels/reduce_core.cuh
+++ b/include/kernels/reduce_core.cuh
@@ -68,9 +68,14 @@ namespace quda
         if (arg.r.read.W) arg.W.load(w, i, parity);
         if (arg.r.read.V) arg.V.load(v, i, parity);
 
+        // if(0<=i&&i<4) printf("i %d  arg.r.read.X %d  arg.r.read.Y %d  arg.r.read.Z %d\n", i, arg.r.read.X, arg.r.read.Y, arg.r.read.Z);
+        // auto s = [](auto x){return reinterpret_cast<double*>(&x)[0];};  // debug purpose, assuming double
         arg.r.pre();
+        // if(0<=i&&i<4) printf("pre  %d sum %g x %g y %g z %g w %g z %g\n", i, s(sum), s(x), s(y), s(z), s(w), s(v));
         arg.r(sum, x, y, z, w, v);
+        // if(0<=i&&i<4) printf("r    %d sum %g x %g y %g z %g w %g z %g\n", i, s(sum), s(x), s(y), s(z), s(w), s(v));
         arg.r.post(sum);
+        // if(0<=i&&i<4) printf("post %d sum %g x %g y %g z %g w %g z %g\n", i, s(sum), s(x), s(y), s(z), s(w), s(v));
 
         if (arg.r.write.X) arg.X.save(x, i, parity);
         if (arg.r.write.Y) arg.Y.save(y, i, parity);
diff --git a/lib/cuda_color_spinor_field.cpp b/lib/cuda_color_spinor_field.cpp
index 93b54ad239..9214e892a3 100644
--- a/lib/cuda_color_spinor_field.cpp
+++ b/lib/cuda_color_spinor_field.cpp
@@ -400,8 +400,10 @@ namespace quda {
   }
 
   void cudaColorSpinorField::loadSpinorField(const ColorSpinorField &src) {
+    // ompwip("DBG: loadSpinorField(const ColorSpinorField &src)");
 
     if ( reorder_location() == QUDA_CPU_FIELD_LOCATION && typeid(src) == typeid(cpuColorSpinorField)) {
+      // ompwip("DBG: reorder_location() == QUDA_CPU_FIELD_LOCATION && typeid(src) == typeid(cpuColorSpinorField)");
       void *buffer = pool_pinned_malloc(bytes + norm_bytes);
       memset(buffer, 0, bytes+norm_bytes); // FIXME (temporary?) bug fix for padding
 
@@ -412,22 +414,28 @@ namespace quda {
 
       pool_pinned_free(buffer);
     } else if (typeid(src) == typeid(cudaColorSpinorField)) {
+      // ompwip("DBG: typeid(src) == typeid(cudaColorSpinorField)");
       copyGenericColorSpinor(*this, src, QUDA_CUDA_FIELD_LOCATION);
     } else {
 
       if (src.FieldOrder() == QUDA_PADDED_SPACE_SPIN_COLOR_FIELD_ORDER) {
+        // ompwip("DBG: src.FieldOrder() == QUDA_PADDED_SPACE_SPIN_COLOR_FIELD_ORDER");
         // special case where we use mapped memory to read/write directly from application's array
         void *src_d = get_mapped_device_pointer(src.V());
         copyGenericColorSpinor(*this, src, QUDA_CUDA_FIELD_LOCATION, v, src_d);
       } else {
         void *Src=nullptr, *srcNorm=nullptr, *buffer=nullptr;
         if (!zeroCopy) {
+          // ompwip("DBG: !zeroCopy");
           buffer = pool_device_malloc(src.Bytes()+src.NormBytes());
           Src = buffer;
           srcNorm = static_cast<char*>(Src) + src.Bytes();
+          // ompwip("DBG: !zeroCopy qudaMemcpy(Src, src.V())");
           qudaMemcpy(Src, src.V(), src.Bytes(), qudaMemcpyDefault);
+          // ompwip("DBG: !zeroCopy qudaMemcpy(srcNorm, src.Norm())");
           qudaMemcpy(srcNorm, src.Norm(), src.NormBytes(), qudaMemcpyDefault);
         } else {
+          // ompwip("DBG: else");
           buffer = pool_pinned_malloc(src.Bytes()+src.NormBytes());
           memcpy(buffer, src.V(), src.Bytes());
           memcpy(static_cast<char*>(buffer)+src.Bytes(), src.Norm(), src.NormBytes());
@@ -436,13 +444,16 @@ namespace quda {
         }
 
         qudaMemsetAsync(v, 0, bytes, device::get_default_stream()); // FIXME (temporary?) bug fix for padding
+        // ompwip("DBG: begin copyGenericColorSpinor(*this, src)");
         copyGenericColorSpinor(*this, src, QUDA_CUDA_FIELD_LOCATION, 0, Src, 0, srcNorm);
+        // ompwip("DBG: end copyGenericColorSpinor(*this, src)");
 
         if (zeroCopy) pool_pinned_free(buffer);
         else pool_device_free(buffer);
       }
     }
 
+    // ompwip("DBG: sync loadSpinorField(const ColorSpinorField &src)");
     qudaDeviceSynchronize(); // include sync here for accurate host-device profiling
   }
 
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 0167e8d615..147a4bda39 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -33,26 +33,92 @@ ompwipMemset(void *p, unsigned char b, std::size_t s)
   for(std::size_t i=0;i<s;++i) *(unsigned char *)p = b;
 }
 
+#if 1
+#define printmem(d,m,h)
+#else
+static inline void
+printmem(void *d, std::size_t m, int host)
+{
+  constexpr size_t loc[] = {0, 1, 2, 3, 5, 7, 11, 13};
+  constexpr size_t nloc = sizeof(loc)/sizeof(loc[0]);
+  unsigned char buf[nloc*8];
+  unsigned char *p = (unsigned char *)d;
+
+  if(host){
+    for(size_t i=0; i<nloc; ++i){
+      if(i*8>=m) break;
+      for(size_t j=0; j<8; ++j)
+        buf[j+i*8] = p[j+loc[i]*8];
+    }
+    printf("host   mem:");
+  }else{
+    #pragma omp target map(tofrom:buf[:8*nloc]) map(to:loc[:nloc]) is_device_ptr(p)
+    {
+    for(size_t i=0; i<nloc; ++i){
+      if(i*8>=m) break;
+      for(size_t j=0; j<8; ++j)
+        buf[j+i*8] = p[j+loc[i]*8];
+    }
+    }
+    printf("target mem:");
+  }
+  for(size_t i=0; i<nloc; ++i){
+    printf(" ");
+    for(size_t j=0; j<8; ++j)
+      printf("%02X", buf[j+i*8]);
+  }
+  printf("\n");
+  fflush(0);
+}
+#endif
+
 static inline void
 ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
 {
   switch(k){
-  case cudaMemcpyHostToHost: memcpy(d,s,c); break;
+  case cudaMemcpyHostToHost:
+    printmem(s,c,1);
+    memcpy(d,s,c);
+    printmem(d,c,1);
+    break;
   case cudaMemcpyHostToDevice:
-    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
-    else{ warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy"); memcpy(d,s,c); }
+    printmem(s,c,1);
+    if(0<omp_get_num_devices()){
+      omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+      printmem(d,c,0);
+    }else{
+      warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy");
+      memcpy(d,s,c);
+      printmem(d,c,1);
+    }
     break;
   case cudaMemcpyDeviceToHost:
-    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
-    else{ warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy"); memcpy(d,s,c); }
+    if(0<omp_get_num_devices()){
+      printmem(s,c,0);
+      omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
+    }else{
+      warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy");
+      printmem(s,c,1);
+      memcpy(d,s,c);
+    }
+    printmem(d,c,1);
     break;
   case cudaMemcpyDeviceToDevice:
-    warningQuda("unimplemented for cudaMemcpyDeviceToDevice");
+    printmem(s,c,0);
+    omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
+    printmem(d,c,0);
     break;
   case cudaMemcpyDefault:
     warningQuda("cudaMemcpyDefault calling host to device");
-    if(0<omp_get_num_devices()) omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
-    else{ warningQuda("cudaMemcpyDefault without a device, calling memcpy"); memcpy(d,s,c); }
+    printmem(s,c,1);
+    if(0<omp_get_num_devices()){
+      omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+      printmem(d,c,0);
+    }else{
+      warningQuda("cudaMemcpyDefault without a device, calling memcpy");
+      memcpy(d,s,c);
+      printmem(d,c,1);
+    }
     break;
   default: errorQuda("Unsupported cudaMemcpyType %d", k);
   }

From d2416702fdf08da89cccdee7be0a5af7fb698468 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 31 Mar 2021 15:14:05 +0000
Subject: [PATCH 030/198] omp: change some errorQuda to warningQuda

---
 lib/comm_common.cpp              | 1 -
 lib/targets/omptarget/device.cpp | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/comm_common.cpp b/lib/comm_common.cpp
index 0adb05dde5..8235f166be 100644
--- a/lib/comm_common.cpp
+++ b/lib/comm_common.cpp
@@ -627,7 +627,6 @@ void comm_init_common(int ndim, const int *dims, QudaCommsMap rank_from_coords,
 
   int device_count = quda::device::get_device_count();
   if (device_count == 0) { warningQuda("No devices found"); }
-  if (device_count == 0) { errorQuda("No devices found"); }
   if (gpuid >= device_count) {
     char *enable_mps_env = getenv("QUDA_ENABLE_MPS");
     if (enable_mps_env && strcmp(enable_mps_env, "1") == 0) {
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 8c9879ee15..a654ae0215 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -166,7 +166,7 @@ namespace quda
       }
 */
 
-      if (!deviceProp.unifiedAddressing) errorQuda("Device %d does not support unified addressing", dev);
+      if (!deviceProp.unifiedAddressing) warningQuda("Device %d does not support unified addressing", dev);
 
       if (getVerbosity() >= QUDA_SUMMARIZE) {
         printfQuda("Using device %d: %s\n", dev, deviceProp.name);

From 1b2efc16e3e87575e0b75d6478b8cf5b638fac2e Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 31 Mar 2021 15:15:09 +0000
Subject: [PATCH 031/198] omp: correctely memcpy arg for kernels, work around
 reduction bug

---
 include/tunable_nd.h        | 20 ++++++++++++++++----
 include/tunable_reduction.h | 27 ++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/include/tunable_nd.h b/include/tunable_nd.h
index 16f4b007e5..124e75b4dc 100644
--- a/include/tunable_nd.h
+++ b/include/tunable_nd.h
@@ -72,11 +72,15 @@ namespace quda {
       const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
       const int ld = tp.block.x*tp.block.y*tp.grid.z;
       const int tx = arg.threads.x;
-      Functor<Arg> f(const_cast<Arg &>(arg));
+      printf("launch parameter: gd %d ld %d tx %d\n", gd, ld, tx);
+      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+      Functor<Arg> f(*dparg);
 #pragma omp target teams distribute parallel for simd num_teams(gd) thread_limit(ld) num_threads(ld)
       for (int i = 0; i < tx; i++) {
         f(i);
       }
+      omp_target_free(dparg, omp_get_default_device());
 #else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel1D), tp, stream, arg, param);
 #endif
@@ -193,13 +197,17 @@ namespace quda {
       const int ld = tp.block.x*tp.block.y*tp.grid.z;
       const int tx = arg.threads.x;
       const int ty = arg.threads.y;
-      Functor<Arg> f(const_cast<Arg &>(arg));
-#pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld)
+      printf("launch parameter: gd %d ld %d tx %d ty %d\n", gd, ld, tx, ty);
+      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+      Functor<Arg> f(*dparg);
+#pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld) map(to:f)
       for (int i = 0; i < tx; i++) {
         for (int j = 0; j < ty; j++) {
           f(i, j);
         }
       }
+      omp_target_free(dparg, omp_get_default_device());
 #else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel2D), tp, stream, arg, param);
 #endif
@@ -381,7 +389,10 @@ clang-12: error: llvm-spirv command failed with exit code 7 (use -v to see invoc
       const int tx = arg.threads.x;
       const int ty = arg.threads.y;
       const int tz = arg.threads.z;
-      Functor<Arg> f(const_cast<Arg &>(arg));
+      printf("launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+      Functor<Arg> f(*dparg);
 #pragma omp target teams distribute parallel for simd collapse(3) num_teams(gd) thread_limit(ld) num_threads(ld)
       for (int i = 0; i < tx; i++) {
         for (int j = 0; j < ty; j++) {
@@ -390,6 +401,7 @@ clang-12: error: llvm-spirv command failed with exit code 7 (use -v to see invoc
           }
         }
       }
+      omp_target_free(dparg, omp_get_default_device());
 #else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel3D), tp, stream, arg, param);
 #endif
diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index f35077a8bc..4ce64cfe26 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -81,17 +81,42 @@ namespace quda {
       const int ld = tp.block.x*tp.block.y*tp.grid.z;
       const int tx = arg.threads.x;
       const int ty = arg.threads.y;
+      printf("launch parameter: gd %d ld %d tx %d ty %d\n", gd, ld, tx, ty);
       using reduce_t = typename Transformer<Arg>::reduce_t;
-      Transformer<Arg> t(arg);
       reduce_t value = arg.init();
+
 #pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(Transformer<Arg>::init_omp())
 
+      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+      // printf("dparg %p\n", dparg);
+      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+      Transformer<Arg> t(*dparg);
+#if 0
+// gives wrong results?!
 #pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld) reduction(OMPReduce_:value)
       for (int j = 0; j < ty; j++) {
         for (int i = 0; i < tx; i++) {
           value = t(value, i, j);
         }
       }
+#endif
+#if 0
+// also wrong results
+      for (int j = 0; j < ty; j++) {
+#pragma omp target teams distribute parallel for reduction(OMPReduce_:value)
+        for (int i = 0; i < tx; i++) {
+          value = t(value, i, j);
+        }
+      }
+#endif
+// NOT really a parallel reduction
+#pragma omp target teams distribute parallel for simd num_teams(gd) thread_limit(ld) num_threads(ld) reduction(OMPReduce_:value)
+      for (int j = 0; j < ty; j++) {
+        for (int i = 0; i < tx; i++) {
+          value = t(value, i, j);
+        }
+      }
+      omp_target_free(dparg, omp_get_default_device());
       int input_size = vec_length<reduce_t>::value;
       int output_size = result.size() * vec_length<T>::value;
       if (output_size != input_size) errorQuda("Input %d and output %d length do not match", input_size, output_size);

From 11a67474ca8362d412638eb209f8e0e3f48f1048 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 7 Apr 2021 22:34:08 +0000
Subject: [PATCH 032/198] omp: fix reduction by properly initializing omp_priv

---
 include/tunable_reduction.h | 21 +--------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index 4ce64cfe26..b0efed32f8 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -85,37 +85,18 @@ namespace quda {
       using reduce_t = typename Transformer<Arg>::reduce_t;
       reduce_t value = arg.init();
 
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(Transformer<Arg>::init_omp())
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
 
       Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
       // printf("dparg %p\n", dparg);
       omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
       Transformer<Arg> t(*dparg);
-#if 0
-// gives wrong results?!
 #pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld) reduction(OMPReduce_:value)
       for (int j = 0; j < ty; j++) {
         for (int i = 0; i < tx; i++) {
           value = t(value, i, j);
         }
       }
-#endif
-#if 0
-// also wrong results
-      for (int j = 0; j < ty; j++) {
-#pragma omp target teams distribute parallel for reduction(OMPReduce_:value)
-        for (int i = 0; i < tx; i++) {
-          value = t(value, i, j);
-        }
-      }
-#endif
-// NOT really a parallel reduction
-#pragma omp target teams distribute parallel for simd num_teams(gd) thread_limit(ld) num_threads(ld) reduction(OMPReduce_:value)
-      for (int j = 0; j < ty; j++) {
-        for (int i = 0; i < tx; i++) {
-          value = t(value, i, j);
-        }
-      }
       omp_target_free(dparg, omp_get_default_device());
       int input_size = vec_length<reduce_t>::value;
       int output_size = result.size() * vec_length<T>::value;

From b76a1edcfe1533b0e66949ef0556a6e889469989 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 7 Apr 2021 22:36:48 +0000
Subject: [PATCH 033/198] omp: allow program to run when offloading is disabled

---
 lib/targets/omptarget/device.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index a654ae0215..2489368cef 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -127,6 +127,8 @@ namespace quda
       if (NVML_SUCCESS != result) errorQuda("NVML Shutdown failed with error %d", result);
 #endif
 
+      if(get_device_count() == 0)  /* TODO: get CPU device properties */
+        cudaGetDeviceProperties(&deviceProp, -1);
       for (int i = 0; i < get_device_count(); i++) {
         CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, i));
         if (getVerbosity() >= QUDA_SUMMARIZE) {

From 7ba3c0cf50c67fa00d702dbf2f0e06985d29c81a Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 21 Apr 2021 20:49:09 +0000
Subject: [PATCH 034/198] omp: fix last merge

---
 include/blas_helper.cuh                       |   1 +
 include/color_spinor_field_order.h            |   2 +-
 include/kernels/coarse_op_kernel.cuh          |   1 +
 .../omptarget/block_reduction_kernel.h        |  16 ++-
 .../targets/omptarget/constant_kernel_arg.h   |  47 ++++++++
 include/targets/omptarget/kernel.h            |  28 ++++-
 include/targets/omptarget/reduction_kernel.h  |  29 ++++-
 include/targets/omptarget/target_device.h     |  65 ++++++----
 include/tunable_reduction.h                   |   2 -
 lib/comm_common.cpp                           |  13 --
 lib/targets/omptarget/blas_lapack_cublas.cpp  |   3 +-
 lib/targets/omptarget/comm_target.cpp         |   8 +-
 lib/targets/omptarget/device.cpp              |  50 ++++----
 lib/targets/omptarget/malloc.cpp              | 111 ++++++++++++++----
 lib/targets/omptarget/quda_api.cpp            |  61 ++++++----
 15 files changed, 315 insertions(+), 122 deletions(-)
 create mode 100644 include/targets/omptarget/constant_kernel_arg.h

diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index 95ef4392e7..da15309421 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -4,6 +4,7 @@
 #include <reduce_helper.h>
 #include <load_store.h>
 #include <convert.h>
+#include <math_helper.cuh>
 
 //#define QUAD_SUM
 #ifdef QUAD_SUM
diff --git a/include/color_spinor_field_order.h b/include/color_spinor_field_order.h
index e6610e65a1..4b8ae61ede 100644
--- a/include/color_spinor_field_order.h
+++ b/include/color_spinor_field_order.h
@@ -750,7 +750,7 @@ namespace quda {
       {
         return fieldorder_wrapper<Float, ghostFloat, block_float_ghost, norm_t>(ghost[2*dim+dir],
                                                                                 ghostAccessor.index(dim,parity,x_cb,s,c,n),
-                                                                                block_float_ghost ? fdividef(1.f, max) : ghost_scale,
+                                                                                block_float_ghost ? fdividef((Float)1., max) : ghost_scale,
                                                                                 block_float_ghost ? max : ghost_scale_inv,
                                                                                 ghost_norm[2 * dim + dir],
                                                                                 parity*ghostAccessor.faceVolumeCB[dim] + x_cb,
diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index af29ea8013..02bf8c349d 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -787,6 +787,7 @@ namespace quda {
     template <typename VUV, typename Pack, typename Arg>
     inline __device__ void operator()(VUV &vuv, bool isDiagonal, int coarse_x_cb, int coarse_parity, int i0, int j0, int parity, const Pack &pack, const Arg &arg)
     {
+      QUDA_RT_CONSTS;
       using Float = typename Arg::Float;
       using TileType = typename Arg::vuvTileType;
       const int dim_index = arg.dim_index % arg.Y_atomic.geometry;
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index da3452e3bf..51db448ad2 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -42,7 +42,7 @@ namespace quda {
      thread dimension is a trivial vectorizable dimension.
   */
   template <unsigned int block_size, template <int, typename> class Transformer, typename Arg>
-  __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0) __global__ void BlockKernel2D(Arg arg)
+  __forceinline__ __device__ void BlockKernel2D_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
     const dim3 block_idx(virtual_block_idx(arg), blockIdx.y, 0);
@@ -54,4 +54,18 @@ namespace quda {
     t(block_idx, thread_idx);
   }
 
+  template <unsigned int block_size, template <int, typename> class Transformer, typename Arg>
+    __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0)
+    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
+  {
+    BlockKernel2D_impl<block_size, Transformer, Arg>(arg);
+  }
+
+  template <unsigned int block_size, template <int, typename> class Transformer, typename Arg>
+    __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0)
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D()
+  {
+    BlockKernel2D_impl<block_size, Transformer, Arg>(device::get_arg<Arg>());
+  }
+
 }
diff --git a/include/targets/omptarget/constant_kernel_arg.h b/include/targets/omptarget/constant_kernel_arg.h
new file mode 100644
index 0000000000..0bdd389bfa
--- /dev/null
+++ b/include/targets/omptarget/constant_kernel_arg.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include <util_quda.h>
+#include <quda_api.h>
+#include <target_device.h>
+
+/**
+   @file constant_kernel_arg.h
+
+   This file should be included in the kernel files for which we wish
+   to utilize __constant__ memory for the kernel parameter struct.
+   This needs to be included before the definition of the kernel,
+   e.g., kernel.h in order for the compiler to do the kernel
+   instantiation correctly.
+ */
+
+namespace quda {
+
+  namespace device {
+    
+    /**
+       @brief The __constant__ buffer used for kernel parameters
+    */
+#if defined(__CUDACC_RDC__) && !defined(QUDA_CONSTANT_DEFINE) // what is the equivalent macro for nvc++?
+    // rdc is enabled when NVSHMEM is enabled, so we need to make the
+    // buffer as extern and define it in one place only
+    extern __constant__ char buffer[max_constant_size()];
+#else
+    __constant__ char buffer[max_constant_size()];
+#endif
+
+    /**
+       @brief Helper function that returns kernel argument from
+       __constant__ memory.
+     */
+    template <typename Arg>
+      constexpr std::enable_if_t<!use_kernel_arg<Arg>(), Arg&> get_arg() { return reinterpret_cast<Arg&>(buffer); }
+
+    /**
+       @brief Helper function that returns a pointer to the
+       __constant__ memory buffer.
+     */
+    template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), void*> get_constant_buffer() { return qudaGetSymbolAddress(buffer); }
+
+  }
+
+}
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 42c966e8dc..c686f75658 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -1,9 +1,11 @@
 #pragma once
 
+#include <kernel_helper.h>
+
 namespace quda {
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ void Kernel1D(Arg arg)
+  __forceinline__ __device__ void Kernel1D_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
     Functor<Arg> f(arg);
@@ -17,7 +19,14 @@ namespace quda {
   }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ void Kernel2D(Arg arg)
+    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg) { Kernel1D_impl<Functor, Arg, grid_stride>(arg); }
+
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D() { Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>()); }
+
+
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  __forceinline__ __device__ void Kernel2D_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
     Functor<Arg> f(arg);
@@ -33,7 +42,14 @@ namespace quda {
   }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ void Kernel3D(Arg arg)
+    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg) { Kernel2D_impl<Functor, Arg, grid_stride>(arg); }
+
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D() { Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>()); }
+
+
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  __forceinline__ __device__ void Kernel3D_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
     Functor<Arg> f(arg);
@@ -50,6 +66,12 @@ namespace quda {
     }
   }
 
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg) { Kernel3D_impl<Functor, Arg, grid_stride>(arg); }
+
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D() { Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>()); }
+
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 15d367b86f..eeead67508 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -5,7 +5,7 @@
 namespace quda {
 
   template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
-  __global__ void Reduction2D(Arg arg)
+  __forceinline__ __device__ void Reduction2D_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
     using reduce_t = typename Transformer<Arg>::reduce_t;
@@ -26,7 +26,20 @@ namespace quda {
   }
 
   template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
-  __global__ void MultiReduction(Arg arg)
+  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
+  {
+    Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(arg);
+  }
+
+  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D()
+  {
+    Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+  }
+
+
+  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  __forceinline__ __device__ void MultiReduction_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
     using reduce_t = typename Transformer<Arg>::reduce_t;
@@ -49,4 +62,16 @@ namespace quda {
     reduce<block_size_x, block_size_y>(arg, t, value, j);
   }
 
+  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
+  {
+    MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(arg);
+  }
+
+  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction()
+  {
+    MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+  }
+
 }
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 33b8ad458f..5bd618797a 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -46,13 +46,16 @@ namespace quda {
     __device__ __host__ inline dim3 block_dim()
     {
       return dim3(omp_get_num_threads());
-/*
-#ifdef __CUDA_ARCH__
-      return dim3(blockDim.x, blockDim.y, blockDim.z);
-#else
-      return dim3(1, 1, 1);
-#endif
-*/
+    }
+
+    /**
+       @brief Helper function that returns the grid dimensions.  On
+       CUDA this returns the intrinsic blockDim, whereas on the host
+       this returns (1, 1, 1).
+    */
+    __device__ __host__ inline dim3 grid_dim()
+    {
+      return dim3(omp_get_num_teams());
     }
 
     /**
@@ -63,13 +66,6 @@ namespace quda {
     __device__ __host__ inline dim3 block_idx()
     {
       return dim3(omp_get_team_num(),0,0);
-/*
-#ifdef __CUDA_ARCH__
-      return dim3(blockIdx.x, blockIdx.y, blockIdx.z);
-#else
-      return dim3(0, 0, 0);
-#endif
-*/
     }
 
     /**
@@ -80,13 +76,6 @@ namespace quda {
     __device__ __host__ inline dim3 thread_idx()
     {
       return dim3(omp_get_thread_num(),0,0);
-/*
-#ifdef __CUDA_ARCH__
-      return dim3(threadIdx.x, threadIdx.y, threadIdx.z);
-#else
-      return dim3(0, 0, 0);
-#endif
-*/
     }
 
   }
@@ -145,10 +134,10 @@ namespace quda {
 
     /**
        @brief Helper function that returns the maximum size of a
-       constant_param_t buffer on the target architecture.  For CUDA,
-       this corresponds to the maximum __constant__ buffer size.
+       __constant__ buffer on the target architecture.  For CUDA,
+       this is set to the somewhat arbitrary limit of 32 KiB for now.
     */
-    constexpr size_t max_constant_param_size() { return 8192; }
+    constexpr size_t max_constant_size() { return 32768; }
 
     /**
        @brief Helper function that returns the maximum static size of
@@ -163,6 +152,34 @@ namespace quda {
     */
     constexpr int shared_memory_bank_width() { return 32; }
 
+    /**
+       @brief Helper function that returns true if we are to pass the
+       kernel parameter struct to the kernel as an explicit kernel
+       argument.  Otherwise the parameter struct is explicitly copied
+       to the device prior to kernel launch.
+    */
+    template <typename Arg> constexpr bool use_kernel_arg()
+    {
+      return (sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg);
+    }
+
+    /**
+       @brief Helper function that returns kernel argument from
+       __constant__ memory.  Note this is the dummy implementation,
+       and is present only to keep the compiler happy in the
+       translation units where constant memory is not used.
+     */
+    template <typename Arg>
+      constexpr std::enable_if_t<use_kernel_arg<Arg>(), const Arg&> get_arg() { return reinterpret_cast<Arg&>(nullptr); }
+
+    /**
+       @brief Helper function that returns a pointer to the
+       __constant__ memory buffer.  Note this is the dummy
+       implementation, and is present only to keep the compiler happy
+       in the translation units where constant memory is not used.
+     */
+    template <typename Arg> constexpr std::enable_if_t<use_kernel_arg<Arg>(), void *> get_constant_buffer() { return nullptr; }
+
   }
 
 }
diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index 5494389186..a27917333d 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -86,8 +86,6 @@ namespace quda {
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg)
     {
 #ifdef QUDA_BACKEND_OMPTARGET
-      for (unsigned int i = 0; i < param.size(); i++)
-        qudaMemcpyAsync(param[i].device_ptr, param[i].host, param[i].bytes, qudaMemcpyHostToDevice, stream);
       ompwip("OFFLOADING...");
       const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
       const int ld = tp.block.x*tp.block.y*tp.grid.z;
diff --git a/lib/comm_common.cpp b/lib/comm_common.cpp
index 441fd8a5f4..1dd5feb3c8 100644
--- a/lib/comm_common.cpp
+++ b/lib/comm_common.cpp
@@ -101,23 +101,10 @@ Topology *comm_create_topology(int ndim, const int *dims, QudaCommsMap rank_from
 
   topo->ndim = ndim;
 
-<<<<<<< HEAD
-  int device_count = quda::device::get_device_count();
-  if (device_count == 0) { warningQuda("No devices found"); }
-  if (gpuid >= device_count) {
-    char *enable_mps_env = getenv("QUDA_ENABLE_MPS");
-    if (enable_mps_env && strcmp(enable_mps_env, "1") == 0) {
-      gpuid = gpuid % device_count;
-      printf("MPS enabled, rank=%d -> gpu=%d\n", comm_rank(), gpuid);
-    } else {
-      warningQuda("Too few GPUs available on %s", comm_hostname());
-    }
-=======
   int nodes = 1;
   for (int i = 0; i < ndim; i++) {
     topo->dims[i] = dims[i];
     nodes *= dims[i];
->>>>>>> feature/generic_kernel
   }
 
   topo->ranks = new int[nodes];
diff --git a/lib/targets/omptarget/blas_lapack_cublas.cpp b/lib/targets/omptarget/blas_lapack_cublas.cpp
index ead59bb383..35f328da86 100644
--- a/lib/targets/omptarget/blas_lapack_cublas.cpp
+++ b/lib/targets/omptarget/blas_lapack_cublas.cpp
@@ -31,7 +31,8 @@ namespace quda
         if (!cublas_init) {
 #ifdef NATIVE_LAPACK_LIB
           cublasStatus_t error = cublasCreate(&handle);
-          if (error != CUBLAS_STATUS_SUCCESS) errorQuda("cublasCreate failed with error %d", error);
+          if (error != CUBLAS_STATUS_SUCCESS)
+            errorQuda("cublasCreate failed with error %d", error);
           else
             printfQuda("cublasCreated successfully\n");
           cublas_init = true;
diff --git a/lib/targets/omptarget/comm_target.cpp b/lib/targets/omptarget/comm_target.cpp
index 7d95c583f6..3556ba5cc3 100644
--- a/lib/targets/omptarget/comm_target.cpp
+++ b/lib/targets/omptarget/comm_target.cpp
@@ -2,6 +2,7 @@
 #include <quda_api.h>
 #include <quda_cuda_api.h>
 #include <algorithm>
+#include <shmem_helper.cuh>
 
 #define CHECK_CUDA_ERROR(func)                                          \
   quda::cuda::set_runtime_error(func, #func, __func__, __FILE__, __STRINGIFY__(__LINE__));
@@ -77,9 +78,10 @@ void comm_create_neighbor_memory(void *remote[QUDA_MAX_DIM][2], void *local)
     if (comm_dim(dim)==1) continue;
     // even if comm_dim(2) == 2, we might not have p2p enabled in both directions, so check this
     const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
-    for (int dir=0; dir<num_dir; ++dir) {
+    for (int dir = 0; dir < num_dir; dir++) {
       remote[dim][dir] = nullptr;
-      if (comm_peer2peer_enabled(dir,dim)) CHECK_CUDA_ERROR(cudaIpcOpenMemHandle(&remote[dim][dir], remote_handle[dir][dim], cudaIpcMemLazyEnablePeerAccess));
+      if (!comm_peer2peer_enabled(dir, dim)) continue;
+      CHECK_CUDA_ERROR(cudaIpcOpenMemHandle(&remote[dim][dir], remote_handle[dir][dim], cudaIpcMemLazyEnablePeerAccess));
     }
     if (num_dir == 1) remote[dim][1] = remote[dim][0];
   }
@@ -106,7 +108,7 @@ void comm_destroy_neighbor_memory(void *remote[QUDA_MAX_DIM][2])
   } // iterate over dim
 */
 }
-  
+
 void comm_create_neighbor_event(qudaEvent_t remote[2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
 {
   ompwip("unimplemented");
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 2489368cef..a49ab929c5 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -148,31 +148,40 @@ namespace quda
       // d) QUDA built for same major compute capability but lower minor: warn
 
       const int my_major = __COMPUTE_CAPABILITY__ / 100;
-      const int my_minor = (__COMPUTE_CAPABILITY__  - my_major * 100) / 10;
+      const int my_minor = (__COMPUTE_CAPABILITY__ - my_major * 100) / 10;
       // b) QUDA was compiled for a higher compute capability
       if (deviceProp.major * 100 + deviceProp.minor * 10 < __COMPUTE_CAPABILITY__)
-        errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. ** \n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
+        errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. ** \n --- "
+                  "Please set the correct QUDA_GPU_ARCH when running cmake.\n",
+                  deviceProp.major, deviceProp.minor, my_major, my_minor);
 
       // c) QUDA was compiled for a lower compute capability
       if (deviceProp.major < my_major) {
         char *allow_jit_env = getenv("QUDA_ALLOW_JIT");
         if (allow_jit_env && strcmp(allow_jit_env, "1") == 0) {
-          if (getVerbosity() > QUDA_SILENT) warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- Jitting the PTX since QUDA_ALLOW_JIT=1 was set. Note that this will take some time.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
+          if (getVerbosity() > QUDA_SILENT)
+            warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- "
+                        "Jitting the PTX since QUDA_ALLOW_JIT=1 was set. Note that this will take some time.\n",
+                        deviceProp.major, deviceProp.minor, my_major, my_minor);
         } else {
-          errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n --- Please set the correct QUDA_GPU_ARCH when running cmake.\n If you want the PTX to be jitted for your current GPU arch please set the enviroment variable QUDA_ALLOW_JIT=1.", deviceProp.major, deviceProp.minor, my_major, my_minor);
+          errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n --- "
+                    "Please set the correct QUDA_GPU_ARCH when running cmake.\n If you want the PTX to be jitted for "
+                    "your current GPU arch please set the enviroment variable QUDA_ALLOW_JIT=1.",
+                    deviceProp.major, deviceProp.minor, my_major, my_minor);
         }
       }
       // d) QUDA built for same major compute capability but lower minor
       if (deviceProp.major == my_major and deviceProp.minor > my_minor) {
-        warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- This might result in a lower performance. Please consider adjusting QUDA_GPU_ARCH when running cmake.\n", deviceProp.major, deviceProp.minor, my_major, my_minor);
+        warningQuda(
+          "** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- This might "
+          "result in a lower performance. Please consider adjusting QUDA_GPU_ARCH when running cmake.\n",
+          deviceProp.major, deviceProp.minor, my_major, my_minor);
       }
 */
 
       if (!deviceProp.unifiedAddressing) warningQuda("Device %d does not support unified addressing", dev);
 
-      if (getVerbosity() >= QUDA_SUMMARIZE) {
-        printfQuda("Using device %d: %s\n", dev, deviceProp.name);
-      }
+      if (getVerbosity() >= QUDA_SUMMARIZE) printfQuda("Using device %d: %s\n", dev, deviceProp.name);
 #ifndef USE_QDPJIT
       CHECK_CUDA_ERROR(cudaSetDevice(dev));
 #endif
@@ -181,7 +190,7 @@ namespace quda
       char *enable_numa_env = getenv("QUDA_ENABLE_NUMA");
       if (enable_numa_env && strcmp(enable_numa_env, "0") == 0) {
         if (getVerbosity() > QUDA_SILENT) printfQuda("Disabling numa_affinity\n");
-      } else{
+      } else {
         setNumaAffinityNVML(dev);
       }
 #endif
@@ -278,13 +287,13 @@ namespace quda
     void destroy()
     {
       if (streams) {
-        for (int i=0; i<Nstream; i++) CHECK_CUDA_ERROR(cudaStreamDestroy(streams[i]));
+        for (int i = 0; i < Nstream; i++) CHECK_CUDA_ERROR(cudaStreamDestroy(streams[i]));
         delete []streams;
         streams = nullptr;
       }
 
       char *device_reset_env = getenv("QUDA_DEVICE_RESET");
-      if (device_reset_env && strcmp(device_reset_env,"1") == 0) {
+      if (device_reset_env && strcmp(device_reset_env, "1") == 0) {
         // end this CUDA context
         CHECK_CUDA_ERROR(cudaDeviceReset());
       }
@@ -391,19 +400,14 @@ namespace quda
 #endif
     }
 
-    namespace profile {
+    namespace profile
+    {
 
-      void start()
-      {
-        // cudaProfilerStart();
-      }
+      void start() { /* cudaProfilerStart(); */ }
 
-      void stop()
-      {
-        // cudaProfilerStop();
-      }
+      void stop() { /* cudaProfilerStop(); */ }
 
-    }
+    } // namespace profile
 
-  }
-}
+  } // namespace device
+} // namespace quda
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index 0a3190d446..fdbbc35c97 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -6,6 +6,7 @@
 #include <execinfo.h> // for backtrace
 #include <quda_internal.h>
 #include <device.h>
+#include <shmem_helper.cuh>
 
 #ifdef USE_QDPJIT
 #include "qdp_quda.h"
@@ -15,10 +16,11 @@
 #ifdef QUDA_BACKWARDSCPP
 #include "backward.hpp"
 #endif
+
 namespace quda
 {
 
-  enum AllocType { DEVICE, DEVICE_PINNED, HOST, PINNED, MAPPED, MANAGED, N_ALLOC_TYPE };
+  enum AllocType { DEVICE, DEVICE_PINNED, HOST, PINNED, MAPPED, MANAGED, SHMEM, N_ALLOC_TYPE };
 
   class MemAlloc
   {
@@ -86,12 +88,11 @@ namespace quda
 
   static void print_alloc(AllocType type)
   {
-    const char *type_str[] = {"Device", "Device Pinned", "Host  ", "Pinned", "Mapped", "Managed"};
-    std::map<void *, MemAlloc>::iterator entry;
+    const char *type_str[] = {"Device", "Device Pinned", "Host  ", "Pinned", "Mapped", "Managed", "Shmem "};
 
-    for (entry = alloc[type].begin(); entry != alloc[type].end(); entry++) {
-      void *ptr = entry->first;
-      MemAlloc a = entry->second;
+    for (auto entry : alloc[type]) {
+      void *ptr = entry.first;
+      MemAlloc a = entry.second;
       printfQuda("%s  %15p  %15lu  %s(), %s:%d\n", type_str[type], ptr, (unsigned long)a.base_size, a.func.c_str(),
                  a.file.c_str(), a.line);
 #ifdef QUDA_BACKWARDSCPP
@@ -107,7 +108,7 @@ namespace quda
   {
     total_bytes[type] += a.base_size;
     if (total_bytes[type] > max_total_bytes[type]) { max_total_bytes[type] = total_bytes[type]; }
-    if (type != DEVICE && type != DEVICE_PINNED) {
+    if (type != DEVICE && type != DEVICE_PINNED && type != SHMEM) {
       total_host_bytes += a.base_size;
       if (total_host_bytes > max_total_host_bytes) { max_total_host_bytes = total_host_bytes; }
     }
@@ -126,7 +127,7 @@ namespace quda
   {
     size_t size = alloc[type][ptr].base_size;
     total_bytes[type] -= size;
-    if (type != DEVICE && type != DEVICE_PINNED) { total_host_bytes -= size; }
+    if (type != DEVICE && type != DEVICE_PINNED && type != SHMEM) { total_host_bytes -= size; }
     if (type == PINNED || type == MAPPED) { total_pinned_bytes -= size; }
 {
     auto a = alloc[type][ptr];
@@ -364,6 +365,42 @@ namespace quda
 #endif
     return ptr;
   }
+  /**
+   * Allocate shemm device memory. This function should only be called via
+   * device_comms_pinned_malloc_()
+   */
+#ifdef NVSHMEM_COMMS
+  void *shmem_malloc_(const char *func, const char *file, int line, size_t size)
+  {
+    MemAlloc a(func, file, line);
+
+    a.size = a.base_size = size;
+
+    auto ptr = nvshmem_malloc(size);
+    if (ptr == nullptr) {
+      printfQuda("ERROR: Failed to allocate shmem memory of size %zu (%s:%d in %s())\n", size, file, line, func);
+      errorQuda("Aborting");
+    }
+    track_malloc(SHMEM, a, ptr);
+#ifdef HOST_DEBUG
+    qudaMemset(ptr, 0xff, size);
+#endif
+    return ptr;
+  }
+#endif
+
+  /**
+   * Allocate pinned or symmetric (shmem) device memory for comms. Should only be called via the
+   * device_comms_pinned_malloc macro, defined in malloc_quda.h
+   */
+  void *device_comms_pinned_malloc_(const char *func, const char *file, int line, size_t size)
+  {
+#ifdef NVSHMEM_COMMS
+    return shmem_malloc_(func, file, line, size);
+#else
+    return device_pinned_malloc_(func, file, line, size);
+#endif
+  }
 
   /**
    * Perform a standard cudaMallocManaged() with error-checking.  This
@@ -515,11 +552,44 @@ namespace quda
     }
   }
 
+#ifdef NVSHMEM_COMMS
+  /**
+   * Free symmetric memory allocated with shmem_malloc_. Should only be called via the device_comms_* functions.
+   */
+  void shmem_free_(const char *func, const char *file, int line, void *ptr)
+  {
+    if (!ptr) {
+      printfQuda("ERROR: Attempt to free NULL shmem pointer (%s:%d in %s())\n", file, line, func);
+      errorQuda("Aborting");
+    }
+    if (!alloc[SHMEM].count(ptr)) {
+      printfQuda("ERROR: Attempt to free invalid shmem pointer (%s:%d in %s())\n", file, line, func);
+      errorQuda("Aborting");
+    }
+    nvshmem_free(ptr);
+    track_free(SHMEM, ptr);
+  }
+#endif
+
+  /**
+   * Free device comms memory allocated with device_comms_pinned_malloc(). This function should only be
+   * called via the device_comms_pinned_free() macro, defined in malloc_quda.h
+   */
+  void device_comms_pinned_free_(const char *func, const char *file, int line, void *ptr)
+  {
+#ifdef NVSHMEM_COMMS
+    shmem_free_(func, file, line, ptr);
+#else
+    device_pinned_free_(func, file, line, ptr);
+#endif
+  }
+
   void printPeakMemUsage()
   {
     printfQuda("Device memory used = %.1f MB\n", max_total_bytes[DEVICE] / (double)(1 << 20));
     printfQuda("Pinned device memory used = %.1f MB\n", max_total_bytes[DEVICE_PINNED] / (double)(1 << 20));
     printfQuda("Managed memory used = %.1f MB\n", max_total_bytes[MANAGED] / (double)(1 << 20));
+    printfQuda("Shmem memory used = %.1f MB\n", max_total_bytes[SHMEM] / (double)(1 << 20));
     printfQuda("Page-locked host memory used = %.1f MB\n", max_total_pinned_bytes / (double)(1 << 20));
     printfQuda("Total host memory used >= %.1f MB\n", max_total_host_bytes / (double)(1 << 20));
   }
@@ -533,6 +603,7 @@ namespace quda
       print_alloc_header();
       print_alloc(DEVICE);
       print_alloc(DEVICE_PINNED);
+      print_alloc(SHMEM);
       print_alloc(HOST);
       print_alloc(PINNED);
       print_alloc(MAPPED);
@@ -585,8 +656,8 @@ namespace quda
     void *device;
     auto error = cudaHostGetDevicePointer(&device, const_cast<void *>(host), 0);
     if (error != cudaSuccess) {
-      errorQuda("cudaHostGetDevicePointer failed with error %s (%s:%d in %s()",
-                cudaGetErrorString(error), file, line, func);
+      errorQuda("cudaHostGetDevicePointer failed with error %s (%s:%d in %s()", cudaGetErrorString(error), file, line,
+                func);
     }
     return device;
 */
@@ -678,12 +749,10 @@ namespace quda
       ompwip("pool::pinned_malloc_");
       void *ptr = nullptr;
       if (pinned_memory_pool) {
-        std::multimap<size_t, void *>::iterator it;
-
         if (pinnedCache.empty()) {
           ptr = quda::pinned_malloc_(func, file, line, nbytes);
         } else {
-          it = pinnedCache.lower_bound(nbytes);
+          auto it = pinnedCache.lower_bound(nbytes);
           if (it != pinnedCache.end()) { // sufficiently large allocation found
             nbytes = it->first;
             ptr = it->second;
@@ -720,12 +789,10 @@ namespace quda
       ompwip("pool::device_malloc_");
       void *ptr = nullptr;
       if (device_memory_pool) {
-        std::multimap<size_t, void *>::iterator it;
-
         if (deviceCache.empty()) {
           ptr = quda::device_malloc_(func, file, line, nbytes);
         } else {
-          it = deviceCache.lower_bound(nbytes);
+          auto it = deviceCache.lower_bound(nbytes);
           if (it != deviceCache.end()) { // sufficiently large allocation found
             nbytes = it->first;
             ptr = it->second;
@@ -762,11 +829,7 @@ namespace quda
     {
       ompwip("pool::flush_pinned");
       if (pinned_memory_pool) {
-        std::multimap<size_t, void *>::iterator it;
-        for (it = pinnedCache.begin(); it != pinnedCache.end(); it++) {
-          void *ptr = it->second;
-          host_free(ptr);
-        }
+        for (auto it : pinnedCache) { host_free(it.second); }
         pinnedCache.clear();
       }
     }
@@ -775,11 +838,7 @@ namespace quda
     {
       ompwip("pool:flush_device");
       if (device_memory_pool) {
-        std::multimap<size_t, void *>::iterator it;
-        for (it = deviceCache.begin(); it != deviceCache.end(); it++) {
-          void *ptr = it->second;
-          device_free(ptr);
-        }
+        for (auto it : deviceCache) { device_free(it.second); }
         deviceCache.clear();
       }
     }
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 147a4bda39..42a0619c5b 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -15,6 +15,7 @@ enum cudaMemcpyKind{cudaMemcpyHostToHost, cudaMemcpyHostToDevice, cudaMemcpyDevi
 static char FIXME[]="OMP FIXME";
 
 #define cudaGetErrorString(a) "OMP FIXME"
+#define cudaGetLastError()
 #define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
 
 #define cudaMemcpy(a,b,c,d) ompwip([&](){printfQuda("memcpy %p <- %p %ld\n",a,b,c);ompwipMemcpy(a,(void*)b,c,d);})
@@ -148,15 +149,16 @@ enum {cudaEventDisableTiming};
 //#define API_PROFILE
 
 #ifdef API_PROFILE
-#define PROFILE(f, idx)                                 \
-  apiTimer.TPSTART(idx);				\
-  f;                                                    \
+#define PROFILE(f, idx)                                                                                                \
+  apiTimer.TPSTART(idx);                                                                                               \
+  f;                                                                                                                   \
   apiTimer.TPSTOP(idx);
 #else
 #define PROFILE(f, idx) f;
 #endif
 
-namespace quda {
+namespace quda
+{
 
   static qudaError_t last_error = QUDA_SUCCESS;
   static std::string last_error_str("CUDA_SUCCESS");
@@ -184,6 +186,7 @@ namespace quda {
       last_error = error == cudaSuccess ? QUDA_SUCCESS : QUDA_ERROR;
       last_error_str = cudaGetErrorString(error);
       if (!allow_error) errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, cudaGetErrorString(error), file, line, func);
+      else cudaGetLastError(); // clear the error state
     }
 
     void set_driver_error(CUresult error, const char *api_func, const char *func, const char *file, const char *line,
@@ -195,6 +198,7 @@ namespace quda {
       cuGetErrorName(error, &str);
       last_error_str = str;
       if (!allow_error) errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, str, file, line, func);
+      else cudaGetLastError(); // clear the error state
     }
 
   }
@@ -301,25 +305,34 @@ namespace quda {
   public:
     inline QudaMem(void *dst, const void *src, size_t count, cudaMemcpyKind kind, const qudaStream_t &stream,
                    bool async, const char *func, const char *file, const char *line) :
-      dst(dst), src(src), count(count), value(0), copy(true), kind(kind), async(async), active_tuning(activeTuning()),
-      func(func), file(file), line(line)
+      dst(dst),
+      src(src),
+      count(count),
+      value(0),
+      copy(true),
+      kind(kind),
+      async(async),
+      active_tuning(activeTuning()),
+      func(func),
+      file(file),
+      line(line)
     {
       if (!async) {
         switch (kind) {
-        case cudaMemcpyDeviceToHost:   name = "cudaMemcpyDeviceToHost";   break;
-        case cudaMemcpyHostToDevice:   name = "cudaMemcpyHostToDevice";   break;
-        case cudaMemcpyHostToHost:     name = "cudaMemcpyHostToHost";     break;
+        case cudaMemcpyDeviceToHost: name = "cudaMemcpyDeviceToHost"; break;
+        case cudaMemcpyHostToDevice: name = "cudaMemcpyHostToDevice"; break;
+        case cudaMemcpyHostToHost: name = "cudaMemcpyHostToHost"; break;
         case cudaMemcpyDeviceToDevice: name = "cudaMemcpyDeviceToDevice"; break;
-        case cudaMemcpyDefault:        name = "cudaMemcpyDefault";        break;
+        case cudaMemcpyDefault: name = "cudaMemcpyDefault"; break;
         default: errorQuda("Unsupported cudaMemcpyKind %d", kind);
         }
       } else {
-        switch(kind) {
-        case cudaMemcpyDeviceToHost:   name = "cudaMemcpyAsyncDeviceToHost";   break;
-        case cudaMemcpyHostToDevice:   name = "cudaMemcpyAsyncHostToDevice";   break;
-        case cudaMemcpyHostToHost:     name = "cudaMemcpyAsyncHostToHost";     break;
+        switch (kind) {
+        case cudaMemcpyDeviceToHost: name = "cudaMemcpyAsyncDeviceToHost"; break;
+        case cudaMemcpyHostToDevice: name = "cudaMemcpyAsyncHostToDevice"; break;
+        case cudaMemcpyHostToHost: name = "cudaMemcpyAsyncHostToHost"; break;
         case cudaMemcpyDeviceToDevice: name = "cudaMemcpyAsyncDeviceToDevice"; break;
-        case cudaMemcpyDefault:        name = "cudaMemcpyAsyncDefault";        break;
+        case cudaMemcpyDefault: name = "cudaMemcpyAsyncDefault"; break;
         default: errorQuda("Unsupported cudaMemcpyKind %d", kind);
         }
       }
@@ -426,19 +439,21 @@ namespace quda {
 
     bool advanceTuneParam(TuneParam &) const { return false; }
 
-    TuneKey tuneKey() const {
+    TuneKey tuneKey() const
+    {
       char vol[128];
-      strcpy(vol,"bytes=");
-      u64toa(vol+6, (uint64_t)count);
+      strcpy(vol, "bytes=");
+      u64toa(vol + 6, (uint64_t)count);
       return TuneKey(vol, name, aux);
     }
 
     long long flops() const { return 0; }
-    long long bytes() const { return kind == cudaMemcpyDeviceToDevice ? 2*count : count; }
+    long long bytes() const { return kind == cudaMemcpyDeviceToDevice ? 2 * count : count; }
   };
 
   void qudaMemcpy_(void *dst, const void *src, size_t count, qudaMemcpyKind kind,
-                   const char *func, const char *file, const char *line) {
+                   const char *func, const char *file, const char *line)
+  {
     if (count == 0) return;
     QudaMem copy(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_default_stream(), false, func, file, line);
   }
@@ -465,8 +480,7 @@ namespace quda {
       case qudaMemcpyDefault:
         PROFILE(cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
         break;
-      default:
-        errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
+      default: errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
       }
 #else
       PROFILE(cudaMemcpyAsync(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)),
@@ -745,7 +759,8 @@ namespace quda {
     set_runtime_error(error, __func__, func, file, line);
   }
 
-  void printAPIProfile() {
+  void printAPIProfile()
+  {
 #ifdef API_PROFILE
     apiTimer.Print();
 #endif

From 3907f296919ed1f728396517571c43cd5fcdc2b7 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 21 Apr 2021 20:49:47 +0000
Subject: [PATCH 035/198] omp: ignore *_ctest

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index e652edd6d9..9f7d2843b5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@
 *.a
 *~
 tests/*_test
+tests/*_ctest
 make.inc
 milc_interface/*
 *#*

From 08f6a5296c4c36e7c3cf1858bf579d42dcfe1a14 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 10 May 2021 21:26:28 +0000
Subject: [PATCH 036/198] omp: add explicit specialization for zero()

---
 include/float_vector.h | 128 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 128 insertions(+)

diff --git a/include/float_vector.h b/include/float_vector.h
index 1adc813379..f9b03d1a24 100644
--- a/include/float_vector.h
+++ b/include/float_vector.h
@@ -163,6 +163,134 @@ namespace quda {
     return v;
   }
 
+  template <> __device__ __host__ inline vector_type<double2, 1> zero<vector_type<double2, 1>>()
+  {
+    vector_type<double2, 1> v;
+#pragma unroll
+    for (int i = 0; i < 1; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 2> zero<vector_type<double2, 2>>()
+  {
+    vector_type<double2, 2> v;
+#pragma unroll
+    for (int i = 0; i < 2; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 3> zero<vector_type<double2, 3>>()
+  {
+    vector_type<double2, 3> v;
+#pragma unroll
+    for (int i = 0; i < 3; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 4> zero<vector_type<double2, 4>>()
+  {
+    vector_type<double2, 4> v;
+#pragma unroll
+    for (int i = 0; i < 4; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 5> zero<vector_type<double2, 5>>()
+  {
+    vector_type<double2, 5> v;
+#pragma unroll
+    for (int i = 0; i < 5; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 6> zero<vector_type<double2, 6>>()
+  {
+    vector_type<double2, 6> v;
+#pragma unroll
+    for (int i = 0; i < 6; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 7> zero<vector_type<double2, 7>>()
+  {
+    vector_type<double2, 7> v;
+#pragma unroll
+    for (int i = 0; i < 7; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 8> zero<vector_type<double2, 8>>()
+  {
+    vector_type<double2, 8> v;
+#pragma unroll
+    for (int i = 0; i < 8; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 9> zero<vector_type<double2, 9>>()
+  {
+    vector_type<double2, 9> v;
+#pragma unroll
+    for (int i = 0; i < 9; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 10> zero<vector_type<double2, 10>>()
+  {
+    vector_type<double2, 10> v;
+#pragma unroll
+    for (int i = 0; i < 10; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 11> zero<vector_type<double2, 11>>()
+  {
+    vector_type<double2, 11> v;
+#pragma unroll
+    for (int i = 0; i < 11; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 12> zero<vector_type<double2, 12>>()
+  {
+    vector_type<double2, 12> v;
+#pragma unroll
+    for (int i = 0; i < 12; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 13> zero<vector_type<double2, 13>>()
+  {
+    vector_type<double2, 13> v;
+#pragma unroll
+    for (int i = 0; i < 13; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 14> zero<vector_type<double2, 14>>()
+  {
+    vector_type<double2, 14> v;
+#pragma unroll
+    for (int i = 0; i < 14; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 15> zero<vector_type<double2, 15>>()
+  {
+    vector_type<double2, 15> v;
+#pragma unroll
+    for (int i = 0; i < 15; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
+  template <> __device__ __host__ inline vector_type<double2, 16> zero<vector_type<double2, 16>>()
+  {
+    vector_type<double2, 16> v;
+#pragma unroll
+    for (int i = 0; i < 16; i++) v.data[i] = zero<double2>();
+    return v;
+  }
+
   template <typename scalar, int n>
   __device__ __host__ inline vector_type<scalar, n> operator+(const vector_type<scalar, n> &a,
                                                               const vector_type<scalar, n> &b)

From e893cebd382f3b0781c0fd07a7ccfe1e7ae4ca99 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 10 May 2021 21:28:23 +0000
Subject: [PATCH 037/198] omp: add more static functions for omp declare
 reduction [WIP]

---
 include/kernels/transform_reduce.cuh | 5 +++++
 include/reducer.h                    | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/include/kernels/transform_reduce.cuh b/include/kernels/transform_reduce.cuh
index c1158e058e..eba79cac9e 100644
--- a/include/kernels/transform_reduce.cuh
+++ b/include/kernels/transform_reduce.cuh
@@ -45,6 +45,11 @@ namespace quda {
 
     static constexpr bool do_sum = Arg::reducer::do_sum;
 
+#ifdef QUDA_BACKEND_OMPTARGET
+    static reduce_t reduce_omp(reduce_t a, reduce_t b) { return Arg::reducer::reduce_omp(a, b); }
+    static reduce_t init_omp() { return Arg::reducer::init_omp(); }
+#endif
+
     __device__ __host__ inline reduce_t operator()(reduce_t a, reduce_t b) const { return arg.r(a, b); }
 
     __device__ __host__ inline reduce_t operator()(reduce_t &value, count_t i, int j, int)
diff --git a/include/reducer.h b/include/reducer.h
index ce734ef6a8..a5cd069ed8 100644
--- a/include/reducer.h
+++ b/include/reducer.h
@@ -14,11 +14,19 @@ namespace quda {
   template <typename T> struct maximum {
     static constexpr bool do_sum = false;
     __device__ __host__ T operator()(T a, T b) const { return a > b ? a : b; }
+#ifdef QUDA_BACKEND_OMPTARGET
+    static T reduce_omp(T a, T b) { return a > b ? a : b; }
+    static T init_omp() { return ::quda::zero<T>(); }  // FIXME wrong for negative values.
+#endif
   };
 
   template <typename T> struct minimum {
     static constexpr bool do_sum = false;
     __device__ __host__ T operator()(T a, T b) const { return a < b ? a : b; }
+#ifdef QUDA_BACKEND_OMPTARGET
+    static T reduce_omp(T a, T b) { return a < b ? a : b; }
+    static T init_omp() { return ::quda::zero<T>(); }  // FIXME wrong for positive values.
+#endif
   };
 
   template <typename T> struct identity {

From b1fae7d8662c12d98b66952280ed4f7e09cfa0b1 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 10 May 2021 21:33:32 +0000
Subject: [PATCH 038/198] omp: use global variable in omp target for emulating
 3D kernels

---
 include/quda_api.h                            |  34 +++-
 .../targets/omptarget/constant_kernel_arg.h   |   4 +-
 include/targets/omptarget/kernel.h            | 146 +++++++++++++++++-
 include/targets/omptarget/target_device.h     |  10 +-
 include/tunable_nd.h                          |  62 --------
 lib/targets/omptarget/quda_api.cpp            |  11 ++
 6 files changed, 189 insertions(+), 78 deletions(-)

diff --git a/include/quda_api.h b/include/quda_api.h
index f53c5c0b5e..665fed11c9 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -13,7 +13,12 @@
 
 #define CUDA_SUCCESS QUDA_SUCCESS
 
-#define QUDA_RT_CONSTS ompwip();const dim3 threadIdx(omp_get_thread_num(),0,0),blockDim(omp_get_num_threads()),blockIdx(omp_get_team_num(),0,0),gridDim(omp_get_num_teams())
+#define QUDA_RT_CONSTS \
+  const dim3\
+    blockDim=launch_param.block,\
+    gridDim=launch_param.grid,\
+    threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y)),\
+    blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y))
 
 #include <functional>
 #include <iostream>
@@ -68,7 +73,11 @@ ompwip_(const char * const file, const size_t line, const char * const func, std
 using cudaStream_t = int;  // device.h:/cudaStream_t
 using CUresult = int;  // ../lib/coarse_op.cuh:/CUresult
 
-#else // QUDA_BACKEND_OMPTARGET
+struct LaunchParam{
+  dim3 block;
+  dim3 grid;
+};
+extern LaunchParam launch_param;
 
 #endif // QUDA_BACKEND_OMPTARGET
 
@@ -111,6 +120,19 @@ namespace quda
   */
   qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, void **args, qudaStream_t stream);
 
+  template <typename... Arg>
+  qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, qudaStream_t stream, const Arg &... arg)
+  {
+#ifdef QUDA_BACKEND_OMPTARGET
+    extern void qudaSetupLaunchParameter(const TuneParam &tp);
+    qudaSetupLaunchParameter(tp);
+    reinterpret_cast<void(*)(Arg...)>(const_cast<void*>(func))(arg...);
+    return QUDA_SUCCESS;
+#else
+    const void *args[] = {&arg...};
+    return qudaLaunchKernel(reinterpret_cast<const void *>(func), tp, const_cast<void **>(args), stream);
+#endif
+  }
   /**
      @brief Templated wrapper around qudaLaunchKernel which can accept
      a templated kernel, and expects a kernel with a single Arg argument
@@ -122,9 +144,15 @@ namespace quda
   template <typename T, typename... Arg>
   qudaError_t qudaLaunchKernel(T *func, const TuneParam &tp, qudaStream_t stream, const Arg &... arg)
   {
-    ompwip("directly calling qudaLaunchKernel is unsupported");
+#ifdef QUDA_BACKEND_OMPTARGET
+    extern void qudaSetupLaunchParameter(const TuneParam &tp);
+    qudaSetupLaunchParameter(tp);
+    func(arg...);
+    return QUDA_SUCCESS;
+#else
     const void *args[] = {&arg...};
     return qudaLaunchKernel(reinterpret_cast<const void *>(func), tp, const_cast<void **>(args), stream);
+#endif
   }
 
   /**
diff --git a/include/targets/omptarget/constant_kernel_arg.h b/include/targets/omptarget/constant_kernel_arg.h
index 0bdd389bfa..3e33c89f89 100644
--- a/include/targets/omptarget/constant_kernel_arg.h
+++ b/include/targets/omptarget/constant_kernel_arg.h
@@ -26,7 +26,9 @@ namespace quda {
     // buffer as extern and define it in one place only
     extern __constant__ char buffer[max_constant_size()];
 #else
+    #pragma omp declare target
     __constant__ char buffer[max_constant_size()];
+    #pragma omp end declare target
 #endif
 
     /**
@@ -40,7 +42,7 @@ namespace quda {
        @brief Helper function that returns a pointer to the
        __constant__ memory buffer.
      */
-    template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), void*> get_constant_buffer() { return qudaGetSymbolAddress(buffer); }
+    template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), void*> get_constant_buffer() { return omp_get_mapped_ptr(buffer, omp_get_default_device()); }
 
   }
 
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index c686f75658..19da2c5494 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -19,10 +19,47 @@ namespace quda {
   }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg) { Kernel1D_impl<Functor, Arg, grid_stride>(arg); }
+    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg)
+    {
+      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      const int tx = arg.threads.x;
+      const int ty = arg.threads.y;
+      const int tz = arg.threads.z;
+      printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+      // printf("dparg %p\n", dparg);
+      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+      #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+      #pragma omp parallel num_threads(ld)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        Kernel1D_impl<Functor, Arg, grid_stride>(*dparg);
+      }
+      omp_target_free(dparg, omp_get_default_device());
+    }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D() { Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>()); }
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D()
+    {
+      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
+      #pragma omp target teams num_teams(gd) thread_limit(ld)
+      #pragma omp parallel num_threads(ld)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+      }
+    }
 
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
@@ -42,10 +79,47 @@ namespace quda {
   }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg) { Kernel2D_impl<Functor, Arg, grid_stride>(arg); }
+    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg)
+    {
+      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      const int tx = arg.threads.x;
+      const int ty = arg.threads.y;
+      const int tz = arg.threads.z;
+      printf("Kernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+      // printf("dparg %p\n", dparg);
+      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+      #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+      #pragma omp parallel num_threads(ld)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        Kernel2D_impl<Functor, Arg, grid_stride>(*dparg);
+      }
+      omp_target_free(dparg, omp_get_default_device());
+    }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D() { Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>()); }
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D()
+    {
+      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+      #pragma omp target teams num_teams(gd) thread_limit(ld)
+      #pragma omp parallel num_threads(ld)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+      }
+    }
 
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
@@ -67,16 +141,72 @@ namespace quda {
   }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg) { Kernel3D_impl<Functor, Arg, grid_stride>(arg); }
+    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg)
+    {
+      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      const int tx = arg.threads.x;
+      const int ty = arg.threads.y;
+      const int tz = arg.threads.z;
+      printf("Kernel3D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+      // printf("dparg %p\n", dparg);
+      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+      #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+      #pragma omp parallel num_threads(ld)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        Kernel3D_impl<Functor, Arg, grid_stride>(*dparg);
+      }
+      omp_target_free(dparg, omp_get_default_device());
+    }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D() { Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>()); }
+    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D()
+    {
+      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
+      #pragma omp target teams num_teams(gd) thread_limit(ld)
+      #pragma omp parallel num_threads(ld)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+      }
+    }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
-    Functor<Arg> f(arg);
-    f();
+    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    const int tx = arg.threads.x;
+    const int ty = arg.threads.y;
+    const int tz = arg.threads.z;
+    printf("raw_kernel: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // printf("dparg %p\n", dparg);
+    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    #pragma omp parallel num_threads(ld)
+    {
+      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+               "omp reports: teams %d threads %d\n",
+               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+               omp_get_num_teams(), omp_get_num_threads());
+      Functor<Arg> f(*dparg);
+      f();
+    }
+    omp_target_free(dparg, omp_get_default_device());
   }
 
 }
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 5bd618797a..42e4f70aea 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -45,7 +45,7 @@ namespace quda {
     */
     __device__ __host__ inline dim3 block_dim()
     {
-      return dim3(omp_get_num_threads());
+      return launch_param.block;
     }
 
     /**
@@ -55,7 +55,7 @@ namespace quda {
     */
     __device__ __host__ inline dim3 grid_dim()
     {
-      return dim3(omp_get_num_teams());
+      return launch_param.grid;
     }
 
     /**
@@ -65,7 +65,8 @@ namespace quda {
     */
     __device__ __host__ inline dim3 block_idx()
     {
-      return dim3(omp_get_team_num(),0,0);
+      const int n = omp_get_team_num();
+      return dim3(n%launch_param.grid.x, (n/launch_param.grid.x)%launch_param.grid.y, n/(launch_param.grid.x*launch_param.grid.y));
     }
 
     /**
@@ -75,7 +76,8 @@ namespace quda {
     */
     __device__ __host__ inline dim3 thread_idx()
     {
-      return dim3(omp_get_thread_num(),0,0);
+      const int n = omp_get_thread_num();
+      return dim3(n%launch_param.block.x, (n/launch_param.block.x)%launch_param.block.y, n/(launch_param.block.x*launch_param.block.y));
     }
 
   }
diff --git a/include/tunable_nd.h b/include/tunable_nd.h
index da233aa0cd..748bf292bd 100644
--- a/include/tunable_nd.h
+++ b/include/tunable_nd.h
@@ -79,22 +79,7 @@ namespace quda {
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
       ompwip("OFFLOADING...");
-#ifdef QUDA_BACKEND_OMPTARGET
-      const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
-      const int ld = tp.block.x*tp.block.y*tp.grid.z;
-      const int tx = arg.threads.x;
-      printf("launch parameter: gd %d ld %d tx %d\n", gd, ld, tx);
-      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-      Functor<Arg> f(*dparg);
-#pragma omp target teams distribute parallel for simd num_teams(gd) thread_limit(ld) num_threads(ld)
-      for (int i = 0; i < tx; i++) {
-        f(i);
-      }
-      omp_target_free(dparg, omp_get_default_device());
-#else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel1D), tp, stream, arg);
-#endif
     }
 
     template <template <typename> class Functor, typename Arg>
@@ -201,25 +186,7 @@ namespace quda {
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
       ompwip("OFFLOADING...");
-#ifdef QUDA_BACKEND_OMPTARGET
-      const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
-      const int ld = tp.block.x*tp.block.y*tp.grid.z;
-      const int tx = arg.threads.x;
-      const int ty = arg.threads.y;
-      printf("launch parameter: gd %d ld %d tx %d ty %d\n", gd, ld, tx, ty);
-      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-      Functor<Arg> f(*dparg);
-#pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld) map(to:f)
-      for (int i = 0; i < tx; i++) {
-        for (int j = 0; j < ty; j++) {
-          f(i, j);
-        }
-      }
-      omp_target_free(dparg, omp_get_default_device());
-#else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel2D), tp, stream, arg);
-#endif
     }
 
     template <template <typename> class Functor, typename Arg>
@@ -382,37 +349,8 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-#if 1
-      ompwip("NOOOO...");
-#else
-/*
-InvalidArraySize: Array size must be at least 1:  [Src: /netbatch/donb884519_00/runDir/72/20210130_000000/llvm/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp:399 T->getArrayNumElements() >= 1 ]
-clang-12: error: llvm-spirv command failed with exit code 7 (use -v to see invocation)
-*/
       ompwip("OFFLOADING...");
-#ifdef QUDA_BACKEND_OMPTARGET
-      const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
-      const int ld = tp.block.x*tp.block.y*tp.grid.z;
-      const int tx = arg.threads.x;
-      const int ty = arg.threads.y;
-      const int tz = arg.threads.z;
-      printf("launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-      Functor<Arg> f(*dparg);
-#pragma omp target teams distribute parallel for simd collapse(3) num_teams(gd) thread_limit(ld) num_threads(ld)
-      for (int i = 0; i < tx; i++) {
-        for (int j = 0; j < ty; j++) {
-          for (int k = 0; k < tz; k++) {
-            f(i, j, k);
-          }
-        }
-      }
-      omp_target_free(dparg, omp_get_default_device());
-#else
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel3D), tp, stream, arg);
-#endif
-#endif
     }
 
     template <template <typename> class Functor, typename Arg>
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 42a0619c5b..03f717d823 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -157,6 +157,9 @@ enum {cudaEventDisableTiming};
 #define PROFILE(f, idx) f;
 #endif
 
+LaunchParam launch_param;
+#pragma omp declare target to(launch_param)
+
 namespace quda
 {
 
@@ -177,6 +180,14 @@ namespace quda
     return rtn;
   }
 
+  void qudaSetupLaunchParameter(const TuneParam &tp)
+  {
+    ompwip("set up global parameter");
+    launch_param.grid = tp.grid;
+    launch_param.block = tp.block;
+    #pragma omp target update to(launch_param)
+  }
+
   namespace cuda {
 
     void set_runtime_error(cudaError_t error, const char *api_func, const char *func, const char *file, const char *line,

From a14a4469e59fbbb5302b47daaf650ef4279ddc6f Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 10 May 2021 21:34:24 +0000
Subject: [PATCH 039/198] omp: use multiple parallel regions in target teams
 region for block-level reduction

---
 include/reduce_helper.h                       |  54 ++-
 .../omptarget/block_reduction_kernel.h        |  35 +-
 include/targets/omptarget/reduction_kernel.h  | 427 +++++++++++++++++-
 include/tunable_reduction.h                   |  43 +-
 4 files changed, 491 insertions(+), 68 deletions(-)

diff --git a/include/reduce_helper.h b/include/reduce_helper.h
index d34277ae6c..6cc838ef36 100644
--- a/include/reduce_helper.h
+++ b/include/reduce_helper.h
@@ -67,7 +67,10 @@ namespace quda
     friend __device__  void reduce(Arg&, const Reducer &, const I &, const int);
     qudaError_t launch_error; // only do complete if no launch error to avoid hang
 
+#ifndef QUDA_BACKEND_OMPTARGET
+  /** Workaround compiler limitations.  We have to inline parallel regions with target teams.  We need public access. **/
   private:
+#endif
     const int n_reduce; // number of reductions of length n_item
     bool reset = false; // reset the counter post completion (required for multiple calls with the same arg instance
 #ifdef HETEROGENEOUS_ATOMIC
@@ -179,7 +182,13 @@ namespace quda
       const int n_element = n_reduce * sizeof(T) / sizeof(device_t);
       if (result.size() != (unsigned)n_element)
         errorQuda("result vector length %lu does not match n_reduce %d", result.size(), n_element);
+#ifdef QUDA_BACKEND_OMPTARGET
+      omp_target_memcpy(result_h, result_d, n_element*sizeof(device_t), 0, 0, omp_get_initial_device(), omp_get_default_device());
+#endif
       for (int i = 0; i < n_element; i++) result[i] = reinterpret_cast<device_t *>(result_h)[i];
+#ifdef QUDA_BACKEND_OMPTARGET
+      for (int i = 0; i < n_element; i++) std::cerr << "complete: result[" << i << "] = " << result[i] << std::endl;
+#endif
     }
 #endif
   };
@@ -266,7 +275,50 @@ namespace quda
     __device__ inline void reduce(Arg &arg, const Reducer &r, const T &in, const int idx = 0)
     {
 #ifdef QUDA_BACKEND_OMPTARGET
-      ompwip("unimplemented");
+      const dim3
+        blockDim=launch_param.block,
+        gridDim=launch_param.grid,
+        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
+
+      // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
+      bool isLastBlockDone;
+      // if (threadIdx.x == 0 && threadIdx.y == 0)
+      { // This is the main thread per team
+        arg.partial[idx * gridDim.x + blockIdx.x] = in;
+
+        // increment global block counter
+        // auto value = atomicInc(&arg.count[idx], gridDim.x);
+        int value = 0;
+        #pragma omp atomic capture
+        { value = arg.count[idx]; arg.count[idx] = ((arg.count[idx] >= gridDim.x) ? 0 : (arg.count[idx]+1)); }
+
+        // determine if last block
+        isLastBlockDone = (value == (gridDim.x - 1));
+      }
+      // finish the reduction if last block
+      if (isLastBlockDone) {
+        T sum = arg.init();
+        const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+#pragma omp declare reduction(OMPReduce_ : T : omp_out=Reducer::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Reducer::init_omp())
+        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+        {
+          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+          auto i = threadIdx.y * block_size_x + threadIdx.x;
+          while (i < gridDim.x) {
+            sum = r(sum, const_cast<T &>(static_cast<volatile T *>(arg.partial)[idx * gridDim.x + i]));
+            i += block_size_x * block_size_y;
+          }
+        }
+
+        // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
+
+        // write out the final reduced value
+        // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+        { // This is the main thread per team
+          arg.result_d[idx] = sum;
+          arg.count[idx] = 0; // set to zero for next time
+        }
+      }
 #else
       using BlockReduce = cub::BlockReduce<T, block_size_x, cub::BLOCK_REDUCE_WARP_REDUCTIONS, block_size_y>;
       __shared__ typename BlockReduce::TempStorage cub_tmp;
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 51db448ad2..b123169d08 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -58,14 +58,45 @@ namespace quda {
     __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0)
     __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
-    BlockKernel2D_impl<block_size, Transformer, Arg>(arg);
+    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    const int tx = arg.threads.x;
+    const int ty = arg.threads.y;
+    const int tz = arg.threads.z;
+    printf("BlockKernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // printf("dparg %p\n", dparg);
+    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    #pragma omp parallel num_threads(ld)
+    {
+      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+               "omp reports: teams %d threads %d\n",
+               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+               omp_get_num_teams(), omp_get_num_threads());
+      BlockKernel2D_impl<block_size, Transformer, Arg>(*dparg);
+    }
+    omp_target_free(dparg, omp_get_default_device());
   }
 
   template <unsigned int block_size, template <int, typename> class Transformer, typename Arg>
     __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0)
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D()
   {
-    BlockKernel2D_impl<block_size, Transformer, Arg>(device::get_arg<Arg>());
+    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp parallel num_threads(ld)
+    {
+      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+               "omp reports: teams %d threads %d\n",
+               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+               omp_get_num_teams(), omp_get_num_threads());
+      BlockKernel2D_impl<block_size, Transformer, Arg>(device::get_arg<Arg>());
+    }
   }
 
 }
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index eeead67508..e749843a04 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -7,71 +7,450 @@ namespace quda {
   template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __forceinline__ __device__ void Reduction2D_impl(const Arg &arg)
   {
-    QUDA_RT_CONSTS;
+    const dim3
+      blockDim=launch_param.block,
+      gridDim=launch_param.grid,
+      blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
+
     using reduce_t = typename Transformer<Arg>::reduce_t;
-    Transformer<Arg> t(arg);
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+    reduce_t value = arg.init();
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
+    {
+      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+               "omp reports: teams %d threads %d\n",
+               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+               omp_get_num_teams(), omp_get_num_threads());
+      dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
-    auto idx = threadIdx.x + blockIdx.x * blockDim.x;
-    auto j = threadIdx.y;
+      Transformer<Arg> t(arg);
 
-    reduce_t value = arg.init();
+      auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+      auto j = threadIdx.y;
 
-    while (idx < arg.threads.x) {
-      value = t(value, idx, j);
-      if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+      while (idx < arg.threads.x) {
+        value = t(value, idx, j);
+        if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+      }
     }
 
     // perform final inter-block reduction and write out result
+    Transformer<Arg> t(arg);
     reduce<block_size_x, block_size_y>(arg, t, value);
   }
 
   template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
   {
-    Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(arg);
+    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    const int tx = arg.threads.x;
+    const int ty = arg.threads.y;
+    const int tz = arg.threads.z;
+    printf("Reduction2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // printf("dparg %p\n", dparg);
+    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    {
+      // Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(*dparg);
+      const dim3
+        blockDim=launch_param.block,
+        gridDim=launch_param.grid,
+        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
+
+      using reduce_t = typename Transformer<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+      reduce_t value = dparg->init();
+      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+
+        Transformer<Arg> t(*dparg);
+
+        auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+        auto j = threadIdx.y;
+
+        while (idx < dparg->threads.x) {
+          value = t(value, idx, j);
+          if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+        }
+      }
+      // perform final inter-block reduction and write out result
+      Transformer<Arg> t(*dparg);
+      // reduce<block_size_x, block_size_y>(*dparg, t, value);
+      // ../../reduce_helper.h:/reduce
+      {
+        const auto idx = 0;
+        // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
+        bool isLastBlockDone;
+        // if (threadIdx.x == 0 && threadIdx.y == 0)
+        { // This is the main thread per team
+          if(blockIdx.x==0) printf("team %d: value: %g\n", omp_get_team_num(), *reinterpret_cast<double *>(&value));
+          dparg->partial[idx * gridDim.x + blockIdx.x] = value;
+
+          // increment global block counter
+          // auto value = atomicInc(&arg.count[idx], gridDim.x);
+          unsigned int cvalue = 0;
+          unsigned int *c = &dparg->count[idx];
+          #pragma omp atomic capture
+          { cvalue = *c; *c = *c + 1; }
+          // { cvalue = dparg->count[idx]; dparg->count[idx] = ((dparg->count[idx] >= gridDim.x) ? 0 : (dparg->count[idx]+1)); }
+
+          // determine if last block
+          isLastBlockDone = (cvalue == (gridDim.x - 1));
+        }
+        // finish the reduction if last block
+        if (isLastBlockDone) {
+          reduce_t sum = dparg->init();
+          #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+          {
+            dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+            auto i = threadIdx.y * block_size_x + threadIdx.x;
+            while (i < gridDim.x) {
+              sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
+              i += block_size_x * block_size_y;
+            }
+          }
+
+          // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
+
+          printf("lastBlock: team %d: idx: %d value: %g\n", omp_get_team_num(), idx, *reinterpret_cast<double *>(&sum));
+          // write out the final reduced value
+          // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+          { // This is the main thread per team
+            dparg->result_d[idx] = sum;
+            dparg->count[idx] = 0; // set to zero for next time
+          }
+        }
+      }
+    }
+    omp_target_free(dparg, omp_get_default_device());
   }
 
   template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D()
   {
-    Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    printf("Reduction2D: launch parameter: gd %d ld %d\n", gd, ld);
+    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    {
+      // Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+      Arg *dparg = &device::get_arg<Arg>();
+      const dim3
+        blockDim=launch_param.block,
+        gridDim=launch_param.grid,
+        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
+
+      using reduce_t = typename Transformer<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+      reduce_t value = dparg->init();
+      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+
+        Transformer<Arg> t(*dparg);
+
+        auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+        auto j = threadIdx.y;
+
+        while (idx < dparg->threads.x) {
+          value = t(value, idx, j);
+          if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+        }
+      }
+      // perform final inter-block reduction and write out result
+      Transformer<Arg> t(*dparg);
+      auto j = blockIdx.y * blockDim.y;
+      // reduce<block_size_x, block_size_y>(*dparg, t, value);
+      // ../../reduce_helper.h:/reduce
+      {
+        const auto idx = 0;
+        // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
+        bool isLastBlockDone;
+        // if (threadIdx.x == 0 && threadIdx.y == 0)
+        { // This is the main thread per team
+          dparg->partial[idx * gridDim.x + blockIdx.x] = value;
+
+          // increment global block counter
+          // auto value = atomicInc(&arg.count[idx], gridDim.x);
+          unsigned int cvalue = 0;
+          unsigned int *c = &dparg->count[idx];
+          #pragma omp atomic capture
+          { cvalue = *c; *c = *c + 1; }
+          // { cvalue = dparg->count[idx]; dparg->count[idx] = ((dparg->count[idx] >= gridDim.x) ? 0 : (dparg->count[idx]+1)); }
+
+          // determine if last block
+          isLastBlockDone = (cvalue == (gridDim.x - 1));
+        }
+        // finish the reduction if last block
+        if (isLastBlockDone) {
+          reduce_t sum = dparg->init();
+          #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+          {
+            dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+            auto i = threadIdx.y * block_size_x + threadIdx.x;
+            while (i < gridDim.x) {
+              sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
+              i += block_size_x * block_size_y;
+            }
+          }
+
+          // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
+
+          // write out the final reduced value
+          // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+          { // This is the main thread per team
+            dparg->result_d[idx] = sum;
+            dparg->count[idx] = 0; // set to zero for next time
+          }
+        }
+      }
+    }
   }
 
 
   template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __forceinline__ __device__ void MultiReduction_impl(const Arg &arg)
   {
-    QUDA_RT_CONSTS;
-    using reduce_t = typename Transformer<Arg>::reduce_t;
-    Transformer<Arg> t(arg);
+    const dim3
+      blockDim=launch_param.block,
+      gridDim=launch_param.grid,
+      blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
 
-    auto idx = threadIdx.x + blockIdx.x * blockDim.x;
-    auto j = threadIdx.y + blockIdx.y * blockDim.y;
-    auto k = threadIdx.z;
+    using reduce_t = typename Transformer<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+    reduce_t value = arg.init();
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
+    {
+      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+               "omp reports: teams %d threads %d\n",
+               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+               omp_get_num_teams(), omp_get_num_threads());
+      dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
-    if (j >= arg.threads.y) return;
+      Transformer<Arg> t(arg);
 
-    reduce_t value = arg.init();
+      auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+      auto j = threadIdx.y + blockIdx.y * blockDim.y;
+      auto k = threadIdx.z;
 
-    while (idx < arg.threads.x) {
-      value = t(value, idx, j, k);
-      if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+      if (j < arg.threads.y){
+        while (idx < arg.threads.x) {
+          value = t(value, idx, j, k);
+          if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+        }
+      }
     }
-
     // perform final inter-block reduction and write out result
+    Transformer<Arg> t(arg);
+    auto j = blockIdx.y * blockDim.y;
     reduce<block_size_x, block_size_y>(arg, t, value, j);
   }
 
+  /** Workaround compiler limitations.  We have to inline parallel regions with target teams. **/
+
   template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
   {
-    MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(arg);
+    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    const int tx = arg.threads.x;
+    const int ty = arg.threads.y;
+    const int tz = arg.threads.z;
+    printf("MultiReduction: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // printf("dparg %p\n", dparg);
+    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    {
+      // MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(*dparg);
+      const dim3
+        blockDim=launch_param.block,
+        gridDim=launch_param.grid,
+        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
+
+      using reduce_t = typename Transformer<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+      reduce_t value = dparg->init();
+      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+
+        Transformer<Arg> t(*dparg);
+
+        auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+        auto j = threadIdx.y + blockIdx.y * blockDim.y;
+        auto k = threadIdx.z;
+
+        if (j < dparg->threads.y){
+          while (idx < dparg->threads.x) {
+            value = t(value, idx, j, k);
+            if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+          }
+        }
+      }
+      // perform final inter-block reduction and write out result
+      Transformer<Arg> t(*dparg);
+      auto j = blockIdx.y * blockDim.y;
+      // reduce<block_size_x, block_size_y>(*dparg, t, value, j);
+      // ../../reduce_helper.h:/reduce
+      {
+        const auto idx = j;
+        // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
+        bool isLastBlockDone;
+        // if (threadIdx.x == 0 && threadIdx.y == 0)
+        { // This is the main thread per team
+          if(blockIdx.x==0) printf("team %d: value: %g\n", omp_get_team_num(), *reinterpret_cast<double *>(&value));
+          dparg->partial[idx * gridDim.x + blockIdx.x] = value;
+
+          // increment global block counter
+          // auto value = atomicInc(&arg.count[idx], gridDim.x);
+          unsigned int cvalue = 0;
+          unsigned int *c = &dparg->count[idx];
+          #pragma omp atomic capture
+          { cvalue = *c; *c = *c + 1; }
+          // { cvalue = dparg->count[idx]; dparg->count[idx] = ((dparg->count[idx] >= gridDim.x) ? 0 : (dparg->count[idx]+1)); }
+
+          // determine if last block
+          isLastBlockDone = (cvalue == (gridDim.x - 1));
+        }
+        // finish the reduction if last block
+        if (isLastBlockDone) {
+          reduce_t sum = dparg->init();
+          #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+          {
+            dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+            auto i = threadIdx.y * block_size_x + threadIdx.x;
+            while (i < gridDim.x) {
+              sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
+              i += block_size_x * block_size_y;
+            }
+          }
+
+          // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
+
+          printf("lastBlock: team %d: idx: %d value: %g\n", omp_get_team_num(), idx, *reinterpret_cast<double *>(&sum));
+          // write out the final reduced value
+          // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+          { // This is the main thread per team
+            dparg->result_d[idx] = sum;
+            dparg->count[idx] = 0; // set to zero for next time
+          }
+        }
+      }
+    }
+    omp_target_free(dparg, omp_get_default_device());
   }
 
   template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction()
   {
-    MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
+    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    printf("MultiReduction: launch parameter: gd %d ld %d\n", gd, ld);
+    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    {
+      // MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+      Arg *dparg = &device::get_arg<Arg>();
+      const dim3
+        blockDim=launch_param.block,
+        gridDim=launch_param.grid,
+        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
+
+      using reduce_t = typename Transformer<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+      reduce_t value = dparg->init();
+      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
+      {
+        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+                 "omp reports: teams %d threads %d\n",
+                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+                 omp_get_num_teams(), omp_get_num_threads());
+        dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+
+        Transformer<Arg> t(*dparg);
+
+        auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+        auto j = threadIdx.y + blockIdx.y * blockDim.y;
+        auto k = threadIdx.z;
+
+        if (j < dparg->threads.y){
+          while (idx < dparg->threads.x) {
+            value = t(value, idx, j, k);
+            if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+          }
+        }
+      }
+      // perform final inter-block reduction and write out result
+      Transformer<Arg> t(*dparg);
+      auto j = blockIdx.y * blockDim.y;
+      // reduce<block_size_x, block_size_y>(*dparg, t, value, j);
+      // ../../reduce_helper.h:/reduce
+      {
+        const auto idx = j;
+        // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
+        bool isLastBlockDone;
+        // if (threadIdx.x == 0 && threadIdx.y == 0)
+        { // This is the main thread per team
+          dparg->partial[idx * gridDim.x + blockIdx.x] = value;
+
+          // increment global block counter
+          // auto value = atomicInc(&arg.count[idx], gridDim.x);
+          unsigned int cvalue = 0;
+          unsigned int *c = &dparg->count[idx];
+          #pragma omp atomic capture
+          { cvalue = *c; *c = *c + 1; }
+          // { cvalue = dparg->count[idx]; dparg->count[idx] = ((dparg->count[idx] >= gridDim.x) ? 0 : (dparg->count[idx]+1)); }
+
+          // determine if last block
+          isLastBlockDone = (cvalue == (gridDim.x - 1));
+        }
+        // finish the reduction if last block
+        if (isLastBlockDone) {
+          reduce_t sum = dparg->init();
+          #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+          {
+            dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+            auto i = threadIdx.y * block_size_x + threadIdx.x;
+            while (i < gridDim.x) {
+              sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
+              i += block_size_x * block_size_y;
+            }
+          }
+
+          // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
+
+          // write out the final reduced value
+          // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+          { // This is the main thread per team
+            dparg->result_d[idx] = sum;
+            dparg->count[idx] = 0; // set to zero for next time
+          }
+        }
+      }
+    }
   }
 
 }
diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index a27917333d..8cfce5454b 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -85,45 +85,7 @@ namespace quda {
     template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg)
     {
-#ifdef QUDA_BACKEND_OMPTARGET
-      ompwip("OFFLOADING...");
-      const int gd = tp.grid.x*tp.grid.y*tp.grid.z;
-      const int ld = tp.block.x*tp.block.y*tp.grid.z;
-      const int tx = arg.threads.x;
-      const int ty = arg.threads.y;
-      printf("launch parameter: gd %d ld %d tx %d ty %d\n", gd, ld, tx, ty);
-      using reduce_t = typename Transformer<Arg>::reduce_t;
-      reduce_t value = arg.init();
-
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
-
-      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-      // printf("dparg %p\n", dparg);
-      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-      Transformer<Arg> t(*dparg);
-#pragma omp target teams distribute parallel for simd collapse(2) num_teams(gd) thread_limit(ld) num_threads(ld) reduction(OMPReduce_:value)
-      for (int j = 0; j < ty; j++) {
-        for (int i = 0; i < tx; i++) {
-          value = t(value, i, j);
-        }
-      }
-      omp_target_free(dparg, omp_get_default_device());
-      int input_size = vec_length<reduce_t>::value;
-      int output_size = result.size() * vec_length<T>::value;
-      if (output_size != input_size) errorQuda("Input %d and output %d length do not match", input_size, output_size);
-
-      // copy element by element to output vector
-      for (int i = 0; i < output_size; i++) {
-        std::cerr << "launch_device: reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[" << i << "]: " << reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i] << std::endl;
-        reinterpret_cast<typename scalar<T>::type*>(result.data())[i] =
-          reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i];
-      }
-
-      if (!activeTuning() && commGlobalReduction()) {
-        // FIXME - this will break when we have non-double reduction types, e.g., double-double on the host
-        comm_allreduce_array((double*)result.data(), result.size() * sizeof(T) / sizeof(double));
-      }
-#else
+      ompwip("Multireduction on device");
 #ifdef JITIFY
       arg.launch_error = launch_jitify<Transformer, true, Arg, true>("Reduction2D", tp, stream, arg);
 #else
@@ -138,7 +100,6 @@ namespace quda {
           comm_allreduce_array((double*)result.data(), result.size() * sizeof(T) / sizeof(double));
         }
       }
-#endif
     }
 
     template <template <typename> class Transformer, typename Arg, typename T>
@@ -345,7 +306,7 @@ namespace quda {
     template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg)
     {
-      ompwip("unimplemented multireduction");
+      ompwip("Multireduction on device");
 #ifdef JITIFY
       arg.launch_error = launch_jitify<Transformer, true, Arg, true>("MultiReduction", tp, stream, arg);
 #else

From a93bf9c35076e309603cab37af332d834f17fc2b Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 10 May 2021 21:56:27 +0000
Subject: [PATCH 040/198] omp: add QUDA_RT_CONSTS in coarse_op_kernel for the
 last merge

---
 include/kernels/coarse_op_kernel.cuh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 11cfc756d8..bd0cd50de3 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -1027,6 +1027,7 @@ namespace quda {
     template <typename Arg> __device__ inline void operator()(int &parity_coarse, int &x_coarse_cb, int &parity, int &x_cb,
                                                               int &parity_c_row, int &c_row, int &c_col, const Arg &arg)
     {
+      QUDA_RT_CONSTS;
       if (arg.coarse_color_wave) {
         int parity_c_row_block_idx_z = blockDim.y*blockIdx.x + threadIdx.y;
         int c_row_block_idx_z = arg.parity_flip ? (parity_c_row_block_idx_z % arg.coarse_color_grid_z ) : (parity_c_row_block_idx_z / 2); // coarse color row index

From 789839ade0ba6e89460276103264ca47188ff50d Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 11 May 2021 01:40:58 +0000
Subject: [PATCH 041/198] omp: remove unused j in reduction_kernel.h

---
 include/targets/omptarget/reduction_kernel.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index e749843a04..3b86be77da 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -174,7 +174,6 @@ namespace quda {
       }
       // perform final inter-block reduction and write out result
       Transformer<Arg> t(*dparg);
-      auto j = blockIdx.y * blockDim.y;
       // reduce<block_size_x, block_size_y>(*dparg, t, value);
       // ../../reduce_helper.h:/reduce
       {

From 45c0f1c4616335f9ec2eb32a1da6bc19579eadd7 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 11 May 2021 16:08:34 +0000
Subject: [PATCH 042/198] omp: fix diagnostic output in reductions

---
 include/tunable_reduction.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index 8cfce5454b..70b7ff2ba0 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -85,7 +85,7 @@ namespace quda {
     template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg)
     {
-      ompwip("Multireduction on device");
+      ompwip("Reduction2D on device");
 #ifdef JITIFY
       arg.launch_error = launch_jitify<Transformer, true, Arg, true>("Reduction2D", tp, stream, arg);
 #else
@@ -306,7 +306,7 @@ namespace quda {
     template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg)
     {
-      ompwip("Multireduction on device");
+      ompwip("MultiReduction on device");
 #ifdef JITIFY
       arg.launch_error = launch_jitify<Transformer, true, Arg, true>("MultiReduction", tp, stream, arg);
 #else

From 027acc814190c6a2d6d973c954f040fd934d3278 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 12 May 2021 21:45:31 +0000
Subject: [PATCH 043/198] omp: revert blas_test debug

---
 tests/blas_test.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/blas_test.cpp b/tests/blas_test.cpp
index ca5a8412b8..a3b1c6fd9b 100644
--- a/tests/blas_test.cpp
+++ b/tests/blas_test.cpp
@@ -666,7 +666,6 @@ double test(Kernel kernel)
     break;
 
   case Kernel::norm2:
-    ompwip("*xD = *xH");
     *xD = *xH;
     error = fabs(blas::norm2(*xD) - blas::norm2(*xH)) / blas::norm2(*xH);
     break;

From 967bf343c6cd366279a0c8c59c01ab2604a0a0b3 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 12 May 2021 21:46:16 +0000
Subject: [PATCH 044/198] omp: add a few commented out printf in reduction

---
 include/targets/omptarget/reduction_kernel.h | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 3b86be77da..1e7356b920 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -78,8 +78,10 @@ namespace quda {
         auto idx = threadIdx.x + blockIdx.x * blockDim.x;
         auto j = threadIdx.y;
 
+        // if(blockIdx.x==0&&threadIdx.x==0) printf("team %d: thread %d: in value: %g\n", omp_get_team_num(), omp_get_thread_num(), *reinterpret_cast<double *>(&value));
         while (idx < dparg->threads.x) {
           value = t(value, idx, j);
+          // if(blockIdx.x==0&&threadIdx.x==0) printf("team %d: thread %d: idx %d: value: %g\n", omp_get_team_num(), omp_get_thread_num(), idx, *reinterpret_cast<double *>(&value));
           if (grid_stride) idx += blockDim.x * gridDim.x; else break;
         }
       }
@@ -93,7 +95,7 @@ namespace quda {
         bool isLastBlockDone;
         // if (threadIdx.x == 0 && threadIdx.y == 0)
         { // This is the main thread per team
-          if(blockIdx.x==0) printf("team %d: value: %g\n", omp_get_team_num(), *reinterpret_cast<double *>(&value));
+          // if(blockIdx.x==0) printf("team %d: value: %g\n", omp_get_team_num(), *reinterpret_cast<double *>(&value));
           dparg->partial[idx * gridDim.x + blockIdx.x] = value;
 
           // increment global block counter
@@ -122,7 +124,9 @@ namespace quda {
 
           // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
 
-          printf("lastBlock: team %d: idx: %d value: %g\n", omp_get_team_num(), idx, *reinterpret_cast<double *>(&sum));
+          // printf("lastBlock: team %d: idx: %d value: ", omp_get_team_num(), idx);
+          // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
+          // printf("\n");
           // write out the final reduced value
           // if (threadIdx.y * block_size_x + threadIdx.x == 0)
           { // This is the main thread per team
@@ -210,6 +214,9 @@ namespace quda {
 
           // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
 
+          // printf("lastBlock: team %d: idx: %d value: ", omp_get_team_num(), idx);
+          // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
+          // printf("\n");
           // write out the final reduced value
           // if (threadIdx.y * block_size_x + threadIdx.x == 0)
           { // This is the main thread per team
@@ -320,7 +327,7 @@ namespace quda {
         bool isLastBlockDone;
         // if (threadIdx.x == 0 && threadIdx.y == 0)
         { // This is the main thread per team
-          if(blockIdx.x==0) printf("team %d: value: %g\n", omp_get_team_num(), *reinterpret_cast<double *>(&value));
+          // if(blockIdx.x==0) printf("team %d: value: %g\n", omp_get_team_num(), *reinterpret_cast<double *>(&value));
           dparg->partial[idx * gridDim.x + blockIdx.x] = value;
 
           // increment global block counter
@@ -349,7 +356,9 @@ namespace quda {
 
           // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
 
-          printf("lastBlock: team %d: idx: %d value: %g\n", omp_get_team_num(), idx, *reinterpret_cast<double *>(&sum));
+          // printf("lastBlock: team %d: idx: %d value: ", omp_get_team_num(), idx);
+          // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
+          // printf("\n");
           // write out the final reduced value
           // if (threadIdx.y * block_size_x + threadIdx.x == 0)
           { // This is the main thread per team
@@ -441,6 +450,9 @@ namespace quda {
 
           // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
 
+          // printf("lastBlock: team %d: idx: %d value: ", omp_get_team_num(), idx);
+          // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
+          // printf("\n");
           // write out the final reduced value
           // if (threadIdx.y * block_size_x + threadIdx.x == 0)
           { // This is the main thread per team

From 80a2535b64745211ba8e947f610c11fa05647a64 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 12 May 2021 21:47:50 +0000
Subject: [PATCH 045/198] omp: copy kernel arg to stack to for arg modifying
 kernels like caxpyXmazMR

---
 include/targets/omptarget/kernel.h | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 19da2c5494..c4be6cbef9 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -38,7 +38,9 @@ namespace quda {
                  "omp reports: teams %d threads %d\n",
                  launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
                  omp_get_num_teams(), omp_get_num_threads());
-        Kernel1D_impl<Functor, Arg, grid_stride>(*dparg);
+        char buffer[sizeof(Arg)];
+        memcpy(buffer, dparg, sizeof(Arg));
+        Kernel1D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
       omp_target_free(dparg, omp_get_default_device());
     }
@@ -98,7 +100,9 @@ namespace quda {
                  "omp reports: teams %d threads %d\n",
                  launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
                  omp_get_num_teams(), omp_get_num_threads());
-        Kernel2D_impl<Functor, Arg, grid_stride>(*dparg);
+        char buffer[sizeof(Arg)];
+        memcpy(buffer, dparg, sizeof(Arg));
+        Kernel2D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
       omp_target_free(dparg, omp_get_default_device());
     }
@@ -160,7 +164,9 @@ namespace quda {
                  "omp reports: teams %d threads %d\n",
                  launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
                  omp_get_num_teams(), omp_get_num_threads());
-        Kernel3D_impl<Functor, Arg, grid_stride>(*dparg);
+        char buffer[sizeof(Arg)];
+        memcpy(buffer, dparg, sizeof(Arg));
+        Kernel3D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
       omp_target_free(dparg, omp_get_default_device());
     }

From 20600b76fdbd6221f46e2e7296e885867b5101df Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 17 May 2021 19:23:52 +0000
Subject: [PATCH 046/198] Revert "omp: workaround rng"

This reverts commit 4a31920d148ff54ea22ab5d335456ecf716c7831.
---
 lib/gauge_random.cu | 3 ---
 lib/random.cu       | 3 ---
 lib/spinor_noise.cu | 3 ---
 3 files changed, 9 deletions(-)

diff --git a/lib/gauge_random.cu b/lib/gauge_random.cu
index 89b597d15b..6bc07409b5 100644
--- a/lib/gauge_random.cu
+++ b/lib/gauge_random.cu
@@ -34,15 +34,12 @@ namespace quda {
 
     void apply(const qudaStream_t &stream)
     {
-      ompwip("skip nonfunctioning RNG.");
-/*
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       if (group) {
         launch<GaussGauge>(tp, stream, GaugeGaussArg<Float, nColor, recon, true>(U, rng.State(), sigma));
       } else {
         launch<GaussGauge>(tp, stream, GaugeGaussArg<Float, nColor, recon, false>(U, rng.State(), sigma));
       }
-*/
     }
 
     long long flops() const { return 0; }
diff --git a/lib/random.cu b/lib/random.cu
index 71f348f5d0..774f60fae7 100644
--- a/lib/random.cu
+++ b/lib/random.cu
@@ -26,11 +26,8 @@ namespace quda {
 
     void apply(const qudaStream_t &stream)
     {
-      ompwip("skip nonfunctioning RNG.");
-/*
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       launch_device<init_random>(tp, stream, rngArg(rng.State(), seed, meta));
-*/
     }
 
     long long flops() const { return 0; }
diff --git a/lib/spinor_noise.cu b/lib/spinor_noise.cu
index 675bd8d48c..37729db098 100644
--- a/lib/spinor_noise.cu
+++ b/lib/spinor_noise.cu
@@ -24,8 +24,6 @@ namespace quda {
     }
 
     void apply(const qudaStream_t &stream) {
-      ompwip("skip nonfunctioning RNG.");
-/*
       TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
       switch (type) {
       case QUDA_NOISE_GAUSS:
@@ -36,7 +34,6 @@ namespace quda {
         break;
       default: errorQuda("Noise type %d not implemented", type);
       }
-*/
     }
 
     long long flops() const { return 0; }

From 06fa0c48735e4132b3f8b2bb49e3ecc7a9e0ba94 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 17 May 2021 19:25:14 +0000
Subject: [PATCH 047/198] omp: use rocrand_mrg32k3a for omp target

---
 include/externals/rocrand/README              |  26 +
 include/externals/rocrand/rocrand_common.h    | 150 +++
 include/externals/rocrand/rocrand_mrg32k3a.h  | 506 +++++++++++
 .../rocrand/rocrand_mrg32k3a_precomputed.h    | 859 ++++++++++++++++++
 include/targets/omptarget/random_helper.h     | 103 ++-
 5 files changed, 1612 insertions(+), 32 deletions(-)
 create mode 100644 include/externals/rocrand/README
 create mode 100644 include/externals/rocrand/rocrand_common.h
 create mode 100644 include/externals/rocrand/rocrand_mrg32k3a.h
 create mode 100644 include/externals/rocrand/rocrand_mrg32k3a_precomputed.h

diff --git a/include/externals/rocrand/README b/include/externals/rocrand/README
new file mode 100644
index 0000000000..7e4a0c8676
--- /dev/null
+++ b/include/externals/rocrand/README
@@ -0,0 +1,26 @@
+This directory contains a standalone rocRAND implementation of mrg32k3a, imported from
+https://github.com/ROCmSoftwarePlatform/rocRAND/commit/56b64021e145e65d3f6bee0fef6ceae15df289b4
+with modifications by Xiao-Yong Jin.
+
+LICENSE
+
+Copyright (c) 2021, QUDA Developers
+Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/include/externals/rocrand/rocrand_common.h b/include/externals/rocrand/rocrand_common.h
new file mode 100644
index 0000000000..a6ee5e68fd
--- /dev/null
+++ b/include/externals/rocrand/rocrand_common.h
@@ -0,0 +1,150 @@
+// Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef ROCRAND_COMMON_H_
+#define ROCRAND_COMMON_H_
+
+#define ROCRAND_2POW16_INV (1.5258789e-05f)
+#define ROCRAND_2POW16_INV_2PI (1.5258789e-05f * 6.2831855f)
+#define ROCRAND_2POW32_INV (2.3283064e-10f)
+#define ROCRAND_2POW32_INV_DOUBLE (2.3283064365386963e-10)
+#define ROCRAND_2POW32_INV_2PI (2.3283064e-10f * 6.2831855f)
+#define ROCRAND_2POW53_INV_DOUBLE (1.1102230246251565e-16)
+#define ROCRAND_PI  (3.1415926f)
+#define ROCRAND_PI_DOUBLE  (3.1415926535897932)
+#define ROCRAND_2PI (6.2831855f)
+#define ROCRAND_SQRT2 (1.4142135f)
+#define ROCRAND_SQRT2_DOUBLE (1.4142135623730951)
+
+#include <math.h>
+
+#ifndef FQUALIFIERS
+#define FQUALIFIERS __forceinline__ __device__
+#endif // FQUALIFIERS
+
+#if __HIP_DEVICE_COMPILE__ && (defined(__HIP_PLATFORM_HCC__) || (defined(__HIP_PLATFORM_NVCC__) && (__CUDA_ARCH__ >= 530)))
+#define ROCRAND_HALF_MATH_SUPPORTED
+#endif
+
+namespace rocrand_device {
+namespace detail {
+
+  #if ( defined(__gfx801__) || \
+        defined(__gfx802__) || \
+        defined(__gfx803__) || \
+        defined(__gfx810__) || \
+        defined(__gfx900__) || \
+        defined(__gfx902__) || \
+        defined(__gfx904__) || \
+        defined(__gfx906__) || \
+        defined(__gfx908__) || \
+        defined(__gfx909__) )
+  #ifndef ROCRAND_ENABLE_INLINE_ASM
+    #define ROCRAND_ENABLE_INLINE_ASM
+  #endif
+#else
+  #ifdef ROCRAND_ENABLE_INLINE_ASM
+    #undef ROCRAND_ENABLE_INLINE_ASM
+    #pragma warning "Disabled inline asm, because the build target does not support it."
+  #endif
+#endif
+
+FQUALIFIERS
+unsigned long long mad_u64_u32(const unsigned int x, const unsigned int y, const unsigned long long z)
+{
+  #if defined(__HIP_PLATFORM_HCC__) && defined(__HIP_DEVICE_COMPILE__) \
+    && defined(ROCRAND_ENABLE_INLINE_ASM)
+
+    unsigned long long r;
+    unsigned long long c; // carry bits, SGPR, unused
+    // x has "r" constraint. This allows to use both VGPR and SGPR
+    // (to save VGPR) as input.
+    // y and z have "v" constraints, because only one SGPR or literal
+    // can be read by the instruction.
+    asm volatile("v_mad_u64_u32 %0, %1, %2, %3, %4"
+      : "=v"(r), "=s"(c) : "r"(x), "v"(y), "v"(z)
+    );
+    return r;
+  #elif defined(__HIP_PLATFORM_NVCC__) && defined(__HIP_DEVICE_COMPILE__) \
+        && defined(ROCRAND_ENABLE_INLINE_ASM)
+
+    unsigned long long r;
+    asm("mad.wide.u32 %0, %1, %2, %3;"
+        : "=l"(r) : "r"(x), "r"(y), "l"(z)
+    );
+    return r;
+
+  #else // host code
+
+    return static_cast<unsigned long long>(x) * static_cast<unsigned long long>(y) + z;
+
+  #endif
+}
+
+// This helps access fields of engine's internal state which
+// saves floats and doubles generated using the Box–Muller transform
+template<typename Engine>
+struct engine_boxmuller_helper
+{
+    static FQUALIFIERS
+    bool has_float(const Engine * engine)
+    {
+        return engine->m_state.boxmuller_float_state != 0;
+    }
+
+    static FQUALIFIERS
+    float get_float(Engine * engine)
+    {
+        engine->m_state.boxmuller_float_state = 0;
+        return engine->m_state.boxmuller_float;
+    }
+
+    static FQUALIFIERS
+    void save_float(Engine * engine, float f)
+    {
+        engine->m_state.boxmuller_float_state = 1;
+        engine->m_state.boxmuller_float = f;
+    }
+
+    static FQUALIFIERS
+    bool has_double(const Engine * engine)
+    {
+        return engine->m_state.boxmuller_double_state != 0;
+    }
+
+    static FQUALIFIERS
+    float get_double(Engine * engine)
+    {
+        engine->m_state.boxmuller_double_state = 0;
+        return engine->m_state.boxmuller_double;
+    }
+
+    static FQUALIFIERS
+    void save_double(Engine * engine, double d)
+    {
+        engine->m_state.boxmuller_double_state = 1;
+        engine->m_state.boxmuller_double = d;
+    }
+};
+
+} // end namespace detail
+} // end namespace rocrand_device
+
+#endif // ROCRAND_COMMON_H_
diff --git a/include/externals/rocrand/rocrand_mrg32k3a.h b/include/externals/rocrand/rocrand_mrg32k3a.h
new file mode 100644
index 0000000000..ffa9ef08e4
--- /dev/null
+++ b/include/externals/rocrand/rocrand_mrg32k3a.h
@@ -0,0 +1,506 @@
+// Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef ROCRAND_MRG32K3A_H_
+#define ROCRAND_MRG32K3A_H_
+
+#ifndef FQUALIFIERS
+#define FQUALIFIERS __forceinline__ __device__
+#endif // FQUALIFIERS_
+
+#include "rocrand_common.h"
+#include "rocrand_mrg32k3a_precomputed.h"
+
+// Thomas Bradley, Parallelisation Techniques for Random Number Generators
+// https://www.nag.co.uk/IndustryArticles/gpu_gems_article.pdf
+
+#define ROCRAND_MRG32K3A_POW32 4294967296
+#define ROCRAND_MRG32K3A_M1 4294967087
+#define ROCRAND_MRG32K3A_M1C 209
+#define ROCRAND_MRG32K3A_M2 4294944443
+#define ROCRAND_MRG32K3A_M2C 22853
+#define ROCRAND_MRG32K3A_A12 1403580
+#define ROCRAND_MRG32K3A_A13 (4294967087 -  810728)
+#define ROCRAND_MRG32K3A_A13N 810728
+#define ROCRAND_MRG32K3A_A21 527612
+#define ROCRAND_MRG32K3A_A23 (4294944443 - 1370589)
+#define ROCRAND_MRG32K3A_A23N 1370589
+#define ROCRAND_MRG32K3A_NORM_DOUBLE (2.3283065498378288e-10) // 1/ROCRAND_MRG32K3A_M1
+#define ROCRAND_MRG32K3A_UINT_NORM (1.000000048661607) // (ROCRAND_MRG32K3A_POW32 - 1)/(ROCRAND_MRG32K3A_M1 - 1)
+
+/** \rocrand_internal \addtogroup rocranddevice
+ *
+ *  @{
+ */
+ /**
+ * \def ROCRAND_MRG32K3A_DEFAULT_SEED
+ * \brief Default seed for MRG32K3A PRNG.
+ */
+ #define ROCRAND_MRG32K3A_DEFAULT_SEED 12345ULL
+ /** @} */ // end of group rocranddevice
+
+namespace rocrand_device {
+
+class mrg32k3a_engine
+{
+public:
+    struct mrg32k3a_state
+    {
+        unsigned int g1[3];
+        unsigned int g2[3];
+
+        #ifndef ROCRAND_DETAIL_MRG32K3A_BM_NOT_IN_STATE
+        // The Box–Muller transform requires two inputs to convert uniformly
+        // distributed real values [0; 1] to normally distributed real values
+        // (with mean = 0, and stddev = 1). Often user wants only one
+        // normally distributed number, to save performance and random
+        // numbers the 2nd value is saved for future requests.
+        unsigned int boxmuller_float_state; // is there a float in boxmuller_float
+        unsigned int boxmuller_double_state; // is there a double in boxmuller_double
+        float boxmuller_float; // normally distributed float
+        double boxmuller_double; // normally distributed double
+        #endif
+
+        FQUALIFIERS
+        ~mrg32k3a_state() { }
+    };
+
+    FQUALIFIERS
+    mrg32k3a_engine()
+    {
+        this->seed(ROCRAND_MRG32K3A_DEFAULT_SEED, 0, 0);
+    }
+
+    /// Initializes the internal state of the PRNG using
+    /// seed value \p seed, goes to \p subsequence -th subsequence,
+    /// and skips \p offset random numbers.
+    ///
+    /// New seed value should not be zero. If \p seed_value is equal
+    /// zero, value \p ROCRAND_MRG32K3A_DEFAULT_SEED is used instead.
+    ///
+    /// A subsequence is 2^67 numbers long.
+    FQUALIFIERS
+    mrg32k3a_engine(const unsigned long long seed,
+                    const unsigned long long subsequence,
+                    const unsigned long long offset)
+    {
+        this->seed(seed, subsequence, offset);
+    }
+
+    FQUALIFIERS
+    ~mrg32k3a_engine() { }
+
+    /// Reinitializes the internal state of the PRNG using new
+    /// seed value \p seed_value, skips \p subsequence subsequences
+    /// and \p offset random numbers.
+    ///
+    /// New seed value should not be zero. If \p seed_value is equal
+    /// zero, value \p ROCRAND_MRG32K3A_DEFAULT_SEED is used instead.
+    ///
+    /// A subsequence is 2^67 numbers long.
+    FQUALIFIERS
+    void seed(unsigned long long seed_value,
+              const unsigned long long subsequence,
+              const unsigned long long offset)
+    {
+        if(seed_value == 0)
+        {
+            seed_value = ROCRAND_MRG32K3A_DEFAULT_SEED;
+        }
+        unsigned int x = (unsigned int) seed_value ^ 0x55555555U;
+        unsigned int y = (unsigned int) ((seed_value >> 32) ^ 0xAAAAAAAAU);
+        m_state.g1[0] = mod_mul_m1(x, seed_value);
+        m_state.g1[1] = mod_mul_m1(y, seed_value);
+        m_state.g1[2] = mod_mul_m1(x, seed_value);
+        m_state.g2[0] = mod_mul_m2(y, seed_value);
+        m_state.g2[1] = mod_mul_m2(x, seed_value);
+        m_state.g2[2] = mod_mul_m2(y, seed_value);
+        this->restart(subsequence, offset);
+    }
+
+    /// Advances the internal state to skip \p offset numbers.
+    FQUALIFIERS
+    void discard(unsigned long long offset)
+    {
+        this->discard_impl(offset);
+    }
+
+    /// Advances the internal state to skip \p subsequence subsequences.
+    /// A subsequence is 2^67 numbers long.
+    FQUALIFIERS
+    void discard_subsequence(unsigned long long subsequence)
+    {
+        this->discard_subsequence_impl(subsequence);
+    }
+
+    /// Advances the internal state to skip \p sequence sequences.
+    /// A sequence is 2^127 numbers long.
+    FQUALIFIERS
+    void discard_sequence(unsigned long long sequence)
+    {
+        this->discard_sequence_impl(sequence);
+    }
+
+    FQUALIFIERS
+    void restart(const unsigned long long subsequence,
+                 const unsigned long long offset)
+    {
+        #ifndef ROCRAND_DETAIL_MRG32K3A_BM_NOT_IN_STATE
+        m_state.boxmuller_float_state = 0;
+        m_state.boxmuller_double_state = 0;
+        #endif
+        this->discard_subsequence_impl(subsequence);
+        this->discard_impl(offset);
+    }
+
+    FQUALIFIERS
+    unsigned int operator()()
+    {
+        return this->next();
+    }
+
+    // Returned value is in range [1, ROCRAND_MRG32K3A_M1],
+    // where ROCRAND_MRG32K3A_M1 < UINT_MAX
+    FQUALIFIERS
+    unsigned int next()
+    {
+        const unsigned int p1 = mod_m1(
+            detail::mad_u64_u32(
+                ROCRAND_MRG32K3A_A12,
+                m_state.g1[1],
+                detail::mad_u64_u32(
+                    ROCRAND_MRG32K3A_A13N,
+                    (ROCRAND_MRG32K3A_M1 - m_state.g1[0]),
+                    0
+                )
+            )
+        );
+
+        m_state.g1[0] = m_state.g1[1]; m_state.g1[1] = m_state.g1[2];
+        m_state.g1[2] = p1;
+
+        const unsigned int p2 = mod_m2(
+            detail::mad_u64_u32(
+                ROCRAND_MRG32K3A_A21,
+                m_state.g2[2],
+                detail::mad_u64_u32(
+                    ROCRAND_MRG32K3A_A23N,
+                    (ROCRAND_MRG32K3A_M2 - m_state.g2[0]),
+                    0
+                )
+            )
+        );
+
+        m_state.g2[0] = m_state.g2[1]; m_state.g2[1] = m_state.g2[2];
+        m_state.g2[2] = p2;
+
+        return (p1 - p2) + (p1 <= p2 ? ROCRAND_MRG32K3A_M1 : 0);
+    }
+
+protected:
+    // Advances the internal state to skip \p offset numbers.
+    // DOES NOT CALCULATE NEW ULONGLONG
+    FQUALIFIERS
+    void discard_impl(unsigned long long offset)
+    {
+        discard_state(offset);
+    }
+
+    // DOES NOT CALCULATE NEW ULONGLONG
+    FQUALIFIERS
+    void discard_subsequence_impl(unsigned long long subsequence)
+    {
+        int i = 0;
+
+        while(subsequence > 0) {
+            if (subsequence & 1) {
+                #if defined(__HIP_DEVICE_COMPILE__)
+                mod_mat_vec_m1(d_A1P67 + i, m_state.g1);
+                mod_mat_vec_m2(d_A2P67 + i, m_state.g2);
+                #else
+                mod_mat_vec_m1(h_A1P67 + i, m_state.g1);
+                mod_mat_vec_m2(h_A2P67 + i, m_state.g2);
+                #endif
+            }
+            subsequence >>= 1;
+            i += 9;
+        }
+    }
+
+    // DOES NOT CALCULATE NEW ULONGLONG
+    FQUALIFIERS
+    void discard_sequence_impl(unsigned long long sequence)
+    {
+        int i = 0;
+
+        while(sequence > 0) {
+            if (sequence & 1) {
+                #if defined(__HIP_DEVICE_COMPILE__)
+                mod_mat_vec_m1(d_A1P127 + i, m_state.g1);
+                mod_mat_vec_m2(d_A2P127 + i, m_state.g2);
+                #else
+                mod_mat_vec_m1(h_A1P127 + i, m_state.g1);
+                mod_mat_vec_m2(h_A2P127 + i, m_state.g2);
+                #endif
+            }
+            sequence >>= 1;
+            i += 9;
+        }
+    }
+
+    // Advances the internal state by offset times.
+    // DOES NOT CALCULATE NEW ULONGLONG
+    FQUALIFIERS
+    void discard_state(unsigned long long offset)
+    {
+        int i = 0;
+
+        while(offset > 0) {
+            if (offset & 1) {
+                #if defined(__HIP_DEVICE_COMPILE__)
+                mod_mat_vec_m1(d_A1 + i, m_state.g1);
+                mod_mat_vec_m2(d_A2 + i, m_state.g2);
+                #else
+                mod_mat_vec_m1(h_A1 + i, m_state.g1);
+                mod_mat_vec_m2(h_A2 + i, m_state.g2);
+                #endif
+            }
+            offset >>= 1;
+            i += 9;
+        }
+    }
+
+    // Advances the internal state to the next state
+    // DOES NOT CALCULATE NEW ULONGLONG
+    FQUALIFIERS
+    void discard_state()
+    {
+        discard_state(1);
+    }
+
+private:
+    FQUALIFIERS
+    void mod_mat_vec_m1(const unsigned long long * A,
+                        unsigned int * s)
+    {
+        unsigned long long x[3];
+
+        x[0] = mod_m1(mod_m1(A[0] * s[0])
+                    + mod_m1(A[1] * s[1])
+                    + mod_m1(A[2] * s[2]));
+
+        x[1] = mod_m1(mod_m1(A[3] * s[0])
+                    + mod_m1(A[4] * s[1])
+                    + mod_m1(A[5] * s[2]));
+
+        x[2] = mod_m1(mod_m1(A[6] * s[0])
+                    + mod_m1(A[7] * s[1])
+                    + mod_m1(A[8] * s[2]));
+
+        s[0] = x[0];
+        s[1] = x[1];
+        s[2] = x[2];
+    }
+
+    FQUALIFIERS
+    void mod_mat_vec_m2(const unsigned long long * A,
+                        unsigned int * s)
+    {
+        unsigned long long x[3];
+
+        x[0] = mod_m2(mod_m2(A[0] * s[0])
+                    + mod_m2(A[1] * s[1])
+                    + mod_m2(A[2] * s[2]));
+
+        x[1] = mod_m2(mod_m2(A[3] * s[0])
+                    + mod_m2(A[4] * s[1])
+                    + mod_m2(A[5] * s[2]));
+
+        x[2] = mod_m2(mod_m2(A[6] * s[0])
+                    + mod_m2(A[7] * s[1])
+                    + mod_m2(A[8] * s[2]));
+
+        s[0] = x[0];
+        s[1] = x[1];
+        s[2] = x[2];
+    }
+
+    FQUALIFIERS
+    unsigned long long mod_mul_m1(unsigned int i,
+                                  unsigned long long j)
+    {
+        long long hi, lo, temp1, temp2;
+
+        hi = i / 131072;
+        lo = i - (hi * 131072);
+        temp1 = mod_m1(hi * j) * 131072;
+        temp2 = mod_m1(lo * j);
+        lo = mod_m1(temp1 + temp2);
+
+        if (lo < 0)
+            lo += ROCRAND_MRG32K3A_M1;
+        return lo;
+    }
+
+    FQUALIFIERS
+    unsigned long long mod_m1(unsigned long long p)
+    {
+        p = detail::mad_u64_u32(ROCRAND_MRG32K3A_M1C, (p >> 32), p & (ROCRAND_MRG32K3A_POW32 - 1));
+        if (p >= ROCRAND_MRG32K3A_M1)
+            p -= ROCRAND_MRG32K3A_M1;
+
+        return p;
+    }
+
+    FQUALIFIERS
+    unsigned long long mod_mul_m2(unsigned int i,
+                                  unsigned long long j)
+    {
+        long long hi, lo, temp1, temp2;
+
+        hi = i / 131072;
+        lo = i - (hi * 131072);
+        temp1 = mod_m2(hi * j) * 131072;
+        temp2 = mod_m2(lo * j);
+        lo = mod_m2(temp1 + temp2);
+
+        if (lo < 0)
+            lo += ROCRAND_MRG32K3A_M2;
+        return lo;
+    }
+
+    FQUALIFIERS
+    unsigned long long mod_m2(unsigned long long p)
+    {
+        p = detail::mad_u64_u32(ROCRAND_MRG32K3A_M2C, (p >> 32), p & (ROCRAND_MRG32K3A_POW32 - 1));
+        p = detail::mad_u64_u32(ROCRAND_MRG32K3A_M2C, (p >> 32), p & (ROCRAND_MRG32K3A_POW32 - 1));
+        if (p >= ROCRAND_MRG32K3A_M2)
+            p -= ROCRAND_MRG32K3A_M2;
+
+        return p;
+    }
+
+protected:
+    // State
+    mrg32k3a_state m_state;
+
+    #ifndef ROCRAND_DETAIL_MRG32K3A_BM_NOT_IN_STATE
+    friend struct detail::engine_boxmuller_helper<mrg32k3a_engine>;
+    #endif
+
+}; // mrg32k3a_engine class
+
+} // end namespace rocrand_device
+
+/** \rocrand_internal \addtogroup rocranddevice
+ *
+ *  @{
+ */
+
+/// \cond ROCRAND_KERNEL_DOCS_TYPEDEFS
+typedef rocrand_device::mrg32k3a_engine rocrand_state_mrg32k3a;
+/// \endcond
+
+/**
+ * \brief Initializes MRG32K3A state.
+ *
+ * Initializes the MRG32K3A generator \p state with the given
+ * \p seed, \p subsequence, and \p offset.
+ *
+ * \param seed - Value to use as a seed
+ * \param subsequence - Subsequence to start at
+ * \param offset - Absolute offset into subsequence
+ * \param state - Pointer to state to initialize
+ */
+FQUALIFIERS
+void rocrand_init(const unsigned long long seed,
+                  const unsigned long long subsequence,
+                  const unsigned long long offset,
+                  rocrand_state_mrg32k3a * state)
+{
+    *state = rocrand_state_mrg32k3a(seed, subsequence, offset);
+}
+
+/**
+ * \brief Returns uniformly distributed random <tt>unsigned int</tt> value
+ * from [0; 2^32 - 1] range.
+ *
+ * Generates and returns uniformly distributed random <tt>unsigned int</tt>
+ * value from [0; 2^32 - 1] range using MRG32K3A generator in \p state.
+ * State is incremented by one position.
+ *
+ * \param state - Pointer to a state to use
+ *
+ * \return Pseudorandom value (32-bit) as an <tt>unsigned int</tt>
+ */
+FQUALIFIERS
+unsigned int rocrand(rocrand_state_mrg32k3a * state)
+{
+    // next() in [1, ROCRAND_MRG32K3A_M1]
+    return static_cast<unsigned int>((state->next() - 1) * ROCRAND_MRG32K3A_UINT_NORM);
+}
+
+/**
+ * \brief Updates MRG32K3A state to skip ahead by \p offset elements.
+ *
+ * Updates the MRG32K3A state in \p state to skip ahead by \p offset elements.
+ *
+ * \param offset - Number of elements to skip
+ * \param state - Pointer to state to update
+ */
+FQUALIFIERS
+void skipahead(unsigned long long offset, rocrand_state_mrg32k3a * state)
+{
+    return state->discard(offset);
+}
+
+/**
+ * \brief Updates MRG32K3A state to skip ahead by \p subsequence subsequences.
+ *
+ * Updates the MRG32K3A state in \p state to skip ahead by \p subsequence subsequences.
+ * Each subsequence is 2^67 numbers long.
+ *
+ * \param subsequence - Number of subsequences to skip
+ * \param state - Pointer to state to update
+ */
+FQUALIFIERS
+void skipahead_subsequence(unsigned long long subsequence, rocrand_state_mrg32k3a * state)
+{
+    return state->discard_subsequence(subsequence);
+}
+
+/**
+ * \brief Updates MRG32K3A state to skip ahead by \p sequence sequences.
+ *
+ * Updates the MRG32K3A state in \p state to skip ahead by \p sequence sequences.
+ * Each sequence is 2^127 numbers long.
+ *
+ * \param sequence - Number of sequences to skip
+ * \param state - Pointer to state to update
+ */
+FQUALIFIERS
+void skipahead_sequence(unsigned long long sequence, rocrand_state_mrg32k3a * state)
+{
+    return state->discard_sequence(sequence);
+}
+
+#endif // ROCRAND_MRG32K3A_H_
+
+/** @} */ // end of group rocranddevice
diff --git a/include/externals/rocrand/rocrand_mrg32k3a_precomputed.h b/include/externals/rocrand/rocrand_mrg32k3a_precomputed.h
new file mode 100644
index 0000000000..25ad28813f
--- /dev/null
+++ b/include/externals/rocrand/rocrand_mrg32k3a_precomputed.h
@@ -0,0 +1,859 @@
+// Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef ROCRAND_MRG32K3A_PRECOMPUTED_H_
+#define ROCRAND_MRG32K3A_PRECOMPUTED_H_
+
+// Auto-generated file. Do not edit!
+// Generated by tools/mrg32k3a_precomputed_generator
+
+#define MRG323A_DIM 64
+#define MRG323A_N 576
+
+static constexpr __device__ unsigned long long d_A1[MRG323A_N] =
+    {
+        0, 1, 0, 0, 0, 1, 4294156359, 1403580, 0,
+        0, 0, 1, 4294156359, 1403580, 0, 0, 4294156359, 1403580,
+        0, 4294156359, 1403580, 244671815, 2941890554, 4294156359, 149925673, 489343630, 2941890554,
+        1527363550, 2758233149, 1831234280, 4072640363, 939574583, 2758233149, 2064391165, 3228066636, 939574583,
+        736416029, 2961816100, 342112271, 387300998, 1062452522, 2961816100, 2955879160, 340793741, 1062452522,
+        1243502014, 2218748291, 1709215645, 2019641772, 3847560959, 2218748291, 3866010231, 2305448679, 3847560959,
+        3241775219, 3453352062, 3721871040, 4062454730, 3015754, 3453352062, 919711945, 613405362, 3015754,
+        1955221006, 1414472808, 1746037714, 3653507277, 1644962013, 1414472808, 3501544776, 2336229602, 1644962013,
+        1170096663, 49135452, 3441537107, 1857945175, 1649398389, 49135452, 333002869, 3109147376, 1649398389,
+        2299034194, 2297111910, 862649200, 1399961132, 996706937, 2297111910, 3439056503, 1481993076, 996706937,
+        4146310528, 458782589, 1007330283, 4241015765, 3979619964, 458782589, 553886495, 2186897562, 3979619964,
+        3630027893, 2130448350, 292773857, 1392525159, 1299285967, 2130448350, 2589171163, 1217405758, 1299285967,
+        892409263, 1999175811, 2979225418, 1996163538, 2148702503, 1999175811, 3922720782, 103819730, 2148702503,
+        1586003016, 2114210471, 3240775579, 2777288607, 1400478398, 2114210471, 3018215420, 535326008, 1400478398,
+        2188531273, 1783231160, 3576659343, 1908318389, 379210133, 1783231160, 554369329, 250053591, 379210133,
+        4022841636, 3951951872, 2143424240, 1046219306, 1591992468, 3951951872, 1510277444, 381333958, 1591992468,
+        2256493727, 3715182130, 642697923, 3615342722, 3975008370, 3715182130, 2405650329, 754337639, 3975008370,
+        1286664224, 627406673, 963516608, 1541344588, 460768826, 627406673, 1089892553, 2717717970, 460768826,
+        2956342842, 3471097641, 2353092905, 2996150472, 420480221, 3471097641, 2221681883, 372736411, 420480221,
+        420492906, 153526651, 3499730988, 2662640502, 3278195133, 153526651, 4086436419, 2510762118, 3278195133,
+        3310184147, 2228376089, 823220763, 3992771814, 1693168425, 2228376089, 2295790366, 1401872772, 1693168425,
+        2529428830, 1497104068, 4253248635, 3746310018, 630867741, 1497104068, 627043435, 721725795, 630867741,
+        2571072593, 3039669025, 1591031831, 526054481, 661344445, 3039669025, 4246010312, 735391270, 661344445,
+        1847312821, 4042890210, 4241772463, 606605705, 2644799309, 4042890210, 2658402822, 1342278931, 2644799309,
+        2409846784, 1096138313, 1416249993, 1501878241, 138013862, 1096138313, 1617749306, 1975136163, 138013862,
+        599453422, 73950522, 2965395603, 55354701, 3855242202, 73950522, 3981734504, 3354399019, 3855242202,
+        4271076381, 813410089, 3461955319, 1044920137, 3029005516, 813410089, 3501837362, 3321539504, 3029005516,
+        3058183515, 941408572, 1783998098, 1546486080, 4116985007, 941408572, 2247500745, 1460625377, 4116985007,
+        4216782514, 3352801941, 2315095646, 639029973, 94451952, 3352801941, 1242898773, 3964593332, 94451952,
+        2264905138, 1926285644, 1108147171, 2390706911, 385258225, 1926285644, 3569882325, 3728744670, 385258225,
+        270679073, 1065683096, 2992662885, 4196917281, 2886425156, 1065683096, 749134119, 1849148167, 2886425156,
+        35689930, 1378151623, 951629713, 673810920, 948843427, 1378151623, 3808868984, 927013635, 948843427,
+        1891490872, 1130489594, 3734864133, 1457450350, 3362920032, 1130489594, 638998846, 1401175590, 3362920032,
+        2254459023, 2384691454, 1730098031, 2844861718, 1807491073, 2384691454, 351423668, 1570264155, 1807491073,
+        3047429268, 4245359555, 2449575498, 1797081212, 1237196477, 4245359555, 143400628, 3663731096, 1237196477,
+        3313321106, 4263819658, 1047529624, 3719941673, 3155049403, 4263819658, 1981313839, 4281524426, 3155049403,
+        2005252417, 3263186729, 1535805957, 2951515865, 1729281525, 3263186729, 1141249417, 2268963059, 1729281525,
+        2367065164, 83908466, 4294308508, 1352516724, 1416676049, 83908466, 1040867745, 1304732377, 1416676049,
+        3214147257, 1434230503, 2944821434, 2753040912, 4041536918, 1434230503, 1317260239, 338830578, 4041536918,
+        300628476, 2054743463, 1499597869, 1762244284, 1422043015, 2054743463, 3581125669, 1207561803, 1422043015,
+        4171745404, 4064983592, 1934508265, 3049723261, 1744636487, 4064983592, 947753516, 3952135907, 1744636487,
+        1625369148, 3577024659, 2778677259, 1729967818, 1049600974, 3577024659, 2089137344, 1569794605, 1049600974,
+        1373068765, 3958611830, 569117280, 410042396, 3551255470, 3958611830, 869476379, 1680625376, 3551255470,
+        2108618602, 2543645250, 913717833, 2111984988, 1012482542, 2543645250, 2545745615, 3141042890, 1012482542,
+        1157293598, 584852249, 2272893205, 1631801979, 3013855247, 584852249, 3977310441, 82049263, 3013855247,
+        3580234334, 3137526662, 2403875621, 3580869206, 3670086228, 3137526662, 656744553, 1764904195, 3670086228,
+        2792496861, 3634185196, 3887031679, 3601823850, 3464838365, 3634185196, 3136165138, 2842987937, 3464838365,
+        1362557480, 3230022138, 4278720212, 3427386258, 3848976950, 3230022138, 2109817045, 2441486578, 3848976950,
+        1198519135, 2007945401, 3868481, 3335076429, 2082683147, 2007945401, 2341088247, 888193479, 2082683147,
+        3473925387, 3193380570, 565138859, 307060547, 782210925, 3193380570, 167617770, 2180014252, 782210925,
+        3811588895, 3303532086, 2766583698, 908630605, 2665400165, 3303532086, 2499994113, 3316180851, 2665400165,
+        4288926968, 3033075037, 1505732852, 1531633406, 645804125, 3033075037, 2942690261, 2205365640, 645804125,
+        3976196483, 3651411522, 1652430357, 1690405883, 1294990760, 3651411522, 209339647, 3088484327, 1294990760,
+        3171589548, 2291131070, 2093793287, 2997812074, 4093879780, 2291131070, 3255666800, 858124816, 4093879780,
+        4113016361, 2999667479, 3995043314, 1333973326, 4007774239, 2999667479, 3322921863, 4278103786, 4007774239,
+        925786347, 2109676036, 1879981040, 1701566570, 1489702270, 2109676036, 2719807628, 158549605, 1489702270,
+        2255405265, 3460246357, 218033453, 2135115875, 359516994, 3460246357, 3568862459, 3114762683, 359516994,
+        773148471, 4117539411, 3073622315, 3807175775, 186466108, 4117539411, 2842197411, 651334129, 186466108,
+        615242951, 1475251263, 3586439101, 1693917167, 3058812486, 1475251263, 568701600, 1164226398, 3058812486,
+        1632636204, 15370275, 2061555515, 4187505695, 1741164221, 15370275, 2882176274, 3978412194, 1741164221,
+        3446066703, 344820524, 74213775, 1008543583, 2579620192, 344820524, 3753911358, 1538453821, 2579620192,
+        3600859892, 1269921024, 4069458760, 2050939727, 2222725697, 1269921024, 3208347646, 690898125, 2222725697,
+        599407451, 2806239788, 1742216102, 975123999, 764869161, 2806239788, 2729710367, 1845257036, 764869161,
+        967330218, 3464884028, 3444447102, 580449578, 1343714307, 3464884028, 1775329096, 4027221761, 1343714307,
+
+    };
+
+static constexpr unsigned long long h_A1[MRG323A_N] =
+    {
+        0, 1, 0, 0, 0, 1, 4294156359, 1403580, 0,
+        0, 0, 1, 4294156359, 1403580, 0, 0, 4294156359, 1403580,
+        0, 4294156359, 1403580, 244671815, 2941890554, 4294156359, 149925673, 489343630, 2941890554,
+        1527363550, 2758233149, 1831234280, 4072640363, 939574583, 2758233149, 2064391165, 3228066636, 939574583,
+        736416029, 2961816100, 342112271, 387300998, 1062452522, 2961816100, 2955879160, 340793741, 1062452522,
+        1243502014, 2218748291, 1709215645, 2019641772, 3847560959, 2218748291, 3866010231, 2305448679, 3847560959,
+        3241775219, 3453352062, 3721871040, 4062454730, 3015754, 3453352062, 919711945, 613405362, 3015754,
+        1955221006, 1414472808, 1746037714, 3653507277, 1644962013, 1414472808, 3501544776, 2336229602, 1644962013,
+        1170096663, 49135452, 3441537107, 1857945175, 1649398389, 49135452, 333002869, 3109147376, 1649398389,
+        2299034194, 2297111910, 862649200, 1399961132, 996706937, 2297111910, 3439056503, 1481993076, 996706937,
+        4146310528, 458782589, 1007330283, 4241015765, 3979619964, 458782589, 553886495, 2186897562, 3979619964,
+        3630027893, 2130448350, 292773857, 1392525159, 1299285967, 2130448350, 2589171163, 1217405758, 1299285967,
+        892409263, 1999175811, 2979225418, 1996163538, 2148702503, 1999175811, 3922720782, 103819730, 2148702503,
+        1586003016, 2114210471, 3240775579, 2777288607, 1400478398, 2114210471, 3018215420, 535326008, 1400478398,
+        2188531273, 1783231160, 3576659343, 1908318389, 379210133, 1783231160, 554369329, 250053591, 379210133,
+        4022841636, 3951951872, 2143424240, 1046219306, 1591992468, 3951951872, 1510277444, 381333958, 1591992468,
+        2256493727, 3715182130, 642697923, 3615342722, 3975008370, 3715182130, 2405650329, 754337639, 3975008370,
+        1286664224, 627406673, 963516608, 1541344588, 460768826, 627406673, 1089892553, 2717717970, 460768826,
+        2956342842, 3471097641, 2353092905, 2996150472, 420480221, 3471097641, 2221681883, 372736411, 420480221,
+        420492906, 153526651, 3499730988, 2662640502, 3278195133, 153526651, 4086436419, 2510762118, 3278195133,
+        3310184147, 2228376089, 823220763, 3992771814, 1693168425, 2228376089, 2295790366, 1401872772, 1693168425,
+        2529428830, 1497104068, 4253248635, 3746310018, 630867741, 1497104068, 627043435, 721725795, 630867741,
+        2571072593, 3039669025, 1591031831, 526054481, 661344445, 3039669025, 4246010312, 735391270, 661344445,
+        1847312821, 4042890210, 4241772463, 606605705, 2644799309, 4042890210, 2658402822, 1342278931, 2644799309,
+        2409846784, 1096138313, 1416249993, 1501878241, 138013862, 1096138313, 1617749306, 1975136163, 138013862,
+        599453422, 73950522, 2965395603, 55354701, 3855242202, 73950522, 3981734504, 3354399019, 3855242202,
+        4271076381, 813410089, 3461955319, 1044920137, 3029005516, 813410089, 3501837362, 3321539504, 3029005516,
+        3058183515, 941408572, 1783998098, 1546486080, 4116985007, 941408572, 2247500745, 1460625377, 4116985007,
+        4216782514, 3352801941, 2315095646, 639029973, 94451952, 3352801941, 1242898773, 3964593332, 94451952,
+        2264905138, 1926285644, 1108147171, 2390706911, 385258225, 1926285644, 3569882325, 3728744670, 385258225,
+        270679073, 1065683096, 2992662885, 4196917281, 2886425156, 1065683096, 749134119, 1849148167, 2886425156,
+        35689930, 1378151623, 951629713, 673810920, 948843427, 1378151623, 3808868984, 927013635, 948843427,
+        1891490872, 1130489594, 3734864133, 1457450350, 3362920032, 1130489594, 638998846, 1401175590, 3362920032,
+        2254459023, 2384691454, 1730098031, 2844861718, 1807491073, 2384691454, 351423668, 1570264155, 1807491073,
+        3047429268, 4245359555, 2449575498, 1797081212, 1237196477, 4245359555, 143400628, 3663731096, 1237196477,
+        3313321106, 4263819658, 1047529624, 3719941673, 3155049403, 4263819658, 1981313839, 4281524426, 3155049403,
+        2005252417, 3263186729, 1535805957, 2951515865, 1729281525, 3263186729, 1141249417, 2268963059, 1729281525,
+        2367065164, 83908466, 4294308508, 1352516724, 1416676049, 83908466, 1040867745, 1304732377, 1416676049,
+        3214147257, 1434230503, 2944821434, 2753040912, 4041536918, 1434230503, 1317260239, 338830578, 4041536918,
+        300628476, 2054743463, 1499597869, 1762244284, 1422043015, 2054743463, 3581125669, 1207561803, 1422043015,
+        4171745404, 4064983592, 1934508265, 3049723261, 1744636487, 4064983592, 947753516, 3952135907, 1744636487,
+        1625369148, 3577024659, 2778677259, 1729967818, 1049600974, 3577024659, 2089137344, 1569794605, 1049600974,
+        1373068765, 3958611830, 569117280, 410042396, 3551255470, 3958611830, 869476379, 1680625376, 3551255470,
+        2108618602, 2543645250, 913717833, 2111984988, 1012482542, 2543645250, 2545745615, 3141042890, 1012482542,
+        1157293598, 584852249, 2272893205, 1631801979, 3013855247, 584852249, 3977310441, 82049263, 3013855247,
+        3580234334, 3137526662, 2403875621, 3580869206, 3670086228, 3137526662, 656744553, 1764904195, 3670086228,
+        2792496861, 3634185196, 3887031679, 3601823850, 3464838365, 3634185196, 3136165138, 2842987937, 3464838365,
+        1362557480, 3230022138, 4278720212, 3427386258, 3848976950, 3230022138, 2109817045, 2441486578, 3848976950,
+        1198519135, 2007945401, 3868481, 3335076429, 2082683147, 2007945401, 2341088247, 888193479, 2082683147,
+        3473925387, 3193380570, 565138859, 307060547, 782210925, 3193380570, 167617770, 2180014252, 782210925,
+        3811588895, 3303532086, 2766583698, 908630605, 2665400165, 3303532086, 2499994113, 3316180851, 2665400165,
+        4288926968, 3033075037, 1505732852, 1531633406, 645804125, 3033075037, 2942690261, 2205365640, 645804125,
+        3976196483, 3651411522, 1652430357, 1690405883, 1294990760, 3651411522, 209339647, 3088484327, 1294990760,
+        3171589548, 2291131070, 2093793287, 2997812074, 4093879780, 2291131070, 3255666800, 858124816, 4093879780,
+        4113016361, 2999667479, 3995043314, 1333973326, 4007774239, 2999667479, 3322921863, 4278103786, 4007774239,
+        925786347, 2109676036, 1879981040, 1701566570, 1489702270, 2109676036, 2719807628, 158549605, 1489702270,
+        2255405265, 3460246357, 218033453, 2135115875, 359516994, 3460246357, 3568862459, 3114762683, 359516994,
+        773148471, 4117539411, 3073622315, 3807175775, 186466108, 4117539411, 2842197411, 651334129, 186466108,
+        615242951, 1475251263, 3586439101, 1693917167, 3058812486, 1475251263, 568701600, 1164226398, 3058812486,
+        1632636204, 15370275, 2061555515, 4187505695, 1741164221, 15370275, 2882176274, 3978412194, 1741164221,
+        3446066703, 344820524, 74213775, 1008543583, 2579620192, 344820524, 3753911358, 1538453821, 2579620192,
+        3600859892, 1269921024, 4069458760, 2050939727, 2222725697, 1269921024, 3208347646, 690898125, 2222725697,
+        599407451, 2806239788, 1742216102, 975123999, 764869161, 2806239788, 2729710367, 1845257036, 764869161,
+        967330218, 3464884028, 3444447102, 580449578, 1343714307, 3464884028, 1775329096, 4027221761, 1343714307,
+
+    };
+
+static constexpr __device__ unsigned long long d_A2[MRG323A_N] =
+    {
+        0, 1, 0, 0, 0, 1, 4293573854, 0, 527612,
+        0, 0, 1, 4293573854, 0, 527612, 2706407399, 4293573854, 3497978192,
+        2706407399, 4293573854, 3497978192, 1431525864, 2706407399, 3281754271, 97673890, 1431525864, 1673476130,
+        3405842137, 2680076935, 893509979, 4035147174, 3405842137, 3280220074, 2623373296, 4035147174, 361718588,
+        818368950, 3790774567, 3542344109, 1817134745, 818368950, 3321940838, 3493477402, 1817134745, 2854655037,
+        498682467, 2928649385, 811441367, 1777037472, 498682467, 479207863, 3058260025, 1777037472, 1528225099,
+        3893311647, 3140922085, 64039185, 82107183, 3893311647, 2655465224, 1674879036, 82107183, 1089381262,
+        28639152, 3496041927, 2231910770, 3174683233, 28639152, 2828785870, 3681140872, 3174683233, 3910194649,
+        1463826069, 300842059, 3313769518, 1799677538, 1463826069, 3174861078, 1882279394, 1799677538, 3509975160,
+        2092194020, 184076987, 2202401252, 3103629604, 2092194020, 3409560232, 4257445059, 3103629604, 2390202783,
+        812917091, 2574011276, 4168802395, 209817750, 812917091, 2974870628, 3238802184, 209817750, 3692836406,
+        477309738, 3314523413, 3442242150, 2755731404, 477309738, 2782713347, 1606221490, 2755731404, 1033463096,
+        2155469603, 3326516116, 3843369786, 288604458, 2155469603, 571673571, 1501677614, 288604458, 2928213494,
+        2082469029, 749754403, 3963963316, 2764859700, 2082469029, 3576428059, 2840894706, 2764859700, 1782279859,
+        3760163766, 1041986082, 1799196192, 1022129134, 3760163766, 1332558840, 276873446, 1022129134, 3979423632,
+        1021313167, 1312544548, 1716381787, 3037868518, 1021313167, 199085085, 2582787611, 3037868518, 3539882179,
+        2569413030, 1631336015, 2594942403, 1030618503, 2569413030, 3467650326, 1998739584, 1030618503, 3174552073,
+        2334639309, 3114094203, 601680947, 2110199318, 2334639309, 678342865, 1649523168, 2110199318, 2154948056,
+        563657176, 191330473, 1641595774, 780563537, 563657176, 3029522338, 2037330914, 780563537, 2084602709,
+        3414769923, 1968799026, 2238126504, 832866376, 3414769923, 3754780168, 2165145850, 832866376, 1594768331,
+        1646861218, 2317984620, 2301581548, 2672536210, 1646861218, 359763062, 2391283983, 2672536210, 1885870777,
+        841254072, 3765813448, 1635365181, 2013240130, 841254072, 605925849, 3743932305, 2013240130, 400681955,
+        1930213004, 2072952279, 3077694794, 3579956569, 1930213004, 2478539210, 1960229502, 3579956569, 1455652656,
+        1097613522, 1784540933, 1194440107, 321747515, 1097613522, 1225209584, 74521379, 321747515, 4288531000,
+        143812745, 3254530816, 3514348856, 769295000, 143812745, 2468210728, 1927161272, 769295000, 522705580,
+        2692035063, 2596905012, 1643240704, 1103432342, 2692035063, 1446182108, 4161111774, 1103432342, 3076435551,
+        2375319030, 1391532370, 3742334018, 1202100604, 2375319030, 4098434768, 2327872488, 1202100604, 1471526950,
+        4269164791, 2795313144, 2507855960, 4245372460, 4269164791, 4094914553, 3873219634, 4245372460, 1473695507,
+        513890845, 1208902926, 2870530442, 1984873167, 513890845, 1257532340, 1212627640, 1984873167, 2354363842,
+        1848364568, 1552116673, 3496528455, 4160778291, 1848364568, 141769900, 3611019106, 4160778291, 596424080,
+        364070020, 3520039729, 837362349, 2544671570, 364070020, 2188646679, 163978331, 2544671570, 672947816,
+        1192700714, 3968150021, 298357363, 635565666, 1192700714, 2589432341, 2548654227, 635565666, 3531570992,
+        2709640529, 676525399, 875361870, 1315499519, 2709640529, 3842690720, 3300994644, 1315499519, 2446760804,
+        2742149264, 1410604392, 3032350755, 3774935330, 2742149264, 597633965, 4085935803, 3774935330, 3952463556,
+        3878579563, 845297523, 1721916511, 2077922420, 3878579563, 3651360351, 2177255734, 2077922420, 3791239282,
+        1570315355, 4252790045, 3522351060, 2324624266, 1570315355, 3594939336, 1725087354, 2324624266, 1338343327,
+        2305761589, 381933244, 3663579047, 1355307047, 2305761589, 313617972, 992174375, 1355307047, 3881593435,
+        1667857811, 1564715297, 2263851601, 3791771273, 1667857811, 4196134923, 3347975047, 3791771273, 615040705,
+        4093947334, 3454015638, 2815567716, 4261953004, 4093947334, 3973733876, 2979573134, 4261953004, 3757047667,
+        250120061, 570149551, 1513430926, 3178644752, 250120061, 1701869032, 4172515680, 3178644752, 4213855850,
+        4158106802, 3062358456, 1815738463, 1379176112, 4158106802, 3926509890, 2842564878, 1379176112, 2852219546,
+        931848746, 256263523, 2633569246, 3284646837, 931848746, 2567084715, 415258465, 3284646837, 2017565947,
+        1648005210, 1032291296, 3987397422, 1831496020, 1648005210, 2829448427, 1821082272, 1831496020, 2917140265,
+        4161327077, 489964129, 3870847744, 1669447863, 4161327077, 4292947198, 1522417114, 1669447863, 2652286672,
+        1270934555, 3136631324, 505612043, 2981474723, 1270934555, 2528619024, 625182639, 2981474723, 1008985039,
+        280996820, 143706137, 3013099060, 1797675893, 280996820, 3743985508, 1123794455, 1797675893, 2460119169,
+        919218027, 4154920441, 1125672685, 3933041881, 919218027, 474242849, 564891116, 3933041881, 2263904321,
+        2920112852, 1965329198, 1177141043, 2135250851, 2920112852, 969184056, 296035385, 2135250851, 4267827987,
+        1481142942, 4120754772, 1088557292, 265491023, 1481142942, 2860005744, 301796252, 265491023, 1935975979,
+        2111859033, 2813610100, 1001476468, 73849832, 2111859033, 3980799998, 3330206241, 73849832, 1933943506,
+        1781286360, 3661231931, 3509383709, 2753158871, 1781286360, 3119883109, 3576525143, 2753158871, 551079002,
+        1185024844, 587779104, 1004942725, 3763632860, 1185024844, 947424568, 3811666068, 3763632860, 2352253462,
+        1310227170, 218138208, 3172947233, 766129426, 1310227170, 1808643264, 2226659371, 766129426, 3853798112,
+        2230902378, 4243560874, 2491962392, 3836629116, 2230902378, 3637515403, 2846140932, 3836629116, 3083355464,
+        999448569, 1464488480, 3344426626, 946166795, 999448569, 340856814, 3686999436, 946166795, 3231079441,
+        1226155368, 3477563770, 550006884, 2378667355, 1226155368, 1493409040, 260364836, 2378667355, 4133888397,
+        1277901832, 310796286, 2818511068, 3088910653, 1277901832, 3303406025, 2507911914, 3088910653, 3712928074,
+        481918378, 339570348, 1728801469, 1623163429, 481918378, 2209094694, 3146982514, 1623163429, 508445538,
+        3138921230, 2381863183, 1992357430, 1024510915, 3138921230, 2122851650, 1453455184, 1024510915, 941946604,
+        2465372719, 1391015357, 3328905025, 1821933605, 2465372719, 1343489680, 3648970313, 1821933605, 1816599716,
+        118634664, 3358712512, 2492792220, 348833376, 118634664, 2495544591, 3235582254, 348833376, 4043157504,
+        2303067090, 3371139074, 1967771133, 598630070, 2303067090, 1819012637, 2049250561, 598630070, 4093044926,
+        3035321857, 3971176093, 226779704, 3361614254, 3035321857, 2807125404, 326640887, 3361614254, 3147308542,
+        1774298149, 4179629947, 3145006948, 1688753503, 1774298149, 94869516, 2327946901, 1688753503, 2786835219,
+
+    };
+
+static constexpr unsigned long long h_A2[MRG323A_N] =
+    {
+        0, 1, 0, 0, 0, 1, 4293573854, 0, 527612,
+        0, 0, 1, 4293573854, 0, 527612, 2706407399, 4293573854, 3497978192,
+        2706407399, 4293573854, 3497978192, 1431525864, 2706407399, 3281754271, 97673890, 1431525864, 1673476130,
+        3405842137, 2680076935, 893509979, 4035147174, 3405842137, 3280220074, 2623373296, 4035147174, 361718588,
+        818368950, 3790774567, 3542344109, 1817134745, 818368950, 3321940838, 3493477402, 1817134745, 2854655037,
+        498682467, 2928649385, 811441367, 1777037472, 498682467, 479207863, 3058260025, 1777037472, 1528225099,
+        3893311647, 3140922085, 64039185, 82107183, 3893311647, 2655465224, 1674879036, 82107183, 1089381262,
+        28639152, 3496041927, 2231910770, 3174683233, 28639152, 2828785870, 3681140872, 3174683233, 3910194649,
+        1463826069, 300842059, 3313769518, 1799677538, 1463826069, 3174861078, 1882279394, 1799677538, 3509975160,
+        2092194020, 184076987, 2202401252, 3103629604, 2092194020, 3409560232, 4257445059, 3103629604, 2390202783,
+        812917091, 2574011276, 4168802395, 209817750, 812917091, 2974870628, 3238802184, 209817750, 3692836406,
+        477309738, 3314523413, 3442242150, 2755731404, 477309738, 2782713347, 1606221490, 2755731404, 1033463096,
+        2155469603, 3326516116, 3843369786, 288604458, 2155469603, 571673571, 1501677614, 288604458, 2928213494,
+        2082469029, 749754403, 3963963316, 2764859700, 2082469029, 3576428059, 2840894706, 2764859700, 1782279859,
+        3760163766, 1041986082, 1799196192, 1022129134, 3760163766, 1332558840, 276873446, 1022129134, 3979423632,
+        1021313167, 1312544548, 1716381787, 3037868518, 1021313167, 199085085, 2582787611, 3037868518, 3539882179,
+        2569413030, 1631336015, 2594942403, 1030618503, 2569413030, 3467650326, 1998739584, 1030618503, 3174552073,
+        2334639309, 3114094203, 601680947, 2110199318, 2334639309, 678342865, 1649523168, 2110199318, 2154948056,
+        563657176, 191330473, 1641595774, 780563537, 563657176, 3029522338, 2037330914, 780563537, 2084602709,
+        3414769923, 1968799026, 2238126504, 832866376, 3414769923, 3754780168, 2165145850, 832866376, 1594768331,
+        1646861218, 2317984620, 2301581548, 2672536210, 1646861218, 359763062, 2391283983, 2672536210, 1885870777,
+        841254072, 3765813448, 1635365181, 2013240130, 841254072, 605925849, 3743932305, 2013240130, 400681955,
+        1930213004, 2072952279, 3077694794, 3579956569, 1930213004, 2478539210, 1960229502, 3579956569, 1455652656,
+        1097613522, 1784540933, 1194440107, 321747515, 1097613522, 1225209584, 74521379, 321747515, 4288531000,
+        143812745, 3254530816, 3514348856, 769295000, 143812745, 2468210728, 1927161272, 769295000, 522705580,
+        2692035063, 2596905012, 1643240704, 1103432342, 2692035063, 1446182108, 4161111774, 1103432342, 3076435551,
+        2375319030, 1391532370, 3742334018, 1202100604, 2375319030, 4098434768, 2327872488, 1202100604, 1471526950,
+        4269164791, 2795313144, 2507855960, 4245372460, 4269164791, 4094914553, 3873219634, 4245372460, 1473695507,
+        513890845, 1208902926, 2870530442, 1984873167, 513890845, 1257532340, 1212627640, 1984873167, 2354363842,
+        1848364568, 1552116673, 3496528455, 4160778291, 1848364568, 141769900, 3611019106, 4160778291, 596424080,
+        364070020, 3520039729, 837362349, 2544671570, 364070020, 2188646679, 163978331, 2544671570, 672947816,
+        1192700714, 3968150021, 298357363, 635565666, 1192700714, 2589432341, 2548654227, 635565666, 3531570992,
+        2709640529, 676525399, 875361870, 1315499519, 2709640529, 3842690720, 3300994644, 1315499519, 2446760804,
+        2742149264, 1410604392, 3032350755, 3774935330, 2742149264, 597633965, 4085935803, 3774935330, 3952463556,
+        3878579563, 845297523, 1721916511, 2077922420, 3878579563, 3651360351, 2177255734, 2077922420, 3791239282,
+        1570315355, 4252790045, 3522351060, 2324624266, 1570315355, 3594939336, 1725087354, 2324624266, 1338343327,
+        2305761589, 381933244, 3663579047, 1355307047, 2305761589, 313617972, 992174375, 1355307047, 3881593435,
+        1667857811, 1564715297, 2263851601, 3791771273, 1667857811, 4196134923, 3347975047, 3791771273, 615040705,
+        4093947334, 3454015638, 2815567716, 4261953004, 4093947334, 3973733876, 2979573134, 4261953004, 3757047667,
+        250120061, 570149551, 1513430926, 3178644752, 250120061, 1701869032, 4172515680, 3178644752, 4213855850,
+        4158106802, 3062358456, 1815738463, 1379176112, 4158106802, 3926509890, 2842564878, 1379176112, 2852219546,
+        931848746, 256263523, 2633569246, 3284646837, 931848746, 2567084715, 415258465, 3284646837, 2017565947,
+        1648005210, 1032291296, 3987397422, 1831496020, 1648005210, 2829448427, 1821082272, 1831496020, 2917140265,
+        4161327077, 489964129, 3870847744, 1669447863, 4161327077, 4292947198, 1522417114, 1669447863, 2652286672,
+        1270934555, 3136631324, 505612043, 2981474723, 1270934555, 2528619024, 625182639, 2981474723, 1008985039,
+        280996820, 143706137, 3013099060, 1797675893, 280996820, 3743985508, 1123794455, 1797675893, 2460119169,
+        919218027, 4154920441, 1125672685, 3933041881, 919218027, 474242849, 564891116, 3933041881, 2263904321,
+        2920112852, 1965329198, 1177141043, 2135250851, 2920112852, 969184056, 296035385, 2135250851, 4267827987,
+        1481142942, 4120754772, 1088557292, 265491023, 1481142942, 2860005744, 301796252, 265491023, 1935975979,
+        2111859033, 2813610100, 1001476468, 73849832, 2111859033, 3980799998, 3330206241, 73849832, 1933943506,
+        1781286360, 3661231931, 3509383709, 2753158871, 1781286360, 3119883109, 3576525143, 2753158871, 551079002,
+        1185024844, 587779104, 1004942725, 3763632860, 1185024844, 947424568, 3811666068, 3763632860, 2352253462,
+        1310227170, 218138208, 3172947233, 766129426, 1310227170, 1808643264, 2226659371, 766129426, 3853798112,
+        2230902378, 4243560874, 2491962392, 3836629116, 2230902378, 3637515403, 2846140932, 3836629116, 3083355464,
+        999448569, 1464488480, 3344426626, 946166795, 999448569, 340856814, 3686999436, 946166795, 3231079441,
+        1226155368, 3477563770, 550006884, 2378667355, 1226155368, 1493409040, 260364836, 2378667355, 4133888397,
+        1277901832, 310796286, 2818511068, 3088910653, 1277901832, 3303406025, 2507911914, 3088910653, 3712928074,
+        481918378, 339570348, 1728801469, 1623163429, 481918378, 2209094694, 3146982514, 1623163429, 508445538,
+        3138921230, 2381863183, 1992357430, 1024510915, 3138921230, 2122851650, 1453455184, 1024510915, 941946604,
+        2465372719, 1391015357, 3328905025, 1821933605, 2465372719, 1343489680, 3648970313, 1821933605, 1816599716,
+        118634664, 3358712512, 2492792220, 348833376, 118634664, 2495544591, 3235582254, 348833376, 4043157504,
+        2303067090, 3371139074, 1967771133, 598630070, 2303067090, 1819012637, 2049250561, 598630070, 4093044926,
+        3035321857, 3971176093, 226779704, 3361614254, 3035321857, 2807125404, 326640887, 3361614254, 3147308542,
+        1774298149, 4179629947, 3145006948, 1688753503, 1774298149, 94869516, 2327946901, 1688753503, 2786835219,
+
+    };
+
+static constexpr __device__ unsigned long long d_A1P67[MRG323A_N] =
+    {
+        82758667, 1871391091, 4127413238, 3672831523, 69195019, 1871391091, 3672091415, 3528743235, 69195019,
+        3361372532, 2329303404, 99651939, 2008671965, 2931758910, 2329303404, 1113529483, 2374097189, 2931758910,
+        1831590873, 1588259595, 1314332382, 2385989343, 2508077280, 1588259595, 1787615788, 661437137, 2508077280,
+        2326052247, 4183591379, 4049009082, 2604529491, 1453913233, 4183591379, 2311925423, 1805360390, 1453913233,
+        3956367490, 604461629, 1257432102, 794711716, 1155867175, 604461629, 1777070788, 429445904, 1155867175,
+        1686241617, 1257046062, 1427609439, 490376081, 387798431, 1257046062, 235551485, 1312672615, 387798431,
+        2362447880, 3445363024, 3160262066, 2426867845, 4194339866, 3445363024, 1046144413, 4177893681, 4194339866,
+        4251175413, 3559576374, 3107663662, 697539134, 1909472435, 3559576374, 280754246, 375835695, 1909472435,
+        1099512970, 712404985, 1571467521, 546519870, 1135109300, 712404985, 3325312332, 2352874613, 1135109300,
+        1945425936, 1653045514, 381988982, 3733376326, 414410025, 1653045514, 1181583679, 1185848176, 414410025,
+        2526336124, 3019211015, 4215964965, 2683163472, 4188191530, 3019211015, 2964651598, 293801056, 4188191530,
+        1444052678, 2253324417, 39719589, 1880267534, 2391992038, 2253324417, 987740265, 3691889508, 2391992038,
+        166599066, 2335494420, 1232261118, 2227597731, 2570600780, 2335494420, 2700034538, 3460843234, 2570600780,
+        2511338360, 1188954576, 1251401239, 2511664974, 292276982, 1188954576, 697844082, 3093661552, 292276982,
+        3624650744, 51993077, 3540268009, 3252828938, 3710319575, 51993077, 2858628849, 3910069381, 3710319575,
+        655966702, 754002362, 1646581402, 1958331075, 475572423, 754002362, 3248619000, 3228514800, 475572423,
+        2760311307, 4166372813, 741596417, 2282679206, 3090782630, 4166372813, 3242468721, 1628442374, 3090782630,
+        4265279407, 3532111852, 1754687396, 500404765, 2603727025, 3532111852, 1428367254, 3149485478, 2603727025,
+        2873769531, 2081104178, 596284397, 4153800443, 1261269623, 2081104178, 3967600061, 1830023157, 1261269623,
+        278611533, 2229285304, 3443204327, 3110641420, 77498444, 2229285304, 3904070810, 1070507239, 77498444,
+        544639534, 568528663, 2177189807, 2475829068, 121482268, 568528663, 876978915, 3116647617, 121482268,
+        1547862823, 2404658587, 4191448009, 2158188804, 2976916793, 2404658587, 168571747, 1691884706, 2976916793,
+        3208213311, 4212638780, 3235157352, 671148556, 2951207765, 4212638780, 2075145516, 2395485231, 2951207765,
+        4080517315, 2133433101, 4043998180, 2044221845, 867670560, 2133433101, 834432416, 3613001199, 867670560,
+        4102885735, 1319434267, 2678775073, 740092580, 607380970, 1319434267, 2198271844, 2610193258, 607380970,
+        1165218048, 1317690360, 1189150958, 399240205, 2507168618, 1317690360, 2988334517, 2687593413, 2507168618,
+        1028861702, 4082006648, 338232527, 1888486946, 1842080991, 4082006648, 3903826366, 3109935091, 1842080991,
+        614134826, 2261996505, 2888080641, 710199359, 2773979788, 2261996505, 1144301620, 2554371815, 2773979788,
+        4056173823, 1285620078, 357420018, 2423072612, 2309408315, 1285620078, 1533175115, 2760088020, 2309408315,
+        4264130267, 815015434, 3142242173, 180649975, 2500813569, 815015434, 3378723563, 829683767, 2500813569,
+        4174387531, 1030729435, 2812778314, 1752988797, 4044178729, 1030729435, 467969301, 554748104, 4044178729,
+        1348429235, 2928743274, 3776082629, 3607529209, 3069812185, 2928743274, 2542432347, 3208181168, 3069812185,
+        4064845753, 668285756, 3816217625, 3713143233, 1380634204, 668285756, 3533700508, 1192551435, 1380634204,
+        1515684518, 1706771705, 728123349, 3174850469, 2057456462, 1706771705, 3410402985, 2897339640, 2057456462,
+        3082272717, 531091457, 1390161328, 3895139973, 2171402857, 531091457, 4030688141, 3049703400, 2171402857,
+        1241147206, 3193892819, 1244284192, 65180262, 4065669017, 3193892819, 1484817937, 3661081858, 4065669017,
+        1438760812, 3491341751, 3414470157, 2805337292, 272266053, 3491341751, 824109230, 3202556526, 272266053,
+        135412706, 3627115412, 2345042216, 1565169824, 2166856449, 3627115412, 1026946745, 3467845248, 2166856449,
+        1889419951, 3256876154, 1240505488, 1254783743, 989966800, 3256876154, 1995297400, 3692472918, 989966800,
+        3206226875, 285700890, 496017472, 2515316194, 2129675196, 285700890, 1863853990, 2673457552, 2129675196,
+        4163770641, 255160418, 772100749, 1987092456, 3237660221, 255160418, 1394381051, 4216039401, 3237660221,
+        2133915627, 2713747584, 627765421, 2300605925, 35690583, 2713747584, 2918902946, 2638220304, 35690583,
+        2587549655, 998684270, 4292130625, 1791772791, 2820705344, 998684270, 124590158, 3831143549, 2820705344,
+        978482299, 3200877282, 497605289, 3717741518, 3737164414, 3200877282, 4046686626, 861393946, 3737164414,
+        2665561897, 300934584, 3179822945, 893043137, 2031413512, 300934584, 3806926970, 2413249929, 2031413512,
+        1417581911, 3071835354, 2575196237, 4101127251, 1375339216, 3071835354, 847617977, 3632503316, 1375339216,
+        2747488994, 3296604805, 898095468, 1742777145, 219265369, 3296604805, 823714885, 667779292, 219265369,
+        2640209692, 3040506537, 3626115220, 161827078, 852668118, 3040506537, 3856381322, 3360242076, 852668118,
+        3734246393, 4151553160, 4177051283, 266522866, 1731798531, 4151553160, 632196679, 3864297722, 1731798531,
+        1694175127, 1087914338, 2384195794, 2764925057, 505782858, 1087914338, 3235634082, 807915248, 505782858,
+        2402749950, 2353776151, 75909174, 890570951, 1752665661, 2353776151, 3120241607, 3862435696, 1752665661,
+        2427906178, 3580155704, 949770784, 226153695, 1230515664, 3580155704, 1988835001, 986791581, 1230515664,
+        1774047142, 3199155377, 3106427820, 1901920839, 4290900039, 3199155377, 4178980191, 280623348, 4290900039,
+        3567524348, 1934119675, 3188270128, 2997767678, 826363896, 1934119675, 262952343, 614326610, 826363896,
+        1625613062, 4288164505, 2481284279, 4273461426, 1177260757, 4288164505, 305959988, 4017252267, 1177260757,
+        337929267, 333342539, 418300166, 2944208672, 379097734, 333342539, 2084056909, 3625475947, 379097734,
+        1189899255, 1307754719, 1214919992, 3736721708, 3514751918, 1307754719, 732435953, 2021244538, 3514751918,
+        4089172695, 1533534334, 525643282, 1497577018, 1335684482, 1533534334, 2079007086, 3977541427, 1335684482,
+        3075256652, 2762754934, 3846844247, 3057872364, 3274545167, 2762754934, 4028573983, 938934351, 3274545167,
+        2597859300, 2880151048, 2523330453, 1121709186, 175667448, 2880151048, 4182510911, 1723133625, 175667448,
+        484148868, 1404283933, 2982534313, 3736767353, 3179865161, 1404283933, 391120388, 3758716888, 3179865161,
+        2138867468, 1128973399, 2133702321, 1613561693, 3622350766, 1128973399, 1500151924, 3759983985, 3622350766,
+        3027706760, 3786576552, 2698781808, 2810527099, 90498489, 3786576552, 4220122612, 1855245979, 90498489,
+        3739389517, 1110440720, 917457922, 2163873618, 3707591763, 1110440720, 2667061910, 2533383962, 3707591763,
+
+    };
+
+static constexpr unsigned long long h_A1P67[MRG323A_N] =
+    {
+        82758667, 1871391091, 4127413238, 3672831523, 69195019, 1871391091, 3672091415, 3528743235, 69195019,
+        3361372532, 2329303404, 99651939, 2008671965, 2931758910, 2329303404, 1113529483, 2374097189, 2931758910,
+        1831590873, 1588259595, 1314332382, 2385989343, 2508077280, 1588259595, 1787615788, 661437137, 2508077280,
+        2326052247, 4183591379, 4049009082, 2604529491, 1453913233, 4183591379, 2311925423, 1805360390, 1453913233,
+        3956367490, 604461629, 1257432102, 794711716, 1155867175, 604461629, 1777070788, 429445904, 1155867175,
+        1686241617, 1257046062, 1427609439, 490376081, 387798431, 1257046062, 235551485, 1312672615, 387798431,
+        2362447880, 3445363024, 3160262066, 2426867845, 4194339866, 3445363024, 1046144413, 4177893681, 4194339866,
+        4251175413, 3559576374, 3107663662, 697539134, 1909472435, 3559576374, 280754246, 375835695, 1909472435,
+        1099512970, 712404985, 1571467521, 546519870, 1135109300, 712404985, 3325312332, 2352874613, 1135109300,
+        1945425936, 1653045514, 381988982, 3733376326, 414410025, 1653045514, 1181583679, 1185848176, 414410025,
+        2526336124, 3019211015, 4215964965, 2683163472, 4188191530, 3019211015, 2964651598, 293801056, 4188191530,
+        1444052678, 2253324417, 39719589, 1880267534, 2391992038, 2253324417, 987740265, 3691889508, 2391992038,
+        166599066, 2335494420, 1232261118, 2227597731, 2570600780, 2335494420, 2700034538, 3460843234, 2570600780,
+        2511338360, 1188954576, 1251401239, 2511664974, 292276982, 1188954576, 697844082, 3093661552, 292276982,
+        3624650744, 51993077, 3540268009, 3252828938, 3710319575, 51993077, 2858628849, 3910069381, 3710319575,
+        655966702, 754002362, 1646581402, 1958331075, 475572423, 754002362, 3248619000, 3228514800, 475572423,
+        2760311307, 4166372813, 741596417, 2282679206, 3090782630, 4166372813, 3242468721, 1628442374, 3090782630,
+        4265279407, 3532111852, 1754687396, 500404765, 2603727025, 3532111852, 1428367254, 3149485478, 2603727025,
+        2873769531, 2081104178, 596284397, 4153800443, 1261269623, 2081104178, 3967600061, 1830023157, 1261269623,
+        278611533, 2229285304, 3443204327, 3110641420, 77498444, 2229285304, 3904070810, 1070507239, 77498444,
+        544639534, 568528663, 2177189807, 2475829068, 121482268, 568528663, 876978915, 3116647617, 121482268,
+        1547862823, 2404658587, 4191448009, 2158188804, 2976916793, 2404658587, 168571747, 1691884706, 2976916793,
+        3208213311, 4212638780, 3235157352, 671148556, 2951207765, 4212638780, 2075145516, 2395485231, 2951207765,
+        4080517315, 2133433101, 4043998180, 2044221845, 867670560, 2133433101, 834432416, 3613001199, 867670560,
+        4102885735, 1319434267, 2678775073, 740092580, 607380970, 1319434267, 2198271844, 2610193258, 607380970,
+        1165218048, 1317690360, 1189150958, 399240205, 2507168618, 1317690360, 2988334517, 2687593413, 2507168618,
+        1028861702, 4082006648, 338232527, 1888486946, 1842080991, 4082006648, 3903826366, 3109935091, 1842080991,
+        614134826, 2261996505, 2888080641, 710199359, 2773979788, 2261996505, 1144301620, 2554371815, 2773979788,
+        4056173823, 1285620078, 357420018, 2423072612, 2309408315, 1285620078, 1533175115, 2760088020, 2309408315,
+        4264130267, 815015434, 3142242173, 180649975, 2500813569, 815015434, 3378723563, 829683767, 2500813569,
+        4174387531, 1030729435, 2812778314, 1752988797, 4044178729, 1030729435, 467969301, 554748104, 4044178729,
+        1348429235, 2928743274, 3776082629, 3607529209, 3069812185, 2928743274, 2542432347, 3208181168, 3069812185,
+        4064845753, 668285756, 3816217625, 3713143233, 1380634204, 668285756, 3533700508, 1192551435, 1380634204,
+        1515684518, 1706771705, 728123349, 3174850469, 2057456462, 1706771705, 3410402985, 2897339640, 2057456462,
+        3082272717, 531091457, 1390161328, 3895139973, 2171402857, 531091457, 4030688141, 3049703400, 2171402857,
+        1241147206, 3193892819, 1244284192, 65180262, 4065669017, 3193892819, 1484817937, 3661081858, 4065669017,
+        1438760812, 3491341751, 3414470157, 2805337292, 272266053, 3491341751, 824109230, 3202556526, 272266053,
+        135412706, 3627115412, 2345042216, 1565169824, 2166856449, 3627115412, 1026946745, 3467845248, 2166856449,
+        1889419951, 3256876154, 1240505488, 1254783743, 989966800, 3256876154, 1995297400, 3692472918, 989966800,
+        3206226875, 285700890, 496017472, 2515316194, 2129675196, 285700890, 1863853990, 2673457552, 2129675196,
+        4163770641, 255160418, 772100749, 1987092456, 3237660221, 255160418, 1394381051, 4216039401, 3237660221,
+        2133915627, 2713747584, 627765421, 2300605925, 35690583, 2713747584, 2918902946, 2638220304, 35690583,
+        2587549655, 998684270, 4292130625, 1791772791, 2820705344, 998684270, 124590158, 3831143549, 2820705344,
+        978482299, 3200877282, 497605289, 3717741518, 3737164414, 3200877282, 4046686626, 861393946, 3737164414,
+        2665561897, 300934584, 3179822945, 893043137, 2031413512, 300934584, 3806926970, 2413249929, 2031413512,
+        1417581911, 3071835354, 2575196237, 4101127251, 1375339216, 3071835354, 847617977, 3632503316, 1375339216,
+        2747488994, 3296604805, 898095468, 1742777145, 219265369, 3296604805, 823714885, 667779292, 219265369,
+        2640209692, 3040506537, 3626115220, 161827078, 852668118, 3040506537, 3856381322, 3360242076, 852668118,
+        3734246393, 4151553160, 4177051283, 266522866, 1731798531, 4151553160, 632196679, 3864297722, 1731798531,
+        1694175127, 1087914338, 2384195794, 2764925057, 505782858, 1087914338, 3235634082, 807915248, 505782858,
+        2402749950, 2353776151, 75909174, 890570951, 1752665661, 2353776151, 3120241607, 3862435696, 1752665661,
+        2427906178, 3580155704, 949770784, 226153695, 1230515664, 3580155704, 1988835001, 986791581, 1230515664,
+        1774047142, 3199155377, 3106427820, 1901920839, 4290900039, 3199155377, 4178980191, 280623348, 4290900039,
+        3567524348, 1934119675, 3188270128, 2997767678, 826363896, 1934119675, 262952343, 614326610, 826363896,
+        1625613062, 4288164505, 2481284279, 4273461426, 1177260757, 4288164505, 305959988, 4017252267, 1177260757,
+        337929267, 333342539, 418300166, 2944208672, 379097734, 333342539, 2084056909, 3625475947, 379097734,
+        1189899255, 1307754719, 1214919992, 3736721708, 3514751918, 1307754719, 732435953, 2021244538, 3514751918,
+        4089172695, 1533534334, 525643282, 1497577018, 1335684482, 1533534334, 2079007086, 3977541427, 1335684482,
+        3075256652, 2762754934, 3846844247, 3057872364, 3274545167, 2762754934, 4028573983, 938934351, 3274545167,
+        2597859300, 2880151048, 2523330453, 1121709186, 175667448, 2880151048, 4182510911, 1723133625, 175667448,
+        484148868, 1404283933, 2982534313, 3736767353, 3179865161, 1404283933, 391120388, 3758716888, 3179865161,
+        2138867468, 1128973399, 2133702321, 1613561693, 3622350766, 1128973399, 1500151924, 3759983985, 3622350766,
+        3027706760, 3786576552, 2698781808, 2810527099, 90498489, 3786576552, 4220122612, 1855245979, 90498489,
+        3739389517, 1110440720, 917457922, 2163873618, 3707591763, 1110440720, 2667061910, 2533383962, 3707591763,
+
+    };
+
+static constexpr __device__ unsigned long long d_A2P67[MRG323A_N] =
+    {
+        1511326704, 3759209742, 1610795712, 4292754251, 1511326704, 3889917532, 3859662829, 4292754251, 3708466080,
+        972103006, 964807713, 878035866, 4248550197, 972103006, 1926628839, 1448629089, 4248550197, 3196114006,
+        3497384788, 3174249442, 3182508868, 3864816447, 3497384788, 3038399593, 2546884738, 3864816447, 2980208068,
+        1776335558, 1189944887, 4095757548, 3813600746, 1776335558, 789475914, 4119698302, 3813600746, 2145357457,
+        4022832294, 4130146837, 1942923647, 1675130777, 4022832294, 916677004, 4089786548, 1675130777, 116540512,
+        165639584, 1205513289, 2037453462, 1444587280, 165639584, 161923120, 2617085459, 1444587280, 2006913311,
+        3458099202, 3062421748, 4052486999, 1064270720, 3458099202, 230768332, 4056228301, 1064270720, 2219267779,
+        296275263, 3452455838, 2081462173, 1789143993, 296275263, 3463234943, 2097389984, 1789143993, 3447191459,
+        2828288883, 3866690251, 410553827, 1587005542, 2828288883, 1469478670, 2766486018, 1587005542, 2627363449,
+        3288027530, 412403981, 2458742268, 4267121909, 3288027530, 138566505, 420803572, 4267121909, 4094554844,
+        3844599430, 2430152838, 3283485436, 2486244684, 3844599430, 4252427633, 3560842909, 2486244684, 3960267499,
+        67933059, 1294996291, 2657888382, 513233413, 67933059, 1379805031, 44564058, 513233413, 86971645,
+        2732588524, 1866530072, 818237694, 2540507736, 2732588524, 3257104212, 1164400003, 2540507736, 1124501551,
+        4199239222, 3155848463, 2121388468, 1135554501, 4199239222, 2056492193, 3251740389, 1135554501, 2343537248,
+        550710036, 500329021, 1075236085, 356444753, 550710036, 1634965500, 58733535, 356444753, 1261552815,
+        708689546, 419139045, 2012018174, 706488081, 708689546, 1113760995, 585555005, 706488081, 76092226,
+        1293182265, 3168473803, 366230236, 3319068849, 1293182265, 1085259665, 1675229290, 3319068849, 3912300371,
+        3186089068, 4188864734, 1211781402, 756122322, 3186089068, 578262892, 2518961174, 756122322, 1658665581,
+        1347291439, 2050427676, 736113023, 4102191254, 1347291439, 878627148, 1293500383, 4102191254, 745646810,
+        4196897331, 3436564969, 1900167098, 3108887846, 4196897331, 2697923227, 1405263476, 3108887846, 314631094,
+        958383622, 3694638688, 1150087061, 3770009830, 958383622, 793326651, 533700213, 3770009830, 1513734026,
+        4119603367, 3479396923, 3534176399, 3765397477, 4119603367, 1458031003, 3380901602, 3765397477, 2684083587,
+        980937351, 2094378936, 448446028, 1421333909, 980937351, 3405683645, 323724368, 1421333909, 338680738,
+        2942968846, 4293637338, 3549906544, 527851489, 2942968846, 3852871282, 4209198933, 527851489, 1091268872,
+        1975983015, 2092556693, 611187071, 3982652344, 1975983015, 3001736262, 2055073597, 3982652344, 1875181995,
+        2970221269, 880904779, 2447465272, 2888742196, 2970221269, 3521651749, 3019977656, 2888742196, 2712717326,
+        419134859, 2976059897, 747864206, 4101695717, 419134859, 4264593116, 2657991148, 4101695717, 2542621682,
+        4043135299, 1612983166, 1149778656, 1267010518, 4043135299, 3496325546, 3094232897, 1267010518, 2949176293,
+        3949395794, 1774568686, 2123036003, 2182983404, 3949395794, 2355671350, 2820933455, 2182983404, 513963325,
+        3046911698, 2576744453, 2492729814, 4277866093, 3046911698, 3146977604, 2249371766, 4277866093, 3622293976,
+        1391529818, 423458502, 2587125255, 3536237833, 1391529818, 985347517, 157623850, 3536237833, 1015566287,
+        48329260, 2599277669, 821961664, 902187690, 48329260, 1716556555, 4019658974, 902187690, 950730510,
+        1318489562, 1530977112, 3713577419, 4270158447, 1318489562, 1654940598, 2679964938, 4270158447, 1337075195,
+        770600793, 3249576224, 3578552768, 2710443459, 770600793, 2990852339, 3098163705, 2710443459, 522138188,
+        2803285489, 1922250286, 3164022812, 477609731, 2803285489, 2140252218, 2252852611, 477609731, 3058519788,
+        208329741, 3633562083, 3548346666, 3892091460, 208329741, 516833304, 3440632377, 3892091460, 1638833719,
+        1816075033, 3570111203, 959489356, 3482051486, 1816075033, 861657108, 3119495098, 3482051486, 2576849579,
+        4240216888, 2891584407, 2102314945, 4064489450, 4240216888, 1427441010, 2441164913, 4064489450, 3558527186,
+        2918371295, 65155283, 3469357011, 3579773554, 2918371295, 3494391959, 3266584309, 3579773554, 3837485479,
+        2959420453, 1365016881, 4082486022, 236489012, 2959420453, 3802558529, 2687043642, 236489012, 2547086826,
+        4185325422, 2762854843, 3200044912, 3664909559, 4185325422, 3543921700, 4240262918, 3664909559, 2853212443,
+        2618500928, 4237264351, 1470046497, 1893990098, 2618500928, 2982567031, 3017062825, 1893990098, 3195556801,
+        1868464655, 3407681142, 1652841784, 1678569574, 1868464655, 4162480901, 1477016185, 1678569574, 4145063890,
+        792188465, 4251338402, 2219407026, 3840340879, 792188465, 3493367465, 2979958414, 3840340879, 2338974139,
+        478845700, 2378167062, 882114621, 1674533845, 478845700, 3572905305, 3571222880, 1674533845, 1242316901,
+        2636090868, 1972761498, 71690719, 1228103463, 2636090868, 1280685025, 3741735502, 1228103463, 994061750,
+        1156725261, 1100755307, 221922891, 2892200461, 1156725261, 1505716533, 2287613563, 2892200461, 3689457190,
+        1387244644, 3135090808, 1243609165, 1724967466, 1387244644, 3296353235, 1064364031, 1724967466, 2107521044,
+        2822471992, 2034317853, 2071407475, 170903528, 2822471992, 1322162887, 2524982332, 170903528, 2656231333,
+        3653936868, 3893194049, 2484299328, 1313746234, 3653936868, 1705346273, 1397638018, 1313746234, 4015529545,
+        4129760842, 1671665759, 1677834656, 3200005334, 4129760842, 3486207172, 2850728736, 3200005334, 3076201597,
+        1464411153, 277697599, 1610723613, 32183930, 1464411153, 1022607788, 2824425944, 32183930, 2093834863,
+        3492361727, 1027004383, 3167429889, 3674905362, 3492361727, 3572939265, 4270409313, 3674905362, 698814233,
+        880482061, 205175925, 4070445105, 2208329119, 880482061, 1933248566, 3741227945, 2208329119, 3962062826,
+        4184605179, 1189429800, 567967482, 107217966, 4184605179, 784865788, 549462420, 107217966, 3134382704,
+        2732536445, 1231107067, 3374588386, 409954030, 2732536445, 1044831206, 3398162498, 409954030, 3505648581,
+        2169560691, 1076348534, 637306236, 3704346564, 2169560691, 293694496, 632453145, 3704346564, 1609425246,
+        372115891, 3928812480, 2830541169, 3056527841, 372115891, 1924239834, 3044937468, 3056527841, 547142630,
+        1660852083, 3635660815, 1389092450, 1025573319, 1660852083, 3276803366, 4036331438, 1025573319, 4092197741,
+        1360732901, 2887812973, 4101068693, 52572783, 1360732901, 112458461, 2636566855, 52572783, 1136777988,
+        3455696508, 536919193, 3978804036, 3094157668, 3455696508, 3821833900, 2278849016, 3094157668, 2531965909,
+        2125991744, 890897326, 3790557569, 1433592392, 2125991744, 3671109604, 808215503, 1433592392, 2446306581,
+        3524411799, 932865240, 1838275365, 1789634890, 3524411799, 4130736474, 2252266098, 1789634890, 3048775967,
+        1773339925, 948403862, 1999624391, 983864203, 1773339925, 3734776305, 314407045, 983864203, 2648614071,
+
+    };
+
+static constexpr unsigned long long h_A2P67[MRG323A_N] =
+    {
+        1511326704, 3759209742, 1610795712, 4292754251, 1511326704, 3889917532, 3859662829, 4292754251, 3708466080,
+        972103006, 964807713, 878035866, 4248550197, 972103006, 1926628839, 1448629089, 4248550197, 3196114006,
+        3497384788, 3174249442, 3182508868, 3864816447, 3497384788, 3038399593, 2546884738, 3864816447, 2980208068,
+        1776335558, 1189944887, 4095757548, 3813600746, 1776335558, 789475914, 4119698302, 3813600746, 2145357457,
+        4022832294, 4130146837, 1942923647, 1675130777, 4022832294, 916677004, 4089786548, 1675130777, 116540512,
+        165639584, 1205513289, 2037453462, 1444587280, 165639584, 161923120, 2617085459, 1444587280, 2006913311,
+        3458099202, 3062421748, 4052486999, 1064270720, 3458099202, 230768332, 4056228301, 1064270720, 2219267779,
+        296275263, 3452455838, 2081462173, 1789143993, 296275263, 3463234943, 2097389984, 1789143993, 3447191459,
+        2828288883, 3866690251, 410553827, 1587005542, 2828288883, 1469478670, 2766486018, 1587005542, 2627363449,
+        3288027530, 412403981, 2458742268, 4267121909, 3288027530, 138566505, 420803572, 4267121909, 4094554844,
+        3844599430, 2430152838, 3283485436, 2486244684, 3844599430, 4252427633, 3560842909, 2486244684, 3960267499,
+        67933059, 1294996291, 2657888382, 513233413, 67933059, 1379805031, 44564058, 513233413, 86971645,
+        2732588524, 1866530072, 818237694, 2540507736, 2732588524, 3257104212, 1164400003, 2540507736, 1124501551,
+        4199239222, 3155848463, 2121388468, 1135554501, 4199239222, 2056492193, 3251740389, 1135554501, 2343537248,
+        550710036, 500329021, 1075236085, 356444753, 550710036, 1634965500, 58733535, 356444753, 1261552815,
+        708689546, 419139045, 2012018174, 706488081, 708689546, 1113760995, 585555005, 706488081, 76092226,
+        1293182265, 3168473803, 366230236, 3319068849, 1293182265, 1085259665, 1675229290, 3319068849, 3912300371,
+        3186089068, 4188864734, 1211781402, 756122322, 3186089068, 578262892, 2518961174, 756122322, 1658665581,
+        1347291439, 2050427676, 736113023, 4102191254, 1347291439, 878627148, 1293500383, 4102191254, 745646810,
+        4196897331, 3436564969, 1900167098, 3108887846, 4196897331, 2697923227, 1405263476, 3108887846, 314631094,
+        958383622, 3694638688, 1150087061, 3770009830, 958383622, 793326651, 533700213, 3770009830, 1513734026,
+        4119603367, 3479396923, 3534176399, 3765397477, 4119603367, 1458031003, 3380901602, 3765397477, 2684083587,
+        980937351, 2094378936, 448446028, 1421333909, 980937351, 3405683645, 323724368, 1421333909, 338680738,
+        2942968846, 4293637338, 3549906544, 527851489, 2942968846, 3852871282, 4209198933, 527851489, 1091268872,
+        1975983015, 2092556693, 611187071, 3982652344, 1975983015, 3001736262, 2055073597, 3982652344, 1875181995,
+        2970221269, 880904779, 2447465272, 2888742196, 2970221269, 3521651749, 3019977656, 2888742196, 2712717326,
+        419134859, 2976059897, 747864206, 4101695717, 419134859, 4264593116, 2657991148, 4101695717, 2542621682,
+        4043135299, 1612983166, 1149778656, 1267010518, 4043135299, 3496325546, 3094232897, 1267010518, 2949176293,
+        3949395794, 1774568686, 2123036003, 2182983404, 3949395794, 2355671350, 2820933455, 2182983404, 513963325,
+        3046911698, 2576744453, 2492729814, 4277866093, 3046911698, 3146977604, 2249371766, 4277866093, 3622293976,
+        1391529818, 423458502, 2587125255, 3536237833, 1391529818, 985347517, 157623850, 3536237833, 1015566287,
+        48329260, 2599277669, 821961664, 902187690, 48329260, 1716556555, 4019658974, 902187690, 950730510,
+        1318489562, 1530977112, 3713577419, 4270158447, 1318489562, 1654940598, 2679964938, 4270158447, 1337075195,
+        770600793, 3249576224, 3578552768, 2710443459, 770600793, 2990852339, 3098163705, 2710443459, 522138188,
+        2803285489, 1922250286, 3164022812, 477609731, 2803285489, 2140252218, 2252852611, 477609731, 3058519788,
+        208329741, 3633562083, 3548346666, 3892091460, 208329741, 516833304, 3440632377, 3892091460, 1638833719,
+        1816075033, 3570111203, 959489356, 3482051486, 1816075033, 861657108, 3119495098, 3482051486, 2576849579,
+        4240216888, 2891584407, 2102314945, 4064489450, 4240216888, 1427441010, 2441164913, 4064489450, 3558527186,
+        2918371295, 65155283, 3469357011, 3579773554, 2918371295, 3494391959, 3266584309, 3579773554, 3837485479,
+        2959420453, 1365016881, 4082486022, 236489012, 2959420453, 3802558529, 2687043642, 236489012, 2547086826,
+        4185325422, 2762854843, 3200044912, 3664909559, 4185325422, 3543921700, 4240262918, 3664909559, 2853212443,
+        2618500928, 4237264351, 1470046497, 1893990098, 2618500928, 2982567031, 3017062825, 1893990098, 3195556801,
+        1868464655, 3407681142, 1652841784, 1678569574, 1868464655, 4162480901, 1477016185, 1678569574, 4145063890,
+        792188465, 4251338402, 2219407026, 3840340879, 792188465, 3493367465, 2979958414, 3840340879, 2338974139,
+        478845700, 2378167062, 882114621, 1674533845, 478845700, 3572905305, 3571222880, 1674533845, 1242316901,
+        2636090868, 1972761498, 71690719, 1228103463, 2636090868, 1280685025, 3741735502, 1228103463, 994061750,
+        1156725261, 1100755307, 221922891, 2892200461, 1156725261, 1505716533, 2287613563, 2892200461, 3689457190,
+        1387244644, 3135090808, 1243609165, 1724967466, 1387244644, 3296353235, 1064364031, 1724967466, 2107521044,
+        2822471992, 2034317853, 2071407475, 170903528, 2822471992, 1322162887, 2524982332, 170903528, 2656231333,
+        3653936868, 3893194049, 2484299328, 1313746234, 3653936868, 1705346273, 1397638018, 1313746234, 4015529545,
+        4129760842, 1671665759, 1677834656, 3200005334, 4129760842, 3486207172, 2850728736, 3200005334, 3076201597,
+        1464411153, 277697599, 1610723613, 32183930, 1464411153, 1022607788, 2824425944, 32183930, 2093834863,
+        3492361727, 1027004383, 3167429889, 3674905362, 3492361727, 3572939265, 4270409313, 3674905362, 698814233,
+        880482061, 205175925, 4070445105, 2208329119, 880482061, 1933248566, 3741227945, 2208329119, 3962062826,
+        4184605179, 1189429800, 567967482, 107217966, 4184605179, 784865788, 549462420, 107217966, 3134382704,
+        2732536445, 1231107067, 3374588386, 409954030, 2732536445, 1044831206, 3398162498, 409954030, 3505648581,
+        2169560691, 1076348534, 637306236, 3704346564, 2169560691, 293694496, 632453145, 3704346564, 1609425246,
+        372115891, 3928812480, 2830541169, 3056527841, 372115891, 1924239834, 3044937468, 3056527841, 547142630,
+        1660852083, 3635660815, 1389092450, 1025573319, 1660852083, 3276803366, 4036331438, 1025573319, 4092197741,
+        1360732901, 2887812973, 4101068693, 52572783, 1360732901, 112458461, 2636566855, 52572783, 1136777988,
+        3455696508, 536919193, 3978804036, 3094157668, 3455696508, 3821833900, 2278849016, 3094157668, 2531965909,
+        2125991744, 890897326, 3790557569, 1433592392, 2125991744, 3671109604, 808215503, 1433592392, 2446306581,
+        3524411799, 932865240, 1838275365, 1789634890, 3524411799, 4130736474, 2252266098, 1789634890, 3048775967,
+        1773339925, 948403862, 1999624391, 983864203, 1773339925, 3734776305, 314407045, 983864203, 2648614071,
+
+    };
+
+static constexpr __device__ unsigned long long d_A1P127[MRG323A_N] =
+    {
+        2427906178, 3580155704, 949770784, 226153695, 1230515664, 3580155704, 1988835001, 986791581, 1230515664,
+        1774047142, 3199155377, 3106427820, 1901920839, 4290900039, 3199155377, 4178980191, 280623348, 4290900039,
+        3567524348, 1934119675, 3188270128, 2997767678, 826363896, 1934119675, 262952343, 614326610, 826363896,
+        1625613062, 4288164505, 2481284279, 4273461426, 1177260757, 4288164505, 305959988, 4017252267, 1177260757,
+        337929267, 333342539, 418300166, 2944208672, 379097734, 333342539, 2084056909, 3625475947, 379097734,
+        1189899255, 1307754719, 1214919992, 3736721708, 3514751918, 1307754719, 732435953, 2021244538, 3514751918,
+        4089172695, 1533534334, 525643282, 1497577018, 1335684482, 1533534334, 2079007086, 3977541427, 1335684482,
+        3075256652, 2762754934, 3846844247, 3057872364, 3274545167, 2762754934, 4028573983, 938934351, 3274545167,
+        2597859300, 2880151048, 2523330453, 1121709186, 175667448, 2880151048, 4182510911, 1723133625, 175667448,
+        484148868, 1404283933, 2982534313, 3736767353, 3179865161, 1404283933, 391120388, 3758716888, 3179865161,
+        2138867468, 1128973399, 2133702321, 1613561693, 3622350766, 1128973399, 1500151924, 3759983985, 3622350766,
+        3027706760, 3786576552, 2698781808, 2810527099, 90498489, 3786576552, 4220122612, 1855245979, 90498489,
+        3739389517, 1110440720, 917457922, 2163873618, 3707591763, 1110440720, 2667061910, 2533383962, 3707591763,
+        1545226000, 1812182123, 3693349190, 3422065122, 3291428549, 1812182123, 1193168720, 2072837757, 3291428549,
+        3230096243, 2131723358, 3262178024, 2882890127, 4088518247, 2131723358, 3991553306, 1282224087, 4088518247,
+        301207261, 1722796810, 3697719854, 3350228505, 3410986694, 1722796810, 3684514720, 2846958957, 3410986694,
+        1532963114, 4236235786, 3871128158, 3540401964, 1285250577, 4236235786, 1105070646, 2764245175, 1285250577,
+        210906218, 3068599594, 3034582784, 340633153, 4004365908, 3068599594, 4238928187, 2299166464, 4004365908,
+        2274701639, 3955606166, 3081246407, 3199954992, 3948054919, 3955606166, 2399101442, 3438340286, 3948054919,
+        504137100, 1182303684, 201533985, 4188299661, 3042453580, 1182303684, 2578519273, 2674782930, 3042453580,
+        1382964588, 2578452047, 3140440866, 261861891, 1076783073, 2578452047, 1634588989, 164438428, 1076783073,
+        2529186343, 526867394, 3102803247, 2687252475, 2908898908, 526867394, 1213100579, 86050422, 2908898908,
+        2690118316, 538108523, 790337895, 4193870709, 1053552056, 538108523, 1635227281, 4002399925, 1053552056,
+        2123712957, 4205383007, 1812304090, 1095349745, 166243972, 4205383007, 428569070, 2128782357, 166243972,
+        1330151766, 3569679412, 4107175982, 3808641551, 3621125056, 3569679412, 4262164578, 1927692878, 3621125056,
+        3606295184, 2442739556, 3894922338, 1629626641, 2729678535, 2442739556, 3379124758, 4279360935, 2729678535,
+        1052092278, 4249024666, 919210106, 3253349463, 3629539480, 4249024666, 852514024, 4025926501, 3629539480,
+        12394571, 1252747620, 2133571953, 4227339509, 3197545170, 1252747620, 1884529704, 1976203831, 3197545170,
+        2986331025, 2671019282, 2847338542, 3173738401, 3542657885, 2671019282, 745203060, 1546667401, 3542657885,
+        2613012997, 2311336951, 2911336433, 1493974713, 92565032, 2311336951, 2786645250, 257065974, 92565032,
+        3424925004, 2776053372, 2204068573, 3770626858, 2509257810, 2776053372, 2979919489, 1146336783, 2509257810,
+        1474384834, 827894421, 515339473, 1373055755, 1949809417, 827894421, 3088339524, 1194193824, 1949809417,
+        1825805135, 1289872272, 3700877161, 3433422861, 4062509844, 1289872272, 3019008744, 2060641859, 4062509844,
+        3842597153, 4253338264, 3424495942, 698444416, 60268595, 4253338264, 4096010585, 47309624, 60268595,
+        2662288323, 2043518992, 1593435980, 1330201507, 3618850300, 2043518992, 2538793204, 271787962, 3618850300,
+        741020448, 997594656, 2398808739, 1160477043, 1522130854, 997594656, 3036916315, 2847712653, 1522130854,
+        2654964886, 1889728930, 53329096, 2042322941, 1621136330, 1889728930, 1553642730, 784545882, 1621136330,
+        1715219514, 2831829177, 929124824, 997274536, 404228189, 2831829177, 1386575385, 4107238699, 404228189,
+        3928131551, 2912523524, 1840499723, 4216003022, 2970489088, 2912523524, 1158689953, 1425511081, 2970489088,
+        2807004452, 2510299562, 271603006, 2505735035, 2370490899, 2510299562, 10873814, 2450376936, 2370490899,
+        2000734342, 1113679064, 2502160539, 1475266926, 2787925323, 1113679064, 1475797635, 3044470744, 2787925323,
+        1457157056, 1252556678, 3073232607, 1926798761, 3639907189, 1252556678, 2067740348, 2256217204, 3639907189,
+        3740999688, 1035400458, 3162437311, 4126312242, 686702830, 1035400458, 1699805291, 667792040, 686702830,
+        2422495016, 3203768688, 1858240466, 848719394, 4092709154, 3203768688, 659945473, 1863075174, 4092709154,
+        246817944, 871751352, 2834051003, 3976202597, 3721214025, 871751352, 783929942, 745295675, 3721214025,
+        3811740424, 3603608092, 2365398362, 3826150877, 2906557036, 3603608092, 2300510686, 966815948, 2906557036,
+        2816329160, 18201123, 3367710570, 437309679, 2220769388, 18201123, 1346863388, 705296543, 2220769388,
+        3310028953, 1662315499, 132645114, 2572908401, 3105849797, 1662315499, 1937586849, 1735620028, 3105849797,
+        461386353, 1359675853, 3599822966, 106675209, 2044154050, 1359675853, 1787730088, 1149892630, 2044154050,
+        3303902397, 345146034, 1417149696, 2231869247, 1116882637, 345146034, 1846832385, 79626976, 1116882637,
+        2765049417, 3117782790, 1805260159, 3796182890, 1101141726, 3117782790, 224270120, 1004001443, 1101141726,
+        89118668, 2494198515, 1356989069, 2490435731, 997151755, 2494198515, 1175528637, 3444341166, 997151755,
+        2340639019, 510225634, 286119182, 2045217287, 1194574818, 510225634, 2662281592, 1728500627, 1194574818,
+        210787847, 1189120688, 2848040407, 1087786165, 2343328484, 1189120688, 3465141330, 2893041005, 2343328484,
+        3438170226, 3236285682, 962036916, 2873263091, 215280489, 3236285682, 730413847, 1474823842, 215280489,
+        1566461658, 133010024, 2886695328, 2835827516, 653809404, 133010024, 3082882924, 3710942807, 653809404,
+        4201558916, 1263786956, 326001602, 762846463, 621546357, 1263786956, 2697142404, 1156650856, 621546357,
+        2655768102, 2339029465, 2430211448, 2669906627, 403962847, 2339029465, 1483118807, 639660658, 403962847,
+        3508595200, 4228486662, 754946994, 1913148390, 3500531602, 4228486662, 24637, 3773159052, 3500531602,
+        4024866227, 1143874914, 3205058469, 2970344133, 2873927273, 1143874914, 2167114735, 4095476435, 2873927273,
+        1479401095, 2958366486, 3027708794, 2704486034, 3574053987, 2958366486, 3630964515, 1276667706, 3574053987,
+        2035927380, 1363628533, 818363998, 3023327955, 3968427114, 1363628533, 1284825950, 2871663372, 3968427114,
+        3827747418, 3897287251, 4106993377, 1527779946, 3221052941, 3897287251, 4178727866, 4281160673, 3221052941,
+        1174358892, 2835476193, 959978619, 850076464, 3774782533, 2835476193, 3880910680, 3237990203, 3774782533,
+
+    };
+
+static constexpr unsigned long long h_A1P127[MRG323A_N] =
+    {
+        2427906178, 3580155704, 949770784, 226153695, 1230515664, 3580155704, 1988835001, 986791581, 1230515664,
+        1774047142, 3199155377, 3106427820, 1901920839, 4290900039, 3199155377, 4178980191, 280623348, 4290900039,
+        3567524348, 1934119675, 3188270128, 2997767678, 826363896, 1934119675, 262952343, 614326610, 826363896,
+        1625613062, 4288164505, 2481284279, 4273461426, 1177260757, 4288164505, 305959988, 4017252267, 1177260757,
+        337929267, 333342539, 418300166, 2944208672, 379097734, 333342539, 2084056909, 3625475947, 379097734,
+        1189899255, 1307754719, 1214919992, 3736721708, 3514751918, 1307754719, 732435953, 2021244538, 3514751918,
+        4089172695, 1533534334, 525643282, 1497577018, 1335684482, 1533534334, 2079007086, 3977541427, 1335684482,
+        3075256652, 2762754934, 3846844247, 3057872364, 3274545167, 2762754934, 4028573983, 938934351, 3274545167,
+        2597859300, 2880151048, 2523330453, 1121709186, 175667448, 2880151048, 4182510911, 1723133625, 175667448,
+        484148868, 1404283933, 2982534313, 3736767353, 3179865161, 1404283933, 391120388, 3758716888, 3179865161,
+        2138867468, 1128973399, 2133702321, 1613561693, 3622350766, 1128973399, 1500151924, 3759983985, 3622350766,
+        3027706760, 3786576552, 2698781808, 2810527099, 90498489, 3786576552, 4220122612, 1855245979, 90498489,
+        3739389517, 1110440720, 917457922, 2163873618, 3707591763, 1110440720, 2667061910, 2533383962, 3707591763,
+        1545226000, 1812182123, 3693349190, 3422065122, 3291428549, 1812182123, 1193168720, 2072837757, 3291428549,
+        3230096243, 2131723358, 3262178024, 2882890127, 4088518247, 2131723358, 3991553306, 1282224087, 4088518247,
+        301207261, 1722796810, 3697719854, 3350228505, 3410986694, 1722796810, 3684514720, 2846958957, 3410986694,
+        1532963114, 4236235786, 3871128158, 3540401964, 1285250577, 4236235786, 1105070646, 2764245175, 1285250577,
+        210906218, 3068599594, 3034582784, 340633153, 4004365908, 3068599594, 4238928187, 2299166464, 4004365908,
+        2274701639, 3955606166, 3081246407, 3199954992, 3948054919, 3955606166, 2399101442, 3438340286, 3948054919,
+        504137100, 1182303684, 201533985, 4188299661, 3042453580, 1182303684, 2578519273, 2674782930, 3042453580,
+        1382964588, 2578452047, 3140440866, 261861891, 1076783073, 2578452047, 1634588989, 164438428, 1076783073,
+        2529186343, 526867394, 3102803247, 2687252475, 2908898908, 526867394, 1213100579, 86050422, 2908898908,
+        2690118316, 538108523, 790337895, 4193870709, 1053552056, 538108523, 1635227281, 4002399925, 1053552056,
+        2123712957, 4205383007, 1812304090, 1095349745, 166243972, 4205383007, 428569070, 2128782357, 166243972,
+        1330151766, 3569679412, 4107175982, 3808641551, 3621125056, 3569679412, 4262164578, 1927692878, 3621125056,
+        3606295184, 2442739556, 3894922338, 1629626641, 2729678535, 2442739556, 3379124758, 4279360935, 2729678535,
+        1052092278, 4249024666, 919210106, 3253349463, 3629539480, 4249024666, 852514024, 4025926501, 3629539480,
+        12394571, 1252747620, 2133571953, 4227339509, 3197545170, 1252747620, 1884529704, 1976203831, 3197545170,
+        2986331025, 2671019282, 2847338542, 3173738401, 3542657885, 2671019282, 745203060, 1546667401, 3542657885,
+        2613012997, 2311336951, 2911336433, 1493974713, 92565032, 2311336951, 2786645250, 257065974, 92565032,
+        3424925004, 2776053372, 2204068573, 3770626858, 2509257810, 2776053372, 2979919489, 1146336783, 2509257810,
+        1474384834, 827894421, 515339473, 1373055755, 1949809417, 827894421, 3088339524, 1194193824, 1949809417,
+        1825805135, 1289872272, 3700877161, 3433422861, 4062509844, 1289872272, 3019008744, 2060641859, 4062509844,
+        3842597153, 4253338264, 3424495942, 698444416, 60268595, 4253338264, 4096010585, 47309624, 60268595,
+        2662288323, 2043518992, 1593435980, 1330201507, 3618850300, 2043518992, 2538793204, 271787962, 3618850300,
+        741020448, 997594656, 2398808739, 1160477043, 1522130854, 997594656, 3036916315, 2847712653, 1522130854,
+        2654964886, 1889728930, 53329096, 2042322941, 1621136330, 1889728930, 1553642730, 784545882, 1621136330,
+        1715219514, 2831829177, 929124824, 997274536, 404228189, 2831829177, 1386575385, 4107238699, 404228189,
+        3928131551, 2912523524, 1840499723, 4216003022, 2970489088, 2912523524, 1158689953, 1425511081, 2970489088,
+        2807004452, 2510299562, 271603006, 2505735035, 2370490899, 2510299562, 10873814, 2450376936, 2370490899,
+        2000734342, 1113679064, 2502160539, 1475266926, 2787925323, 1113679064, 1475797635, 3044470744, 2787925323,
+        1457157056, 1252556678, 3073232607, 1926798761, 3639907189, 1252556678, 2067740348, 2256217204, 3639907189,
+        3740999688, 1035400458, 3162437311, 4126312242, 686702830, 1035400458, 1699805291, 667792040, 686702830,
+        2422495016, 3203768688, 1858240466, 848719394, 4092709154, 3203768688, 659945473, 1863075174, 4092709154,
+        246817944, 871751352, 2834051003, 3976202597, 3721214025, 871751352, 783929942, 745295675, 3721214025,
+        3811740424, 3603608092, 2365398362, 3826150877, 2906557036, 3603608092, 2300510686, 966815948, 2906557036,
+        2816329160, 18201123, 3367710570, 437309679, 2220769388, 18201123, 1346863388, 705296543, 2220769388,
+        3310028953, 1662315499, 132645114, 2572908401, 3105849797, 1662315499, 1937586849, 1735620028, 3105849797,
+        461386353, 1359675853, 3599822966, 106675209, 2044154050, 1359675853, 1787730088, 1149892630, 2044154050,
+        3303902397, 345146034, 1417149696, 2231869247, 1116882637, 345146034, 1846832385, 79626976, 1116882637,
+        2765049417, 3117782790, 1805260159, 3796182890, 1101141726, 3117782790, 224270120, 1004001443, 1101141726,
+        89118668, 2494198515, 1356989069, 2490435731, 997151755, 2494198515, 1175528637, 3444341166, 997151755,
+        2340639019, 510225634, 286119182, 2045217287, 1194574818, 510225634, 2662281592, 1728500627, 1194574818,
+        210787847, 1189120688, 2848040407, 1087786165, 2343328484, 1189120688, 3465141330, 2893041005, 2343328484,
+        3438170226, 3236285682, 962036916, 2873263091, 215280489, 3236285682, 730413847, 1474823842, 215280489,
+        1566461658, 133010024, 2886695328, 2835827516, 653809404, 133010024, 3082882924, 3710942807, 653809404,
+        4201558916, 1263786956, 326001602, 762846463, 621546357, 1263786956, 2697142404, 1156650856, 621546357,
+        2655768102, 2339029465, 2430211448, 2669906627, 403962847, 2339029465, 1483118807, 639660658, 403962847,
+        3508595200, 4228486662, 754946994, 1913148390, 3500531602, 4228486662, 24637, 3773159052, 3500531602,
+        4024866227, 1143874914, 3205058469, 2970344133, 2873927273, 1143874914, 2167114735, 4095476435, 2873927273,
+        1479401095, 2958366486, 3027708794, 2704486034, 3574053987, 2958366486, 3630964515, 1276667706, 3574053987,
+        2035927380, 1363628533, 818363998, 3023327955, 3968427114, 1363628533, 1284825950, 2871663372, 3968427114,
+        3827747418, 3897287251, 4106993377, 1527779946, 3221052941, 3897287251, 4178727866, 4281160673, 3221052941,
+        1174358892, 2835476193, 959978619, 850076464, 3774782533, 2835476193, 3880910680, 3237990203, 3774782533,
+
+    };
+
+static constexpr __device__ unsigned long long d_A2P127[MRG323A_N] =
+    {
+        1464411153, 277697599, 1610723613, 32183930, 1464411153, 1022607788, 2824425944, 32183930, 2093834863,
+        3492361727, 1027004383, 3167429889, 3674905362, 3492361727, 3572939265, 4270409313, 3674905362, 698814233,
+        880482061, 205175925, 4070445105, 2208329119, 880482061, 1933248566, 3741227945, 2208329119, 3962062826,
+        4184605179, 1189429800, 567967482, 107217966, 4184605179, 784865788, 549462420, 107217966, 3134382704,
+        2732536445, 1231107067, 3374588386, 409954030, 2732536445, 1044831206, 3398162498, 409954030, 3505648581,
+        2169560691, 1076348534, 637306236, 3704346564, 2169560691, 293694496, 632453145, 3704346564, 1609425246,
+        372115891, 3928812480, 2830541169, 3056527841, 372115891, 1924239834, 3044937468, 3056527841, 547142630,
+        1660852083, 3635660815, 1389092450, 1025573319, 1660852083, 3276803366, 4036331438, 1025573319, 4092197741,
+        1360732901, 2887812973, 4101068693, 52572783, 1360732901, 112458461, 2636566855, 52572783, 1136777988,
+        3455696508, 536919193, 3978804036, 3094157668, 3455696508, 3821833900, 2278849016, 3094157668, 2531965909,
+        2125991744, 890897326, 3790557569, 1433592392, 2125991744, 3671109604, 808215503, 1433592392, 2446306581,
+        3524411799, 932865240, 1838275365, 1789634890, 3524411799, 4130736474, 2252266098, 1789634890, 3048775967,
+        1773339925, 948403862, 1999624391, 983864203, 1773339925, 3734776305, 314407045, 983864203, 2648614071,
+        321802921, 1099164995, 2112167358, 3760936985, 321802921, 1003573324, 3758858458, 3760936985, 4014658840,
+        2196438580, 805386227, 4266375092, 4124675351, 2196438580, 2527961345, 94452540, 4124675351, 2825656399,
+        66735368, 2228005807, 4186703168, 2624855312, 66735368, 2708679078, 4098470056, 2624855312, 1773862183,
+        3072642883, 2746897053, 2690305546, 1105106652, 3072642883, 4047666135, 2862886282, 1105106652, 3597347398,
+        232906611, 3873338256, 4051554873, 3027413363, 232906611, 3159432673, 3872967050, 3027413363, 987156327,
+        1160686753, 3676603152, 1635979789, 1447386846, 1160686753, 2670438424, 816212890, 1447386846, 4288868534,
+        3825238244, 1445162354, 2362389441, 3440193648, 3825238244, 3520937545, 2652790808, 3440193648, 405299994,
+        1984094858, 532165989, 2027397575, 1455977136, 1984094858, 2433255524, 1039994763, 1455977136, 2069333087,
+        3680843319, 2332949611, 3516795313, 2033851810, 3680843319, 3843367307, 3686294589, 2033851810, 3912995069,
+        967423689, 1724183394, 635932799, 641380480, 967423689, 2145297779, 1723000412, 641380480, 455633660,
+        2130938335, 1534972306, 2511584766, 273828453, 2130938335, 3112810093, 4084843716, 273828453, 1399334152,
+        168278549, 541167592, 190177712, 403188859, 168278549, 2092073970, 58789558, 403188859, 2777887189,
+        634843389, 4082275720, 2092828966, 351187677, 634843389, 1312056270, 3347241070, 351187677, 2417192332,
+        443276110, 1113643788, 271102234, 3083745876, 443276110, 3370743767, 4200577503, 3083745876, 3298601960,
+        3533393557, 764977733, 3400275098, 144639933, 3533393557, 2646475951, 77963866, 144639933, 3794766611,
+        4064854722, 1198665008, 2872196602, 3274748603, 4064854722, 4164637970, 4238693771, 3274748603, 1981721347,
+        2279220396, 2355957139, 1417574285, 885864931, 2279220396, 1344421653, 1895527787, 885864931, 3726919367,
+        2898100178, 2427331008, 348923199, 3175444953, 2898100178, 4290541487, 246118669, 3175444953, 3410622769,
+        284442065, 4064194676, 2295560707, 4182706556, 284442065, 3696899246, 1201342255, 4182706556, 1145356382,
+        656615546, 442908965, 3724738272, 1624967553, 656615546, 798014134, 1157949454, 1624967553, 496247378,
+        265689579, 675056541, 3009083380, 3820679930, 265689579, 2961990151, 562287964, 3820679930, 1853486796,
+        1675739167, 2319843005, 760605578, 4161492847, 1675739167, 226142150, 1017447188, 4161492847, 3431158427,
+        1759873736, 2334568602, 2154570180, 1812793060, 1759873736, 2111094408, 1168460586, 1812793060, 2495653141,
+        317621194, 868104288, 664971082, 2340275074, 317621194, 2168960688, 725706104, 2340275074, 3532023115,
+        3926931954, 2907684453, 615601328, 1132340715, 3926931954, 676995757, 1154819290, 1132340715, 1662727700,
+        3921782078, 3376494857, 2969567377, 475345024, 3921782078, 4206379953, 1795936544, 475345024, 934679595,
+        3119292228, 741613041, 2083352304, 1047885963, 3119292228, 1581078542, 1065969969, 1047885963, 661718928,
+        3643472111, 2870554228, 3995474529, 3804264051, 3643472111, 1366457944, 1246805564, 3804264051, 993186530,
+        796711791, 3878204845, 3160293932, 255632881, 796711791, 3778927111, 3472564181, 255632881, 388382377,
+        1776984101, 1742284034, 3449763933, 1349354417, 1776984101, 1264780832, 715722511, 1349354417, 1213319489,
+        4261866865, 1914382786, 201872335, 614207188, 4261866865, 1853554849, 2046042882, 614207188, 3193186353,
+        2210205512, 2847073169, 3324925707, 1251969297, 2210205512, 3491451503, 470400916, 1251969297, 2184392547,
+        1523590942, 2391111113, 68341529, 295466806, 1523590942, 4143310876, 3527253079, 295466806, 4059123142,
+        1406902110, 3735012720, 1774518130, 1814959027, 1406902110, 1560544267, 346472965, 1814959027, 964257199,
+        855309653, 4208503105, 1518467541, 2025248418, 855309653, 4148125749, 1349947330, 2025248418, 1168504873,
+        2375338156, 3629519168, 409696181, 252401654, 2375338156, 3992097193, 2793725401, 252401654, 1350184085,
+        873141039, 3885583138, 361604799, 3554143374, 873141039, 894746180, 1919765327, 3554143374, 876210854,
+        246368794, 1703793169, 2317362874, 2300930144, 246368794, 2560214589, 2016163623, 2300930144, 1504276775,
+        1574610921, 2147546631, 4103450226, 107416526, 1574610921, 1773803959, 1402542742, 107416526, 550063800,
+        363388665, 592194244, 1746615522, 2637234667, 363388665, 4031408742, 2895130475, 2637234667, 296510335,
+        3997368560, 3047771871, 3178383826, 1160174754, 3997368560, 4027094919, 1234984211, 1160174754, 4226264344,
+        3303179301, 4243968063, 3235964171, 1776841674, 3303179301, 2867287469, 1500495759, 1776841674, 1708226553,
+        1482944153, 3192311574, 354466071, 3932773012, 1482944153, 389193591, 3350181058, 3932773012, 3398059015,
+        640968550, 3226860971, 922372912, 1254989667, 640968550, 2383815228, 2027371896, 1254989667, 2925300409,
+        2313146046, 3910187183, 1377591475, 1689291784, 2313146046, 4255405993, 1650609719, 1689291784, 1897624297,
+        3656310954, 882924050, 2702189958, 3185020283, 3656310954, 1923190496, 2449669145, 3185020283, 4235849984,
+        377232416, 1498446142, 4229103619, 3926377906, 377232416, 600268838, 511317726, 3926377906, 216160452,
+        1969399344, 3273966859, 4220943579, 3952111894, 1969399344, 575096961, 3815277103, 3952111894, 792177412,
+        2957238169, 1410010554, 1523740068, 3949237584, 2957238169, 74149658, 2564746147, 3949237584, 2557663578,
+        3377318569, 1927835240, 2556102508, 3022040116, 3377318569, 2549406364, 2387074241, 3022040116, 1477293711,
+        257306870, 1748489735, 547809226, 3708493374, 257306870, 4183546362, 4435502, 3708493374, 1607696753,
+
+    };
+
+static constexpr unsigned long long h_A2P127[MRG323A_N] =
+    {
+        1464411153, 277697599, 1610723613, 32183930, 1464411153, 1022607788, 2824425944, 32183930, 2093834863,
+        3492361727, 1027004383, 3167429889, 3674905362, 3492361727, 3572939265, 4270409313, 3674905362, 698814233,
+        880482061, 205175925, 4070445105, 2208329119, 880482061, 1933248566, 3741227945, 2208329119, 3962062826,
+        4184605179, 1189429800, 567967482, 107217966, 4184605179, 784865788, 549462420, 107217966, 3134382704,
+        2732536445, 1231107067, 3374588386, 409954030, 2732536445, 1044831206, 3398162498, 409954030, 3505648581,
+        2169560691, 1076348534, 637306236, 3704346564, 2169560691, 293694496, 632453145, 3704346564, 1609425246,
+        372115891, 3928812480, 2830541169, 3056527841, 372115891, 1924239834, 3044937468, 3056527841, 547142630,
+        1660852083, 3635660815, 1389092450, 1025573319, 1660852083, 3276803366, 4036331438, 1025573319, 4092197741,
+        1360732901, 2887812973, 4101068693, 52572783, 1360732901, 112458461, 2636566855, 52572783, 1136777988,
+        3455696508, 536919193, 3978804036, 3094157668, 3455696508, 3821833900, 2278849016, 3094157668, 2531965909,
+        2125991744, 890897326, 3790557569, 1433592392, 2125991744, 3671109604, 808215503, 1433592392, 2446306581,
+        3524411799, 932865240, 1838275365, 1789634890, 3524411799, 4130736474, 2252266098, 1789634890, 3048775967,
+        1773339925, 948403862, 1999624391, 983864203, 1773339925, 3734776305, 314407045, 983864203, 2648614071,
+        321802921, 1099164995, 2112167358, 3760936985, 321802921, 1003573324, 3758858458, 3760936985, 4014658840,
+        2196438580, 805386227, 4266375092, 4124675351, 2196438580, 2527961345, 94452540, 4124675351, 2825656399,
+        66735368, 2228005807, 4186703168, 2624855312, 66735368, 2708679078, 4098470056, 2624855312, 1773862183,
+        3072642883, 2746897053, 2690305546, 1105106652, 3072642883, 4047666135, 2862886282, 1105106652, 3597347398,
+        232906611, 3873338256, 4051554873, 3027413363, 232906611, 3159432673, 3872967050, 3027413363, 987156327,
+        1160686753, 3676603152, 1635979789, 1447386846, 1160686753, 2670438424, 816212890, 1447386846, 4288868534,
+        3825238244, 1445162354, 2362389441, 3440193648, 3825238244, 3520937545, 2652790808, 3440193648, 405299994,
+        1984094858, 532165989, 2027397575, 1455977136, 1984094858, 2433255524, 1039994763, 1455977136, 2069333087,
+        3680843319, 2332949611, 3516795313, 2033851810, 3680843319, 3843367307, 3686294589, 2033851810, 3912995069,
+        967423689, 1724183394, 635932799, 641380480, 967423689, 2145297779, 1723000412, 641380480, 455633660,
+        2130938335, 1534972306, 2511584766, 273828453, 2130938335, 3112810093, 4084843716, 273828453, 1399334152,
+        168278549, 541167592, 190177712, 403188859, 168278549, 2092073970, 58789558, 403188859, 2777887189,
+        634843389, 4082275720, 2092828966, 351187677, 634843389, 1312056270, 3347241070, 351187677, 2417192332,
+        443276110, 1113643788, 271102234, 3083745876, 443276110, 3370743767, 4200577503, 3083745876, 3298601960,
+        3533393557, 764977733, 3400275098, 144639933, 3533393557, 2646475951, 77963866, 144639933, 3794766611,
+        4064854722, 1198665008, 2872196602, 3274748603, 4064854722, 4164637970, 4238693771, 3274748603, 1981721347,
+        2279220396, 2355957139, 1417574285, 885864931, 2279220396, 1344421653, 1895527787, 885864931, 3726919367,
+        2898100178, 2427331008, 348923199, 3175444953, 2898100178, 4290541487, 246118669, 3175444953, 3410622769,
+        284442065, 4064194676, 2295560707, 4182706556, 284442065, 3696899246, 1201342255, 4182706556, 1145356382,
+        656615546, 442908965, 3724738272, 1624967553, 656615546, 798014134, 1157949454, 1624967553, 496247378,
+        265689579, 675056541, 3009083380, 3820679930, 265689579, 2961990151, 562287964, 3820679930, 1853486796,
+        1675739167, 2319843005, 760605578, 4161492847, 1675739167, 226142150, 1017447188, 4161492847, 3431158427,
+        1759873736, 2334568602, 2154570180, 1812793060, 1759873736, 2111094408, 1168460586, 1812793060, 2495653141,
+        317621194, 868104288, 664971082, 2340275074, 317621194, 2168960688, 725706104, 2340275074, 3532023115,
+        3926931954, 2907684453, 615601328, 1132340715, 3926931954, 676995757, 1154819290, 1132340715, 1662727700,
+        3921782078, 3376494857, 2969567377, 475345024, 3921782078, 4206379953, 1795936544, 475345024, 934679595,
+        3119292228, 741613041, 2083352304, 1047885963, 3119292228, 1581078542, 1065969969, 1047885963, 661718928,
+        3643472111, 2870554228, 3995474529, 3804264051, 3643472111, 1366457944, 1246805564, 3804264051, 993186530,
+        796711791, 3878204845, 3160293932, 255632881, 796711791, 3778927111, 3472564181, 255632881, 388382377,
+        1776984101, 1742284034, 3449763933, 1349354417, 1776984101, 1264780832, 715722511, 1349354417, 1213319489,
+        4261866865, 1914382786, 201872335, 614207188, 4261866865, 1853554849, 2046042882, 614207188, 3193186353,
+        2210205512, 2847073169, 3324925707, 1251969297, 2210205512, 3491451503, 470400916, 1251969297, 2184392547,
+        1523590942, 2391111113, 68341529, 295466806, 1523590942, 4143310876, 3527253079, 295466806, 4059123142,
+        1406902110, 3735012720, 1774518130, 1814959027, 1406902110, 1560544267, 346472965, 1814959027, 964257199,
+        855309653, 4208503105, 1518467541, 2025248418, 855309653, 4148125749, 1349947330, 2025248418, 1168504873,
+        2375338156, 3629519168, 409696181, 252401654, 2375338156, 3992097193, 2793725401, 252401654, 1350184085,
+        873141039, 3885583138, 361604799, 3554143374, 873141039, 894746180, 1919765327, 3554143374, 876210854,
+        246368794, 1703793169, 2317362874, 2300930144, 246368794, 2560214589, 2016163623, 2300930144, 1504276775,
+        1574610921, 2147546631, 4103450226, 107416526, 1574610921, 1773803959, 1402542742, 107416526, 550063800,
+        363388665, 592194244, 1746615522, 2637234667, 363388665, 4031408742, 2895130475, 2637234667, 296510335,
+        3997368560, 3047771871, 3178383826, 1160174754, 3997368560, 4027094919, 1234984211, 1160174754, 4226264344,
+        3303179301, 4243968063, 3235964171, 1776841674, 3303179301, 2867287469, 1500495759, 1776841674, 1708226553,
+        1482944153, 3192311574, 354466071, 3932773012, 1482944153, 389193591, 3350181058, 3932773012, 3398059015,
+        640968550, 3226860971, 922372912, 1254989667, 640968550, 2383815228, 2027371896, 1254989667, 2925300409,
+        2313146046, 3910187183, 1377591475, 1689291784, 2313146046, 4255405993, 1650609719, 1689291784, 1897624297,
+        3656310954, 882924050, 2702189958, 3185020283, 3656310954, 1923190496, 2449669145, 3185020283, 4235849984,
+        377232416, 1498446142, 4229103619, 3926377906, 377232416, 600268838, 511317726, 3926377906, 216160452,
+        1969399344, 3273966859, 4220943579, 3952111894, 1969399344, 575096961, 3815277103, 3952111894, 792177412,
+        2957238169, 1410010554, 1523740068, 3949237584, 2957238169, 74149658, 2564746147, 3949237584, 2557663578,
+        3377318569, 1927835240, 2556102508, 3022040116, 3377318569, 2549406364, 2387074241, 3022040116, 1477293711,
+        257306870, 1748489735, 547809226, 3708493374, 257306870, 4183546362, 4435502, 3708493374, 1607696753,
+
+    };
+
+
+#endif // ROCRAND_MRG32K3A_PRECOMPUTED_H_
diff --git a/include/targets/omptarget/random_helper.h b/include/targets/omptarget/random_helper.h
index b9d06882ac..7ea84123f7 100644
--- a/include/targets/omptarget/random_helper.h
+++ b/include/targets/omptarget/random_helper.h
@@ -1,22 +1,15 @@
 #pragma once
 
-#include <random>
-// #include <curand_kernel.h>
+#include <random_quda.h>
+
+// We don't use the internal Box-Muller state.
+#define ROCRAND_DETAIL_MRG32K3A_BM_NOT_IN_STATE
+#include <externals/rocrand/rocrand_mrg32k3a.h>
 
 namespace quda {
-/*
-#if defined(XORWOW)
-  using rng_state_t = curandStateXORWOW;
-#elif defined(MRG32k3a)
-  using rng_state_t = curandStateMRG32k3a;
-#else
-  using rng_state_t = curandStateMRG32k3a;
-#endif
-*/
-  using rng_state_t = std::mt19937_64;
 
   struct RNGState {
-    rng_state_t state;
+    rocrand_state_mrg32k3a state;
   };
 
   /**
@@ -26,26 +19,26 @@ namespace quda {
    * @param [in] offset -- the offset
    * @param [in,out] state - the RNG State
    */
-  __device__ inline void random_init(unsigned long long seed, unsigned long long sequence, unsigned long long offset, RNGState &state)
+  inline void random_init(unsigned long long seed, unsigned long long sequence,
+			  unsigned long long offset, RNGState &state)
   {
-    ompwip();
-    std::seed_seq ss{seed, sequence, offset};
-    state.state.seed(ss);
-    // curand_init(seed, sequence, offset, &state.state);
+    //curand_init(seed, sequence, offset, &state.state);
+    state.state.seed(seed, sequence, offset);
   }
 
   template<class Real>
-    struct uniform {
+    struct uniform { };
+  template<>
+    struct uniform<float> {
 
     /**
      * \brief Return a uniform deviate between 0 and 1
      * @param [in,out] the RNG State
      */
-    __device__ static inline Real rand(RNGState &state)
+    static inline float rand(RNGState &state)
     {
-      ompwip();
-      std::uniform_real_distribution<Real> u;
-      return u(state.state);
+      //curand_uniform(&state.state);
+      return ROCRAND_2POW32_INV + (rocrand(&state.state) * ROCRAND_2POW32_INV);
     }
 
     /**
@@ -54,25 +47,71 @@ namespace quda {
      * @param [in] a (the lower end of the range)
      * @param [in] b (the upper end of the range)
      */
-    __device__ static inline Real rand(RNGState &state, Real a, Real b)
+    static inline float rand(RNGState &state, float a, float b)
+    {
+      //return a + (b - a) * rand(state);
+      return a + (b - a) * (ROCRAND_2POW32_INV + (rocrand(&state.state) * ROCRAND_2POW32_INV));
+    }
+
+  };
+
+  template<>
+    struct uniform<double> {
+    /**
+     * \brief Return a uniform deviate between 0 and 1
+     * @param [in,out] the RNG State
+     */
+    static inline double rand(RNGState &state)
     {
-      ompwip();
-      std::uniform_real_distribution<Real> u(a,b);
-      return u(state.state);
+      //curand_uniform_double(&state.state);
+      return ROCRAND_2POW32_INV_DOUBLE + (rocrand(&state.state) * ROCRAND_2POW32_INV_DOUBLE);
     }
 
+    /**
+     * \brief Return a uniform deviate between a and b
+     * @param [in,out] the RNG State
+     * @param [in] a -- the lower end of the range
+     * @param [in] b -- the high end of the range
+     */
+    static inline double rand(RNGState &state, double a, double b)
+    {
+      //return a + (b - a) * rand(state);
+      return a + (b - a) * (ROCRAND_2POW32_INV_DOUBLE + (rocrand(&state.state) * ROCRAND_2POW32_INV_DOUBLE));
+    }
   };
 
   template<class Real>
-    struct normal {    /**
+    struct normal { };
+
+  template<>
+    struct normal<float> {
+    /**
      * \brief return a gaussian normal deviate with mean of 0
      * @param [in,out] state
      */
-    __device__ static inline Real rand(RNGState &state)
+    static inline float rand(RNGState &state)
+    {
+      //curand_normal(&state.state);
+      float u = ROCRAND_2POW32_INV + (rocrand(&state.state) * ROCRAND_2POW32_INV);
+      float v = (ROCRAND_2POW32_INV + (rocrand(&state.state) * ROCRAND_2POW32_INV)) * ROCRAND_2PI;
+      float s = sqrtf(-2.0f * logf(u));
+      return sinf(v) * s;
+    }
+  };
+
+  template<>
+    struct normal<double> {
+    /**
+     * \brief return a gaussian (normal) deviate with a mean of 0
+     * @param [in,out] state
+     */
+    static inline double rand(RNGState &state)
     {
-      ompwip();
-      std::normal_distribution<Real> n;
-      return n(state.state);
+      //curand_normal_double(&state.state);
+      double u = ROCRAND_2POW32_INV_DOUBLE + (rocrand(&state.state) * ROCRAND_2POW32_INV_DOUBLE);
+      double v = (ROCRAND_2POW32_INV_DOUBLE + (rocrand(&state.state) * ROCRAND_2POW32_INV_DOUBLE)) * ROCRAND_PI_DOUBLE * 2.;
+      double s = sqrt(-2. * log(u));
+      return sin(v) * s;
     }
   };
 

From 7616d7d8ab9e3ce39fb4bee9010512910f2dd437 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 17 May 2021 19:26:06 +0000
Subject: [PATCH 048/198] omp: better handling of memcpyDefault

---
 lib/targets/omptarget/malloc.cpp   | 23 ++++++++++++-------
 lib/targets/omptarget/quda_api.cpp | 36 ++++++++++++++++++++++++++----
 2 files changed, 47 insertions(+), 12 deletions(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index fdbbc35c97..a15828bc0a 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -137,6 +137,14 @@ namespace quda
     alloc[type].erase(ptr);
   }
 
+  namespace target {
+    static std::map<const void*,void*> omp_mapped_ptr;  // host -> device
+    bool is_device_ptr(void *p)
+    {
+      // DEVICE, DEVICE_PINNED, HOST, PINNED, MAPPED, MANAGED, SHMEM
+      return alloc[DEVICE].count(p)>0 || alloc[DEVICE_PINNED].count(p)>0;
+    }
+  }
   /**
    * Under CUDA 4.0, cudaHostRegister seems to require that both the
    * beginning and end of the buffer be aligned on page boundaries.
@@ -322,7 +330,6 @@ namespace quda
     return ptr;
   }
 
-  static std::map<const void*,void*> omp_mapped_ptr;  // host -> device
   /**
    * Allocate page-locked ("pinned") host memory, and map it into the
    * GPU address space.  This function should only be called via the
@@ -347,16 +354,16 @@ namespace quda
     print_trace();
     if(0<omp_get_num_devices()){
       int d = omp_get_default_device();
-      void *dp = omp_target_alloc(a.base_size, d);  // FIXME keep it somewhere so we can free it
+      void *dp = omp_target_alloc(a.base_size, d);
       ompwip("require special memcpy",[=](){printfQuda("mapped_malloc_ device: %p\n",dp);});
       if(!dp)
         errorQuda("%s:%d %s() Failed to allocate device memory of size %zu for mapped malloc\n", file, line, func, size);
       if(omp_target_associate_ptr(ptr, dp, a.base_size, 0, d))
         errorQuda("%s:%d %s() Failed to assocaite device memory to host pointer\n", file, line, func);
-      omp_mapped_ptr[ptr] = dp;
+      target::omp_mapped_ptr[ptr] = dp;
     }else{
       warningQuda("%s:%d %s() mapped malloc without a device", file, line, func);
-      omp_mapped_ptr[ptr] = ptr;
+      target::omp_mapped_ptr[ptr] = ptr;
     }
 #endif
     track_malloc(MAPPED, a, ptr);
@@ -513,13 +520,13 @@ namespace quda
       track_free(HOST, ptr);
       free(ptr);
     } else if (alloc[PINNED].count(ptr)) {
-      ompwip("free PINNED ptr unimplemented");
+      ompwip("free PINNED ptr");
 /*
       cudaError_t err = cudaHostUnregister(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to unregister pinned memory (%s:%d in %s())\n", file, line, func); }
+*/
       track_free(PINNED, ptr);
       free(ptr);
-*/
     } else if (alloc[MAPPED].count(ptr)) {
 #ifdef HOST_ALLOC
       ompwip("HOST_ALLOC alloc[MAPPED].count(ptr) untested code path"); print_trace();
@@ -530,7 +537,7 @@ namespace quda
 */
 #else
       ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr)");
-      void *dp = omp_mapped_ptr[ptr];
+      void *dp = target::omp_mapped_ptr[ptr];
       if(dp!=ptr){
         omp_target_disassociate_ptr(ptr, omp_get_default_device());
         omp_target_free(dp, omp_get_default_device());
@@ -651,7 +658,7 @@ namespace quda
   {
     ompwip("get_mapped_device_pointer_");
     print_trace();
-    return omp_mapped_ptr[host];
+    return target::omp_mapped_ptr[host];
 /*
     void *device;
     auto error = cudaHostGetDevicePointer(&device, const_cast<void *>(host), 0);
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 03f717d823..11adcfec99 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -73,6 +73,12 @@ printmem(void *d, std::size_t m, int host)
 }
 #endif
 
+namespace quda {
+  namespace target {
+    extern bool is_device_ptr(void *);
+  }
+}
+
 static inline void
 ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
 {
@@ -110,13 +116,35 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
     printmem(d,c,0);
     break;
   case cudaMemcpyDefault:
-    warningQuda("cudaMemcpyDefault calling host to device");
-    printmem(s,c,1);
     if(0<omp_get_num_devices()){
-      omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
-      printmem(d,c,0);
+      if(quda::target::is_device_ptr(d)){
+        if(quda::target::is_device_ptr(s)){
+          warningQuda("cudaMemcpyDefault calling device to device");
+          printmem(s,c,0);
+          omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
+          printmem(d,c,0);
+        }else{
+          warningQuda("cudaMemcpyDefault calling host to device");
+          printmem(s,c,1);
+          omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+          printmem(d,c,0);
+        }
+      }else{
+        if(quda::target::is_device_ptr(s)){
+          warningQuda("cudaMemcpyDefault calling device to host");
+          printmem(s,c,0);
+          omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
+          printmem(d,c,1);
+        }else{
+          warningQuda("cudaMemcpyDefault calling host to host");
+          printmem(s,c,1);
+          memcpy(d,s,c);
+          printmem(d,c,1);
+        }
+      }
     }else{
       warningQuda("cudaMemcpyDefault without a device, calling memcpy");
+      printmem(s,c,1);
       memcpy(d,s,c);
       printmem(d,c,1);
     }

From 0dfb5c7b403ff4ad799069d6b136a024158c9ec8 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 17 May 2021 19:27:05 +0000
Subject: [PATCH 049/198] omp: update default device parameters

---
 lib/targets/omptarget/device.cpp | 42 ++++++++++++++++----------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index a49ab929c5..173295839f 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -42,35 +42,35 @@ static int cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
     p->major = 7;
     p->minor = 0;
     p->sharedMemPerBlock = 0;
-    p->maxThreadsPerBlock = 65536;
-    p->maxThreadsPerMultiProcessor = 65536;
-    p->maxThreadsDim[0] = 65536;
-    p->maxThreadsDim[1] = 2;
-    p->maxThreadsDim[2] = 1;
+    p->maxThreadsPerBlock = 1024;
+    p->maxThreadsPerMultiProcessor = 1024;
+    p->maxThreadsDim[0] = 1024;
+    p->maxThreadsDim[1] = 1024;
+    p->maxThreadsDim[2] = 64;
     p->maxGridSize[0] = 65536;
-    p->maxGridSize[1] = 8;
-    p->maxGridSize[2] = 8;
-    p->multiProcessorCount = 4096;
+    p->maxGridSize[1] = 65536;
+    p->maxGridSize[2] = 65536;
+    p->multiProcessorCount = 16;
   }else{
     p->name = omphostname;
-    p->major = 3;
+    p->major = 7;
     p->minor = 0;
     p->sharedMemPerBlock = 0;
-    p->maxThreadsPerBlock = 4096;
-    p->maxThreadsPerMultiProcessor = 4096;
-    p->maxThreadsDim[0] = 4096;
-    p->maxThreadsDim[1] = 2;
-    p->maxThreadsDim[2] = 1;
-    p->maxGridSize[0] = 128;
-    p->maxGridSize[1] = 8;
-    p->maxGridSize[2] = 8;
-    p->multiProcessorCount = 64;
+    p->maxThreadsPerBlock = 1024;
+    p->maxThreadsPerMultiProcessor = 1024;
+    p->maxThreadsDim[0] = 1024;
+    p->maxThreadsDim[1] = 1024;
+    p->maxThreadsDim[2] = 64;
+    p->maxGridSize[0] = 65536;
+    p->maxGridSize[1] = 65536;
+    p->maxGridSize[2] = 65536;
+    p->multiProcessorCount = 16;
   }
   p->warpSize = 32;
   p->totalGlobalMem = 1u<34;
-  p->totalConstMem = 1u<34;
-  p->memPitch = 1u<34;
-  p->regsPerBlock = 1024;
+  p->totalConstMem = 65536;
+  p->memPitch = 1u<31;
+  p->regsPerBlock = 32768;
   p->unifiedAddressing = 1;
   p->deviceOverlap = 0;
   return 0;

From 736600da563cae6c658738853e5e28fdee051cc3 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 17 May 2021 19:27:58 +0000
Subject: [PATCH 050/198] omp: try allocator(omp_pteam_mem_alloc) for shared
 memory [WIP]

---
 include/gauge_fix_ovr_hit_devf.cuh | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/include/gauge_fix_ovr_hit_devf.cuh b/include/gauge_fix_ovr_hit_devf.cuh
index c4d5625eee..576b7981a1 100644
--- a/include/gauge_fix_ovr_hit_devf.cuh
+++ b/include/gauge_fix_ovr_hit_devf.cuh
@@ -53,7 +53,12 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
+#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
+#else
+    Float elems[2048];
+    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
+#endif
 
     //initialize shared memory
     if (mu < 4) elems[mu * blockSize + tid] = 0.0;
@@ -148,7 +153,12 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
+#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
+#else
+    Float elems[2048];
+    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
+#endif
 
     //Loop over all SU(2) subroups of SU(N)
     //#pragma unroll
@@ -236,7 +246,12 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
+#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
+#else
+    Float elems[2048];
+    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
+#endif
 
     //Loop over all SU(2) subroups of SU(N)
     //#pragma unroll
@@ -364,7 +379,12 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
+#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
+#else
+    Float elems[2048];
+    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
+#endif
 
     //initialize shared memory
     if (mu < 4) elems[mu * blockSize + tid] = 0.0;
@@ -447,7 +467,12 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
+#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
+#else
+    Float elems[2048];
+    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
+#endif
 
     //Loop over all SU(2) subroups of SU(N)
     //#pragma unroll
@@ -523,7 +548,12 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
+#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
+#else
+    Float elems[2048];
+    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
+#endif
 
     //Loop over all SU(2) subroups of SU(N)
     //#pragma unroll

From f299358375672ef9f45112014cd2501ea247853d Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:21:41 +0000
Subject: [PATCH 051/198] omp: only warning if no device

---
 include/communicator_quda.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/communicator_quda.h b/include/communicator_quda.h
index 3b40cb1e34..e204563e82 100644
--- a/include/communicator_quda.h
+++ b/include/communicator_quda.h
@@ -524,14 +524,14 @@ struct Communicator {
       }
 
       int device_count = quda::device::get_device_count();
-      if (device_count == 0) { errorQuda("No devices found"); }
+      if (device_count == 0) { warningQuda("No devices found"); }
       if (gpuid >= device_count) {
         char *enable_mps_env = getenv("QUDA_ENABLE_MPS");
         if (enable_mps_env && strcmp(enable_mps_env, "1") == 0) {
           gpuid = gpuid % device_count;
           printf("MPS enabled, rank=%3d -> gpu=%d\n", comm_rank(), gpuid);
         } else {
-          errorQuda("Too few GPUs available on %s", comm_hostname());
+          warningQuda("Too few GPUs available on %s", comm_hostname());
         }
       }
     }

From a19a5b88aa601d355643243164d10eb18499d66c Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:22:38 +0000
Subject: [PATCH 052/198] omp: add omp_init/reduce to MomUpdate

---
 include/kernels/momentum.cuh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/include/kernels/momentum.cuh b/include/kernels/momentum.cuh
index a13a2d443b..6c292de9c3 100644
--- a/include/kernels/momentum.cuh
+++ b/include/kernels/momentum.cuh
@@ -95,6 +95,17 @@ namespace quda {
 
     static constexpr bool do_sum = false;
 
+#ifdef QUDA_BACKEND_OMPTARGET
+    static reduce_t reduce_omp(const reduce_t &a, const reduce_t &b)
+    {
+      auto c = a;
+      if (b[0] > a[0]) c[0] = b[0];
+      if (b[1] > a[1]) c[1] = b[1];
+      return c;
+    }
+    static reduce_t init_omp() { return reduce_t(); }  // see UpdateMomArg::init().
+#endif
+
     /**
        @brief Functor for finding the maximum over a vec2 field.  Each
        lane is evaluated separately.

From 06ca380a7c529cdfd104ee9a28f9822ad315d4eb Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:23:26 +0000
Subject: [PATCH 053/198] omp: update ompwip functions

---
 include/quda_api.h | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/include/quda_api.h b/include/quda_api.h
index 665fed11c9..933896f170 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -58,16 +58,29 @@ static inline double3 make_double3(double x,double y,double z){return double3{x,
 struct double4 {double x,y,z,w;};
 static inline double4 make_double4(double x,double y,double z,double w){return double4{x,y,z,w};}
 
+template <typename... Fmt>
 static inline int
-ompwip_(const char * const file, const size_t line, const char * const func, const char * const msg, std::function<void(void)>f = [](){})
+ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f, Fmt... fmt)
 {
-  if(0==omp_get_team_num()&&0==omp_get_thread_num()) std::cerr<<"OMP WIP:"<<msg<<": "<<file<<':'<<line<<' '<<func<<std::endl;
+  if(0==omp_get_team_num()&&0==omp_get_thread_num()){
+    printf("OMP WIP: ");
+    printf(fmt...);
+    printf(" %s:%ld %s\n", file, line, func);
+  }
   f();
   return 0;
 }
+template <typename... Fmt>
+static inline int
+ompwip_(const char * const file, const size_t line, const char * const func, const char * const msg, Fmt... fmt)
+{
+  return ompwip_(file,line,func,[](){},msg,fmt...);
+}
 static inline int
-ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f = [](){})
-{return ompwip_(file,line,func,"",f);}
+ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f=[](){})
+{
+  return ompwip_(file,line,func,f,"");
+}
 #define ompwip(...) ompwip_(__FILE__,__LINE__,__PRETTY_FUNCTION__,##__VA_ARGS__)
 
 using cudaStream_t = int;  // device.h:/cudaStream_t

From 6a576054c60706e25f7f37de578edc541a166a20 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:23:59 +0000
Subject: [PATCH 054/198] omp: remove cpu side debug print in reduce_helper

---
 include/reduce_helper.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/include/reduce_helper.h b/include/reduce_helper.h
index 6cc838ef36..3e130c6e7d 100644
--- a/include/reduce_helper.h
+++ b/include/reduce_helper.h
@@ -186,9 +186,6 @@ namespace quda
       omp_target_memcpy(result_h, result_d, n_element*sizeof(device_t), 0, 0, omp_get_initial_device(), omp_get_default_device());
 #endif
       for (int i = 0; i < n_element; i++) result[i] = reinterpret_cast<device_t *>(result_h)[i];
-#ifdef QUDA_BACKEND_OMPTARGET
-      for (int i = 0; i < n_element; i++) std::cerr << "complete: result[" << i << "] = " << result[i] << std::endl;
-#endif
     }
 #endif
   };

From 83b93dcd1154cc73376c6616c568efd774a4e6ed Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:24:51 +0000
Subject: [PATCH 055/198] omp: remove debug print in tunable_nd/reduction

---
 include/tunable_nd.h        | 13 -------------
 include/tunable_reduction.h |  3 ---
 2 files changed, 16 deletions(-)

diff --git a/include/tunable_nd.h b/include/tunable_nd.h
index c32300e468..e6a49676b7 100644
--- a/include/tunable_nd.h
+++ b/include/tunable_nd.h
@@ -24,7 +24,6 @@ namespace quda {
     std::enable_if_t<device::use_kernel_arg<Arg>(), void>
       launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip();
 #ifdef JITIFY
       launch_error = launch_jitify<Functor, grid_stride, Arg, false>(kernel.name, tp, stream, arg);
 #else
@@ -55,7 +54,6 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_cuda(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg) const
     {
-      ompwip();
       constexpr bool grid_stride = false;
       const_cast<TunableKernel*>(this)->launch_device<Functor, grid_stride>(KERNEL(raw_kernel), tp, stream, arg);
     }
@@ -78,14 +76,12 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip("OFFLOADING...");
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel1D), tp, stream, arg);
     }
 
     template <template <typename> class Functor, typename Arg>
     void launch_host(const TuneParam &, const qudaStream_t &, const Arg &arg)
     {
-      ompwip();
       Functor<Arg> f(const_cast<Arg &>(arg));
       for (int i = 0; i < (int)arg.threads.x; i++) {
         f(i);
@@ -106,7 +102,6 @@ namespace quda {
     template <template <typename> class Functor, bool enable_host = false, typename Arg>
       std::enable_if_t<enable_host, void> launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip();
       if (location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg);
       } else {
@@ -185,7 +180,6 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip("OFFLOADING...");
       const_cast<Arg &>(arg).threads.y = vector_length_y;
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel2D), tp, stream, arg);
     }
@@ -193,7 +187,6 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_host(const TuneParam &, const qudaStream_t &, const Arg &arg)
     {
-      ompwip();
       const_cast<Arg &>(arg).threads.y = vector_length_y;
       Functor<Arg> f(const_cast<Arg &>(arg));
       for (int i = 0; i < (int)arg.threads.x; i++) {
@@ -206,7 +199,6 @@ namespace quda {
     template <template <typename> class Functor, bool enable_host = false, typename Arg>
     std::enable_if_t<!enable_host, void> launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip();
       if (TunableKernel1D_base<grid_stride>::location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg);
       } else {
@@ -217,7 +209,6 @@ namespace quda {
     template <template <typename> class Functor, bool enable_host = false, typename Arg>
     std::enable_if_t<enable_host, void> launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip();
       if (TunableKernel1D_base<grid_stride>::location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg);
       } else {
@@ -349,7 +340,6 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_device(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip("OFFLOADING...");
       const_cast<Arg &>(arg).threads.y = vector_length_y;
       const_cast<Arg &>(arg).threads.z = vector_length_z;
       TunableKernel::launch_device<Functor, grid_stride>(KERNEL(Kernel3D), tp, stream, arg);
@@ -358,7 +348,6 @@ namespace quda {
     template <template <typename> class Functor, typename Arg>
     void launch_host(const TuneParam &, const qudaStream_t &, const Arg &arg)
     {
-      ompwip();
       const_cast<Arg &>(arg).threads.y = vector_length_y;
       const_cast<Arg &>(arg).threads.z = vector_length_z;
       Functor<Arg> f(const_cast<Arg &>(arg));
@@ -374,7 +363,6 @@ namespace quda {
     template <template <typename> class Functor, bool enable_host = false, typename Arg>
   std::enable_if_t<!enable_host, void> launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip();
       if (TunableKernel2D_base<grid_stride>::location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg);
       } else {
@@ -385,7 +373,6 @@ namespace quda {
     template <template <typename> class Functor, bool enable_host = false, typename Arg>
   std::enable_if_t<enable_host, void> launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip();
       if (TunableKernel2D_base<grid_stride>::location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg);
       } else {
diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index 70b7ff2ba0..572afff869 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -85,7 +85,6 @@ namespace quda {
     template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg)
     {
-      ompwip("Reduction2D on device");
 #ifdef JITIFY
       arg.launch_error = launch_jitify<Transformer, true, Arg, true>("Reduction2D", tp, stream, arg);
 #else
@@ -130,7 +129,6 @@ namespace quda {
 
       // copy element by element to output vector
       for (int i = 0; i < output_size; i++) {
-        std::cerr << "launch_host: reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[" << i << "]: " << reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i] << std::endl;
         reinterpret_cast<typename scalar<T>::type*>(result.data())[i] =
           reinterpret_cast<typename scalar<reduce_t>::type*>(&value)[i];
       }
@@ -306,7 +304,6 @@ namespace quda {
     template <template <typename> class Transformer, typename Arg, typename T>
     void launch_device(std::vector<T> &result, const TuneParam &tp, const qudaStream_t &stream, Arg &arg)
     {
-      ompwip("MultiReduction on device");
 #ifdef JITIFY
       arg.launch_error = launch_jitify<Transformer, true, Arg, true>("MultiReduction", tp, stream, arg);
 #else

From ca6939e5349c04a65069381e4f2f289a6df89bb1 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:25:28 +0000
Subject: [PATCH 056/198] omp: remove debug print in cuda_color_spinor_field

---
 lib/cuda_color_spinor_field.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/lib/cuda_color_spinor_field.cpp b/lib/cuda_color_spinor_field.cpp
index ca11123fd2..e9db1dd4ff 100644
--- a/lib/cuda_color_spinor_field.cpp
+++ b/lib/cuda_color_spinor_field.cpp
@@ -401,10 +401,8 @@ namespace quda {
   }
 
   void cudaColorSpinorField::loadSpinorField(const ColorSpinorField &src) {
-    // ompwip("DBG: loadSpinorField(const ColorSpinorField &src)");
 
     if ( reorder_location() == QUDA_CPU_FIELD_LOCATION && typeid(src) == typeid(cpuColorSpinorField)) {
-      // ompwip("DBG: reorder_location() == QUDA_CPU_FIELD_LOCATION && typeid(src) == typeid(cpuColorSpinorField)");
       void *buffer = pool_pinned_malloc(bytes + norm_bytes);
       memset(buffer, 0, bytes+norm_bytes); // FIXME (temporary?) bug fix for padding
 
@@ -415,28 +413,22 @@ namespace quda {
 
       pool_pinned_free(buffer);
     } else if (typeid(src) == typeid(cudaColorSpinorField)) {
-      // ompwip("DBG: typeid(src) == typeid(cudaColorSpinorField)");
       copyGenericColorSpinor(*this, src, QUDA_CUDA_FIELD_LOCATION);
     } else {
 
       if (src.FieldOrder() == QUDA_PADDED_SPACE_SPIN_COLOR_FIELD_ORDER) {
-        // ompwip("DBG: src.FieldOrder() == QUDA_PADDED_SPACE_SPIN_COLOR_FIELD_ORDER");
         // special case where we use mapped memory to read/write directly from application's array
         void *src_d = get_mapped_device_pointer(src.V());
         copyGenericColorSpinor(*this, src, QUDA_CUDA_FIELD_LOCATION, v, src_d);
       } else {
         void *Src=nullptr, *srcNorm=nullptr, *buffer=nullptr;
         if (!zeroCopy) {
-          // ompwip("DBG: !zeroCopy");
           buffer = pool_device_malloc(src.Bytes()+src.NormBytes());
           Src = buffer;
           srcNorm = static_cast<char*>(Src) + src.Bytes();
-          // ompwip("DBG: !zeroCopy qudaMemcpy(Src, src.V())");
           qudaMemcpy(Src, src.V(), src.Bytes(), qudaMemcpyDefault);
-          // ompwip("DBG: !zeroCopy qudaMemcpy(srcNorm, src.Norm())");
           qudaMemcpy(srcNorm, src.Norm(), src.NormBytes(), qudaMemcpyDefault);
         } else {
-          // ompwip("DBG: else");
           buffer = pool_pinned_malloc(src.Bytes()+src.NormBytes());
           memcpy(buffer, src.V(), src.Bytes());
           memcpy(static_cast<char*>(buffer)+src.Bytes(), src.Norm(), src.NormBytes());
@@ -445,16 +437,13 @@ namespace quda {
         }
 
         qudaMemsetAsync(v, 0, bytes, device::get_default_stream()); // FIXME (temporary?) bug fix for padding
-        // ompwip("DBG: begin copyGenericColorSpinor(*this, src)");
         copyGenericColorSpinor(*this, src, QUDA_CUDA_FIELD_LOCATION, 0, Src, 0, srcNorm);
-        // ompwip("DBG: end copyGenericColorSpinor(*this, src)");
 
         if (zeroCopy) pool_pinned_free(buffer);
         else pool_device_free(buffer);
       }
     }
 
-    // ompwip("DBG: sync loadSpinorField(const ColorSpinorField &src)");
     qudaDeviceSynchronize(); // include sync here for accurate host-device profiling
   }
 

From bf353b33ba8f5fc52bdc62ac5d43650082d51315 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:26:36 +0000
Subject: [PATCH 057/198] omp: target/kernel cast pointer to void* before
 memcpy

---
 include/targets/omptarget/kernel.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index c4be6cbef9..5bd3449378 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -39,7 +39,7 @@ namespace quda {
                  launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
                  omp_get_num_teams(), omp_get_num_threads());
         char buffer[sizeof(Arg)];
-        memcpy(buffer, dparg, sizeof(Arg));
+        memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel1D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
       omp_target_free(dparg, omp_get_default_device());
@@ -101,7 +101,7 @@ namespace quda {
                  launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
                  omp_get_num_teams(), omp_get_num_threads());
         char buffer[sizeof(Arg)];
-        memcpy(buffer, dparg, sizeof(Arg));
+        memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel2D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
       omp_target_free(dparg, omp_get_default_device());
@@ -165,7 +165,7 @@ namespace quda {
                  launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
                  omp_get_num_teams(), omp_get_num_threads());
         char buffer[sizeof(Arg)];
-        memcpy(buffer, dparg, sizeof(Arg));
+        memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel3D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
       omp_target_free(dparg, omp_get_default_device());

From 5302454260348d89850026f433af7b84bcaa4d5f Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:27:30 +0000
Subject: [PATCH 058/198] omp: target/math_helper updates and uses generic
 versions

---
 include/targets/omptarget/math_helper.cuh | 89 +++--------------------
 1 file changed, 12 insertions(+), 77 deletions(-)

diff --git a/include/targets/omptarget/math_helper.cuh b/include/targets/omptarget/math_helper.cuh
index 4d8034d4c5..d47a587f49 100644
--- a/include/targets/omptarget/math_helper.cuh
+++ b/include/targets/omptarget/math_helper.cuh
@@ -12,46 +12,16 @@ namespace quda {
    * @param c pointer to the storage for the result of the cos
    */
   template<typename T>
-  inline __host__ __device__ void sincos(const T& a, T* s, T* c)
-    {
-      *s = std::sin(a);
-      *c = std::cos(a);
-    }
-
- 
-  /**
-   * @brief Combined sin and cos colculation in QUDA NAMESPACE
-   *
-   * @param a the angle
-   * @param s pointer to the storage for the result of the sin
-   * @param c pointer to the storage for the result of the cos
-   *
-   * Specialization to float arguments. Device function calls CUDA intrinsic
-   *
-  template<>
-  inline __host__ __device__ void sincos(const float& a, float * s, float *c)
-  {
-#ifdef __CUDA_ARCH__
-    __sincosf(a,s,c);
-#else
-    ::sincosf(a,s,c);
-#endif
-  }
-   */ 
+  inline __host__ __device__ void sincos(const T& a, T* s, T* c) { *s = std::sin(a); *c = std::cos(a); }
 
   /**
    * @brief Reciprocal square root function (rsqrt)
    * @param a the argument  (In|out)
    *
    * some math libraries provide a fast inverse sqrt() function.
-   * this implementation uses the CUDA builtins
+   * this implementation trusts the compiler.
    */
-  template<typename T>
-  inline __host__ __device__ T rsqrt(T a)
-    {
-      return 1.0/std::sqrt(a);
-    }
-
+  template<typename T> inline __host__ __device__ T rsqrt(T a) { return 1.0/std::sqrt(a); }
 
     /**
      Generic wrapper for Trig functions -- used in gauge field order 
@@ -63,27 +33,6 @@ namespace quda {
     __device__ __host__ static T Cos( const T &a ) { return std::cos(a); }
     __device__ __host__ static void SinCos(const T &a, T *s, T *c) { sincos(a, s, c); }
   };
-  
-  /**
-     Specialization of Trig functions using floats
-   */
-  template <>
-    struct Trig<false,float> {
-    __device__ __host__ static float Atan2( const float &a, const float &b) { return std::atan2(a,b); }
-    __device__ __host__ static float Sin(const float &a)
-    {
-      return std::sin(a);
-    }
-    __device__ __host__ static float Cos(const float &a)
-    {
-      return std::cos(a); 
-    }
-
-    __device__ __host__ static void SinCos(const float &a, float *s, float *c)
-    {
-       sincos(a, s, c);
-    }
-  };
 
   /**
      Specialization of Trig functions using fixed b/c gauge reconstructs are -1 -> 1 instead of -Pi -> Pi
@@ -91,36 +40,22 @@ namespace quda {
   template <>
     struct Trig<true,float> {
     __device__ __host__ static float Atan2( const float &a, const float &b) { return std::atan2(a,b)/M_PI; }
-    __device__ __host__ static float Sin(const float &a)
-    {
-      return std::sin(a * static_cast<float>(M_PI));
-    }
-
-    __device__ __host__ static float Cos(const float &a)
-    {
-      return std::cos(a * static_cast<float>(M_PI));
-    }
-
-    __device__ __host__ static void SinCos(const float &a, float *s, float *c)
-    {
-      sincos(a * static_cast<float>(M_PI), s, c);
-    }
+    __device__ __host__ static float Sin(const float &a) { return std::sin(a * static_cast<float>(M_PI)); }
+    __device__ __host__ static float Cos(const float &a) { return std::cos(a * static_cast<float>(M_PI)); }
+    __device__ __host__ static void SinCos(const float &a, float *s, float *c) { sincos(a * static_cast<float>(M_PI), s, c); }
   };
 
-/*
+  /*
     @brief Fast power function that works for negative "a" argument
     @param a argument we want to raise to some power
     @param b power that we want to raise a to
     @return pow(a,b)
   */
-  template <typename real> __device__ __host__ inline real __fast_pow(real a, int b)
-  {
-    return std::pow(a, b);
-  }
+  template <typename real> __device__ __host__ inline real fpow(real a, int b) { return std::pow(a, b); }
 
-  template <typename real> __device__ __host__ inline real fdividef(real a, real b)
-  {
-    return a / b;
-  }
+  /**
+     @brief Optimized division routine on the device
+  */
+  __device__ __host__ inline float fdividef(float a, float b) { return a / b; }
 
 }

From 0dc1f61cc8561d751de075db3da4b2dadcb22d59 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:29:01 +0000
Subject: [PATCH 059/198] omp: target/quda_api.cpp fix memset and add 2D
 versions

---
 lib/targets/omptarget/quda_api.cpp | 137 +++++++++++++++++++++++------
 1 file changed, 111 insertions(+), 26 deletions(-)

diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 11adcfec99..2ce95a510c 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -18,20 +18,31 @@ static char FIXME[]="OMP FIXME";
 #define cudaGetLastError()
 #define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
 
-#define cudaMemcpy(a,b,c,d) ompwip([&](){printfQuda("memcpy %p <- %p %ld\n",a,b,c);ompwipMemcpy(a,(void*)b,c,d);})
-#define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){printfQuda("memcpy %p <- %p %ld\n",a,b,c);ompwipMemcpy(a,(void*)b,c,d);})
-#define cudaMemcpy2D(a,b,c,d,e,f,g) ompwip("unimplemented")
-#define cudaMemcpy2DAsync(a,b,c,d,e,f,g,h) ompwip("unimplemented")
-#define cudaMemset(a,b,c) ompwip([&](){printfQuda("memset %p %d %ld\n",a,b,c);ompwipMemset(a,b,c);})
-#define cudaMemsetAsync(a,b,c,d) ompwip([&](){printfQuda("memset %p %d %ld\n",a,b,c);ompwipMemset(a,b,c);})
-#define cudaMemset2D(a,b,c,d,e) ompwip("unimplemented")
-#define cudaMemset2DAsync(a,b,c,d,e,f) ompwip("unimplemented")
+#define cudaMemcpy(a,b,c,d) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld\n",a,b,c)
+#define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld\n",a,b,c)
+#define cudaMemcpy2D(d,dp,s,sp,w,h,k) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld\n",d,dp,s,sp,w,h)
+#define cudaMemcpy2DAsync(d,dp,s,sp,w,h,k,st) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld\n",d,dp,s,sp,w,h)
+#define cudaMemset(a,b,c) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld\n",a,b,c)
+#define cudaMemsetAsync(a,b,c,d) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld\n",a,b,c)
+#define cudaMemset2D(a,b,c,d,e) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld\n",a,b,c,d,e)
+#define cudaMemset2DAsync(a,b,c,d,e,f) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld\n",a,b,c,d,e)
 
 static inline void
 ompwipMemset(void *p, unsigned char b, std::size_t s)
 {
-#pragma omp target teams distribute parallel for simd is_device_ptr(p)
-  for(std::size_t i=0;i<s;++i) *(unsigned char *)p = b;
+  unsigned char *c = reinterpret_cast<unsigned char *>(p);
+#pragma omp target teams distribute parallel for simd is_device_ptr(c)
+  for(std::size_t i=0;i<s;++i) c[i] = b;
+}
+
+static inline void
+ompwipMemset2D(void *p, size_t pitch, unsigned char b, size_t w, size_t h)
+{
+  unsigned char *c = reinterpret_cast<unsigned char *>(p);
+#pragma omp target teams distribute parallel for simd is_device_ptr(c) collapse(2)
+  for(std::size_t i=0;i<h;++i)
+    for(std::size_t j=0;j<w;++j)
+      c[j+i*pitch] = b;
 }
 
 #if 1
@@ -73,12 +84,6 @@ printmem(void *d, std::size_t m, int host)
 }
 #endif
 
-namespace quda {
-  namespace target {
-    extern bool is_device_ptr(void *);
-  }
-}
-
 static inline void
 ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
 {
@@ -117,26 +122,26 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
     break;
   case cudaMemcpyDefault:
     if(0<omp_get_num_devices()){
-      if(quda::target::is_device_ptr(d)){
-        if(quda::target::is_device_ptr(s)){
-          warningQuda("cudaMemcpyDefault calling device to device");
+      if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(d)){
+        if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
+          ompwip("cudaMemcpyDefault calling device to device");
           printmem(s,c,0);
           omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
           printmem(d,c,0);
         }else{
-          warningQuda("cudaMemcpyDefault calling host to device");
+          ompwip("cudaMemcpyDefault calling host to device");
           printmem(s,c,1);
           omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
           printmem(d,c,0);
         }
       }else{
-        if(quda::target::is_device_ptr(s)){
-          warningQuda("cudaMemcpyDefault calling device to host");
+        if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
+          ompwip("cudaMemcpyDefault calling device to host");
           printmem(s,c,0);
           omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
           printmem(d,c,1);
         }else{
-          warningQuda("cudaMemcpyDefault calling host to host");
+          ompwip("cudaMemcpyDefault calling host to host");
           printmem(s,c,1);
           memcpy(d,s,c);
           printmem(d,c,1);
@@ -153,6 +158,87 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
   }
 }
 
+static inline void
+ompwipMemcpy2D(void *d, std::size_t dp, void *s, std::size_t sp, std::size_t w, std::size_t h, cudaMemcpyKind k)
+{
+  std::size_t v[2] = {w,h}, z[2] = {0,0}, dd[2] = {dp,h}, sd[2] = {sp,h};
+  unsigned char *cd = reinterpret_cast<unsigned char *>(d), *cs = reinterpret_cast<unsigned char *>(s);
+  switch(k){
+  case cudaMemcpyHostToHost:
+    printmem(s,c,1);
+    #pragma omp parallel for
+    for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
+    printmem(d,c,1);
+    break;
+  case cudaMemcpyHostToDevice:
+    printmem(s,c,1);
+    if(0<omp_get_num_devices()){
+      omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_initial_device());
+      printmem(d,c,0);
+    }else{
+      warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy");
+      #pragma omp parallel for
+      for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
+      printmem(d,c,1);
+    }
+    break;
+  case cudaMemcpyDeviceToHost:
+    if(0<omp_get_num_devices()){
+      printmem(s,c,0);
+      omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_initial_device(),omp_get_default_device());
+    }else{
+      warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy");
+      printmem(s,c,1);
+      #pragma omp parallel for
+      for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
+    }
+    printmem(d,c,1);
+    break;
+  case cudaMemcpyDeviceToDevice:
+    printmem(s,c,0);
+    omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_default_device());
+    printmem(d,c,0);
+    break;
+  case cudaMemcpyDefault:
+    if(0<omp_get_num_devices()){
+      if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(d)){
+        if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
+          ompwip("cudaMemcpyDefault calling device to device");
+          printmem(s,c,0);
+          omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_default_device());
+          printmem(d,c,0);
+        }else{
+          ompwip("cudaMemcpyDefault calling host to device");
+          printmem(s,c,1);
+          omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_initial_device());
+          printmem(d,c,0);
+        }
+      }else{
+        if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
+          ompwip("cudaMemcpyDefault calling device to host");
+          printmem(s,c,0);
+          omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_initial_device(),omp_get_default_device());
+          printmem(d,c,1);
+        }else{
+          ompwip("cudaMemcpyDefault calling host to host");
+          printmem(s,c,1);
+          #pragma omp parallel for
+          for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
+          printmem(d,c,1);
+        }
+      }
+    }else{
+      warningQuda("cudaMemcpyDefault without a device, calling memcpy");
+      printmem(s,c,1);
+      #pragma omp parallel for
+      for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
+      printmem(d,c,1);
+    }
+    break;
+  default: errorQuda("Unsupported cudaMemcpyType %d", k);
+  }
+}
+
 using cudaEvent_t = int*;
 enum {cudaEventDisableTiming};
 #define cudaEventCreate(a) ompwip("cudaEventCreate")
@@ -210,7 +296,6 @@ namespace quda
 
   void qudaSetupLaunchParameter(const TuneParam &tp)
   {
-    ompwip("set up global parameter");
     launch_param.grid = tp.grid;
     launch_param.block = tp.block;
     #pragma omp target update to(launch_param)
@@ -295,10 +380,10 @@ namespace quda
 
   qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, void **args, qudaStream_t stream)
   {
-    ompwip("PRETENDING",[&](){std::cerr<<"launch "<<func<<' '<<tp<<' '<<*args<<' '<<device::get_cuda_stream(stream)<<std::endl;});
+    ompwip([&](){std::cerr<<"ERROR: unimplemented launch "<<func<<' '<<tp<<' '<<*args<<' '<<device::get_cuda_stream(stream)<<std::endl;});
     // if launch requests the maximum shared memory and the device supports it then opt in
     if (tp.set_max_shared_bytes && device::max_dynamic_shared_memory() > device::max_default_shared_memory()) {
-      ompwip([](){warningQuda("Unimplemented for maximum shared memory");});
+      ompwip("Unimplemented for maximum shared memory");
 /*
       static std::unordered_set<const void *> cache;
       auto search = cache.find(func);

From 081a68559fb1ed1dea46b06048e3ea2c86b8b83b Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:29:42 +0000
Subject: [PATCH 060/198] omp: update malloc debug print

---
 lib/targets/omptarget/malloc.cpp | 53 ++++++++++++++------------------
 1 file changed, 23 insertions(+), 30 deletions(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index a15828bc0a..90090829fb 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -139,11 +139,6 @@ namespace quda
 
   namespace target {
     static std::map<const void*,void*> omp_mapped_ptr;  // host -> device
-    bool is_device_ptr(void *p)
-    {
-      // DEVICE, DEVICE_PINNED, HOST, PINNED, MAPPED, MANAGED, SHMEM
-      return alloc[DEVICE].count(p)>0 || alloc[DEVICE_PINNED].count(p)>0;
-    }
   }
   /**
    * Under CUDA 4.0, cudaHostRegister seems to require that both the
@@ -292,7 +287,7 @@ namespace quda
     a.size = a.base_size = size;
 
     void *ptr = malloc(size);
-    ompwip([=](){printfQuda("malloc: %p\n",ptr);});
+    ompwip("malloc: %p\n",ptr);
     if (!ptr) { errorQuda("Failed to allocate host memory of size %zu (%s:%d in %s())\n", size, file, line, func); }
     track_malloc(HOST, a, ptr);
 #ifdef HOST_DEBUG
@@ -350,12 +345,12 @@ namespace quda
     }
 #else
     void *ptr = aligned_malloc(a, size);
-    ompwip([=](){printfQuda("mapped_malloc_ host: %p\n",ptr);});
+    ompwip("mapped_malloc_ host: %p\n",ptr);
     print_trace();
     if(0<omp_get_num_devices()){
       int d = omp_get_default_device();
       void *dp = omp_target_alloc(a.base_size, d);
-      ompwip("require special memcpy",[=](){printfQuda("mapped_malloc_ device: %p\n",dp);});
+      ompwip("require special memcpy, mapped_malloc_ device: %p\n",dp);
       if(!dp)
         errorQuda("%s:%d %s() Failed to allocate device memory of size %zu for mapped malloc\n", file, line, func, size);
       if(omp_target_associate_ptr(ptr, dp, a.base_size, 0, d))
@@ -415,7 +410,7 @@ namespace quda
    * defined in malloc_quda.h
    */
   void *managed_malloc_(const char *func, const char *file, int line, size_t size)
-  {ompwip("managed_malloc_ unimplemented return nullptr");return nullptr;/*
+  {ompwip("ERROR: managed_malloc_ unimplemented return nullptr");return nullptr;/*
     MemAlloc a(func, file, line);
     void *ptr;
 
@@ -439,13 +434,13 @@ namespace quda
    */
   void device_free_(const char *func, const char *file, int line, void *ptr)
   {
-    if (use_managed_memory()) {ompwip();
+    if (use_managed_memory()) {ompwip("unimplemented");
       managed_free_(func, file, line, ptr);
       return;
     }
 
 #ifndef QDP_USE_CUDA_MANAGED_MEMORY
-    ompwip();
+    ompwip("device free: %p\n",ptr);
     if (!ptr) { errorQuda("Attempt to free NULL device pointer (%s:%d in %s())\n", file, line, func); }
     if (!alloc[DEVICE].count(ptr)) {
       errorQuda("Attempt to free invalid device pointer (%s:%d in %s())\n", file, line, func);
@@ -471,7 +466,7 @@ namespace quda
    */
   void device_pinned_free_(const char *func, const char *file, int line, void *ptr)
   {
-    ompwip("device_pinned_free_ untested code path"); print_trace();
+    ompwip("device_pinned_free_ untested code path: %p",ptr); print_trace();
     if (!comm_peer2peer_present()) {
       device_free_(func, file, line, ptr);
       return;
@@ -498,7 +493,7 @@ namespace quda
    * malloc_quda.h
    */
   void managed_free_(const char *func, const char *file, int line, void *ptr)
-  {ompwip("managed_free_ unimplemented doing nothing");/*
+  {ompwip("managed_free_ unimplemented doing nothing: %p",ptr);/*
     if (!ptr) { errorQuda("Attempt to free NULL managed pointer (%s:%d in %s())\n", file, line, func); }
     if (!alloc[MANAGED].count(ptr)) {
       errorQuda("Attempt to free invalid managed pointer (%s:%d in %s())\n", file, line, func);
@@ -520,7 +515,7 @@ namespace quda
       track_free(HOST, ptr);
       free(ptr);
     } else if (alloc[PINNED].count(ptr)) {
-      ompwip("free PINNED ptr");
+      ompwip("free PINNED ptr: %p",ptr);
 /*
       cudaError_t err = cudaHostUnregister(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to unregister pinned memory (%s:%d in %s())\n", file, line, func); }
@@ -529,17 +524,18 @@ namespace quda
       free(ptr);
     } else if (alloc[MAPPED].count(ptr)) {
 #ifdef HOST_ALLOC
-      ompwip("HOST_ALLOC alloc[MAPPED].count(ptr) untested code path"); print_trace();
+      ompwip("HOST_ALLOC alloc[MAPPED].count(ptr) untested code path: %p",ptr); print_trace();
       free(ptr);
 /*
       cudaError_t err = cudaFreeHost(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to free host memory (%s:%d in %s())\n", file, line, func); }
 */
 #else
-      ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr)");
+      ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr) free: %p",ptr);
       void *dp = target::omp_mapped_ptr[ptr];
       if(dp!=ptr){
         omp_target_disassociate_ptr(ptr, omp_get_default_device());
+        ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr) free mapped device: %p",dp);
         omp_target_free(dp, omp_get_default_device());
       }
       free(ptr);
@@ -620,17 +616,15 @@ namespace quda
 
   QudaFieldLocation get_pointer_location(const void *ptr)
   {
-    ompwip("get_pointer_location");
-    QudaFieldLocation loc = QUDA_INVALID_FIELD_LOCATION;
     if(alloc[DEVICE].count(const_cast<void*>(ptr)) || alloc[DEVICE_PINNED].count(const_cast<void*>(ptr))){
-      ompwip("get_pointer_location returns QUDA_CUDA_FIELD_LOCATION");
+      ompwip("get_pointer_location %p returns QUDA_CUDA_FIELD_LOCATION",ptr);
       return QUDA_CUDA_FIELD_LOCATION;
     }else if(alloc[HOST].count(const_cast<void*>(ptr)) || alloc[PINNED].count(const_cast<void*>(ptr)) || alloc[MAPPED].count(const_cast<void*>(ptr))){
-      ompwip("get_pointer_location returns QUDA_CPU_FIELD_LOCATION");
+      ompwip("get_pointer_location %p returns QUDA_CPU_FIELD_LOCATION",ptr);
       return QUDA_CPU_FIELD_LOCATION;
     }
-    ompwip("WARNING: get_pointer_location returns QUDA_INVALID_FIELD_LOCATION");
-    return QUDA_INVALID_FIELD_LOCATION;
+    ompwip("WARNING: get_pointer_location assumes %p to be QUDA_CPU_FIELD_LOCATION",ptr);
+    return QUDA_CPU_FIELD_LOCATION;
 /*
     CUpointer_attribute attribute[] = {CU_POINTER_ATTRIBUTE_MEMORY_TYPE};
     CUmemorytype mem_type;
@@ -656,7 +650,7 @@ namespace quda
 
   void *get_mapped_device_pointer_(const char *func, const char *file, int line, const void *host)
   {
-    ompwip("get_mapped_device_pointer_");
+    ompwip("get_mapped_device_pointer_: %p",host);
     print_trace();
     return target::omp_mapped_ptr[host];
 /*
@@ -672,7 +666,7 @@ namespace quda
 
   void register_pinned_(const char *func, const char *file, int line, void *ptr, size_t bytes)
   {
-    ompwip("register_pinned_ unimplemented");
+    ompwip("register_pinned_ unimplemented: %p",ptr);
 /*
     auto error = cudaHostRegister(ptr, bytes, cudaHostRegisterDefault);
     if (error != cudaSuccess) {
@@ -684,7 +678,7 @@ namespace quda
 
   void unregister_pinned_(const char *func, const char *file, int line, void *ptr)
   {
-    ompwip("unregister_pinned_ unimplemented");
+    ompwip("unregister_pinned_ unimplemented: %p",ptr);
 /*
     auto error = cudaHostUnregister(ptr);
     if (error != cudaSuccess) {
@@ -753,7 +747,6 @@ namespace quda
 
     void *pinned_malloc_(const char *func, const char *file, int line, size_t nbytes)
     {
-      ompwip("pool::pinned_malloc_");
       void *ptr = nullptr;
       if (pinned_memory_pool) {
         if (pinnedCache.empty()) {
@@ -776,12 +769,13 @@ namespace quda
       } else {
         ptr = quda::pinned_malloc_(func, file, line, nbytes);
       }
+      ompwip("pool::pinned_malloc_: %p",ptr);
       return ptr;
     }
 
     void pinned_free_(const char *func, const char *file, int line, void *ptr)
     {
-      ompwip("pool::pinnd_free_");
+      ompwip("pool::pinnd_free_: %p",ptr);
       if (pinned_memory_pool) {
         if (!pinnedSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
         pinnedCache.insert(std::make_pair(pinnedSize[ptr], ptr));
@@ -793,7 +787,6 @@ namespace quda
 
     void *device_malloc_(const char *func, const char *file, int line, size_t nbytes)
     {
-      ompwip("pool::device_malloc_");
       void *ptr = nullptr;
       if (device_memory_pool) {
         if (deviceCache.empty()) {
@@ -816,13 +809,13 @@ namespace quda
       } else {
         ptr = quda::device_malloc_(func, file, line, nbytes);
       }
-      ompwip([=](){printfQuda("device malloc: %p\n",ptr);});
+      ompwip("pool::device_malloc_: %p",ptr);
       return ptr;
     }
 
     void device_free_(const char *func, const char *file, int line, void *ptr)
     {
-      ompwip("pool::device_free_");
+      ompwip("pool::device_free_: %p",ptr);
       if (device_memory_pool) {
         if (!deviceSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
         deviceCache.insert(std::make_pair(deviceSize[ptr], ptr));

From c08b764160c0877fb7cacc17475dd658e96b46c3 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 21 May 2021 15:30:23 +0000
Subject: [PATCH 061/198] omp: try turning off shared in dslash_domain_wall_m5

---
 include/kernels/dslash_domain_wall_m5.cuh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/include/kernels/dslash_domain_wall_m5.cuh b/include/kernels/dslash_domain_wall_m5.cuh
index e307aee66c..4508819f63 100644
--- a/include/kernels/dslash_domain_wall_m5.cuh
+++ b/include/kernels/dslash_domain_wall_m5.cuh
@@ -13,7 +13,11 @@ namespace quda
   /** Whether to use a shared memory scratch pad to store the input
      field acrosss the Ls dimension to minimize global memory
      reads. */
+#ifdef QUDA_BACKEND_OMPTARGET
+  constexpr bool shared() { return false; }
+#else
   constexpr bool shared() { return true; }
+#endif
 
   /** Whether to use variable or fixed coefficient algorithm.  Must be
       true if using ZMOBIUS */

From a80465b52faf5e6fb9ccb3c63d174aa0cbbb9970 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 24 May 2021 15:25:54 +0000
Subject: [PATCH 062/198] omp: fix index_helper

---
 include/index_helper.cuh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index 8b586e9cf8..cc90cfd4fb 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -876,6 +876,7 @@ namespace quda {
 #ifdef SWIZZLE
   template <typename T> __device__ inline int block_idx(const T &swizzle)
   {
+    QUDA_RT_CONSTS;
     // the portion of the grid that is exactly divisible by the number of SMs
     const int gridp = gridDim.x - gridDim.x % swizzle;
 
@@ -893,11 +894,8 @@ namespace quda {
 #else
   template <typename T> __device__ inline int block_idx(const T &)
   {
-#ifdef QUDA_BACKEND_OMPTARGET
-    return omp_get_thread_num();
-#else
+    QUDA_RT_CONSTS;
     return blockIdx.x;
-#endif
   }
 #endif
 

From f318f1a6dc89ab9b9379379cf8ccbe947f4680ea Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 24 May 2021 15:29:36 +0000
Subject: [PATCH 063/198] omp: cleanup malloc and device, add event timing

---
 include/quda_api.h                 |   4 +-
 lib/targets/omptarget/device.cpp   |  14 ++
 lib/targets/omptarget/malloc.cpp   |  10 +-
 lib/targets/omptarget/quda_api.cpp | 309 ++++-------------------------
 4 files changed, 57 insertions(+), 280 deletions(-)

diff --git a/include/quda_api.h b/include/quda_api.h
index 933896f170..e1328353ff 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -304,8 +304,8 @@ namespace quda
   /**
      @brief Return elapsed time in seconds between two events
   */
-  float qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
-                              const char *func, const char *file, const char *line);
+  double qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
+                               const char *func, const char *file, const char *line);
 
   /**
      @brief Destroy the event
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 173295839f..7c9d5b0ecc 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -73,6 +73,20 @@ static int cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
   p->regsPerBlock = 32768;
   p->unifiedAddressing = 1;
   p->deviceOverlap = 0;
+
+  /* Now actually querying the device in some way. */
+  int m = 0;
+  #pragma omp target teams map(tofrom:m)
+  if(omp_get_team_num()==0)
+    m = omp_get_max_threads();
+  p->maxThreadsPerMultiProcessor = m;
+  p->maxThreadsPerBlock = m;
+  p->maxThreadsDim[0] = m;
+  p->maxThreadsDim[1] = m;
+  p->maxThreadsDim[2] = m;
+  #pragma omp target map(tofrom:m)
+  m = omp_get_num_procs();
+  p->multiProcessorCount = m/32;
   return 0;
 }
 
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index 90090829fb..bad00a312e 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -68,7 +68,7 @@ namespace quda
 
   long host_allocated_peak() { return max_total_bytes[HOST]; }
 
-  static void print_trace(void)
+  void print_trace(void)
   {
     void *array[10];
     size_t size;
@@ -287,7 +287,7 @@ namespace quda
     a.size = a.base_size = size;
 
     void *ptr = malloc(size);
-    ompwip("malloc: %p\n",ptr);
+    ompwip("malloc: %p",ptr);
     if (!ptr) { errorQuda("Failed to allocate host memory of size %zu (%s:%d in %s())\n", size, file, line, func); }
     track_malloc(HOST, a, ptr);
 #ifdef HOST_DEBUG
@@ -345,12 +345,12 @@ namespace quda
     }
 #else
     void *ptr = aligned_malloc(a, size);
-    ompwip("mapped_malloc_ host: %p\n",ptr);
+    ompwip("mapped_malloc_ host: %p",ptr);
     print_trace();
     if(0<omp_get_num_devices()){
       int d = omp_get_default_device();
       void *dp = omp_target_alloc(a.base_size, d);
-      ompwip("require special memcpy, mapped_malloc_ device: %p\n",dp);
+      ompwip("require special memcpy, mapped_malloc_ device: %p",dp);
       if(!dp)
         errorQuda("%s:%d %s() Failed to allocate device memory of size %zu for mapped malloc\n", file, line, func, size);
       if(omp_target_associate_ptr(ptr, dp, a.base_size, 0, d))
@@ -440,7 +440,7 @@ namespace quda
     }
 
 #ifndef QDP_USE_CUDA_MANAGED_MEMORY
-    ompwip("device free: %p\n",ptr);
+    ompwip("device free: %p",ptr);
     if (!ptr) { errorQuda("Attempt to free NULL device pointer (%s:%d in %s())\n", file, line, func); }
     if (!alloc[DEVICE].count(ptr)) {
       errorQuda("Attempt to free invalid device pointer (%s:%d in %s())\n", file, line, func);
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 2ce95a510c..2446e16145 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -5,11 +5,8 @@
 #include <timer.h>
 #include <device.h>
 
-enum cudaFuncAttribute{cudaFuncAttributePreferredSharedMemoryCarveout};
-using cudaFuncAttributes = int;
-using CUdeviceptr = void*;
 using cudaError_t = int;
-enum {cudaSuccess,cudaErrorNotReady};
+enum {cudaSuccess};
 enum cudaMemcpyKind{cudaMemcpyHostToHost, cudaMemcpyHostToDevice, cudaMemcpyDeviceToHost, cudaMemcpyDeviceToDevice, cudaMemcpyDefault};
 
 static char FIXME[]="OMP FIXME";
@@ -18,14 +15,14 @@ static char FIXME[]="OMP FIXME";
 #define cudaGetLastError()
 #define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
 
-#define cudaMemcpy(a,b,c,d) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld\n",a,b,c)
-#define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld\n",a,b,c)
-#define cudaMemcpy2D(d,dp,s,sp,w,h,k) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld\n",d,dp,s,sp,w,h)
-#define cudaMemcpy2DAsync(d,dp,s,sp,w,h,k,st) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld\n",d,dp,s,sp,w,h)
-#define cudaMemset(a,b,c) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld\n",a,b,c)
-#define cudaMemsetAsync(a,b,c,d) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld\n",a,b,c)
-#define cudaMemset2D(a,b,c,d,e) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld\n",a,b,c,d,e)
-#define cudaMemset2DAsync(a,b,c,d,e,f) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld\n",a,b,c,d,e)
+#define cudaMemcpy(a,b,c,d) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld",a,b,c)
+#define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld",a,b,c)
+#define cudaMemcpy2D(d,dp,s,sp,w,h,k) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld",d,dp,s,sp,w,h)
+#define cudaMemcpy2DAsync(d,dp,s,sp,w,h,k,st) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld",d,dp,s,sp,w,h)
+#define cudaMemset(a,b,c) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld",a,b,c)
+#define cudaMemsetAsync(a,b,c,d) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld",a,b,c)
+#define cudaMemset2D(a,b,c,d,e) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld",a,b,c,d,e)
+#define cudaMemset2DAsync(a,b,c,d,e,f) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld",a,b,c,d,e)
 
 static inline void
 ompwipMemset(void *p, unsigned char b, std::size_t s)
@@ -239,25 +236,7 @@ ompwipMemcpy2D(void *d, std::size_t dp, void *s, std::size_t sp, std::size_t w,
   }
 }
 
-using cudaEvent_t = int*;
-enum {cudaEventDisableTiming};
-#define cudaEventCreate(a) ompwip("cudaEventCreate")
-#define cudaEventCreateWithFlags(a,b) ompwip("cudaEventCreateWithFlags")
-#define cudaEventElapsedTime(a,b,c) ompwip([&](){(*(a))=0;})
-#define cudaEventQuery(a) ompwip("cudaEventQuery")
-#define cudaEventRecord(a,b) ompwip("cudaEventRecord")
-#define cudaEventSynchronize(a) ompwip("cudaEventSynchronize")
-#define cudaEventDestroy(a) ompwip("cudaEventDestroy")
-#define cudaStreamWaitEvent(a,b,c) ompwip("cudaStreamWaitEvent")
-#define cudaStreamSynchronize(a) ompwip("cudaStreamSynchronize")
-#define cudaDeviceSynchronize() ompwip("cudaDeviceSynchronize")
 #define cudaGetSymbolAddress(a,b) ompwip("cudaGetSymbolAddress")
-#define cudaFuncGetAttributes(a,b) ompwip("cudaFuncGetAttributes")
-#define cudaFuncSetAttribute(a,b,c) ompwip("cudaFuncSetAttribute")
-
-// if this macro is defined then we use the driver API, else use the
-// runtime API.  Typically the driver API has 10-20% less overhead
-// #define USE_DRIVER_API
 
 // if this macro is defined then we profile the CUDA API calls
 //#define API_PROFILE
@@ -347,31 +326,6 @@ namespace quda
     }
   }
 
-  // No need to abstract these across the library so keep these definitions local to CUDA target
-
-  /**
-     @brief Wrapper around cudaFuncSetAttribute with built-in error checking
-     @param[in] kernel Kernel function for which we are setting the attribute
-     @param[in] attr Attribute to set
-     @param[in] value Value to set
-  */
-  void qudaFuncSetAttribute_(const void *kernel, cudaFuncAttribute attr, int value, const char *func, const char *file,
-                             const char *line);
-
-  /**
-     @brief Wrapper around cudaFuncGetAttributes with built-in error checking
-     @param[in] attr the cudaFuncGetAttributes object to store the output
-     @param[in] kernel Kernel function for which we are setting the attribute
-  */
-  void qudaFuncGetAttributes_(cudaFuncAttributes &attr, const void *kernel, const char *func, const char *file,
-                              const char *line);
-
-#define qudaFuncSetAttribute(kernel, attr, value)                                                                      \
-  ::quda::qudaFuncSetAttribute_(kernel, attr, value, __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
-
-#define qudaFuncGetAttributes(attr, kernel)                                                                            \
-  ::quda::qudaFuncGetAttributes_(attr, kernel, __func__, quda::file_name(__FILE__), __STRINGIFY__(__LINE__))
-
 #ifdef USE_DRIVER_API
   static TimeProfile apiTimer("CUDA API calls (driver)");
 #else
@@ -384,27 +338,7 @@ namespace quda
     // if launch requests the maximum shared memory and the device supports it then opt in
     if (tp.set_max_shared_bytes && device::max_dynamic_shared_memory() > device::max_default_shared_memory()) {
       ompwip("Unimplemented for maximum shared memory");
-/*
-      static std::unordered_set<const void *> cache;
-      auto search = cache.find(func);
-      if (search == cache.end()) {
-        cache.insert(func);
-        qudaFuncSetAttribute(func, cudaFuncAttributePreferredSharedMemoryCarveout, (int)cudaSharedmemCarveoutMaxShared);
-        cudaFuncAttributes attributes;
-        qudaFuncGetAttributes(attributes, func);
-        qudaFuncSetAttribute(func, cudaFuncAttributeMaxDynamicSharedMemorySize,
-                             device::max_dynamic_shared_memory() - attributes.sharedSizeBytes);
-      }
-*/
     }
-
-/*
-    // no driver API variant here since we have C++ functions
-    PROFILE(cudaError_t error = cudaLaunchKernel(func, tp.grid, tp.block, args, tp.shared_bytes, device::get_cuda_stream(stream)),
-            QUDA_PROFILE_LAUNCH_KERNEL);
-    set_runtime_error(error, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
-    return error == cudaSuccess ? QUDA_SUCCESS : QUDA_ERROR;
-*/
     return QUDA_SUCCESS;
   }
 
@@ -496,25 +430,6 @@ namespace quda
 
       if (copy) {
         if (async) {
-#ifdef USE_DRIVER_API
-          CUresult error = CUDA_SUCCESS;
-          switch (kind) {
-          case cudaMemcpyDeviceToHost:
-            PROFILE(error = cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
-            break;
-          case cudaMemcpyHostToDevice:
-            PROFILE(error = cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
-            break;
-          case cudaMemcpyDeviceToDevice:
-            PROFILE(error = cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
-            break;
-          case cudaMemcpyDefault:
-            PROFILE(error = cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
-            break;
-          default: errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
-          }
-          set_driver_error(error, "cuMemcpyAsync", func, file, line, active_tuning);
-#else
 #ifdef API_PROFILE
           QudaProfileType type = QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC;
           switch (kind) {
@@ -528,36 +443,15 @@ namespace quda
           cudaError_t error;
           PROFILE(error = cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)), type);
           set_runtime_error(error, "cudaMemcpyAsync", func, file, line, active_tuning);
-#endif
         } else {
-#ifdef USE_DRIVER_API
-          CUresult error = CUDA_SUCCESS;
-          switch (kind) {
-          case cudaMemcpyDeviceToHost: error = cuMemcpyDtoH(dst, (CUdeviceptr)src, count); break;
-          case cudaMemcpyHostToDevice: error = cuMemcpyHtoD((CUdeviceptr)dst, src, count); break;
-          case cudaMemcpyHostToHost: memcpy(dst, src, count); break;
-          case cudaMemcpyDeviceToDevice: error = cuMemcpyDtoD((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
-          case cudaMemcpyDefault: error = cuMemcpy((CUdeviceptr)dst, (CUdeviceptr)src, count); break;
-          default: errorQuda("Unsupported cudaMemcpyType %d", kind);
-          }
-          set_driver_error(error, "cuMemcpy", func, file, line, active_tuning);
-#else
           cudaError_t error = cudaMemcpy(dst, src, count, kind);
           set_runtime_error(error, "cudaMemcpy", func, file, line, active_tuning);
-#endif
         }
       } else {
-#ifdef USE_DRIVER_API
-        CUresult error = async ?
-          cuMemsetD32Async((CUdeviceptr)dst, value, count / 4, device::get_cuda_stream(stream)) :
-          cuMemsetD32((CUdeviceptr)dst, value, count / 4);
-        set_driver_error(error, "cuMemset", func, file, line, active_tuning);
-#else
         cudaError_t error = async ?
           cudaMemsetAsync(dst, value, count, device::get_cuda_stream(stream)) :
           cudaMemset(dst, value, count);
         set_runtime_error(error, "cudaMemset", func, file, line, active_tuning);
-#endif
       }
     }
 
@@ -590,26 +484,8 @@ namespace quda
     if (kind == qudaMemcpyDeviceToDevice) {
       QudaMem copy(dst, src, count, qudaMemcpyKindToAPI(kind), stream, true, func, file, line);
     } else {
-#ifdef USE_DRIVER_API
-      switch (kind) {
-      case qudaMemcpyDeviceToHost:
-        PROFILE(cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2H_ASYNC);
-        break;
-      case qudaMemcpyHostToDevice:
-        PROFILE(cuMemcpyHtoDAsync((CUdeviceptr)dst, src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_H2D_ASYNC);
-        break;
-      case qudaMemcpyDeviceToDevice:
-        PROFILE(cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_D2D_ASYNC);
-        break;
-      case qudaMemcpyDefault:
-        PROFILE(cuMemcpyAsync((CUdeviceptr)dst, (CUdeviceptr)src, count, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC);
-        break;
-      default: errorQuda("Unsupported cuMemcpyTypeAsync %d", kind);
-      }
-#else
       PROFILE(cudaMemcpyAsync(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)),
               kind == qudaMemcpyDeviceToHost ? QUDA_PROFILE_MEMCPY_D2H_ASYNC : QUDA_PROFILE_MEMCPY_H2D_ASYNC);
-#endif
     }
   }
 
@@ -625,70 +501,17 @@ namespace quda
   void qudaMemcpy2D_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
                      qudaMemcpyKind kind, const char *func, const char *file, const char *line)
   {
-#ifdef USE_DRIVER_API
-    CUDA_MEMCPY2D param;
-    param.srcPitch = spitch;
-    param.srcY = 0;
-    param.srcXInBytes = 0;
-    param.dstPitch = dpitch;
-    param.dstY = 0;
-    param.dstXInBytes = 0;
-    param.WidthInBytes = width;
-    param.Height = height;
-
-    switch (kind) {
-    case qudaMemcpyDeviceToHost:
-      param.srcDevice = (CUdeviceptr)src;
-      param.srcMemoryType = CU_MEMORYTYPE_DEVICE;
-      param.dstHost = dst;
-      param.dstMemoryType = CU_MEMORYTYPE_HOST;
-      break;
-    default: errorQuda("Unsupported cuMemcpyType2DAsync %d", qudaMemcpyKindToAPI(kind));
-    }
-    PROFILE(auto error = cuMemcpy2D(&param), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
-    if (error != CUDA_SUCCESS) {
-      const char *str;
-      cuGetErrorName(error, &str);
-      errorQuda("cuMemcpy2D returned error %s\n (%s:%s in %s())", str, file, line, func);
-    }
-#else
     PROFILE(auto error = cudaMemcpy2D(dst, dpitch, src, spitch, width, height, qudaMemcpyKindToAPI(kind)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
     if (error != cudaSuccess)
       errorQuda("cudaMemcpy2D returned error %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
-#endif
   }
 
   void qudaMemcpy2DAsync_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
                           qudaMemcpyKind kind, const qudaStream_t &stream, const char *func, const char *file,
                           const char *line)
   {
-#ifdef USE_DRIVER_API
-    CUDA_MEMCPY2D param;
-    param.srcPitch = spitch;
-    param.srcY = 0;
-    param.srcXInBytes = 0;
-    param.dstPitch = dpitch;
-    param.dstY = 0;
-    param.dstXInBytes = 0;
-    param.WidthInBytes = width;
-    param.Height = height;
-
-    switch (kind) {
-    case qudaMemcpyDeviceToHost:
-      param.srcDevice = (CUdeviceptr)src;
-      param.srcMemoryType = CU_MEMORYTYPE_DEVICE;
-      param.dstHost = dst;
-      param.dstMemoryType = CU_MEMORYTYPE_HOST;
-      break;
-    default:
-      errorQuda("Unsupported cuMemcpyType2DAsync %d", qudaMemcpyKindToAPI(kind));
-    }
-    PROFILE(auto error = cuMemcpy2DAsync(&param, device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
-    set_driver_error(error, "cuMemcpy2DAsync", func, file, line);
-#else
     PROFILE(auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
     set_runtime_error(error, "cudaMemcpy2DAsync", func, file, line);
-#endif
   }
 
   void qudaMemcpy2DP2PAsync_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
@@ -743,120 +566,76 @@ namespace quda
 */
   }
 
+  constexpr int max_quda_event = 16;
+  struct QudaEvent { bool active; double time; };
+  static QudaEvent global_quda_event[max_quda_event];
+
   bool qudaEventQuery_(qudaEvent_t &quda_event, const char *func, const char *file, const char *line)
   {
-    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
-#ifdef USE_DRIVER_API
-    PROFILE(CUresult error = cuEventQuery(event), QUDA_PROFILE_EVENT_QUERY);
-    switch (error) {
-    case CUDA_SUCCESS: return true;
-    case CUDA_ERROR_NOT_READY: return false;
-    default: set_driver_error(error, __func__, func, file, line);
-    }
-#else
-    PROFILE(cudaError_t error = cudaEventQuery(event), QUDA_PROFILE_EVENT_QUERY);
-    switch (error) {
-    case cudaSuccess: return true;
-    case cudaErrorNotReady: return false;
-    default: set_runtime_error(error, __func__, func, file, line);
-    }
-#endif
-    return false;
+    return true;
   }
 
   void qudaEventRecord_(qudaEvent_t &quda_event, qudaStream_t stream, const char *func, const char *file, const char *line)
   {
-    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
-#ifdef USE_DRIVER_API
-    PROFILE(CUresult error = cuEventRecord(event, device::get_cuda_stream(stream)), QUDA_PROFILE_EVENT_RECORD);
-    set_driver_error(error, __func__, func, file, line);
-#else
-    PROFILE(cudaError_t error = cudaEventRecord(event, device::get_cuda_stream(stream)), QUDA_PROFILE_EVENT_RECORD);
-    set_runtime_error(error, __func__, func, file, line);
-#endif
+    QudaEvent *e = reinterpret_cast<QudaEvent *>(quda_event.event);
+    if(e!=nullptr)
+      e->time = omp_get_wtime();
   }
 
   void qudaStreamWaitEvent_(qudaStream_t stream, qudaEvent_t quda_event, unsigned int flags, const char *func,
                             const char *file, const char *line)
   {
-    cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(quda_event.event);
-#ifdef USE_DRIVER_API
-    PROFILE(CUresult error = cuStreamWaitEvent(device::get_cuda_stream(stream), event, flags), QUDA_PROFILE_STREAM_WAIT_EVENT);
-    set_driver_error(error, __func__, func, file, line);
-#else
-    PROFILE(cudaError_t error = cudaStreamWaitEvent(device::get_cuda_stream(stream), event, flags), QUDA_PROFILE_STREAM_WAIT_EVENT);
-    set_runtime_error(error, __func__, func, file, line);
-#endif
   }
 
   qudaEvent_t qudaEventCreate_(const char *func, const char *file, const char *line)
   {
-    cudaEvent_t cuda_event;
-    cudaError_t error = cudaEventCreateWithFlags(&cuda_event, cudaEventDisableTiming);
-    set_runtime_error(error, __func__, func, file, line);
     qudaEvent_t quda_event;
-    quda_event.event = reinterpret_cast<void*>(cuda_event);
+    quda_event.event = nullptr;
     return quda_event;
   }
 
   qudaEvent_t qudaChronoEventCreate_(const char *func, const char *file, const char *line)
   {
-    cudaEvent_t cuda_event;
-    cudaError_t error = cudaEventCreate(&cuda_event);
-    set_runtime_error(error, __func__, func, file, line);
     qudaEvent_t quda_event;
-    quda_event.event = reinterpret_cast<void*>(cuda_event);
+    int i;
+    for(i=0;i<max_quda_event;++i)
+      if(!global_quda_event[i].active)
+        break;
+    if(i<max_quda_event){
+      global_quda_event[i].active = true;
+      global_quda_event[i].time = 0.;
+      quda_event.event = reinterpret_cast<void*>(&global_quda_event[i]);
+    }else{
+      errorQuda("global_quda_event exhausted.");
+    }
     return quda_event;
   }
 
-  float qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
+  double qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
                               const char *func, const char *file, const char *line)
   {
-    float elapsed_time;
-    cudaError_t error = cudaEventElapsedTime(&elapsed_time, reinterpret_cast<cudaEvent_t>(start.event),
-                                             reinterpret_cast<cudaEvent_t>(stop.event));
-    set_runtime_error(error, __func__, func, file, line);
-    return elapsed_time / 1000;
+    return reinterpret_cast<QudaEvent *>(stop.event)->time - reinterpret_cast<QudaEvent *>(start.event)->time;
   }
 
   void qudaEventDestroy_(qudaEvent_t &event, const char *func, const char *file, const char *line)
   {
-    cudaError_t error = cudaEventDestroy(reinterpret_cast<cudaEvent_t&>(event.event));
-    set_runtime_error(error, __func__, func, file, line);
+    QudaEvent *e = reinterpret_cast<QudaEvent *>(event.event);
+    if(e!=nullptr){
+      e->active = false;
+      e->time = 0.;
+    }
   }
 
   void qudaEventSynchronize_(const qudaEvent_t &quda_event, const char *func, const char *file, const char *line)
   {
-    const cudaEvent_t &event = reinterpret_cast<const cudaEvent_t&>(quda_event.event);
-#ifdef USE_DRIVER_API
-    PROFILE(CUresult error = cuEventSynchronize(event), QUDA_PROFILE_EVENT_SYNCHRONIZE);
-    set_driver_error(error, __func__, func, file, line);
-#else
-    PROFILE(cudaError_t error = cudaEventSynchronize(event), QUDA_PROFILE_EVENT_SYNCHRONIZE);
-    set_runtime_error(error, __func__, func, file, line);
-#endif
   }
 
   void qudaStreamSynchronize_(const qudaStream_t &stream, const char *func, const char *file, const char *line)
   {
-#ifdef USE_DRIVER_API
-    PROFILE(CUresult error = cuStreamSynchronize(device::get_cuda_stream(stream)), QUDA_PROFILE_STREAM_SYNCHRONIZE);
-    set_driver_error(error, __func__, file, line, func, activeTuning());
-#else
-    PROFILE(cudaError_t error = cudaStreamSynchronize(device::get_cuda_stream(stream)), QUDA_PROFILE_STREAM_SYNCHRONIZE);
-    set_runtime_error(error, __func__, file, line, func, activeTuning());
-#endif
   }
 
   void qudaDeviceSynchronize_(const char *func, const char *file, const char *line)
   {
-#ifdef USE_DRIVER_API
-    PROFILE(CUresult error = cuCtxSynchronize(), QUDA_PROFILE_DEVICE_SYNCHRONIZE);
-    set_driver_error(error, __func__, func, file, line, activeTuning());
-#else
-    PROFILE(cudaError_t error = cudaDeviceSynchronize(), QUDA_PROFILE_DEVICE_SYNCHRONIZE);
-    set_runtime_error(error, __func__, func, file, line, activeTuning());
-#endif
   }
 
   void* qudaGetSymbolAddress_(const char *symbol, const char *func, const char *file, const char *line)
@@ -867,22 +646,6 @@ namespace quda
     return ptr;
   }
 
-  void qudaFuncSetAttribute_(const void *kernel, cudaFuncAttribute attr, int value, const char *func, const char *file,
-                             const char *line)
-  {
-    // no driver API variant here since we have C++ functions
-    PROFILE(cudaError_t error = cudaFuncSetAttribute(kernel, attr, value), QUDA_PROFILE_FUNC_SET_ATTRIBUTE);
-    set_runtime_error(error, __func__, func, file, line);
-  }
-
-  void qudaFuncGetAttributes_(cudaFuncAttributes &attr, const void *kernel, const char *func, const char *file,
-                              const char *line)
-  {
-    // no driver API variant here since we have C++ functions
-    PROFILE(cudaError_t error = cudaFuncGetAttributes(&attr, kernel), QUDA_PROFILE_FUNC_SET_ATTRIBUTE);
-    set_runtime_error(error, __func__, func, file, line);
-  }
-
   void printAPIProfile()
   {
 #ifdef API_PROFILE

From 34279412f2fe16998ff2e1be2a7f91db3bd3c901 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 24 May 2021 15:30:55 +0000
Subject: [PATCH 064/198] omp: remove debug print in kernels

---
 .../omptarget/block_reduction_kernel.h        | 24 +++---
 include/targets/omptarget/kernel.h            | 84 +++++++++----------
 include/targets/omptarget/reduction_kernel.h  | 68 +++++++--------
 3 files changed, 88 insertions(+), 88 deletions(-)

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index b123169d08..a25dee6757 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -63,18 +63,18 @@ namespace quda {
     const int tx = arg.threads.x;
     const int ty = arg.threads.y;
     const int tz = arg.threads.z;
-    printf("BlockKernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    // printf("BlockKernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     #pragma omp parallel num_threads(ld)
     {
-      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-               "omp reports: teams %d threads %d\n",
-               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-               omp_get_num_teams(), omp_get_num_threads());
+      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+      //          "omp reports: teams %d threads %d\n",
+      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+      //          omp_get_num_teams(), omp_get_num_threads());
       BlockKernel2D_impl<block_size, Transformer, Arg>(*dparg);
     }
     omp_target_free(dparg, omp_get_default_device());
@@ -86,15 +86,15 @@ namespace quda {
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+    // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     #pragma omp parallel num_threads(ld)
     {
-      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-               "omp reports: teams %d threads %d\n",
-               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-               omp_get_num_teams(), omp_get_num_threads());
+      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+      //          "omp reports: teams %d threads %d\n",
+      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+      //          omp_get_num_teams(), omp_get_num_threads());
       BlockKernel2D_impl<block_size, Transformer, Arg>(device::get_arg<Arg>());
     }
   }
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 5bd3449378..3df3d14570 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -26,18 +26,18 @@ namespace quda {
       const int tx = arg.threads.x;
       const int ty = arg.threads.y;
       const int tz = arg.threads.z;
-      printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+      // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
       Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
       // printf("dparg %p\n", dparg);
       omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
       #pragma omp parallel num_threads(ld)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         char buffer[sizeof(Arg)];
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel1D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
@@ -50,15 +50,15 @@ namespace quda {
     {
       const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
       const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-      printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
+      // printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
       #pragma omp parallel num_threads(ld)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
     }
@@ -88,18 +88,18 @@ namespace quda {
       const int tx = arg.threads.x;
       const int ty = arg.threads.y;
       const int tz = arg.threads.z;
-      printf("Kernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+      // printf("Kernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
       Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
       // printf("dparg %p\n", dparg);
       omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
       #pragma omp parallel num_threads(ld)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         char buffer[sizeof(Arg)];
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel2D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
@@ -112,15 +112,15 @@ namespace quda {
     {
       const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
       const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-      printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+      // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
       #pragma omp parallel num_threads(ld)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
     }
@@ -152,18 +152,18 @@ namespace quda {
       const int tx = arg.threads.x;
       const int ty = arg.threads.y;
       const int tz = arg.threads.z;
-      printf("Kernel3D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+      // printf("Kernel3D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
       Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
       // printf("dparg %p\n", dparg);
       omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
       #pragma omp parallel num_threads(ld)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         char buffer[sizeof(Arg)];
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel3D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
@@ -176,15 +176,15 @@ namespace quda {
     {
       const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
       const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-      printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
+      // printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
       #pragma omp parallel num_threads(ld)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
     }
@@ -197,18 +197,18 @@ namespace quda {
     const int tx = arg.threads.x;
     const int ty = arg.threads.y;
     const int tz = arg.threads.z;
-    printf("raw_kernel: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    // printf("raw_kernel: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     #pragma omp parallel num_threads(ld)
     {
-      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-               "omp reports: teams %d threads %d\n",
-               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-               omp_get_num_teams(), omp_get_num_threads());
+      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+      //          "omp reports: teams %d threads %d\n",
+      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+      //          omp_get_num_teams(), omp_get_num_threads());
       Functor<Arg> f(*dparg);
       f();
     }
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 1e7356b920..7d6b24a3de 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -18,11 +18,11 @@ namespace quda {
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
     #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
     {
-      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-               "omp reports: teams %d threads %d\n",
-               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-               omp_get_num_teams(), omp_get_num_threads());
+      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+      //          "omp reports: teams %d threads %d\n",
+      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+      //          omp_get_num_teams(), omp_get_num_threads());
       dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
       Transformer<Arg> t(arg);
@@ -49,7 +49,7 @@ namespace quda {
     const int tx = arg.threads.x;
     const int ty = arg.threads.y;
     const int tz = arg.threads.z;
-    printf("Reduction2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    // printf("Reduction2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
@@ -66,11 +66,11 @@ namespace quda {
       reduce_t value = dparg->init();
       #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
         Transformer<Arg> t(*dparg);
@@ -144,7 +144,7 @@ namespace quda {
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    printf("Reduction2D: launch parameter: gd %d ld %d\n", gd, ld);
+    // printf("Reduction2D: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       // Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
@@ -159,11 +159,11 @@ namespace quda {
       reduce_t value = dparg->init();
       #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
         Transformer<Arg> t(*dparg);
@@ -243,11 +243,11 @@ namespace quda {
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
     #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
     {
-      if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-               "omp reports: teams %d threads %d\n",
-               launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-               omp_get_num_teams(), omp_get_num_threads());
+      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+      //          "omp reports: teams %d threads %d\n",
+      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+      //          omp_get_num_teams(), omp_get_num_threads());
       dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
       Transformer<Arg> t(arg);
@@ -279,7 +279,7 @@ namespace quda {
     const int tx = arg.threads.x;
     const int ty = arg.threads.y;
     const int tz = arg.threads.z;
-    printf("MultiReduction: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    // printf("MultiReduction: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
@@ -296,11 +296,11 @@ namespace quda {
       reduce_t value = dparg->init();
       #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
         Transformer<Arg> t(*dparg);
@@ -376,7 +376,7 @@ namespace quda {
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    printf("MultiReduction: launch parameter: gd %d ld %d\n", gd, ld);
+    // printf("MultiReduction: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       // MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
@@ -391,11 +391,11 @@ namespace quda {
       reduce_t value = dparg->init();
       #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
       {
-        if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-          printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-                 "omp reports: teams %d threads %d\n",
-                 launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-                 omp_get_num_teams(), omp_get_num_threads());
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
         dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
         Transformer<Arg> t(*dparg);

From 0d656b53e07eea9f3600ebb65a6213f97c9853e5 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 24 May 2021 15:31:15 +0000
Subject: [PATCH 065/198] omp: update math_helper

---
 include/targets/omptarget/math_helper.cuh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/targets/omptarget/math_helper.cuh b/include/targets/omptarget/math_helper.cuh
index d47a587f49..4891d8f7ac 100644
--- a/include/targets/omptarget/math_helper.cuh
+++ b/include/targets/omptarget/math_helper.cuh
@@ -6,7 +6,7 @@
 namespace quda {
 
   /**
-   * @brief Combined sin and cos colculation in QUDA NAMESPACE
+   * @brief Combined sin and cos calculation in QUDA NAMESPACE
    * @param a the angle
    * @param s pointer to the storage for the result of the sin
    * @param c pointer to the storage for the result of the cos
@@ -21,7 +21,7 @@ namespace quda {
    * some math libraries provide a fast inverse sqrt() function.
    * this implementation trusts the compiler.
    */
-  template<typename T> inline __host__ __device__ T rsqrt(T a) { return 1.0/std::sqrt(a); }
+  template<typename T> inline __host__ __device__ T rsqrt(T a) { return static_cast<T>(1.0) / std::sqrt(a); }
 
     /**
      Generic wrapper for Trig functions -- used in gauge field order 

From c33546bb6e45a7a76b009b4078268906c1717cfe Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 24 May 2021 15:33:36 +0000
Subject: [PATCH 066/198] omp: remove debug print in tunable_nd

---
 include/tunable_nd.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/include/tunable_nd.h b/include/tunable_nd.h
index e6a49676b7..26647a41fa 100644
--- a/include/tunable_nd.h
+++ b/include/tunable_nd.h
@@ -91,7 +91,6 @@ namespace quda {
     template <template <typename> class Functor, bool enable_host = false, typename Arg>
       std::enable_if_t<!enable_host, void> launch(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-      ompwip();
       if (location == QUDA_CUDA_FIELD_LOCATION) {
         launch_device<Functor, Arg>(tp, stream, arg);
       } else {

From 67bc5b813e11433bde8553e20769d09beeba630a Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 24 May 2021 15:34:52 +0000
Subject: [PATCH 067/198] omp: test kernel launches to make sure the runtime
 allows the specified parameters

---
 include/tune_quda.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/include/tune_quda.h b/include/tune_quda.h
index 8677d120d5..740aafd42b 100644
--- a/include/tune_quda.h
+++ b/include/tune_quda.h
@@ -75,7 +75,27 @@ namespace quda {
       if (tuneGridDim()) {
         const int step = gridStep();
         param.grid.x += step;
+#ifdef QUDA_BACKEND_OMPTARGET
+        /* Makes sure the runtime allows us to set the parameters. */
+        bool failed = false;
+        {
+          const int gd = param.grid.x*param.grid.y*param.grid.z;
+          const int ld = param.block.x*param.block.y*param.block.z;
+          int gn = 0, ln = 0;
+          #pragma omp target teams num_teams(gd) thread_limit(ld) map(tofrom:gn,ln)
+          #pragma omp parallel num_threads(ld)
+          {
+            if(omp_get_team_num()==0 && omp_get_thread_num()==0){
+              gn = omp_get_num_teams();
+              ln = omp_get_num_threads();
+            }
+          }
+          failed = gn!=gd||ln!=ld;
+        }
+        if (failed || param.grid.x > maxGridSize()) {
+#else
         if (param.grid.x > maxGridSize()) {
+#endif
           param.grid.x = minGridSize();
           return false;
         } else {

From 7c168dc78103e1bb9403a4e4159288203af12e68 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 24 May 2021 15:36:28 +0000
Subject: [PATCH 068/198] tunable: correctly respect constraints on launch
 parameters

- update param.block/grid before calling Tunable::advanceBlockDim(param)
- add test to make sure nthreads <= device::max_threads_per_block()
---
 include/tunable_block_reduction.h | 2 +-
 include/tunable_nd.h              | 6 +++---
 include/tunable_reduction.h       | 2 +-
 include/tune_quda.h               | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/include/tunable_block_reduction.h b/include/tunable_block_reduction.h
index 647a3cf5fc..22e6c1660f 100644
--- a/include/tunable_block_reduction.h
+++ b/include/tunable_block_reduction.h
@@ -173,9 +173,9 @@ namespace quda {
     {
       dim3 block = param.block;
       dim3 grid = param.grid;
-      bool ret = tune_block_x ? Tunable::advanceBlockDim(param) : false;
       param.block.y = block.y;
       param.grid.y = grid.y;
+      bool ret = tune_block_x ? Tunable::advanceBlockDim(param) : false;
 
       if (ret) {
 	return true;
diff --git a/include/tunable_nd.h b/include/tunable_nd.h
index 26647a41fa..18850af32a 100644
--- a/include/tunable_nd.h
+++ b/include/tunable_nd.h
@@ -236,9 +236,9 @@ namespace quda {
     {
       dim3 block = param.block;
       dim3 grid = param.grid;
-      bool ret = tune_block_x ? Tunable::advanceBlockDim(param) : false;
       param.block.y = block.y;
       param.grid.y = grid.y;
+      bool ret = tune_block_x ? Tunable::advanceBlockDim(param) : false;
 
       if (ret) {
 	return true;
@@ -398,10 +398,10 @@ namespace quda {
     {
       dim3 block = param.block;
       dim3 grid = param.grid;
-      bool ret = tune_block_y ? TunableKernel2D_base<grid_stride>::advanceBlockDim(param) :
-      TunableKernel2D_base<grid_stride>::tune_block_x ? Tunable::advanceBlockDim(param) : false;
       param.block.z = block.z;
       param.grid.z = grid.z;
+      bool ret = tune_block_y ? TunableKernel2D_base<grid_stride>::advanceBlockDim(param) :
+      TunableKernel2D_base<grid_stride>::tune_block_x ? Tunable::advanceBlockDim(param) : false;
 
       if (ret) {
 	// we advanced the block.x / block.y so we're done
diff --git a/include/tunable_reduction.h b/include/tunable_reduction.h
index 572afff869..7de7a8b844 100644
--- a/include/tunable_reduction.h
+++ b/include/tunable_reduction.h
@@ -210,8 +210,8 @@ namespace quda {
     }
 
     virtual bool advanceBlockDim(TuneParam &param) const {
-      bool rtn = Tunable::advanceBlockDim(param);
       param.block.y = block_size_y;
+      bool rtn = Tunable::advanceBlockDim(param);
       return rtn;
     }
 
diff --git a/include/tune_quda.h b/include/tune_quda.h
index 740aafd42b..83b1dd4fe3 100644
--- a/include/tune_quda.h
+++ b/include/tune_quda.h
@@ -164,7 +164,7 @@ namespace quda {
       int nthreads = param.block.x * param.block.y * param.block.z;
       param.shared_bytes = std::max(sharedBytesPerThread() * nthreads, sharedBytesPerBlock(param));
 
-      if (param.block.x > max_threads || param.shared_bytes > max_shared) {
+      if (param.block.x > max_threads || param.shared_bytes > max_shared || nthreads > device::max_threads_per_block()) {
         resetBlockDim(param);
         int nthreads = param.block.x * param.block.y * param.block.z;
         param.shared_bytes = std::max(sharedBytesPerThread() * nthreads, sharedBytesPerBlock(param));

From 8345546b5581a3fbefb42fb308f0f19e1bb781df Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 25 May 2021 18:53:52 +0000
Subject: [PATCH 069/198] omp: clean up OMP WIP print messages

---
 lib/targets/omptarget/device.cpp   | 62 ++------------------------
 lib/targets/omptarget/malloc.cpp   | 59 +++++++++++--------------
 lib/targets/omptarget/quda_api.cpp | 71 +++++++++++++++++++-----------
 3 files changed, 75 insertions(+), 117 deletions(-)

diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 7c9d5b0ecc..b3be12573f 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -98,8 +98,6 @@ static int cudaDeviceGetStreamPriorityRange(int*lo,int*hi){ompwip();lo=0;hi=0;re
 static int cudaStreamCreateWithPriority(cudaStream_t*s,int lo,int hi){ompwip();*s=0;return 0;}
 static int cudaStreamDestroy(cudaStream_t s){ompwip();return 0;}
 static int cudaDeviceReset(){ompwip();return 0;}
-enum {cudaDevAttrMaxSharedMemoryPerBlockOptin};
-static int cudaDeviceGetAttribute(int*b,int o,int i){ompwip();*b=98304;return 0;}
 
 static cudaDeviceProp deviceProp;
 static cudaStream_t *streams;
@@ -342,27 +340,13 @@ namespace quda
       return Nstream - 1;
     }
 
-    bool managed_memory_supported()
-    {
-      // managed memory is supported on Pascal and up
-      return deviceProp.major >= 6;
-    }
+    bool managed_memory_supported() { return false; }
 
-    bool shared_memory_atomic_supported()
-    {
-      // shared memory atomics are supported on Maxwell and up
-      return deviceProp.major >= 5;
-    }
+    bool shared_memory_atomic_supported() { return false; }
 
     size_t max_default_shared_memory() { return deviceProp.sharedMemPerBlock; }
 
-    size_t max_dynamic_shared_memory()
-    {
-      static int max_shared_bytes = 0;
-      if (!max_shared_bytes)
-        CHECK_CUDA_ERROR(cudaDeviceGetAttribute(&max_shared_bytes, cudaDevAttrMaxSharedMemoryPerBlockOptin, comm_gpuid()));
-      return max_shared_bytes;
-    }
+    size_t max_dynamic_shared_memory() { return 98304; }
 
     unsigned int max_threads_per_block() { return deviceProp.maxThreadsPerBlock; }
 
@@ -374,45 +358,7 @@ namespace quda
 
     unsigned int processor_count() { return deviceProp.multiProcessorCount; }
 
-    unsigned int max_blocks_per_processor()
-    {
-#if CUDA_VERSION >= 11000
-      static int max_blocks_per_sm = 0;
-      if (!max_blocks_per_sm)
-        CHECK_CUDA_ERROR(cudaDeviceGetAttribute(&max_blocks_per_sm, cudaDevAttrMaxBlocksPerMultiprocessor, comm_gpuid()));
-      return max_blocks_per_sm;
-#else
-      // these variables are taken from Table 14 of the CUDA 10.2 prgramming guide
-      switch (deviceProp.major) {
-      case 2:
-	return 8;
-	break;
-      case 3:
-	return 16;
-	break;
-      case 5:
-	 return 32;
-	 break;
-      case 6: 
-	 return 32;
-	 break;
-      case 7:
-	{
-          switch (deviceProp.minor) {
-            case 0: return 32; break;
-            case 2: return 32; break;
-            case 5: return 16; break;
-	    default: return 32; break;
-	  };
-        }
-	break;
-      default:
-        warningQuda("Unknown SM architecture %d.%d - assuming limit of 32 blocks per SM\n",
-                    deviceProp.major, deviceProp.minor);
-        return 32;
-      }
-#endif
-    }
+    unsigned int max_blocks_per_processor() { return 32; }
 
     namespace profile
     {
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index bad00a312e..b592017389 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -117,10 +117,6 @@ namespace quda
       if (total_pinned_bytes > max_total_pinned_bytes) { max_total_pinned_bytes = total_pinned_bytes; }
     }
     alloc[type][ptr] = a;
-{
-    const char *type_str[] = {"Device", "Device Pinned", "Host  ", "Pinned", "Mapped", "Managed"};
-    printfQuda("MALLOC: %s  %15p  %15lu  %s(), %s:%d\n", type_str[type], ptr, (unsigned long)a.base_size, a.func.c_str(), a.file.c_str(), a.line);
-}
   }
 
   static void track_free(const AllocType &type, void *ptr)
@@ -129,11 +125,6 @@ namespace quda
     total_bytes[type] -= size;
     if (type != DEVICE && type != DEVICE_PINNED && type != SHMEM) { total_host_bytes -= size; }
     if (type == PINNED || type == MAPPED) { total_pinned_bytes -= size; }
-{
-    auto a = alloc[type][ptr];
-    const char *type_str[] = {"Device", "Device Pinned", "Host  ", "Pinned", "Mapped", "Managed"};
-    printfQuda("FREE: %s  %15p  %15lu  %s(), %s:%d\n", type_str[type], ptr, (unsigned long)a.base_size, a.func.c_str(), a.file.c_str(), a.line);
-}
     alloc[type].erase(ptr);
   }
 
@@ -255,7 +246,7 @@ namespace quda
   void *device_pinned_malloc_(const char *func, const char *file, int line, size_t size)
   {
     if (!comm_peer2peer_present()) return device_malloc_(func, file, line, size);
-    ompwip("device_pinned_malloc_ with !comm_peer2peer_present() unimplemented");
+    ompwip("ERROR: device_pinned_malloc_ with !comm_peer2peer_present() unimplemented");
     return nullptr;
 /*
 
@@ -287,7 +278,7 @@ namespace quda
     a.size = a.base_size = size;
 
     void *ptr = malloc(size);
-    ompwip("malloc: %p",ptr);
+    // ompwip("malloc: %p",ptr);
     if (!ptr) { errorQuda("Failed to allocate host memory of size %zu (%s:%d in %s())\n", size, file, line, func); }
     track_malloc(HOST, a, ptr);
 #ifdef HOST_DEBUG
@@ -309,7 +300,7 @@ namespace quda
   {
     MemAlloc a(func, file, line);
     void *ptr = aligned_malloc(a, size);
-    ompwip("pinned_malloc_ uses aligned_malloc");
+    ompwip("ERROR: pinned_malloc_ uses aligned_malloc");
     if(!ptr)
       errorQuda("Failed to register pinned memory of size %zu (%s:%d in %s())\n", size, file, line, func);
 /*
@@ -345,12 +336,11 @@ namespace quda
     }
 #else
     void *ptr = aligned_malloc(a, size);
-    ompwip("mapped_malloc_ host: %p",ptr);
     print_trace();
     if(0<omp_get_num_devices()){
       int d = omp_get_default_device();
       void *dp = omp_target_alloc(a.base_size, d);
-      ompwip("require special memcpy, mapped_malloc_ device: %p",dp);
+      ompwip("WARNING: require special memcpy, mapped_malloc_ host: %p  device: %p",ptr,dp);
       if(!dp)
         errorQuda("%s:%d %s() Failed to allocate device memory of size %zu for mapped malloc\n", file, line, func, size);
       if(omp_target_associate_ptr(ptr, dp, a.base_size, 0, d))
@@ -434,13 +424,13 @@ namespace quda
    */
   void device_free_(const char *func, const char *file, int line, void *ptr)
   {
-    if (use_managed_memory()) {ompwip("unimplemented");
+    if (use_managed_memory()) {ompwip("ERROR: unimplemented");
       managed_free_(func, file, line, ptr);
       return;
     }
 
 #ifndef QDP_USE_CUDA_MANAGED_MEMORY
-    ompwip("device free: %p",ptr);
+    // ompwip("device free: %p",ptr);
     if (!ptr) { errorQuda("Attempt to free NULL device pointer (%s:%d in %s())\n", file, line, func); }
     if (!alloc[DEVICE].count(ptr)) {
       errorQuda("Attempt to free invalid device pointer (%s:%d in %s())\n", file, line, func);
@@ -466,7 +456,7 @@ namespace quda
    */
   void device_pinned_free_(const char *func, const char *file, int line, void *ptr)
   {
-    ompwip("device_pinned_free_ untested code path: %p",ptr); print_trace();
+    ompwip("ERROR: device_pinned_free_ untested code path: %p",ptr); print_trace();
     if (!comm_peer2peer_present()) {
       device_free_(func, file, line, ptr);
       return;
@@ -493,7 +483,7 @@ namespace quda
    * malloc_quda.h
    */
   void managed_free_(const char *func, const char *file, int line, void *ptr)
-  {ompwip("managed_free_ unimplemented doing nothing: %p",ptr);/*
+  {ompwip("ERROR: managed_free_ unimplemented doing nothing: %p",ptr);/*
     if (!ptr) { errorQuda("Attempt to free NULL managed pointer (%s:%d in %s())\n", file, line, func); }
     if (!alloc[MANAGED].count(ptr)) {
       errorQuda("Attempt to free invalid managed pointer (%s:%d in %s())\n", file, line, func);
@@ -515,7 +505,7 @@ namespace quda
       track_free(HOST, ptr);
       free(ptr);
     } else if (alloc[PINNED].count(ptr)) {
-      ompwip("free PINNED ptr: %p",ptr);
+      ompwip("WARNING: free PINNED ptr: %p",ptr);
 /*
       cudaError_t err = cudaHostUnregister(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to unregister pinned memory (%s:%d in %s())\n", file, line, func); }
@@ -524,18 +514,18 @@ namespace quda
       free(ptr);
     } else if (alloc[MAPPED].count(ptr)) {
 #ifdef HOST_ALLOC
-      ompwip("HOST_ALLOC alloc[MAPPED].count(ptr) untested code path: %p",ptr); print_trace();
+      ompwip("ERROR: HOST_ALLOC alloc[MAPPED].count(ptr) untested code path: %p",ptr); print_trace();
       free(ptr);
 /*
       cudaError_t err = cudaFreeHost(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to free host memory (%s:%d in %s())\n", file, line, func); }
 */
 #else
-      ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr) free: %p",ptr);
+      // ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr) free: %p",ptr);
       void *dp = target::omp_mapped_ptr[ptr];
       if(dp!=ptr){
         omp_target_disassociate_ptr(ptr, omp_get_default_device());
-        ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr) free mapped device: %p",dp);
+        ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr) free mapped host: %p  device: %p",ptr,dp);
         omp_target_free(dp, omp_get_default_device());
       }
       free(ptr);
@@ -617,10 +607,10 @@ namespace quda
   QudaFieldLocation get_pointer_location(const void *ptr)
   {
     if(alloc[DEVICE].count(const_cast<void*>(ptr)) || alloc[DEVICE_PINNED].count(const_cast<void*>(ptr))){
-      ompwip("get_pointer_location %p returns QUDA_CUDA_FIELD_LOCATION",ptr);
+      // ompwip("get_pointer_location %p returns QUDA_CUDA_FIELD_LOCATION",ptr);
       return QUDA_CUDA_FIELD_LOCATION;
     }else if(alloc[HOST].count(const_cast<void*>(ptr)) || alloc[PINNED].count(const_cast<void*>(ptr)) || alloc[MAPPED].count(const_cast<void*>(ptr))){
-      ompwip("get_pointer_location %p returns QUDA_CPU_FIELD_LOCATION",ptr);
+      // ompwip("get_pointer_location %p returns QUDA_CPU_FIELD_LOCATION",ptr);
       return QUDA_CPU_FIELD_LOCATION;
     }
     ompwip("WARNING: get_pointer_location assumes %p to be QUDA_CPU_FIELD_LOCATION",ptr);
@@ -650,9 +640,10 @@ namespace quda
 
   void *get_mapped_device_pointer_(const char *func, const char *file, int line, const void *host)
   {
-    ompwip("get_mapped_device_pointer_: %p",host);
     print_trace();
-    return target::omp_mapped_ptr[host];
+    auto dp = target::omp_mapped_ptr[host];
+    ompwip("WARNING: get_mapped_device_pointer_ host: %p  device: %p  make sure to copy to host before reading from host",host,dp);
+    return dp;
 /*
     void *device;
     auto error = cudaHostGetDevicePointer(&device, const_cast<void *>(host), 0);
@@ -666,7 +657,7 @@ namespace quda
 
   void register_pinned_(const char *func, const char *file, int line, void *ptr, size_t bytes)
   {
-    ompwip("register_pinned_ unimplemented: %p",ptr);
+    ompwip("ERROR: register_pinned_ unimplemented: %p",ptr);
 /*
     auto error = cudaHostRegister(ptr, bytes, cudaHostRegisterDefault);
     if (error != cudaSuccess) {
@@ -678,7 +669,7 @@ namespace quda
 
   void unregister_pinned_(const char *func, const char *file, int line, void *ptr)
   {
-    ompwip("unregister_pinned_ unimplemented: %p",ptr);
+    ompwip("ERROR: unregister_pinned_ unimplemented: %p",ptr);
 /*
     auto error = cudaHostUnregister(ptr);
     if (error != cudaSuccess) {
@@ -769,13 +760,13 @@ namespace quda
       } else {
         ptr = quda::pinned_malloc_(func, file, line, nbytes);
       }
-      ompwip("pool::pinned_malloc_: %p",ptr);
+      ompwip("ERROR: untested pool::pinned_malloc_: %p",ptr);
       return ptr;
     }
 
     void pinned_free_(const char *func, const char *file, int line, void *ptr)
     {
-      ompwip("pool::pinnd_free_: %p",ptr);
+      ompwip("ERROR: untested pool::pinnd_free_: %p",ptr);
       if (pinned_memory_pool) {
         if (!pinnedSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
         pinnedCache.insert(std::make_pair(pinnedSize[ptr], ptr));
@@ -809,13 +800,13 @@ namespace quda
       } else {
         ptr = quda::device_malloc_(func, file, line, nbytes);
       }
-      ompwip("pool::device_malloc_: %p",ptr);
+      // ompwip("pool::device_malloc_: %p",ptr);
       return ptr;
     }
 
     void device_free_(const char *func, const char *file, int line, void *ptr)
     {
-      ompwip("pool::device_free_: %p",ptr);
+      // ompwip("pool::device_free_: %p",ptr);
       if (device_memory_pool) {
         if (!deviceSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
         deviceCache.insert(std::make_pair(deviceSize[ptr], ptr));
@@ -827,7 +818,7 @@ namespace quda
 
     void flush_pinned()
     {
-      ompwip("pool::flush_pinned");
+      ompwip("ERROR: untested pool::flush_pinned");
       if (pinned_memory_pool) {
         for (auto it : pinnedCache) { host_free(it.second); }
         pinnedCache.clear();
@@ -836,7 +827,7 @@ namespace quda
 
     void flush_device()
     {
-      ompwip("pool:flush_device");
+      ompwip("ERROR: untested pool:flush_device");
       if (device_memory_pool) {
         for (auto it : deviceCache) { device_free(it.second); }
         deviceCache.clear();
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 2446e16145..e64eec8aa4 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -15,6 +15,8 @@ static char FIXME[]="OMP FIXME";
 #define cudaGetLastError()
 #define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
 
+#if 0
+
 #define cudaMemcpy(a,b,c,d) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld",a,b,c)
 #define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld",a,b,c)
 #define cudaMemcpy2D(d,dp,s,sp,w,h,k) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld",d,dp,s,sp,w,h)
@@ -24,15 +26,29 @@ static char FIXME[]="OMP FIXME";
 #define cudaMemset2D(a,b,c,d,e) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld",a,b,c,d,e)
 #define cudaMemset2DAsync(a,b,c,d,e,f) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld",a,b,c,d,e)
 
-static inline void
+#else
+
+#define cudaMemcpy(a,b,c,d) ompwipMemcpy(a,(void*)b,c,d)
+#define cudaMemcpyAsync(a,b,c,d,e) ompwipMemcpy(a,(void*)b,c,d)
+#define cudaMemcpy2D(d,dp,s,sp,w,h,k) ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k)
+#define cudaMemcpy2DAsync(d,dp,s,sp,w,h,k,st) ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k)
+#define cudaMemset(a,b,c) ompwipMemset(a,b,c)
+#define cudaMemsetAsync(a,b,c,d) ompwipMemset(a,b,c)
+#define cudaMemset2D(a,b,c,d,e) ompwipMemset2D(a,b,c,d,e)
+#define cudaMemset2DAsync(a,b,c,d,e,f) ompwipMemset2D(a,b,c,d,e)
+
+#endif
+
+static inline int
 ompwipMemset(void *p, unsigned char b, std::size_t s)
 {
   unsigned char *c = reinterpret_cast<unsigned char *>(p);
 #pragma omp target teams distribute parallel for simd is_device_ptr(c)
   for(std::size_t i=0;i<s;++i) c[i] = b;
+  return 0;
 }
 
-static inline void
+static inline int
 ompwipMemset2D(void *p, size_t pitch, unsigned char b, size_t w, size_t h)
 {
   unsigned char *c = reinterpret_cast<unsigned char *>(p);
@@ -40,6 +56,7 @@ ompwipMemset2D(void *p, size_t pitch, unsigned char b, size_t w, size_t h)
   for(std::size_t i=0;i<h;++i)
     for(std::size_t j=0;j<w;++j)
       c[j+i*pitch] = b;
+  return 0;
 }
 
 #if 1
@@ -81,9 +98,10 @@ printmem(void *d, std::size_t m, int host)
 }
 #endif
 
-static inline void
+static inline int
 ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
 {
+  int r = 0;  // return value from omp_target_memcpy, note that no return value is reserved for memcpy.
   switch(k){
   case cudaMemcpyHostToHost:
     printmem(s,c,1);
@@ -93,7 +111,7 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
   case cudaMemcpyHostToDevice:
     printmem(s,c,1);
     if(0<omp_get_num_devices()){
-      omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+      r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
       printmem(d,c,0);
     }else{
       warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy");
@@ -104,7 +122,7 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
   case cudaMemcpyDeviceToHost:
     if(0<omp_get_num_devices()){
       printmem(s,c,0);
-      omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
+      r = omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
     }else{
       warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy");
       printmem(s,c,1);
@@ -114,31 +132,31 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
     break;
   case cudaMemcpyDeviceToDevice:
     printmem(s,c,0);
-    omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
+    r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
     printmem(d,c,0);
     break;
   case cudaMemcpyDefault:
     if(0<omp_get_num_devices()){
       if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(d)){
         if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
-          ompwip("cudaMemcpyDefault calling device to device");
+          // ompwip("cudaMemcpyDefault calling device to device");
           printmem(s,c,0);
-          omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
+          r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
           printmem(d,c,0);
         }else{
-          ompwip("cudaMemcpyDefault calling host to device");
+          // ompwip("cudaMemcpyDefault calling host to device");
           printmem(s,c,1);
-          omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+          r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
           printmem(d,c,0);
         }
       }else{
         if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
-          ompwip("cudaMemcpyDefault calling device to host");
+          // ompwip("cudaMemcpyDefault calling device to host");
           printmem(s,c,0);
-          omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
+          r = omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
           printmem(d,c,1);
         }else{
-          ompwip("cudaMemcpyDefault calling host to host");
+          // ompwip("cudaMemcpyDefault calling host to host");
           printmem(s,c,1);
           memcpy(d,s,c);
           printmem(d,c,1);
@@ -153,11 +171,13 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
     break;
   default: errorQuda("Unsupported cudaMemcpyType %d", k);
   }
+  return r;
 }
 
-static inline void
+static inline int
 ompwipMemcpy2D(void *d, std::size_t dp, void *s, std::size_t sp, std::size_t w, std::size_t h, cudaMemcpyKind k)
 {
+  int r = 0;  // return value from omp_target_memcpy, note that no return value is reserved for memcpy.
   std::size_t v[2] = {w,h}, z[2] = {0,0}, dd[2] = {dp,h}, sd[2] = {sp,h};
   unsigned char *cd = reinterpret_cast<unsigned char *>(d), *cs = reinterpret_cast<unsigned char *>(s);
   switch(k){
@@ -170,7 +190,7 @@ ompwipMemcpy2D(void *d, std::size_t dp, void *s, std::size_t sp, std::size_t w,
   case cudaMemcpyHostToDevice:
     printmem(s,c,1);
     if(0<omp_get_num_devices()){
-      omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_initial_device());
+      r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_initial_device());
       printmem(d,c,0);
     }else{
       warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy");
@@ -182,7 +202,7 @@ ompwipMemcpy2D(void *d, std::size_t dp, void *s, std::size_t sp, std::size_t w,
   case cudaMemcpyDeviceToHost:
     if(0<omp_get_num_devices()){
       printmem(s,c,0);
-      omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_initial_device(),omp_get_default_device());
+      r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_initial_device(),omp_get_default_device());
     }else{
       warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy");
       printmem(s,c,1);
@@ -193,31 +213,31 @@ ompwipMemcpy2D(void *d, std::size_t dp, void *s, std::size_t sp, std::size_t w,
     break;
   case cudaMemcpyDeviceToDevice:
     printmem(s,c,0);
-    omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_default_device());
+    r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_default_device());
     printmem(d,c,0);
     break;
   case cudaMemcpyDefault:
     if(0<omp_get_num_devices()){
       if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(d)){
         if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
-          ompwip("cudaMemcpyDefault calling device to device");
+          // ompwip("cudaMemcpyDefault calling device to device");
           printmem(s,c,0);
-          omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_default_device());
+          r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_default_device());
           printmem(d,c,0);
         }else{
-          ompwip("cudaMemcpyDefault calling host to device");
+          // ompwip("cudaMemcpyDefault calling host to device");
           printmem(s,c,1);
-          omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_initial_device());
+          r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_initial_device());
           printmem(d,c,0);
         }
       }else{
         if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
-          ompwip("cudaMemcpyDefault calling device to host");
+          // ompwip("cudaMemcpyDefault calling device to host");
           printmem(s,c,0);
-          omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_initial_device(),omp_get_default_device());
+          r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_initial_device(),omp_get_default_device());
           printmem(d,c,1);
         }else{
-          ompwip("cudaMemcpyDefault calling host to host");
+          // ompwip("cudaMemcpyDefault calling host to host");
           printmem(s,c,1);
           #pragma omp parallel for
           for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
@@ -234,9 +254,10 @@ ompwipMemcpy2D(void *d, std::size_t dp, void *s, std::size_t sp, std::size_t w,
     break;
   default: errorQuda("Unsupported cudaMemcpyType %d", k);
   }
+  return r;
 }
 
-#define cudaGetSymbolAddress(a,b) ompwip("cudaGetSymbolAddress")
+#define cudaGetSymbolAddress(a,b) ompwip("WARNING: unimplemented cudaGetSymbolAddress")
 
 // if this macro is defined then we profile the CUDA API calls
 //#define API_PROFILE

From c6d4f17c322bd4fdc7a6b208cdc9da25937054c6 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 25 May 2021 18:54:30 +0000
Subject: [PATCH 070/198] omp: reduce constexpr max_block_size to 512

---
 include/targets/omptarget/target_device.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 42e4f70aea..248133a10e 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -102,7 +102,7 @@ namespace quda {
     template <int block_size_y = 1, int block_size_z = 1>
       constexpr unsigned int max_block_size()
       {
-        return std::max(warp_size(), 1024 / (block_size_y * block_size_z));
+        return std::max(warp_size(), 512 / (block_size_y * block_size_z));
       }
 
     /**

From 45c070389501b5b87118e24b7e6a45c0052b280c Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 11 Jun 2021 22:05:53 +0000
Subject: [PATCH 071/198] omp: incorporate kernel launch changes

---
 .../omptarget/block_reduction_kernel.h        | 37 ++++++---
 include/targets/omptarget/kernel.h            |  1 +
 include/targets/omptarget/reduction_kernel.h  | 82 +++++++++++--------
 include/targets/omptarget/target_device.h     |  5 +-
 include/targets/omptarget/tunable_kernel.h    | 73 +++++++++++++++++
 lib/targets/omptarget/device.cpp              |  2 +-
 lib/tune.cpp                                  |  4 +-
 7 files changed, 154 insertions(+), 50 deletions(-)
 create mode 100644 include/targets/omptarget/tunable_kernel.h

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index a25dee6757..9674a48e83 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <target_device.h>
 #include <reduce_helper.h>
 
 namespace quda {
@@ -34,6 +35,18 @@ namespace quda {
     return block_idx;
   }
 
+  /**
+     @brief This class is derived from the arg class that the functor
+     creates and curries in the block size.  This allows the block
+     size to be set statically at launch time in the actual argument
+     class that is passed to the kernel.
+   */
+  template <unsigned int block_size_, typename Arg_> struct BlockKernelArg : Arg_ {
+    using Arg = Arg_;
+    static constexpr unsigned int block_size = block_size_;
+    BlockKernelArg(const Arg &arg) : Arg(arg) { }
+  };
+
   /**
      @brief Generic block kernel.  Here, we split the block and thread
      indices in the x and y dimension and pass these indices
@@ -41,7 +54,7 @@ namespace quda {
      templated, e.g., for efficient reductions, and typically the y
      thread dimension is a trivial vectorizable dimension.
   */
-  template <unsigned int block_size, template <int, typename> class Transformer, typename Arg>
+  template <template <typename> class Transformer, typename Arg>
   __forceinline__ __device__ void BlockKernel2D_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
@@ -50,19 +63,20 @@ namespace quda {
     auto j = blockDim.y*blockIdx.y + threadIdx.y;
     if (j >= arg.threads.y) return;
 
-    Transformer<block_size, Arg> t(arg);
+    Transformer<Arg> t(arg);
     t(block_idx, thread_idx);
   }
 
-  template <unsigned int block_size, template <int, typename> class Transformer, typename Arg>
-    __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0)
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = false>
+    __launch_bounds__(Arg::launch_bounds || Arg::block_size > 512 ? Arg::block_size : 0)
     __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
+    static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    const int tx = arg.threads.x;
-    const int ty = arg.threads.y;
-    const int tz = arg.threads.z;
+    // const int tx = arg.threads.x;
+    // const int ty = arg.threads.y;
+    // const int tz = arg.threads.z;
     // printf("BlockKernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
@@ -75,15 +89,16 @@ namespace quda {
       //          "omp reports: teams %d threads %d\n",
       //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
       //          omp_get_num_teams(), omp_get_num_threads());
-      BlockKernel2D_impl<block_size, Transformer, Arg>(*dparg);
+      BlockKernel2D_impl<Transformer, Arg>(arg);
     }
     omp_target_free(dparg, omp_get_default_device());
   }
 
-  template <unsigned int block_size, template <int, typename> class Transformer, typename Arg>
-    __launch_bounds__(Arg::launch_bounds || block_size > 512 ? block_size : 0)
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = false>
+    __launch_bounds__(Arg::launch_bounds || Arg::block_size > 512 ? Arg::block_size : 0)
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D()
   {
+    static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
     // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
@@ -95,7 +110,7 @@ namespace quda {
       //          "omp reports: teams %d threads %d\n",
       //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
       //          omp_get_num_teams(), omp_get_num_threads());
-      BlockKernel2D_impl<block_size, Transformer, Arg>(device::get_arg<Arg>());
+      BlockKernel2D_impl<Transformer, Arg>(device::get_arg<Arg>());
     }
   }
 
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 3df3d14570..f1c66d7463 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <target_device.h>
 #include <kernel_helper.h>
 
 namespace quda {
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 7d6b24a3de..71783a5ec6 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -1,10 +1,24 @@
 #pragma once
 
+#include <target_device.h>
 #include <reduce_helper.h>
 
 namespace quda {
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  /**
+     @brief This class is derived from the arg class that the functor
+     creates and curries in the block size.  This allows the block
+     size to be set statically at launch time in the actual argument
+     class that is passed to the kernel.
+  */
+  template <int block_size_x_, int block_size_y_, typename Arg_> struct ReduceKernelArg : Arg_ {
+    using Arg = Arg_;
+    static constexpr int block_size_x = block_size_x_;
+    static constexpr int block_size_y = block_size_y_;
+    ReduceKernelArg(const Arg &arg) : Arg(arg) { }
+  };
+
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __forceinline__ __device__ void Reduction2D_impl(const Arg &arg)
   {
     const dim3
@@ -38,24 +52,24 @@ namespace quda {
 
     // perform final inter-block reduction and write out result
     Transformer<Arg> t(arg);
-    reduce<block_size_x, block_size_y>(arg, t, value);
+    reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value);
   }
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    const int tx = arg.threads.x;
-    const int ty = arg.threads.y;
-    const int tz = arg.threads.z;
+    // const int tx = arg.threads.x;
+    // const int ty = arg.threads.y;
+    // const int tz = arg.threads.z;
     // printf("Reduction2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      // Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(*dparg);
+      // Reduction2D_impl<Transformer, Arg, grid_stride>(*dparg);
       const dim3
         blockDim=launch_param.block,
         gridDim=launch_param.grid,
@@ -87,7 +101,7 @@ namespace quda {
       }
       // perform final inter-block reduction and write out result
       Transformer<Arg> t(*dparg);
-      // reduce<block_size_x, block_size_y>(*dparg, t, value);
+      // reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value);
       // ../../reduce_helper.h:/reduce
       {
         const auto idx = 0;
@@ -115,10 +129,10 @@ namespace quda {
           #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
           {
             dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-            auto i = threadIdx.y * block_size_x + threadIdx.x;
+            auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
             while (i < gridDim.x) {
               sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-              i += block_size_x * block_size_y;
+              i += Arg::block_size_x * Arg::block_size_y;
             }
           }
 
@@ -128,7 +142,7 @@ namespace quda {
           // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
           // printf("\n");
           // write out the final reduced value
-          // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+          // if (threadIdx.y * Arg::block_size_x + threadIdx.x == 0)
           { // This is the main thread per team
             dparg->result_d[idx] = sum;
             dparg->count[idx] = 0; // set to zero for next time
@@ -139,7 +153,7 @@ namespace quda {
     omp_target_free(dparg, omp_get_default_device());
   }
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D()
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
@@ -147,7 +161,7 @@ namespace quda {
     // printf("Reduction2D: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      // Reduction2D_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+      // Reduction2D_impl<Transformer, Arg, grid_stride>(device::get_arg<Arg>());
       Arg *dparg = &device::get_arg<Arg>();
       const dim3
         blockDim=launch_param.block,
@@ -178,7 +192,7 @@ namespace quda {
       }
       // perform final inter-block reduction and write out result
       Transformer<Arg> t(*dparg);
-      // reduce<block_size_x, block_size_y>(*dparg, t, value);
+      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value);
       // ../../reduce_helper.h:/reduce
       {
         const auto idx = 0;
@@ -205,10 +219,10 @@ namespace quda {
           #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
           {
             dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-            auto i = threadIdx.y * block_size_x + threadIdx.x;
+            auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
             while (i < gridDim.x) {
               sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-              i += block_size_x * block_size_y;
+              i += Arg::block_size_x * Arg::block_size_y;
             }
           }
 
@@ -218,7 +232,7 @@ namespace quda {
           // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
           // printf("\n");
           // write out the final reduced value
-          // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+          // if (threadIdx.y * Arg::block_size_x + threadIdx.x == 0)
           { // This is the main thread per team
             dparg->result_d[idx] = sum;
             dparg->count[idx] = 0; // set to zero for next time
@@ -229,7 +243,7 @@ namespace quda {
   }
 
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __forceinline__ __device__ void MultiReduction_impl(const Arg &arg)
   {
     const dim3
@@ -266,26 +280,26 @@ namespace quda {
     // perform final inter-block reduction and write out result
     Transformer<Arg> t(arg);
     auto j = blockIdx.y * blockDim.y;
-    reduce<block_size_x, block_size_y>(arg, t, value, j);
+    reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value, j);
   }
 
   /** Workaround compiler limitations.  We have to inline parallel regions with target teams. **/
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    const int tx = arg.threads.x;
-    const int ty = arg.threads.y;
-    const int tz = arg.threads.z;
+    // const int tx = arg.threads.x;
+    // const int ty = arg.threads.y;
+    // const int tz = arg.threads.z;
     // printf("MultiReduction: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      // MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(*dparg);
+      // MultiReduction_impl<Transformer, Arg, grid_stride>(*dparg);
       const dim3
         blockDim=launch_param.block,
         gridDim=launch_param.grid,
@@ -319,7 +333,7 @@ namespace quda {
       // perform final inter-block reduction and write out result
       Transformer<Arg> t(*dparg);
       auto j = blockIdx.y * blockDim.y;
-      // reduce<block_size_x, block_size_y>(*dparg, t, value, j);
+      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value, j);
       // ../../reduce_helper.h:/reduce
       {
         const auto idx = j;
@@ -347,10 +361,10 @@ namespace quda {
           #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
           {
             dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-            auto i = threadIdx.y * block_size_x + threadIdx.x;
+            auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
             while (i < gridDim.x) {
               sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-              i += block_size_x * block_size_y;
+              i += Arg::block_size_x * Arg::block_size_y;
             }
           }
 
@@ -360,7 +374,7 @@ namespace quda {
           // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
           // printf("\n");
           // write out the final reduced value
-          // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+          // if (threadIdx.y * Arg::block_size_x + threadIdx.x == 0)
           { // This is the main thread per team
             dparg->result_d[idx] = sum;
             dparg->count[idx] = 0; // set to zero for next time
@@ -371,7 +385,7 @@ namespace quda {
     omp_target_free(dparg, omp_get_default_device());
   }
 
-  template <int block_size_x, int block_size_y, template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction()
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
@@ -379,7 +393,7 @@ namespace quda {
     // printf("MultiReduction: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      // MultiReduction_impl<block_size_x, block_size_y, Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+      // MultiReduction_impl<Transformer, Arg, grid_stride>(device::get_arg<Arg>());
       Arg *dparg = &device::get_arg<Arg>();
       const dim3
         blockDim=launch_param.block,
@@ -414,7 +428,7 @@ namespace quda {
       // perform final inter-block reduction and write out result
       Transformer<Arg> t(*dparg);
       auto j = blockIdx.y * blockDim.y;
-      // reduce<block_size_x, block_size_y>(*dparg, t, value, j);
+      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value, j);
       // ../../reduce_helper.h:/reduce
       {
         const auto idx = j;
@@ -441,10 +455,10 @@ namespace quda {
           #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
           {
             dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-            auto i = threadIdx.y * block_size_x + threadIdx.x;
+            auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
             while (i < gridDim.x) {
               sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-              i += block_size_x * block_size_y;
+              i += Arg::block_size_x * Arg::block_size_y;
             }
           }
 
@@ -454,7 +468,7 @@ namespace quda {
           // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
           // printf("\n");
           // write out the final reduced value
-          // if (threadIdx.y * block_size_x + threadIdx.x == 0)
+          // if (threadIdx.y * Arg::block_size_x + threadIdx.x == 0)
           { // This is the main thread per team
             dparg->result_d[idx] = sum;
             dparg->count[idx] = 0; // set to zero for next time
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 248133a10e..bfedaab91b 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -110,7 +110,7 @@ namespace quda {
        in a block in the x dimension for reduction kernels.
     */
     template <int block_size_y = 1, int block_size_z = 1>
-      constexpr unsigned int max_reduce_block_size()
+    constexpr unsigned int max_reduce_block_size()
       {
 #ifdef QUDA_FAST_COMPILE_REDUCE
         // This is the specialized variant used when we have fast-compilation mode enabled
@@ -124,13 +124,14 @@ namespace quda {
        @brief Helper function that returns the maximum number of threads
        in a block in the x dimension for reduction kernels.
     */
+    template <int block_size_y = 1, int block_size_z = 1>
     constexpr unsigned int max_multi_reduce_block_size()
     {
 #ifdef QUDA_FAST_COMPILE_REDUCE
       // This is the specialized variant used when we have fast-compilation mode enabled
       return warp_size();
 #else
-      return 128;
+      return max_block_size<block_size_y, block_size_z>();
 #endif
     }
 
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
new file mode 100644
index 0000000000..b0d0f55016
--- /dev/null
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -0,0 +1,73 @@
+#pragma once
+
+#include <tune_quda.h>
+#include <target_device.h>
+#include <kernel_helper.h>
+#include <kernel.h>
+
+#ifdef JITIFY
+#include <jitify_helper2.cuh>
+#endif
+
+namespace quda {
+
+  class TunableKernel : public Tunable
+  {
+
+  protected:
+    QudaFieldLocation location;
+
+    virtual unsigned int sharedBytesPerThread() const { return 0; }
+    virtual unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
+
+    template <template <typename> class Functor, bool grid_stride, typename Arg>
+    std::enable_if_t<device::use_kernel_arg<Arg>(), qudaError_t>
+      launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
+    {
+#ifdef JITIFY
+      launch_error = launch_jitify<Functor, grid_stride, Arg>(kernel.name, tp, stream, arg);
+#else
+      launch_error = qudaLaunchKernel(kernel.func, tp, stream, arg);
+#endif
+      return launch_error;
+    }
+
+    template <template <typename> class Functor, bool grid_stride, typename Arg>
+    std::enable_if_t<!device::use_kernel_arg<Arg>(), qudaError_t>
+      launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
+    {
+#ifdef JITIFY
+      launch_error = launch_jitify<Functor, grid_stride, Arg>(kernel.name, tp, stream, arg);
+#else
+      static_assert(sizeof(Arg) <= device::max_constant_size(), "Parameter struct is greater than max constant size");
+      qudaMemcpyAsync(device::get_constant_buffer<Arg>(), &arg, sizeof(Arg), qudaMemcpyHostToDevice, stream);
+      launch_error = qudaLaunchKernel(kernel.func, tp, stream, arg);
+#endif
+      return launch_error;
+    }
+
+  public:
+    /**
+       @brief Special kernel launcher used for raw CUDA kernels with no
+       assumption made about shape of parallelism.  Kernels launched
+       using this must take responsibility of bounds checking and
+       assignment of threads.
+     */
+    template <template <typename> class Functor, typename Arg>
+    void launch_cuda(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg) const
+    {
+      constexpr bool grid_stride = false;
+      const_cast<TunableKernel*>(this)->launch_device<Functor, grid_stride>(KERNEL(raw_kernel), tp, stream, arg);
+    }
+
+    TunableKernel(QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) : location(location) { } 
+
+    virtual bool advanceTuneParam(TuneParam &param) const
+    {
+      return location == QUDA_CPU_FIELD_LOCATION ? false : Tunable::advanceTuneParam(param);
+    }
+
+    TuneKey tuneKey() const { return TuneKey(vol, typeid(*this).name(), aux); }
+  };
+
+}
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index b3be12573f..d18266c23e 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -358,7 +358,7 @@ namespace quda
 
     unsigned int processor_count() { return deviceProp.multiProcessorCount; }
 
-    unsigned int max_blocks_per_processor() { return 32; }
+    unsigned int max_blocks_per_processor() { return 1; }
 
     namespace profile
     {
diff --git a/lib/tune.cpp b/lib/tune.cpp
index 417b226a46..de1c128544 100644
--- a/lib/tune.cpp
+++ b/lib/tune.cpp
@@ -803,8 +803,8 @@ namespace quda
           }
           if ((verbosity >= QUDA_DEBUG_VERBOSE)) {
             if (error == QUDA_SUCCESS && tunable.launchError() == QUDA_SUCCESS) {
-              printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(),
-                         tunable.perfString(elapsed_time).c_str());
+              printfQuda("    %s gives %s (time %.3f ms)\n", tunable.paramString(param).c_str(),
+                         tunable.perfString(elapsed_time).c_str(), elapsed_time);
             } else {
               printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), qudaGetLastErrorString().c_str());
             }

From 40fc8a60336a4bbdb2a1ab2f91d51a105c76dc42 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 16 Jun 2021 22:51:05 +0000
Subject: [PATCH 072/198] omp: temporary workaround for compiler bugs

---
 lib/copy_gauge_extended.cu          | 12 ++++++++++++
 lib/copy_gauge_offset.cu            |  6 ++++++
 lib/extract_gauge_ghost_extended.cu |  6 ++++++
 3 files changed, 24 insertions(+)

diff --git a/lib/copy_gauge_extended.cu b/lib/copy_gauge_extended.cu
index f3f9025201..842c1d14c5 100644
--- a/lib/copy_gauge_extended.cu
+++ b/lib/copy_gauge_extended.cu
@@ -86,8 +86,11 @@ namespace quda {
     } else if (out.Order() == QUDA_QDP_GAUGE_ORDER) {
 
 #ifdef BUILD_QDP_INTERFACE
+      ompwip("unsupported for now");
+/*
       using G = QDPOrder<FloatOut,length>;
       CopyGaugeEx<FloatOut,FloatIn,length, G, InOrder>(out, in, location, Out, In);
+*/
 #else
       errorQuda("QDP interface has not been built\n");
 #endif
@@ -95,8 +98,11 @@ namespace quda {
     } else if (out.Order() == QUDA_MILC_GAUGE_ORDER) {
 
 #ifdef BUILD_MILC_INTERFACE
+      ompwip("unsupported for now");
+/*
       using G = MILCOrder<FloatOut, length>;
       CopyGaugeEx<FloatOut,FloatIn,length, G, InOrder>(out, in, location, Out, In);
+*/
 #else
       errorQuda("MILC interface has not been built\n");
 #endif
@@ -160,8 +166,11 @@ namespace quda {
     } else if (in.Order() == QUDA_QDP_GAUGE_ORDER) {
 
 #ifdef BUILD_QDP_INTERFACE
+      ompwip("unsupported for now");
+/*
       using G = QDPOrder<FloatIn, length>;
       copyGaugeEx<FloatOut,FloatIn,length, G>(out, in, location, Out, In);
+*/
 #else
       errorQuda("QDP interface has not been built\n");
 #endif
@@ -169,8 +178,11 @@ namespace quda {
     } else if (in.Order() == QUDA_MILC_GAUGE_ORDER) {
 
 #ifdef BUILD_MILC_INTERFACE
+      ompwip("unsupported for now");
+/*
       using G = MILCOrder<FloatIn, length>;
       copyGaugeEx<FloatOut,FloatIn,length, G>(out, in, location, Out, In);
+*/
 #else
       errorQuda("MILC interface has not been built\n");
 #endif
diff --git a/lib/copy_gauge_offset.cu b/lib/copy_gauge_offset.cu
index 9a8494c3a2..6956285232 100644
--- a/lib/copy_gauge_offset.cu
+++ b/lib/copy_gauge_offset.cu
@@ -60,8 +60,11 @@ namespace quda
         }
       } else if (in.Order() == QUDA_QDP_GAUGE_ORDER) {
 #ifdef BUILD_QDP_INTERFACE
+        ompwip("unsupported for now");
+/*
         using G = typename gauge_order_mapper<Float, QUDA_QDP_GAUGE_ORDER, nColor>::type;
         copy_gauge_offset<Field, Element, G>(out, in, offset);
+*/
 #else
         errorQuda("QDP interface has not been built\n");
 #endif
@@ -75,8 +78,11 @@ namespace quda
 
       } else if (in.Order() == QUDA_MILC_GAUGE_ORDER) {
 #ifdef BUILD_MILC_INTERFACE
+        ompwip("unsupported for now");
+/*
         using G = typename gauge_order_mapper<Float, QUDA_MILC_GAUGE_ORDER, nColor>::type;
         copy_gauge_offset<Field, Element, G>(out, in, offset);
+*/
 #else
         errorQuda("MILC interface has not been built\n");
 #endif
diff --git a/lib/extract_gauge_ghost_extended.cu b/lib/extract_gauge_ghost_extended.cu
index 55d9e59d1d..0bfe504d62 100644
--- a/lib/extract_gauge_ghost_extended.cu
+++ b/lib/extract_gauge_ghost_extended.cu
@@ -97,7 +97,10 @@ namespace quda {
       } else if (u.Order() == QUDA_QDP_GAUGE_ORDER) {
 
 #ifdef BUILD_QDP_INTERFACE
+        ompwip("unsupported for now");
+/*
         ExtractGhostEx<QDPOrder<Float,length>>(u, dim, R, ghost, extract);
+*/
 #else
         errorQuda("QDP interface has not been built\n");
 #endif
@@ -121,7 +124,10 @@ namespace quda {
       } else if (u.Order() == QUDA_MILC_GAUGE_ORDER) {
 
 #ifdef BUILD_MILC_INTERFACE
+        ompwip("unsupported for now");
+/*
         ExtractGhostEx<MILCOrder<Float,length>>(u, dim, R, ghost, extract);
+*/
 #else
         errorQuda("MILC interface has not been built\n");
 #endif

From 5cce322594e6a351eb5ec95ed399f04ea82cc603 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 16 Jun 2021 22:51:59 +0000
Subject: [PATCH 073/198] omp: extend omptarget tuning parameters

---
 include/targets/omptarget/target_device.h | 2 +-
 lib/targets/omptarget/device.cpp          | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index bfedaab91b..b36e0e8a18 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -88,7 +88,7 @@ namespace quda {
        @brief Helper function that returns the warp-size of the
        architecture we are running on.
     */
-    constexpr int warp_size() { return 32; }
+    constexpr int warp_size() { return 8; }
 
     /**
        @brief Return the thread mask for a converged warp.
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index d18266c23e..6d48272441 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -66,7 +66,7 @@ static int cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
     p->maxGridSize[2] = 65536;
     p->multiProcessorCount = 16;
   }
-  p->warpSize = 32;
+  p->warpSize = 8;
   p->totalGlobalMem = 1u<34;
   p->totalConstMem = 65536;
   p->memPitch = 1u<31;
@@ -86,7 +86,7 @@ static int cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
   p->maxThreadsDim[2] = m;
   #pragma omp target map(tofrom:m)
   m = omp_get_num_procs();
-  p->multiProcessorCount = m/32;
+  p->multiProcessorCount = m/8;
   return 0;
 }
 
@@ -346,7 +346,7 @@ namespace quda
 
     size_t max_default_shared_memory() { return deviceProp.sharedMemPerBlock; }
 
-    size_t max_dynamic_shared_memory() { return 98304; }
+    size_t max_dynamic_shared_memory() { return 65536; }
 
     unsigned int max_threads_per_block() { return deviceProp.maxThreadsPerBlock; }
 

From 79689cd0dab13132437e6abaf7bff579a3bff8be Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 17 Jun 2021 05:06:07 +0000
Subject: [PATCH 074/198] omp: remove unused qudaLaunchKernel

---
 lib/targets/omptarget/quda_api.cpp | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index e64eec8aa4..8b60323863 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -353,16 +353,6 @@ namespace quda
   static TimeProfile apiTimer("CUDA API calls (runtime)");
 #endif
 
-  qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, void **args, qudaStream_t stream)
-  {
-    ompwip([&](){std::cerr<<"ERROR: unimplemented launch "<<func<<' '<<tp<<' '<<*args<<' '<<device::get_cuda_stream(stream)<<std::endl;});
-    // if launch requests the maximum shared memory and the device supports it then opt in
-    if (tp.set_max_shared_bytes && device::max_dynamic_shared_memory() > device::max_default_shared_memory()) {
-      ompwip("Unimplemented for maximum shared memory");
-    }
-    return QUDA_SUCCESS;
-  }
-
   class QudaMem : public Tunable
   {
     void *dst;

From e4919e0bc24e37bb1b60865c2d78082b0a580dbd Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 17 Jun 2021 05:28:26 +0000
Subject: [PATCH 075/198] omp: add declaration

---
 include/targets/omptarget/tunable_kernel.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index 1716a17bc4..fe2464f750 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -9,6 +9,9 @@
 #include <jitify_helper2.cuh>
 #endif
 
+// defined in ../../../lib/targets/omptarget/quda_api.cpp:/qudaSetupLaunchParameter
+void qudaSetupLaunchParameter(const TuneParam &);
+
 namespace quda {
 
   template <typename... Arg>

From 2286ff0bded68535f5e2152b89df28dbd6db8076 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 17 Jun 2021 05:32:30 +0000
Subject: [PATCH 076/198] omp: fix verbose timing

---
 lib/tune.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tune.cpp b/lib/tune.cpp
index de1c128544..9e9cc61d19 100644
--- a/lib/tune.cpp
+++ b/lib/tune.cpp
@@ -804,7 +804,7 @@ namespace quda
           if ((verbosity >= QUDA_DEBUG_VERBOSE)) {
             if (error == QUDA_SUCCESS && tunable.launchError() == QUDA_SUCCESS) {
               printfQuda("    %s gives %s (time %.3f ms)\n", tunable.paramString(param).c_str(),
-                         tunable.perfString(elapsed_time).c_str(), elapsed_time);
+                         tunable.perfString(elapsed_time).c_str(), elapsed_time*1e3);
             } else {
               printfQuda("    %s gives %s\n", tunable.paramString(param).c_str(), qudaGetLastErrorString().c_str());
             }

From 1f1f67fad5d2b1c05dd6c27ded727a1c76fc12e8 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 17 Jun 2021 06:09:58 +0000
Subject: [PATCH 077/198] omp: fix qudaSetupLauchParam's namespace

---
 include/targets/omptarget/tunable_kernel.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index fe2464f750..ef4183cf60 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -9,11 +9,11 @@
 #include <jitify_helper2.cuh>
 #endif
 
-// defined in ../../../lib/targets/omptarget/quda_api.cpp:/qudaSetupLaunchParameter
-void qudaSetupLaunchParameter(const TuneParam &);
-
 namespace quda {
 
+  // defined in ../../../lib/targets/omptarget/quda_api.cpp:/qudaSetupLaunchParameter
+  void qudaSetupLaunchParameter(const TuneParam &);
+
   template <typename... Arg>
   qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, qudaStream_t stream, const Arg &... arg)
   {

From c024c2a682d3eeb3f796dc61aaa0830b2e2416ca Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 18 Aug 2021 17:00:40 +0000
Subject: [PATCH 078/198] omp: omptarget conforms to cuda from last merge

---
 lib/targets/omptarget/comm_target.cpp |   2 +-
 lib/targets/omptarget/malloc.cpp      |  40 ++++++----
 lib/targets/omptarget/quda_api.cpp    | 110 --------------------------
 3 files changed, 26 insertions(+), 126 deletions(-)

diff --git a/lib/targets/omptarget/comm_target.cpp b/lib/targets/omptarget/comm_target.cpp
index 3556ba5cc3..ecb6837d38 100644
--- a/lib/targets/omptarget/comm_target.cpp
+++ b/lib/targets/omptarget/comm_target.cpp
@@ -58,8 +58,8 @@ void comm_create_neighbor_memory(void *remote[QUDA_MAX_DIM][2], void *local)
       }
       // now send
       cudaIpcMemHandle_t local_handle;
-      CHECK_CUDA_ERROR(cudaIpcGetMemHandle(&local_handle, local));
       if (comm_peer2peer_enabled(dir,dim)) {
+        CHECK_CUDA_ERROR(cudaIpcGetMemHandle(&local_handle, local));
         sendHandle = comm_declare_send_relative(&local_handle, dim, disp, sizeof(local_handle));
       }
       if (receiveHandle) comm_start(receiveHandle);
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index b592017389..b9958e3930 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -53,20 +53,30 @@ namespace quda
   };
 
   static std::map<void *, MemAlloc> alloc[N_ALLOC_TYPE];
-  static long total_bytes[N_ALLOC_TYPE] = {0};
-  static long max_total_bytes[N_ALLOC_TYPE] = {0};
-  static long total_host_bytes, max_total_host_bytes;
-  static long total_pinned_bytes, max_total_pinned_bytes;
+  static size_t total_bytes[N_ALLOC_TYPE] = {0};
+  static size_t max_total_bytes[N_ALLOC_TYPE] = {0};
+  static size_t total_host_bytes, max_total_host_bytes;
+  static size_t total_pinned_bytes, max_total_pinned_bytes;
 
-  long device_allocated_peak() { return max_total_bytes[DEVICE]; }
+  size_t device_allocated() { return total_bytes[DEVICE]; }
 
-  long pinned_allocated_peak() { return max_total_bytes[PINNED]; }
+  size_t pinned_allocated() { return total_bytes[PINNED]; }
 
-  long mapped_allocated_peak() { return max_total_bytes[MAPPED]; }
+  size_t mapped_allocated() { return total_bytes[MAPPED]; }
 
-  long managed_allocated_peak() { return max_total_bytes[MANAGED]; }
+  size_t managed_allocated() { return total_bytes[MANAGED]; }
 
-  long host_allocated_peak() { return max_total_bytes[HOST]; }
+  size_t host_allocated() { return total_bytes[HOST]; }
+
+  size_t device_allocated_peak() { return max_total_bytes[DEVICE]; }
+
+  size_t pinned_allocated_peak() { return max_total_bytes[PINNED]; }
+
+  size_t mapped_allocated_peak() { return max_total_bytes[MAPPED]; }
+
+  size_t managed_allocated_peak() { return max_total_bytes[MANAGED]; }
+
+  size_t host_allocated_peak() { return max_total_bytes[HOST]; }
 
   void print_trace(void)
   {
@@ -579,12 +589,12 @@ namespace quda
 
   void printPeakMemUsage()
   {
-    printfQuda("Device memory used = %.1f MB\n", max_total_bytes[DEVICE] / (double)(1 << 20));
-    printfQuda("Pinned device memory used = %.1f MB\n", max_total_bytes[DEVICE_PINNED] / (double)(1 << 20));
-    printfQuda("Managed memory used = %.1f MB\n", max_total_bytes[MANAGED] / (double)(1 << 20));
-    printfQuda("Shmem memory used = %.1f MB\n", max_total_bytes[SHMEM] / (double)(1 << 20));
-    printfQuda("Page-locked host memory used = %.1f MB\n", max_total_pinned_bytes / (double)(1 << 20));
-    printfQuda("Total host memory used >= %.1f MB\n", max_total_host_bytes / (double)(1 << 20));
+    printfQuda("Device memory used = %.1f MiB\n", max_total_bytes[DEVICE] / (double)(1 << 20));
+    printfQuda("Pinned device memory used = %.1f MiB\n", max_total_bytes[DEVICE_PINNED] / (double)(1 << 20));
+    printfQuda("Managed memory used = %.1f MiB\n", max_total_bytes[MANAGED] / (double)(1 << 20));
+    printfQuda("Shmem memory used = %.1f MiB\n", max_total_bytes[SHMEM] / (double)(1 << 20));
+    printfQuda("Page-locked host memory used = %.1f MiB\n", max_total_pinned_bytes / (double)(1 << 20));
+    printfQuda("Total host memory used >= %.1f MiB\n", max_total_host_bytes / (double)(1 << 20));
   }
 
   void assertAllMemFree()
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 8b60323863..ec33a3c27f 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -19,8 +19,6 @@ static char FIXME[]="OMP FIXME";
 
 #define cudaMemcpy(a,b,c,d) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld",a,b,c)
 #define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld",a,b,c)
-#define cudaMemcpy2D(d,dp,s,sp,w,h,k) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld",d,dp,s,sp,w,h)
-#define cudaMemcpy2DAsync(d,dp,s,sp,w,h,k,st) ompwip([&](){ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k);},"memcpy %p(%ld) <- %p(%ld) %ld %ld",d,dp,s,sp,w,h)
 #define cudaMemset(a,b,c) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld",a,b,c)
 #define cudaMemsetAsync(a,b,c,d) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld",a,b,c)
 #define cudaMemset2D(a,b,c,d,e) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld",a,b,c,d,e)
@@ -30,8 +28,6 @@ static char FIXME[]="OMP FIXME";
 
 #define cudaMemcpy(a,b,c,d) ompwipMemcpy(a,(void*)b,c,d)
 #define cudaMemcpyAsync(a,b,c,d,e) ompwipMemcpy(a,(void*)b,c,d)
-#define cudaMemcpy2D(d,dp,s,sp,w,h,k) ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k)
-#define cudaMemcpy2DAsync(d,dp,s,sp,w,h,k,st) ompwipMemcpy2D(d,dp,(void*)s,sp,w,h,k)
 #define cudaMemset(a,b,c) ompwipMemset(a,b,c)
 #define cudaMemsetAsync(a,b,c,d) ompwipMemset(a,b,c)
 #define cudaMemset2D(a,b,c,d,e) ompwipMemset2D(a,b,c,d,e)
@@ -174,89 +170,6 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
   return r;
 }
 
-static inline int
-ompwipMemcpy2D(void *d, std::size_t dp, void *s, std::size_t sp, std::size_t w, std::size_t h, cudaMemcpyKind k)
-{
-  int r = 0;  // return value from omp_target_memcpy, note that no return value is reserved for memcpy.
-  std::size_t v[2] = {w,h}, z[2] = {0,0}, dd[2] = {dp,h}, sd[2] = {sp,h};
-  unsigned char *cd = reinterpret_cast<unsigned char *>(d), *cs = reinterpret_cast<unsigned char *>(s);
-  switch(k){
-  case cudaMemcpyHostToHost:
-    printmem(s,c,1);
-    #pragma omp parallel for
-    for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
-    printmem(d,c,1);
-    break;
-  case cudaMemcpyHostToDevice:
-    printmem(s,c,1);
-    if(0<omp_get_num_devices()){
-      r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_initial_device());
-      printmem(d,c,0);
-    }else{
-      warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy");
-      #pragma omp parallel for
-      for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
-      printmem(d,c,1);
-    }
-    break;
-  case cudaMemcpyDeviceToHost:
-    if(0<omp_get_num_devices()){
-      printmem(s,c,0);
-      r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_initial_device(),omp_get_default_device());
-    }else{
-      warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy");
-      printmem(s,c,1);
-      #pragma omp parallel for
-      for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
-    }
-    printmem(d,c,1);
-    break;
-  case cudaMemcpyDeviceToDevice:
-    printmem(s,c,0);
-    r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_default_device());
-    printmem(d,c,0);
-    break;
-  case cudaMemcpyDefault:
-    if(0<omp_get_num_devices()){
-      if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(d)){
-        if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
-          // ompwip("cudaMemcpyDefault calling device to device");
-          printmem(s,c,0);
-          r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_default_device());
-          printmem(d,c,0);
-        }else{
-          // ompwip("cudaMemcpyDefault calling host to device");
-          printmem(s,c,1);
-          r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_default_device(),omp_get_initial_device());
-          printmem(d,c,0);
-        }
-      }else{
-        if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
-          // ompwip("cudaMemcpyDefault calling device to host");
-          printmem(s,c,0);
-          r = omp_target_memcpy_rect(d,s,1,2,v,z,z,dd,sd,omp_get_initial_device(),omp_get_default_device());
-          printmem(d,c,1);
-        }else{
-          // ompwip("cudaMemcpyDefault calling host to host");
-          printmem(s,c,1);
-          #pragma omp parallel for
-          for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
-          printmem(d,c,1);
-        }
-      }
-    }else{
-      warningQuda("cudaMemcpyDefault without a device, calling memcpy");
-      printmem(s,c,1);
-      #pragma omp parallel for
-      for(std::size_t i=0;i<h;++i) memcpy(cd+i*dp,cs+i*sp,w);
-      printmem(d,c,1);
-    }
-    break;
-  default: errorQuda("Unsupported cudaMemcpyType %d", k);
-  }
-  return r;
-}
-
 #define cudaGetSymbolAddress(a,b) ompwip("WARNING: unimplemented cudaGetSymbolAddress")
 
 // if this macro is defined then we profile the CUDA API calls
@@ -509,29 +422,6 @@ namespace quda
       errorQuda("cudaMemcpyAsync returned %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
   }
 
-  void qudaMemcpy2D_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
-                     qudaMemcpyKind kind, const char *func, const char *file, const char *line)
-  {
-    PROFILE(auto error = cudaMemcpy2D(dst, dpitch, src, spitch, width, height, qudaMemcpyKindToAPI(kind)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
-    if (error != cudaSuccess)
-      errorQuda("cudaMemcpy2D returned error %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
-  }
-
-  void qudaMemcpy2DAsync_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
-                          qudaMemcpyKind kind, const qudaStream_t &stream, const char *func, const char *file,
-                          const char *line)
-  {
-    PROFILE(auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)), QUDA_PROFILE_MEMCPY2D_D2H_ASYNC);
-    set_runtime_error(error, "cudaMemcpy2DAsync", func, file, line);
-  }
-
-  void qudaMemcpy2DP2PAsync_(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height,
-                             const qudaStream_t &stream, const char *func, const char *file, const char *line)
-  {
-    auto error = cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, device::get_cuda_stream(stream));
-    set_runtime_error(error, "cudaMemcpy2DAsync", func, file, line);
-  }
-
   void qudaMemset_(void *ptr, int value, size_t count, const char *func, const char *file, const char *line)
   {
     if (count == 0) return;

From 9d7a0d11282d5ad8956705744ec4f3e7e9f0fa32 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 10 Sep 2021 22:48:36 +0000
Subject: [PATCH 079/198] omp: finish merging target changes

---
 lib/targets/omptarget/malloc.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index b9958e3930..8de065c49b 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -385,7 +385,7 @@ namespace quda
     }
     track_malloc(SHMEM, a, ptr);
 #ifdef HOST_DEBUG
-    qudaMemset(ptr, 0xff, size);
+    cudaMemset(ptr, 0xff, size);
 #endif
     return ptr;
   }

From c6fa60240f184980a365dc3b7dc7e4ae5f91cb4e Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 10 Sep 2021 23:05:06 +0000
Subject: [PATCH 080/198] Revert "omp: temporary workaround for compiler bugs"

This reverts commit 40fc8a60336a4bbdb2a1ab2f91d51a105c76dc42.
---
 lib/copy_gauge_extended.cu          | 12 ------------
 lib/copy_gauge_offset.cu            |  6 ------
 lib/extract_gauge_ghost_extended.cu |  6 ------
 3 files changed, 24 deletions(-)

diff --git a/lib/copy_gauge_extended.cu b/lib/copy_gauge_extended.cu
index 842c1d14c5..f3f9025201 100644
--- a/lib/copy_gauge_extended.cu
+++ b/lib/copy_gauge_extended.cu
@@ -86,11 +86,8 @@ namespace quda {
     } else if (out.Order() == QUDA_QDP_GAUGE_ORDER) {
 
 #ifdef BUILD_QDP_INTERFACE
-      ompwip("unsupported for now");
-/*
       using G = QDPOrder<FloatOut,length>;
       CopyGaugeEx<FloatOut,FloatIn,length, G, InOrder>(out, in, location, Out, In);
-*/
 #else
       errorQuda("QDP interface has not been built\n");
 #endif
@@ -98,11 +95,8 @@ namespace quda {
     } else if (out.Order() == QUDA_MILC_GAUGE_ORDER) {
 
 #ifdef BUILD_MILC_INTERFACE
-      ompwip("unsupported for now");
-/*
       using G = MILCOrder<FloatOut, length>;
       CopyGaugeEx<FloatOut,FloatIn,length, G, InOrder>(out, in, location, Out, In);
-*/
 #else
       errorQuda("MILC interface has not been built\n");
 #endif
@@ -166,11 +160,8 @@ namespace quda {
     } else if (in.Order() == QUDA_QDP_GAUGE_ORDER) {
 
 #ifdef BUILD_QDP_INTERFACE
-      ompwip("unsupported for now");
-/*
       using G = QDPOrder<FloatIn, length>;
       copyGaugeEx<FloatOut,FloatIn,length, G>(out, in, location, Out, In);
-*/
 #else
       errorQuda("QDP interface has not been built\n");
 #endif
@@ -178,11 +169,8 @@ namespace quda {
     } else if (in.Order() == QUDA_MILC_GAUGE_ORDER) {
 
 #ifdef BUILD_MILC_INTERFACE
-      ompwip("unsupported for now");
-/*
       using G = MILCOrder<FloatIn, length>;
       copyGaugeEx<FloatOut,FloatIn,length, G>(out, in, location, Out, In);
-*/
 #else
       errorQuda("MILC interface has not been built\n");
 #endif
diff --git a/lib/copy_gauge_offset.cu b/lib/copy_gauge_offset.cu
index 6956285232..9a8494c3a2 100644
--- a/lib/copy_gauge_offset.cu
+++ b/lib/copy_gauge_offset.cu
@@ -60,11 +60,8 @@ namespace quda
         }
       } else if (in.Order() == QUDA_QDP_GAUGE_ORDER) {
 #ifdef BUILD_QDP_INTERFACE
-        ompwip("unsupported for now");
-/*
         using G = typename gauge_order_mapper<Float, QUDA_QDP_GAUGE_ORDER, nColor>::type;
         copy_gauge_offset<Field, Element, G>(out, in, offset);
-*/
 #else
         errorQuda("QDP interface has not been built\n");
 #endif
@@ -78,11 +75,8 @@ namespace quda
 
       } else if (in.Order() == QUDA_MILC_GAUGE_ORDER) {
 #ifdef BUILD_MILC_INTERFACE
-        ompwip("unsupported for now");
-/*
         using G = typename gauge_order_mapper<Float, QUDA_MILC_GAUGE_ORDER, nColor>::type;
         copy_gauge_offset<Field, Element, G>(out, in, offset);
-*/
 #else
         errorQuda("MILC interface has not been built\n");
 #endif
diff --git a/lib/extract_gauge_ghost_extended.cu b/lib/extract_gauge_ghost_extended.cu
index 0bfe504d62..55d9e59d1d 100644
--- a/lib/extract_gauge_ghost_extended.cu
+++ b/lib/extract_gauge_ghost_extended.cu
@@ -97,10 +97,7 @@ namespace quda {
       } else if (u.Order() == QUDA_QDP_GAUGE_ORDER) {
 
 #ifdef BUILD_QDP_INTERFACE
-        ompwip("unsupported for now");
-/*
         ExtractGhostEx<QDPOrder<Float,length>>(u, dim, R, ghost, extract);
-*/
 #else
         errorQuda("QDP interface has not been built\n");
 #endif
@@ -124,10 +121,7 @@ namespace quda {
       } else if (u.Order() == QUDA_MILC_GAUGE_ORDER) {
 
 #ifdef BUILD_MILC_INTERFACE
-        ompwip("unsupported for now");
-/*
         ExtractGhostEx<MILCOrder<Float,length>>(u, dim, R, ghost, extract);
-*/
 #else
         errorQuda("MILC interface has not been built\n");
 #endif

From 871d034f2f78162570d673b3382fdf2440c6c0dc Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Sat, 11 Sep 2021 00:59:42 +0000
Subject: [PATCH 081/198] omp: fix return type of __uint_as_float

---
 include/atomic.cuh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/atomic.cuh b/include/atomic.cuh
index 71a9662192..2f64548a68 100644
--- a/include/atomic.cuh
+++ b/include/atomic.cuh
@@ -16,7 +16,7 @@ __float_as_uint(float x)
 {
   return reinterpret_cast<unsigned int &>(x);  // FIXME UB?
 }
-static inline unsigned int
+static inline float
 __uint_as_float(unsigned int x)
 {
   return reinterpret_cast<float &>(x);  // FIXME UB?

From e738528360b6af7afea9b059d0f933bb1372ae80 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 15 Sep 2021 19:38:00 +0000
Subject: [PATCH 082/198] Revert "omp: try allocator(omp_pteam_mem_alloc) for
 shared memory [WIP]"

This reverts commit 736600da563cae6c658738853e5e28fdee051cc3.
---
 include/gauge_fix_ovr_hit_devf.cuh | 30 ------------------------------
 1 file changed, 30 deletions(-)

diff --git a/include/gauge_fix_ovr_hit_devf.cuh b/include/gauge_fix_ovr_hit_devf.cuh
index 7bb1e1dba0..209ef41567 100644
--- a/include/gauge_fix_ovr_hit_devf.cuh
+++ b/include/gauge_fix_ovr_hit_devf.cuh
@@ -53,12 +53,7 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
-#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
-#else
-    Float elems[2048];
-    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
-#endif
 
     //initialize shared memory
     if (mu < 4) elems[mu * blockSize + tid] = 0.0;
@@ -153,12 +148,7 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
-#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
-#else
-    Float elems[2048];
-    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
-#endif
 
     //Loop over all SU(2) subroups of SU(N)
     //#pragma unroll
@@ -246,12 +236,7 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
-#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
-#else
-    Float elems[2048];
-    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
-#endif
 
     //Loop over all SU(2) subroups of SU(N)
     //#pragma unroll
@@ -349,12 +334,7 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
-#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
-#else
-    Float elems[2048];
-    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
-#endif
 
     //initialize shared memory
     if (mu < 4) elems[mu * blockSize + tid] = 0.0;
@@ -437,12 +417,7 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
-#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
-#else
-    Float elems[2048];
-    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
-#endif
 
     //Loop over all SU(2) subroups of SU(N)
     //#pragma unroll
@@ -518,12 +493,7 @@ namespace quda {
 
     //Container for the four real parameters of SU(2) subgroup in shared memory
     SharedMemoryCache<Float> cache;
-#ifndef QUDA_BACKEND_OMPTARGET
     auto elems = cache.data();
-#else
-    Float elems[2048];
-    #pragma omp allocate(elems) allocator(omp_pteam_mem_alloc)
-#endif
 
     //Loop over all SU(2) subroups of SU(N)
     //#pragma unroll

From 28633d6f9f669a6d6523fbb6449e39c27f7853c8 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 15 Sep 2021 19:39:07 +0000
Subject: [PATCH 083/198] Revert "omp: try turning off shared in
 dslash_domain_wall_m5"

This reverts commit c08b764160c0877fb7cacc17475dd658e96b46c3.
---
 include/kernels/dslash_domain_wall_m5.cuh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/include/kernels/dslash_domain_wall_m5.cuh b/include/kernels/dslash_domain_wall_m5.cuh
index 8d971bc0c2..e10f1f4f2c 100644
--- a/include/kernels/dslash_domain_wall_m5.cuh
+++ b/include/kernels/dslash_domain_wall_m5.cuh
@@ -13,11 +13,7 @@ namespace quda
   /** Whether to use a shared memory scratch pad to store the input
      field acrosss the Ls dimension to minimize global memory
      reads. */
-#ifdef QUDA_BACKEND_OMPTARGET
-  constexpr bool shared() { return false; }
-#else
   constexpr bool shared() { return true; }
-#endif
 
   /** Whether to use variable or fixed coefficient algorithm.  Must be
       true if using ZMOBIUS */

From 110d26770f4e0a6b0dd5fa4ec6d7d1b1742113f4 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 17 Sep 2021 19:03:49 +0000
Subject: [PATCH 084/198] omp: only define atomics for limited types

---
 include/atomic.cuh | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/include/atomic.cuh b/include/atomic.cuh
index 2f64548a68..b99dcd2f06 100644
--- a/include/atomic.cuh
+++ b/include/atomic.cuh
@@ -22,26 +22,44 @@ __uint_as_float(unsigned int x)
   return reinterpret_cast<float &>(x);  // FIXME UB?
 }
 
-template<typename T>
-static inline T atomicAdd(T*x,T v)
+static inline int atomicAdd(int *x, int v)
 {
-  T z;
+  int z;
 #pragma omp atomic capture
   {z=*x; *x+=v;}
   return z;
 }
-template<typename T>
-static inline T atomicCAS(T*x,T c,T v)
+static inline unsigned int atomicAdd(unsigned int *x, unsigned int v)
 {
-  T z;
+  unsigned int z;
+#pragma omp atomic capture
+  {z=*x; *x+=v;}
+  return z;
+}
+static inline float atomicAdd(float *x, float v)
+{
+  float z;
+#pragma omp atomic capture
+  {z=*x; *x+=v;}
+  return z;
+}
+static inline double atomicAdd(double *x, double v)
+{
+  double z;
+#pragma omp atomic capture
+  {z=*x; *x+=v;}
+  return z;
+}
+static inline unsigned int atomicCAS(unsigned int*x, unsigned int c, unsigned int v)
+{
+  unsigned int z;
 #pragma omp atomic capture
   {z=*x; *x=z==c?v:z;}
   return z;
 }
-template<typename T>
-static inline T atomicMax(T*x,T v)
+static inline uint32_t atomicMax(uint32_t *x, uint32_t v)
 {
-  T z;
+  uint32_t z;
 #pragma omp atomic capture
   {z=*x; *x=z<v?v:z;}
   return z;

From 556ffab8d497602ef276f18edcb42055548b7e14 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 17 Sep 2021 19:08:12 +0000
Subject: [PATCH 085/198] omp: implement shared local memory using stack array
 inside teams

---
 include/kernels/coarse_op_kernel.cuh          |  6 ++++
 include/quda_api.h                            |  6 +++-
 include/shared_memory_cache_helper.cuh        |  8 ++++++
 .../omptarget/block_reduction_kernel.h        |  8 ++++++
 include/targets/omptarget/kernel.h            | 28 +++++++++++++++++++
 include/targets/omptarget/reduction_kernel.h  |  8 ++++++
 include/targets/omptarget/target_device.h     |  6 ++++
 lib/targets/omptarget/device.cpp              |  3 +-
 lib/targets/omptarget/quda_api.cpp            |  4 +++
 9 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index ae5b612f62..60ec927159 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -896,8 +896,14 @@ namespace quda {
       using real = typename Arg::Float;
       using TileType = typename Arg::vuvTileType;
       const int dim_index = arg.dim_index % arg.Y_atomic.geometry;
+#ifndef QUDA_BACKEND_OMPTARGET
       __shared__ complex<storeType> X[Arg::max_color_height_per_block][Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
       __shared__ complex<storeType> Y[Arg::max_color_height_per_block][Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
+#else
+      typedef complex<storeType> TyX0[Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
+      TyX0 *X = reinterpret_cast<TyX0 *>(shared_cache.addr);
+      TyX0 *Y = X+Arg::max_color_height_per_block;
+#endif
 
       int x_ = coarse_x_cb % arg.aggregates_per_block;
       int tx = virtualThreadIdx(arg);
diff --git a/include/quda_api.h b/include/quda_api.h
index 311b613c57..82c90d0093 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -4,7 +4,6 @@
 // OMPTARGET SPECIFIC workarounds
 #define __host__
 #define __device__
-#define __shared__
 #define __global__
 #define __constant__ static
 #define __launch_bounds__(...)
@@ -86,6 +85,11 @@ ompwip_(const char * const file, const size_t line, const char * const func, std
 using cudaStream_t = int;  // device.h:/cudaStream_t
 using CUresult = int;  // ../lib/coarse_op.cuh:/CUresult
 
+struct SharedCache{
+  int *addr;
+};
+extern SharedCache shared_cache;
+
 struct LaunchParam{
   dim3 block;
   dim3 grid;
diff --git a/include/shared_memory_cache_helper.cuh b/include/shared_memory_cache_helper.cuh
index a0eee20c27..59c3a928b0 100644
--- a/include/shared_memory_cache_helper.cuh
+++ b/include/shared_memory_cache_helper.cuh
@@ -72,8 +72,12 @@ namespace quda
     template <typename dummy> struct cache_dynamic<true, dummy> {
       __device__ inline atom_t* operator()()
       {
+#ifndef QUDA_BACKEND_OMPTARGET
         extern __shared__ int cache_[];
         return reinterpret_cast<atom_t*>(cache_);
+#else
+        return reinterpret_cast<atom_t*>(shared_cache.addr);
+#endif
       }
     };
 
@@ -84,8 +88,12 @@ namespace quda
     template <typename dummy> struct cache_static<true, dummy> {
       __device__ inline atom_t* operator()()
       {
+#ifndef QUDA_BACKEND_OMPTARGET
         static __shared__ atom_t cache_[n_element * block_size_x * block_size_y * block_size_z];
         return reinterpret_cast<atom_t*>(cache_);
+#else
+        return reinterpret_cast<atom_t*>(shared_cache.addr);
+#endif
       }
     };
 
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 9674a48e83..0c01ceb219 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -82,6 +82,9 @@ namespace quda {
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    {
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      shared_cache.addr = cache;
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -91,6 +94,7 @@ namespace quda {
       //          omp_get_num_teams(), omp_get_num_threads());
       BlockKernel2D_impl<Transformer, Arg>(arg);
     }
+    }
     omp_target_free(dparg, omp_get_default_device());
   }
 
@@ -103,6 +107,9 @@ namespace quda {
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
     // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
+    {
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      shared_cache.addr = cache;
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -112,6 +119,7 @@ namespace quda {
       //          omp_get_num_teams(), omp_get_num_threads());
       BlockKernel2D_impl<Transformer, Arg>(device::get_arg<Arg>());
     }
+    }
   }
 
 }
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index f1c66d7463..9912eb4f59 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -32,6 +32,9 @@ namespace quda {
       // printf("dparg %p\n", dparg);
       omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+      {
+        int cache[device::max_shared_memory_size()/sizeof(int)];
+        shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -43,6 +46,7 @@ namespace quda {
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel1D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
+      }
       omp_target_free(dparg, omp_get_default_device());
     }
 
@@ -53,6 +57,9 @@ namespace quda {
       const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
       // printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
+      {
+        int cache[device::max_shared_memory_size()/sizeof(int)];
+        shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -62,6 +69,7 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
+      }
     }
 
 
@@ -94,6 +102,9 @@ namespace quda {
       // printf("dparg %p\n", dparg);
       omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+      {
+        int cache[device::max_shared_memory_size()/sizeof(int)];
+        shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -105,6 +116,7 @@ namespace quda {
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel2D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
+      }
       omp_target_free(dparg, omp_get_default_device());
     }
 
@@ -115,6 +127,9 @@ namespace quda {
       const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
       // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
+      {
+        int cache[device::max_shared_memory_size()/sizeof(int)];
+        shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -124,6 +139,7 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
+      }
     }
 
 
@@ -158,6 +174,9 @@ namespace quda {
       // printf("dparg %p\n", dparg);
       omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+      {
+        int cache[device::max_shared_memory_size()/sizeof(int)];
+        shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -169,6 +188,7 @@ namespace quda {
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel3D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
+      }
       omp_target_free(dparg, omp_get_default_device());
     }
 
@@ -179,6 +199,9 @@ namespace quda {
       const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
       // printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
+      {
+        int cache[device::max_shared_memory_size()/sizeof(int)];
+        shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -188,6 +211,7 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
+      }
     }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
@@ -203,6 +227,9 @@ namespace quda {
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+      {
+        int cache[device::max_shared_memory_size()/sizeof(int)];
+        shared_cache.addr = cache;
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -213,6 +240,7 @@ namespace quda {
       Functor<Arg> f(*dparg);
       f();
     }
+    }
     omp_target_free(dparg, omp_get_default_device());
   }
 
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 71783a5ec6..c9d72d0794 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -69,6 +69,8 @@ namespace quda {
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      shared_cache.addr = cache;
       // Reduction2D_impl<Transformer, Arg, grid_stride>(*dparg);
       const dim3
         blockDim=launch_param.block,
@@ -161,6 +163,8 @@ namespace quda {
     // printf("Reduction2D: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      shared_cache.addr = cache;
       // Reduction2D_impl<Transformer, Arg, grid_stride>(device::get_arg<Arg>());
       Arg *dparg = &device::get_arg<Arg>();
       const dim3
@@ -299,6 +303,8 @@ namespace quda {
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      shared_cache.addr = cache;
       // MultiReduction_impl<Transformer, Arg, grid_stride>(*dparg);
       const dim3
         blockDim=launch_param.block,
@@ -393,6 +399,8 @@ namespace quda {
     // printf("MultiReduction: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      shared_cache.addr = cache;
       // MultiReduction_impl<Transformer, Arg, grid_stride>(device::get_arg<Arg>());
       Arg *dparg = &device::get_arg<Arg>();
       const dim3
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index b36e0e8a18..c3253b666e 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -155,6 +155,12 @@ namespace quda {
     */
     constexpr int shared_memory_bank_width() { return 32; }
 
+    /**
+       @brief Use a compile time fixed size for the shared local memory,
+       until we can find a way to set it dynamically.
+    */
+    constexpr unsigned int max_shared_memory_size() { return 48*1024; /* Some amount below maximum */ }
+
     /**
        @brief Helper function that returns true if we are to pass the
        kernel parameter struct to the kernel as an explicit kernel
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 6d48272441..d7c892ed81 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -2,6 +2,7 @@
 #include <util_quda.h>
 #include <quda_internal.h>
 #include <quda_cuda_api.h>
+#include <target_device.h>
 
 #ifdef QUDA_NVML
 #include <nvml.h>
@@ -346,7 +347,7 @@ namespace quda
 
     size_t max_default_shared_memory() { return deviceProp.sharedMemPerBlock; }
 
-    size_t max_dynamic_shared_memory() { return 65536; }
+    size_t max_dynamic_shared_memory() { return device::max_shared_memory_size(); }
 
     unsigned int max_threads_per_block() { return deviceProp.maxThreadsPerBlock; }
 
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index ec33a3c27f..07d85c7355 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -184,6 +184,10 @@ ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
 #define PROFILE(f, idx) f;
 #endif
 
+#pragma omp declare target
+SharedCache shared_cache;
+#pragma omp end declare target
+
 LaunchParam launch_param;
 #pragma omp declare target to(launch_param)
 

From 0fffba458acda4b0a436a4e056218067e4d1adb6 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 17 Sep 2021 19:12:32 +0000
Subject: [PATCH 086/198] omp: implement block/warpreduction pretending to be
 cub

---
 include/cub_helper.cuh                        | 18 +---
 .../omptarget/cub/block/block_reduce.cuh      | 88 +++++++++++++++++++
 2 files changed, 92 insertions(+), 14 deletions(-)
 create mode 100644 include/targets/omptarget/cub/block/block_reduce.cuh

diff --git a/include/cub_helper.cuh b/include/cub_helper.cuh
index bbff623fa2..0fa8883279 100644
--- a/include/cub_helper.cuh
+++ b/include/cub_helper.cuh
@@ -17,9 +17,7 @@
 
 using namespace quda;
 
-#ifndef QUDA_BACKEND_OMPTARGET
 #include <cub/block/block_reduce.cuh>
-#endif
 
 namespace quda {
 
@@ -30,13 +28,7 @@ namespace quda {
   template <typename T, int width> struct WarpReduce
   {
     static_assert(width <= device::warp_size(), "WarpReduce logical width must not be greater than the warp size");
-#ifdef QUDA_BACKEND_OMPTARGET
-    struct warp_reduce_t {  // FIXME THIS IS COMPLETELY WRONG, ONLY TO MAKE IT COMPILE
-      using TempStorage = int;
-    };
-#else
     using warp_reduce_t = cub::WarpReduce<T, width, __COMPUTE_CAPABILITY__>;
-#endif
 
     __device__ __host__ inline WarpReduce() {}
 
@@ -59,13 +51,7 @@ namespace quda {
   */
   template <typename T, int block_size_x, int batch_size = 1> struct BlockReduce
   {
-#ifdef QUDA_BACKEND_OMPTARGET
-    struct block_reduce_t {  // FIXME THIS IS COMPLETELY WRONG, ONLY TO MAKE IT COMPILE
-      using TempStorage = int;
-    };
-#else
     using block_reduce_t = cub::BlockReduce<T, block_size_x, cub::BLOCK_REDUCE_WARP_REDUCTIONS, 1, 1, __COMPUTE_CAPABILITY__>;
-#endif
     const int batch;
 
     __device__ __host__ inline BlockReduce(int batch = 0) : batch(batch) {}
@@ -76,7 +62,11 @@ namespace quda {
       __device__ inline T operator()(const T &value_, bool pipeline, int batch, bool all_sum)
       {
         QUDA_RT_CONSTS;
+#ifndef QUDA_BACKEND_OMPTARGET
         static __shared__ typename block_reduce_t::TempStorage storage[batch_size];
+#else
+        typename block_reduce_t::TempStorage *storage = reinterpret_cast<typename block_reduce_t::TempStorage *>(shared_cache.addr);
+#endif
         block_reduce_t block_reduce(storage[batch]);
         if (!pipeline) {
           __syncthreads(); // only synchronize if we are not pipelining
diff --git a/include/targets/omptarget/cub/block/block_reduce.cuh b/include/targets/omptarget/cub/block/block_reduce.cuh
new file mode 100644
index 0000000000..30876a4424
--- /dev/null
+++ b/include/targets/omptarget/cub/block/block_reduce.cuh
@@ -0,0 +1,88 @@
+#include <target_device.h>
+
+namespace cub {
+
+  // Naive implementation of cub collectives in OpenMP target.
+
+  enum BlockReduceAlgorithm {BLOCK_REDUCE_WARP_REDUCTIONS};
+
+  constexpr int get_warp_steps(int s)
+  {
+    // log2(s)
+    return s==1?0:s==2?1:s==4?2:s==8?3:s==16?4:s==32?5:s==64?6:-1;
+  }
+
+// https://github.com/NVlabs/cub/blob/1.8.0/cub/warp/specializations/warp_reduce_smem.cuh
+template <typename T, int warp_threads, int ptx_arch = __COMPUTE_CAPABILITY__>
+struct WarpReduce
+{
+  static constexpr int warp_steps = get_warp_steps(warp_threads);
+  static_assert(warp_steps>=0, "unsupported warp_threads");
+  static constexpr int half_warp_threads = 1 << (warp_steps-1);
+  static constexpr int warp_smem_elments = warp_threads + half_warp_threads;
+  struct TempStorage
+  {
+    T reduce[warp_smem_elments];
+  };
+  TempStorage &temp_storage;
+  int tid,lane_id;
+  inline WarpReduce(TempStorage &temp_storage) :
+    temp_storage(temp_storage),
+    tid(omp_get_thread_num()),
+    lane_id(tid%warp_threads)
+  {}
+  inline T Sum(T input, int valid_items = warp_threads)  // FIXME warp_num_valid < warp_threads
+  {
+    #pragma unroll
+    for(int i=0;i<warp_steps;++i){
+      temp_storage.reduce[lane_id] = input;
+      const int offset = 1 << i;
+      // this is bad, but we don't have warp barrier
+      #pragma omp barrier
+      if((lane_id + offset) < valid_items)
+        input += temp_storage.reduce[lane_id+offset];
+    }
+    return input;
+  }
+};
+
+// https://github.com/NVlabs/cub/blob/1.8.0/cub/block/specializations/block_reduce_warp_reductions.cuh
+template <typename T, int block_dim_x, BlockReduceAlgorithm alg = BLOCK_REDUCE_WARP_REDUCTIONS, int block_dim_y = 1, int block_dim_z = 1, int ptx_arch = __COMPUTE_CAPABILITY__>
+struct BlockReduce
+{
+  static constexpr int block_threads = block_dim_x*block_dim_y*block_dim_z;
+  static constexpr int warp_threads = device::warp_size();
+  static constexpr int warps = (block_threads + warp_threads - 1) / warp_threads;
+  static constexpr int logical_warp_size = std::min(block_threads, warp_threads);
+  static constexpr bool even_warp_multiple = block_threads%logical_warp_size == 0;
+  typedef WarpReduce<T, logical_warp_size, ptx_arch> WarpReduce_;
+  struct TempStorage
+  {
+    typename WarpReduce_::TempStorage warp_reduce[warps];
+    T warp_aggregates[warps];
+  };
+  TempStorage &temp_storage;
+  int tid,lane_id,warp_id;
+  inline BlockReduce(TempStorage &temp_storage) :
+    temp_storage(temp_storage),
+    tid(omp_get_thread_num()),
+    lane_id(tid%logical_warp_size),
+    warp_id(tid/logical_warp_size)
+  {}
+  inline T Sum(T input)
+  {
+    int warp_offset = warp_id * logical_warp_size;
+    int warp_num_valid = (even_warp_multiple || (warp_offset + logical_warp_size <= block_threads)) ? logical_warp_size : block_threads - warp_offset;
+    T warp_aggregate = WarpReduce_(temp_storage.warp_reduce[warp_id]).Sum(input, warp_num_valid);
+
+    if (lane_id == 0) temp_storage.warp_aggregates[warp_id] = warp_aggregate;
+    #pragma omp barrier
+    if (tid == 0) {
+#pragma unroll
+      for(int i=1;i<warps;++i) warp_aggregate += temp_storage.warp_aggregates[i];
+    }
+    return warp_aggregate;
+  }
+};
+
+}

From c32cdd9e6af16af61c9e477e95172fc71c5b494a Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 17 Sep 2021 21:21:34 +0000
Subject: [PATCH 087/198] omp: enable cub_helper and block_orthogonalize

---
 include/kernels/block_orthogonalize.cuh | 4 ----
 include/kernels/reduce_core.cuh         | 5 -----
 include/reduce_helper.h                 | 2 --
 3 files changed, 11 deletions(-)

diff --git a/include/kernels/block_orthogonalize.cuh b/include/kernels/block_orthogonalize.cuh
index 797b78888c..56e86dddb3 100644
--- a/include/kernels/block_orthogonalize.cuh
+++ b/include/kernels/block_orthogonalize.cuh
@@ -127,9 +127,6 @@ namespace quda {
 
     __device__ __host__ inline void operator()(dim3 block, dim3 thread)
     {
-#ifdef QUDA_BACKEND_OMPTARGET
-      ompwip("unimplemented");
-#else
       int x_coarse = block.x;
       int x_fine_offset = thread.x;
       int chirality = block.y;
@@ -242,7 +239,6 @@ namespace quda {
           }
         } // j
       }   // n
-#endif
     }
   };
 
diff --git a/include/kernels/reduce_core.cuh b/include/kernels/reduce_core.cuh
index 0f9bf3b295..6eaa5ec35b 100644
--- a/include/kernels/reduce_core.cuh
+++ b/include/kernels/reduce_core.cuh
@@ -80,14 +80,9 @@ namespace quda
         if (arg.r.read.W) arg.W.load(w, i, parity);
         if (arg.r.read.V) arg.V.load(v, i, parity);
 
-        // if(0<=i&&i<4) printf("i %d  arg.r.read.X %d  arg.r.read.Y %d  arg.r.read.Z %d\n", i, arg.r.read.X, arg.r.read.Y, arg.r.read.Z);
-        // auto s = [](auto x){return reinterpret_cast<double*>(&x)[0];};  // debug purpose, assuming double
         arg.r.pre();
-        // if(0<=i&&i<4) printf("pre  %d sum %g x %g y %g z %g w %g z %g\n", i, s(sum), s(x), s(y), s(z), s(w), s(v));
         arg.r(sum, x, y, z, w, v);
-        // if(0<=i&&i<4) printf("r    %d sum %g x %g y %g z %g w %g z %g\n", i, s(sum), s(x), s(y), s(z), s(w), s(v));
         arg.r.post(sum);
-        // if(0<=i&&i<4) printf("post %d sum %g x %g y %g z %g w %g z %g\n", i, s(sum), s(x), s(y), s(z), s(w), s(v));
 
         if (arg.r.write.X) arg.X.save(x, i, parity);
         if (arg.r.write.Y) arg.Y.save(y, i, parity);
diff --git a/include/reduce_helper.h b/include/reduce_helper.h
index 2a887d9f40..8f02e7fc25 100644
--- a/include/reduce_helper.h
+++ b/include/reduce_helper.h
@@ -1,9 +1,7 @@
 #pragma once
 
 #include <float_vector.h>
-#ifndef QUDA_BACKEND_OMPTARGET
 #include <cub_helper.cuh>
-#endif
 #include <target_device.h>
 #include <reducer.h>
 #include <kernel_helper.h>

From d33e67d8204a052ad0ef8f2f159a0c45418a5d98 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 21 Sep 2021 21:03:27 +0000
Subject: [PATCH 088/198] omp: raise warp_size to 16

---
 include/targets/omptarget/target_device.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index c3253b666e..2675bae7f2 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -88,7 +88,7 @@ namespace quda {
        @brief Helper function that returns the warp-size of the
        architecture we are running on.
     */
-    constexpr int warp_size() { return 8; }
+    constexpr int warp_size() { return 16; }
 
     /**
        @brief Return the thread mask for a converged warp.

From 8b840f5c5479862b21bd415dbc3c979d1ceabb7f Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 4 Oct 2021 21:51:29 +0000
Subject: [PATCH 089/198] Introduce QUDA_UNROLL for setting #pragma unroll

---
 include/blas_helper.cuh                       |  16 +-
 include/clover_field_order.h                  |  16 +-
 include/color_spinor.h                        | 184 ++++++++---------
 include/color_spinor_field_order.h            |  52 ++---
 include/float_vector.h                        |  40 ++--
 include/gauge_field_order.h                   |  56 +++---
 include/gauge_fix_ovr_hit_devf.cuh            |  10 +-
 include/index_helper.cuh                      |  12 +-
 include/kernels/blas_core.cuh                 |  28 +--
 include/kernels/block_orthogonalize.cuh       |  42 ++--
 include/kernels/clover_compute.cuh            |   6 +-
 include/kernels/clover_outer_product.cuh      |   2 +-
 .../kernels/clover_sigma_outer_product.cuh    |   2 +-
 include/kernels/coarse_op_kernel.cuh          | 188 +++++++++---------
 include/kernels/coarse_op_preconditioned.cuh  |   6 +-
 include/kernels/color_spinor_pack.cuh         |  10 +-
 include/kernels/contraction.cuh               |   4 +-
 include/kernels/copy_clover.cuh               |   2 +-
 include/kernels/copy_field_offset.cuh         |   6 +-
 include/kernels/dslash_clover_helper.cuh      |   4 +-
 include/kernels/dslash_coarse.cuh             |  40 ++--
 include/kernels/dslash_mdw_fused.cuh          |  16 +-
 include/kernels/dslash_staggered.cuh          |   2 +-
 .../dslash_twisted_clover_preconditioned.cuh  |   2 +-
 .../dslash_twisted_mass_preconditioned.cuh    |   2 +-
 include/kernels/dslash_wilson.cuh             |   2 +-
 include/kernels/dslash_wilson_clover.cuh      |   2 +-
 .../dslash_wilson_clover_hasenbusch_twist.cuh |   2 +-
 ...clover_hasenbusch_twist_preconditioned.cuh |   4 +-
 .../dslash_wilson_clover_preconditioned.cuh   |   2 +-
 include/kernels/gauge_det_trace.cuh           |   6 +-
 include/kernels/gauge_fix_fft.cuh             |  12 +-
 include/kernels/gauge_fix_ovr.cuh             |   8 +-
 include/kernels/gauge_heatbath.cuh            |   4 +-
 include/kernels/gauge_plaq.cuh                |   6 +-
 include/kernels/gauge_qcharge.cuh             |   4 +-
 include/kernels/gauge_utils.cuh               |   2 +-
 include/kernels/hisq_paths_force.cuh          |   6 +-
 include/kernels/laplace.cuh                   |   2 +-
 include/kernels/momentum.cuh                  |   2 +-
 include/kernels/multi_blas_core.cuh           |  12 +-
 include/kernels/multi_reduce_core.cuh         |   8 +-
 include/kernels/pgauge_init.cuh               |  12 +-
 include/kernels/prolongator.cuh               |  16 +-
 include/kernels/reduce_core.cuh               |  38 ++--
 include/kernels/restrictor.cuh                |  24 +--
 .../kernels/staggered_coarse_op_kernel.cuh    |   6 +-
 .../staggered_kd_apply_xinv_kernel.cuh        |  14 +-
 include/kernels/staggered_outer_product.cuh   |   2 +-
 include/linalg.cuh                            |  26 +--
 include/matrix_tile.cuh                       |  62 +++---
 include/quda_api.h                            |   9 +
 include/quda_matrix.h                         | 122 ++++++------
 include/reduce_helper.h                       |   2 +-
 include/shared_memory_cache_helper.cuh        |   4 +-
 include/su3_project.cuh                       |   8 +-
 include/targets/generic/aos.h                 |   4 +-
 .../omptarget/cub/block/block_reduce.cuh      |   4 +-
 include/targets/omptarget/warp_collective.h   |   4 +-
 59 files changed, 599 insertions(+), 590 deletions(-)

diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index 4b67431a97..30580474cd 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -207,10 +207,10 @@ namespace quda
       {
         norm_t max_[n];
         // two-pass to increase ILP (assumes length divisible by two, e.g. complex-valued)
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < n; i++) max_[i] = fmaxf(fabsf((norm_t)v[i].real()), fabsf((norm_t)v[i].imag()));
         norm_t scale = 0.0;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < n; i++) scale = fmaxf(max_[i], scale);
         data.norm[x + parity * data.cb_norm_offset] = scale;
 
@@ -235,12 +235,12 @@ namespace quda
         vector_type<real, len> v_;
 
         constexpr int M = len / N;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < M; i++) {
           // first load from memory
           Vector vecTmp = vector_load<Vector>(data.spinor, parity * data.cb_offset + x + data.stride * i);
           // now copy into output and scale
-#pragma unroll
+QUDA_UNROLL
           for (int j = 0; j < N; j++) copy_and_scale(v_[i * N + j], reinterpret_cast<store_t *>(&vecTmp)[j], nrm);
         }
 
@@ -263,13 +263,13 @@ namespace quda
 
         if (isFixed<store_t>::value) {
           real scale_inv = store_norm<isFixed<store_t>::value, real, n>(v, x, parity);
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < n; i++) {
             v_[2 * i + 0] = scale_inv * v[i].real();
             v_[2 * i + 1] = scale_inv * v[i].imag();
           }
         } else {
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < n; i++) {
             v_[2 * i + 0] = v[i].real();
             v_[2 * i + 1] = v[i].imag();
@@ -277,11 +277,11 @@ namespace quda
         }
 
         constexpr int M = len / N;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < M; i++) {
           Vector vecTmp;
           // first do scalar copy converting into storage type
-#pragma unroll
+QUDA_UNROLL
           for (int j = 0; j < N; j++) copy_scaled(reinterpret_cast<store_t *>(&vecTmp)[j], v_[i * N + j]);
           // second do vectorized copy into memory
           vector_store(data.spinor, parity * data.cb_offset + x + data.stride * i, vecTmp);
diff --git a/include/clover_field_order.h b/include/clover_field_order.h
index 6371720d16..f7eb62448b 100644
--- a/include/clover_field_order.h
+++ b/include/clover_field_order.h
@@ -591,12 +591,12 @@ namespace quda {
         {
           norm_type nrm = isFixed<Float>::value ? vector_load<float>(norm, parity * norm_offset + chirality * stride + x) : 0;
 
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<M; i++) {
             // first load from memory
             Vector vecTmp = vector_load<Vector>(clover, parity * offset + x + stride * (chirality * M + i));
             // second do scalar copy converting into register type
-#pragma unroll
+QUDA_UNROLL
             for (int j = 0; j < N; j++) { copy_and_scale(v[i * N + j], reinterpret_cast<Float *>(&vecTmp)[j], nrm); }
           }
 
@@ -617,19 +617,19 @@ namespace quda {
           // find the norm of each chiral block
           if (isFixed<Float>::value) {
             norm_type scale = static_cast<norm_type>(1.0);
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < block; i++) scale = fabsf((norm_type)v[i]) > scale ? fabsf((norm_type)v[i]) : scale;
             norm[parity*norm_offset + chirality*stride + x] = scale;
 
             real scale_inv = fdividef(fixedMaxValue<Float>::value, scale);
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < block; i++) tmp[i] = v[i] * scale_inv;
           } else {
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < block; i++) tmp[i] = v[i];
           }
 
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < M; i++) {
             Vector vecTmp;
             // first do scalar copy converting into storage type
@@ -647,7 +647,7 @@ namespace quda {
 	   @param[in] chirality Chiral block index
 	 */
 	__device__ __host__ inline void load(real v[length], int x, int parity) const {
-#pragma unroll
+QUDA_UNROLL
           for (int chirality = 0; chirality < 2; chirality++) load(&v[chirality * block], x, parity, chirality);
         }
 
@@ -659,7 +659,7 @@ namespace quda {
 	   @param[in] chirality Chiral block index
 	 */
 	__device__ __host__ inline void save(const real v[length], int x, int parity) const {
-#pragma unroll
+QUDA_UNROLL
           for (int chirality = 0; chirality < 2; chirality++) save(&v[chirality * block], x, parity, chirality);
         }
 
diff --git a/include/color_spinor.h b/include/color_spinor.h
index 22fb200b99..fa9ef81de5 100644
--- a/include/color_spinor.h
+++ b/include/color_spinor.h
@@ -28,18 +28,18 @@ namespace quda {
 
     __device__ __host__ inline ColorSpinor<Float, Nc, Ns>()
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] = 0; }
       }
 
       __device__ __host__ inline ColorSpinor<Float, Nc, Ns>(const ColorSpinor<Float, Nc, Ns> &a) {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < size; i++) { data[i] = a.data[i]; }
       }
 
       __device__ __host__ inline ColorSpinor<Float, Nc, Ns>& operator=(const ColorSpinor<Float, Nc, Ns> &a) {
 	if (this != &a) {
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < size; i++) { data[i] = a.data[i]; }
         }
 	return *this;
@@ -48,20 +48,20 @@ namespace quda {
       __device__ __host__ inline ColorSpinor<Float, Nc, Ns> operator-() const
       {
         ColorSpinor<Float, Nc, Ns> a;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < size; i++) { a.data[i] = -data[i]; }
         return a;
       }
 
       __device__ __host__ inline ColorSpinor<Float, Nc, Ns>& operator+=(const ColorSpinor<Float, Nc, Ns> &a) {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < size; i++) { data[i] += a.data[i]; }
         return *this;
       }
 
       template <typename T> __device__ __host__ inline ColorSpinor<Float, Nc, Ns> &operator*=(const T &a)
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < size; i++) { data[i] *= a; }
         return *this;
       }
@@ -69,7 +69,7 @@ namespace quda {
       __device__ __host__ inline ColorSpinor<Float, Nc, Ns> &operator-=(const ColorSpinor<Float, Nc, Ns> &a)
       {
         if (this != &a) {
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < Nc * Ns; i++) { data[i] -= a.data[i]; }
         }
         return *this;
@@ -141,32 +141,32 @@ namespace quda {
 
       __device__ __host__ inline ColorSpinor<Float, Nc, 4>()
       {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] = 0; }
       }
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 4>(const ColorSpinor<Float, Nc, 4> &a) {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] = a.data[i]; }
     }
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 4>& operator=(const ColorSpinor<Float, Nc, 4> &a) {
       if (this != &a) {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < size; i++) { data[i] = a.data[i]; }
       }
       return *this;
     }
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 4>& operator+=(const ColorSpinor<Float, Nc, 4> &a) {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] += a.data[i]; }
       return *this;
     }
 
     template <typename T> __device__ __host__ inline ColorSpinor<Float, Nc, 4> &operator*=(const T &a)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] *= a; }
       return *this;
     }
@@ -182,7 +182,7 @@ namespace quda {
 
       switch (dim) {
       case 0: // x dimension
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(3, i));
           a(1, i) = i_(t(2, i));
@@ -191,7 +191,7 @@ namespace quda {
         }
         break;
       case 1: // y dimension
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = t(3, i);
           a(1, i) = -t(2, i);
@@ -200,7 +200,7 @@ namespace quda {
         }
         break;
       case 2: // z dimension
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(2, i));
           a(1, i) = -i_(t(3, i));
@@ -209,7 +209,7 @@ namespace quda {
         }
         break;
       case 3: // t dimension
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = t(0, i);
           a(1, i) = t(1, i);
@@ -218,7 +218,7 @@ namespace quda {
         }
         break;
       case 4: // gamma_5
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = t(2, i);
           a(1, i) = t(3, i);
@@ -242,7 +242,7 @@ namespace quda {
 
       switch (dim) {
       case 0: // x dimension
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = -t(3, i);
           a(1, i) = -t(2, i);
@@ -251,7 +251,7 @@ namespace quda {
         }
         break;
       case 1: // y dimension
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(3, i));
           a(1, i) = -i_(t(2, i));
@@ -260,7 +260,7 @@ namespace quda {
         }
         break;
       case 2: // z dimension
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = -t(2, i);
           a(1, i) = t(3, i);
@@ -269,7 +269,7 @@ namespace quda {
         }
         break;
       case 3: // t dimension
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(0, i));
           a(1, i) = i_(t(1, i));
@@ -278,7 +278,7 @@ namespace quda {
         }
         break;
       case 4: // gamma_5
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(2, i));
           a(1, i) = i_(t(3, i));
@@ -297,9 +297,9 @@ namespace quda {
     */
     __device__ __host__ inline ColorSpinor<Float,Nc,2> chiral_project(int chirality) const {
       ColorSpinor<Float,Nc,2> proj;
-#pragma unroll
+QUDA_UNROLL
       for (int s=0; s<Ns/2; s++) {
-#pragma unroll
+QUDA_UNROLL
 	for (int c=0; c<Nc; c++) {
 	  proj(s,c) = (*this)(chirality*Ns/2+s,c);
 	}
@@ -321,14 +321,14 @@ namespace quda {
       case 0: // x dimension
 	switch (sign) {
 	case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) + i_(t(3, i));
             proj(1, i) = t(1, i) + i_(t(2, i));
           }
           break;
 	case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) - i_(t(3, i));
             proj(1, i) = t(1, i) - i_(t(2, i));
@@ -339,14 +339,14 @@ namespace quda {
       case 1: // y dimension
 	switch (sign) {
 	case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) + t(3, i);
             proj(1, i) = t(1, i) - t(2, i);
           }
           break;
 	case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) - t(3, i);
             proj(1, i) = t(1, i) + t(2, i);
@@ -357,14 +357,14 @@ namespace quda {
       case 2: // z dimension
 	switch (sign) {
 	case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) + i_(t(2, i));
             proj(1, i) = t(1, i) - i_(t(3, i));
           }
           break;
         case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) - i_(t(2, i));
             proj(1, i) = t(1, i) + i_(t(3, i));
@@ -375,14 +375,14 @@ namespace quda {
       case 3: // t dimension
 	switch (sign) {
 	case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = static_cast<Float>(2.0) * t(0, i);
             proj(1, i) = static_cast<Float>(2.0) * t(1, i);
           }
           break;
 	case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = static_cast<Float>(2.0) * t(2, i);
             proj(1, i) = static_cast<Float>(2.0) * t(3, i);
@@ -393,14 +393,14 @@ namespace quda {
       case 4:
         switch (sign) {
         case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < Nc; i++) {
             proj(0, i) = t(0, i) + t(2, i);
             proj(1, i) = t(1, i) + t(3, i);
           }
           break;
         case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < Nc; i++) {
             proj(0, i) = t(0, i) - t(2, i);
             proj(1, i) = t(1, i) - t(3, i);
@@ -457,7 +457,7 @@ namespace quda {
       case 0:
 	switch(nu) {
 	case 1:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) =  j*b(0,i);
 	    a(1,i) = -j*b(1,i);
@@ -466,7 +466,7 @@ namespace quda {
 	  }
 	  break;
 	case 2:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -b(1,i);
 	    a(1,i) =  b(0,i);
@@ -475,7 +475,7 @@ namespace quda {
 	  }
 	  break;
 	case 3:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -j*b(3,i);
 	    a(1,i) = -j*b(2,i);
@@ -488,7 +488,7 @@ namespace quda {
       case 1:
 	switch(nu) {
 	case 0:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -j*b(0,i);
 	    a(1,i) =  j*b(1,i);
@@ -497,7 +497,7 @@ namespace quda {
 	  }
 	  break;
 	case 2:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = j*b(1,i);
 	    a(1,i) = j*b(0,i);
@@ -506,7 +506,7 @@ namespace quda {
 	  }
 	  break;
 	case 3:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -b(3,i);
 	    a(1,i) =  b(2,i);
@@ -519,7 +519,7 @@ namespace quda {
       case 2:
 	switch(nu) {
 	case 0:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) =  b(1,i);
 	    a(1,i) = -b(0,i);
@@ -528,7 +528,7 @@ namespace quda {
 	  }
 	  break;
 	case 1:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -j*b(1,i);
 	    a(1,i) = -j*b(0,i);
@@ -537,7 +537,7 @@ namespace quda {
 	  }
 	  break;
 	case 3:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -j*b(2,i);
 	    a(1,i) =  j*b(3,i);
@@ -550,7 +550,7 @@ namespace quda {
       case 3:
 	switch(nu) {
 	case 0:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = j*b(3,i);
 	    a(1,i) = j*b(2,i);
@@ -559,7 +559,7 @@ namespace quda {
 	  }
 	  break;
 	case 1:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) =  b(3,i);
 	    a(1,i) = -b(2,i);
@@ -568,7 +568,7 @@ namespace quda {
 	  }
 	  break;
 	case 2:
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) =  j*b(2,i);
 	    a(1,i) = -j*b(3,i);
@@ -631,7 +631,7 @@ namespace quda {
     */
     __device__ __host__ inline void toNonRel() {
       ColorSpinor<Float,Nc,Ns> a;
-#pragma unroll
+QUDA_UNROLL
       for (int c=0; c<Nc; c++) {
 	a(0,c) =  (*this)(1,c)+(*this)(3,c);
 	a(1,c) = -(*this)(2,c)-(*this)(0,c);
@@ -646,7 +646,7 @@ namespace quda {
     */
     __device__ __host__ inline void toRel() {
       ColorSpinor<Float,Nc,Ns> a;
-#pragma unroll
+QUDA_UNROLL
       for (int c=0; c<Nc; c++) {
 	a(0,c) = -(*this)(1,c)-(*this)(3,c);
 	a(1,c) =  (*this)(2,c)+(*this)(0,c);
@@ -677,33 +677,33 @@ namespace quda {
     complex<Float> data[size];
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 2>() {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] = 0; }
     }
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 2>(const ColorSpinor<Float, Nc, 2> &a) {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] = a.data[i]; }
     }
 
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 2>& operator=(const ColorSpinor<Float, Nc, 2> &a) {
       if (this != &a) {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < size; i++) { data[i] = a.data[i]; }
       }
       return *this;
     }
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 2>& operator+=(const ColorSpinor<Float, Nc, 2> &a) {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] += a.data[i]; }
       return *this;
     }
 
     template <typename T> __device__ __host__ inline ColorSpinor<Float, Nc, 2> &operator*=(const T &a)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < size; i++) { data[i] *= a; }
       return *this;
     }
@@ -714,9 +714,9 @@ namespace quda {
     */
     __device__ __host__ inline ColorSpinor<Float,Nc,4> chiral_reconstruct(int chirality) const {
       ColorSpinor<Float,Nc,4> recon;
-#pragma unroll
+QUDA_UNROLL
       for (int s=0; s<Ns; s++) {
-#pragma unroll
+QUDA_UNROLL
 	for (int c=0; c<Nc; c++) {
 	  recon(chirality*Ns+s,c) = (*this)(s,c);
 	}
@@ -739,7 +739,7 @@ namespace quda {
       case 0: // x dimension
 	switch (sign) {
 	case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -748,7 +748,7 @@ namespace quda {
           }
           break;
 	case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -761,7 +761,7 @@ namespace quda {
       case 1: // y dimension
 	switch (sign) {
 	case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -770,7 +770,7 @@ namespace quda {
           }
           break;
 	case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -783,7 +783,7 @@ namespace quda {
       case 2: // z dimension
 	switch (sign) {
 	case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -792,7 +792,7 @@ namespace quda {
           }
           break;
 	case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -805,7 +805,7 @@ namespace quda {
       case 3: // t dimension
 	switch (sign) {
 	case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -814,7 +814,7 @@ namespace quda {
 	  }
 	  break;
 	case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<Nc; i++) {
 	    recon(0,i) = 0;
 	    recon(1,i) = 0;
@@ -827,7 +827,7 @@ namespace quda {
       case 4:
         switch (sign) {
         case 1: // positive projector
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -836,7 +836,7 @@ namespace quda {
           }
           break;
         case -1: // negative projector
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -912,7 +912,7 @@ namespace quda {
   __device__ __host__ inline void caxpy(const complex<Float> &a, const ColorSpinor<Float, Nc, Ns> &x,
                                         ColorSpinor<Float, Nc, Ns> &y)
   {
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < Nc * Ns; i++) {
       y(i).real( a.real() * x(i).real() + y(i).real());
       y(i).real(-a.imag() * x(i).imag() + y(i).real());
@@ -931,7 +931,7 @@ namespace quda {
   __device__ __host__ inline Float norm2(const ColorSpinor<Float, Nc, Ns> &a)
   {
     Float nrm = 0.0;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < Nc * Ns; i++) {
       nrm += a(i).real() * a(i).real();
       nrm += a(i).imag() * a(i).imag();
@@ -951,7 +951,7 @@ namespace quda {
                                                           const ColorSpinor<Float, Nc, Ns> &b, int sa, int sb)
   {
     complex<Float> dot = cmul(a(sa, 0), b(sb, 0));
-#pragma unroll
+QUDA_UNROLL
     for (int c = 1; c < Nc; c++) dot = cmac(a(sa, c), b(sb, c), dot);
     return dot;
   }
@@ -968,9 +968,9 @@ namespace quda {
                                                          const ColorSpinor<Float, Nc, Ns> &b)
   {
     complex<Float> dot;
-#pragma unroll
+QUDA_UNROLL
     for (int s = 0; s < Ns; s++) {
-#pragma unroll
+QUDA_UNROLL
       for (int c = 0; c < Nc; c++) {
         if (s == 0 && c == 0) dot = cmul(conj(a(s, c)), b(s, c));
         else dot = cmac(conj(a(s, c)), b(s, c), dot);
@@ -992,7 +992,7 @@ namespace quda {
                                                          const ColorSpinor<Float, Nc, Nsb> &b, int sa, int sb)
   {
     complex<Float> dot = cmul(conj(a(sa, 0)), b(sb, 0));
-#pragma unroll
+QUDA_UNROLL
     for (int c = 1; c < Nc; c++) dot = cmac(conj(a(sa, c)), b(sb, c), dot);
     return dot;
   }
@@ -1047,9 +1047,9 @@ namespace quda {
     Matrix<complex<Float>, Nc> out;
 
     // outer product over color
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < Nc; i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int j = 0; j < Nc; j++) {
         // trace over spin (manual unroll for perf)
         out(j, i).real(a(0, j).real() * b(0, i).real());
@@ -1058,7 +1058,7 @@ namespace quda {
         out(j, i).imag(out(j, i).imag() - a(0, j).real() * b(0, i).imag());
         // out(j,i) = a(0,j) * conj(b(0,i));
 
-#pragma unroll
+QUDA_UNROLL
 	for (int s=1; s<Ns; s++) {
 	  out(j,i).real( out(j,i).real() + a(s,j).real() * b(s,i).real() );
 	  out(j,i).real( out(j,i).real() + a(s,j).imag() * b(s,i).imag() );
@@ -1085,9 +1085,9 @@ namespace quda {
     Matrix<complex<Float>, Nc> out;
 
     // outer product over color
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < Nc; i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int j = 0; j < Nc; j++) {
         // trace over spin (manual unroll for perf)
         out(j, i).real(a(0, j).real() * b(0, i).real());
@@ -1111,9 +1111,9 @@ namespace quda {
 
     ColorSpinor<Float,Nc,Ns> z;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<Nc; i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int s=0; s<Ns; s++) {
 	z.data[s*Nc + i] = x.data[s*Nc + i] + y.data[s*Nc + i];
       }
@@ -1133,9 +1133,9 @@ namespace quda {
 
     ColorSpinor<Float,Nc,Ns> z;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<Nc; i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int s=0; s<Ns; s++) {
 	z.data[s*Nc + i] = x.data[s*Nc + i] - y.data[s*Nc + i];
       }
@@ -1155,9 +1155,9 @@ namespace quda {
 
     ColorSpinor<Float,Nc,Ns> y;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<Nc; i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int s=0; s<Ns; s++) {
 	y.data[s*Nc + i] = a * x.data[s*Nc + i];
       }
@@ -1177,18 +1177,18 @@ namespace quda {
 
     ColorSpinor<Float,Nc,Ns> y;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<Nc; i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int s=0; s<Ns; s++) {
 	y.data[s*Nc + i].x  = A(i,0).real() * x.data[s*Nc + 0].real();
 	y.data[s*Nc + i].x -= A(i,0).imag() * x.data[s*Nc + 0].imag();
 	y.data[s*Nc + i].y  = A(i,0).real() * x.data[s*Nc + 0].imag();
 	y.data[s*Nc + i].y += A(i,0).imag() * x.data[s*Nc + 0].real();
       }
-#pragma unroll
+QUDA_UNROLL
       for (int j=1; j<Nc; j++) {
-#pragma unroll
+QUDA_UNROLL
 	for (int s=0; s<Ns; s++) {
 	  y.data[s*Nc + i].x += A(i,j).real() * x.data[s*Nc + j].real();
 	  y.data[s*Nc + i].x -= A(i,j).imag() * x.data[s*Nc + j].imag();
@@ -1213,18 +1213,18 @@ namespace quda {
   {
     ColorSpinor<Float,Nc,Ns> z;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<Nc; i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int s=0; s<Ns; s++) {
 	z.data[s*Nc + i].x  = y.data[s*Nc + i].real() + A(i,0).real() * x.data[s*Nc + 0].real();
 	z.data[s*Nc + i].x -= A(i,0).imag() * x.data[s*Nc + 0].imag();
 	z.data[s*Nc + i].y  = y.data[s*Nc + i].imag() + A(i,0).real() * x.data[s*Nc + 0].imag();
 	z.data[s*Nc + i].y += A(i,0).imag() * x.data[s*Nc + 0].real();
       }
-#pragma unroll
+QUDA_UNROLL
       for (int j=1; j<Nc; j++) {
-#pragma unroll
+QUDA_UNROLL
 	for (int s=0; s<Ns; s++) {
 	  z.data[s*Nc + i].x += A(i,j).real() * x.data[s*Nc + j].real();
 	  z.data[s*Nc + i].x -= A(i,j).imag() * x.data[s*Nc + j].imag();
@@ -1249,7 +1249,7 @@ namespace quda {
     ColorSpinor<Float,Nc,Ns> y;
     constexpr int N = Ns * Nc;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<N; i++) {
       if (i==0) {
 	y.data[i].x  = A(i,0).real() * x.data[0].real();
@@ -1260,7 +1260,7 @@ namespace quda {
 	y.data[i].y  = A(i,0).real() * x.data[0].imag();
 	y.data[i].y += A(i,0).imag() * x.data[0].real();
       }
-#pragma unroll
+QUDA_UNROLL
       for (int j=1; j<N; j++) {
 	if (i==j) {
 	  y.data[i].x += A(i,j).real() * x.data[j].real();
diff --git a/include/color_spinor_field_order.h b/include/color_spinor_field_order.h
index 26c3e02d53..b356392bd7 100644
--- a/include/color_spinor_field_order.h
+++ b/include/color_spinor_field_order.h
@@ -222,7 +222,7 @@ namespace quda {
         using vec_t = typename VectorType<Float, 2>::type;
         constexpr int N = nSpin * nColor * nVec;
         constexpr int M = nSpinBlock * nColor * nVec;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < M; i++) {
           vec_t tmp = vector_load<vec_t>(reinterpret_cast<const vec_t *>(in + parity * offset_cb), x_cb * N + chi * M + i);
           memcpy(&out[i], &tmp, sizeof(vec_t));
@@ -278,7 +278,7 @@ namespace quda {
       {
         using vec_t = typename VectorType<Float, 2>::type;
         constexpr int M = nSpinBlock * nColor * nVec;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < M; i++) {
           vec_t tmp = vector_load<vec_t>(reinterpret_cast<const vec_t *>(in + parity * offset_cb),
                                          (chi * M + i) * stride + x_cb);
@@ -326,7 +326,7 @@ namespace quda {
       {
         using vec_t = typename VectorType<Float, 4>::type;
         constexpr int M = (nSpinBlock * nColor * nVec * 2) / 4;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < M; i++) {
           vec_t tmp = vector_load<vec_t>(reinterpret_cast<const vec_t *>(in + parity * offset_cb),
                                          (chi * M + i) * stride + x_cb);
@@ -379,13 +379,13 @@ namespace quda {
         constexpr int N = nSpin * nColor * nVec * 2; // real numbers in the loaded vector
         constexpr int M = N / 8;
         Float tmp[N];
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < M; i++) {
           vec_t ld_tmp = vector_load<vec_t>(reinterpret_cast<const vec_t *>(in + parity * offset_cb), i * stride + x_cb);
           memcpy(&tmp[i * 8], &ld_tmp, sizeof(vec_t));
         }
         constexpr int N_chi = N / (nSpin / nSpinBlock);
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < N_chi; i++)
           out[i] = complex<Float>(tmp[chi * N_chi + 2 * i + 0], tmp[chi * N_chi + 2 * i + 1]);
       }
@@ -719,11 +719,11 @@ namespace quda {
           complex<storeFloat> tmp[nSpinBlock * nColor * nVec];
           accessor.template load<nSpinBlock>(tmp, v, parity, x_cb, chi);
           Float norm_ = block_float ? norm[parity * norm_offset + x_cb] : scale_inv;
-#pragma unroll
+QUDA_UNROLL
           for (int s = 0; s < nSpinBlock; s++) {
-#pragma unroll
+QUDA_UNROLL
             for (int c = 0; c < nColor; c++) {
-#pragma unroll
+QUDA_UNROLL
               for (int v = 0; v < nVec; v++) {
                 int k = (s * nColor + c) * nVec + v;
                 out[k] = norm_ * complex<Float>(static_cast<Float>(tmp[k].real()), static_cast<Float>(tmp[k].imag()));
@@ -985,16 +985,16 @@ namespace quda {
     real v[length];
     norm_type nrm = isFixed<Float>::value ? vector_load<float>(norm, x + parity * norm_offset) : 0.0;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<M; i++) {
       // first load from memory
       Vector vecTmp = vector_load<Vector>(field, parity * offset + x + stride * i);
       // now copy into output and scale
-#pragma unroll
+QUDA_UNROLL
       for (int j = 0; j < N; j++) copy_and_scale(v[i * N + j], reinterpret_cast<Float *>(&vecTmp)[j], nrm);
     }
 
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < length / 2; i++) out[i] = complex(v[2 * i + 0], v[2 * i + 1]);
   }
 
@@ -1002,7 +1002,7 @@ namespace quda {
   {
     real v[length];
 
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < length / 2; i++) {
       v[2 * i + 0] = in[i].real();
       v[2 * i + 1] = in[i].imag();
@@ -1011,23 +1011,23 @@ namespace quda {
     if (isFixed<Float>::value) {
       norm_type max_[length / 2];
       // two-pass to increase ILP (assumes length divisible by two, e.g. complex-valued)
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < length / 2; i++) max_[i] = fmaxf(fabsf((norm_type)v[i]), fabsf((norm_type)v[i + length / 2]));
       norm_type scale = 0.0;
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < length / 2; i++) scale = fmaxf(max_[i], scale);
       norm[x+parity*norm_offset] = scale;
 
       real scale_inv = fdividef(fixedMaxValue<Float>::value, scale);
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < length; i++) v[i] = v[i] * scale_inv;
     }
 
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<M; i++) {
       Vector vecTmp;
       // first do scalar copy converting into storage type
-#pragma unroll
+QUDA_UNROLL
       for (int j = 0; j < N; j++) copy_scaled(reinterpret_cast<Float *>(&vecTmp)[j], v[i * N + j]);
       // second do vectorized copy into memory
       vector_store(field, parity * offset + x + stride * i, vecTmp);
@@ -1053,15 +1053,15 @@ namespace quda {
     real v[length_ghost];
     norm_type nrm = isFixed<Float>::value ? vector_load<float>(ghost_norm[2 * dim + dir], parity * faceVolumeCB[dim] + x) : 0.0;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < M_ghost; i++) {
       GhostVector vecTmp = vector_load<GhostVector>(ghost[2 * dim + dir],
                                                     parity * faceVolumeCB[dim] * M_ghost + i * faceVolumeCB[dim] + x);
-#pragma unroll
+QUDA_UNROLL
       for (int j = 0; j < N_ghost; j++) copy_and_scale(v[i * N_ghost + j], reinterpret_cast<Float *>(&vecTmp)[j], nrm);
     }
 
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < length_ghost / 2; i++) out[i] = complex(v[2 * i + 0], v[2 * i + 1]);
   }
 
@@ -1069,7 +1069,7 @@ namespace quda {
                                             int parity = 0) const
   {
     real v[length_ghost];
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < length_ghost / 2; i++) {
       v[2 * i + 0] = in[i].real();
       v[2 * i + 1] = in[i].imag();
@@ -1078,25 +1078,25 @@ namespace quda {
     if (isFixed<Float>::value) {
       norm_type max_[length_ghost / 2];
       // two-pass to increase ILP (assumes length divisible by two, e.g. complex-valued)
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < length_ghost / 2; i++)
         max_[i] = fmaxf( (norm_type)fabsf( (norm_type)v[i] ),
                          (norm_type)fabsf( (norm_type)v[i + length_ghost / 2] ) );
       norm_type scale = 0.0;
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < length_ghost / 2; i++) scale = fmaxf(max_[i], scale);
       ghost_norm[2 * dim + dir][parity * faceVolumeCB[dim] + x] = scale;
 
       real scale_inv = fdividef(fixedMaxValue<Float>::value, scale);
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < length_ghost; i++) v[i] = v[i] * scale_inv;
     }
 
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < M_ghost; i++) {
       GhostVector vecTmp;
       // first do scalar copy converting into storage type
-#pragma unroll
+QUDA_UNROLL
       for (int j = 0; j < N_ghost; j++) copy_scaled(reinterpret_cast<Float *>(&vecTmp)[j], v[i * N_ghost + j]);
       // second do vectorized copy into memory
       vector_store(ghost[2 * dim + dir], parity * faceVolumeCB[dim] * M_ghost + i * faceVolumeCB[dim] + x, vecTmp);
diff --git a/include/float_vector.h b/include/float_vector.h
index 83f9f54863..47b23dd0ea 100644
--- a/include/float_vector.h
+++ b/include/float_vector.h
@@ -125,12 +125,12 @@ namespace quda {
     constexpr int size() const { return n; }
     __device__ __host__ inline void operator+=(const vector_type &a)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < n; i++) data[i] += a[i];
     }
     __device__ __host__ vector_type()
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < n; i++) data[i] = zero<scalar>();
     }
 
@@ -158,7 +158,7 @@ namespace quda {
   template <typename scalar, int n> __device__ __host__ inline vector_type<scalar, n> zero()
   {
     vector_type<scalar, n> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < n; i++) v.data[i] = zero<scalar>();
     return v;
   }
@@ -166,7 +166,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 1> zero<vector_type<double2, 1>>()
   {
     vector_type<double2, 1> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 1; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -174,7 +174,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 2> zero<vector_type<double2, 2>>()
   {
     vector_type<double2, 2> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 2; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -182,7 +182,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 3> zero<vector_type<double2, 3>>()
   {
     vector_type<double2, 3> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 3; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -190,7 +190,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 4> zero<vector_type<double2, 4>>()
   {
     vector_type<double2, 4> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 4; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -198,7 +198,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 5> zero<vector_type<double2, 5>>()
   {
     vector_type<double2, 5> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 5; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -206,7 +206,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 6> zero<vector_type<double2, 6>>()
   {
     vector_type<double2, 6> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 6; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -214,7 +214,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 7> zero<vector_type<double2, 7>>()
   {
     vector_type<double2, 7> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 7; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -222,7 +222,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 8> zero<vector_type<double2, 8>>()
   {
     vector_type<double2, 8> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 8; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -230,7 +230,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 9> zero<vector_type<double2, 9>>()
   {
     vector_type<double2, 9> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 9; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -238,7 +238,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 10> zero<vector_type<double2, 10>>()
   {
     vector_type<double2, 10> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 10; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -246,7 +246,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 11> zero<vector_type<double2, 11>>()
   {
     vector_type<double2, 11> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 11; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -254,7 +254,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 12> zero<vector_type<double2, 12>>()
   {
     vector_type<double2, 12> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 12; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -262,7 +262,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 13> zero<vector_type<double2, 13>>()
   {
     vector_type<double2, 13> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 13; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -270,7 +270,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 14> zero<vector_type<double2, 14>>()
   {
     vector_type<double2, 14> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 14; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -278,7 +278,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 15> zero<vector_type<double2, 15>>()
   {
     vector_type<double2, 15> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 15; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -286,7 +286,7 @@ namespace quda {
   template <> __device__ __host__ inline vector_type<double2, 16> zero<vector_type<double2, 16>>()
   {
     vector_type<double2, 16> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < 16; i++) v.data[i] = zero<double2>();
     return v;
   }
@@ -296,7 +296,7 @@ namespace quda {
                                                               const vector_type<scalar, n> &b)
   {
     vector_type<scalar, n> c;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < n; i++) c[i] = a[i] + b[i];
     return c;
   }
diff --git a/include/gauge_field_order.h b/include/gauge_field_order.h
index 189dbd41e4..edd7a403ef 100644
--- a/include/gauge_field_order.h
+++ b/include/gauge_field_order.h
@@ -908,13 +908,13 @@ namespace quda {
         __device__ __host__ inline void Pack(real out[N], const complex in[N / 2]) const
         {
           if (isFixed<Float>::value) {
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < N / 2; i++) {
               out[2 * i + 0] = scale_inv * in[i].real();
               out[2 * i + 1] = scale_inv * in[i].imag();
             }
           } else {
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < N / 2; i++) {
               out[2 * i + 0] = in[i].real();
               out[2 * i + 1] = in[i].imag();
@@ -927,10 +927,10 @@ namespace quda {
                                                const I *, const int *) const
         {
           if (isFixed<Float>::value) {
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < N / 2; i++) { out[i] = scale * complex(in[2 * i + 0], in[2 * i + 1]); }
           } else {
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < N / 2; i++) { out[i] = complex(in[2 * i + 0], in[2 * i + 1]); }
           }
         }
@@ -1024,7 +1024,7 @@ namespace quda {
 
         __device__ __host__ inline void Pack(real out[12], const complex in[9]) const
         {
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < 6; i++) {
             out[2 * i + 0] = in[i].real();
             out[2 * i + 1] = in[i].imag();
@@ -1035,7 +1035,7 @@ namespace quda {
         __device__ __host__ inline void Unpack(complex out[9], const real in[12], int idx, int dir, real,
                                                const I *X, const int *R) const
         {
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < 6; i++) out[i] = complex(in[2 * i + 0], in[2 * i + 1]);
 
           const real u0 = dir < 3 ?
@@ -1080,7 +1080,7 @@ namespace quda {
 
         __device__ __host__ inline void Pack(real out[10], const complex in[9]) const
         {
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < 2; i++) {
             out[2 * i + 0] = in[i + 1].real();
             out[2 * i + 1] = in[i + 1].imag();
@@ -1138,7 +1138,7 @@ namespace quda {
         __device__ __host__ inline void Unpack(complex out[9], const real in[12], int, int, real phase,
                                                const I *, const int *) const
         {
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < 6; i++) out[i] = complex(in[2 * i + 0], in[2 * i + 1]);
 
           out[6] = cmul(out[2], out[4]);
@@ -1182,7 +1182,7 @@ namespace quda {
           }
 #else // phase from determinant
           Matrix<complex, 3> a;
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < 9; i++) a(i) = scale_inv * in[i];
           const complex det = getDeterminant(a);
           return phase = arg(det) / 3;
@@ -1244,7 +1244,7 @@ namespace quda {
           real u0 = u.real();
           real u0_inv = u.imag();
 
-#pragma unroll
+QUDA_UNROLL
           for (int i = 1; i <= 3; i++)
             out[i] = complex(in[2 * i + 0], in[2 * i + 1]); // these elements are copied directly
 
@@ -1310,7 +1310,7 @@ namespace quda {
 
           // Rearrange {{b1,b2,b3},{a1,a2,a3},{-c1,-c2,-c3}} back
           // to {{a1,a2,a3},{b1,b2,b3},{c1,c2,c3}}
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < 3; i++) {
             const auto tmp = out[i];
             out[i] = out[i + 3];
@@ -1366,7 +1366,7 @@ namespace quda {
           }
 #else // phase from determinant
           Matrix<complex, 3> a;
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < 9; i++) a(i) = scale_inv * in[i];
           const complex det = getDeterminant(a);
           real phase = arg(det) / 3;
@@ -1385,10 +1385,10 @@ namespace quda {
             Trig<isFixed<real>::value, real>::SinCos(static_cast<real>(-phase), &cos_sin[1], &cos_sin[0]);
             complex z(cos_sin[0], cos_sin[1]);
             z *= scale_inv;
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < 9; i++) su3[i] = cmul(z, in[i]);
           } else {
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < 9; i++) { su3[i] = phase * in[i]; }
           }
           reconstruct_8.Pack(out, su3);
@@ -1406,10 +1406,10 @@ namespace quda {
             Trig<isFixed<real>::value, real>::SinCos(static_cast<real>(phase), &cos_sin[1], &cos_sin[0]);
             complex z(cos_sin[0], cos_sin[1]);
             z *= scale;
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < 9; i++) out[i] = cmul(z, out[i]);
           } else { // stagic phase
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < 18; i++) { out[i] *= phase; }
           }
         }
@@ -1499,12 +1499,12 @@ namespace quda {
         const int M = reconLen / N;
         real tmp[reconLen];
 
-#pragma unroll
+QUDA_UNROLL
         for (int i=0; i<M; i++){
           // first load from memory
           Vector vecTmp = vector_load<Vector>(gauge, parity * offset + (dir * M + i) * stride + x);
           // second do copy converting into register type
-#pragma unroll
+QUDA_UNROLL
           for (int j = 0; j < N; j++) copy(tmp[i * N + j], reinterpret_cast<Float *>(&vecTmp)[j]);
         }
 
@@ -1527,11 +1527,11 @@ namespace quda {
         real tmp[reconLen];
         reconstruct.Pack(tmp, v);
 
-#pragma unroll
+QUDA_UNROLL
         for (int i=0; i<M; i++){
 	  Vector vecTmp;
 	  // first do copy converting into storage type
-#pragma unroll
+QUDA_UNROLL
 	  for (int j=0; j<N; j++) copy(reinterpret_cast<Float*>(&vecTmp)[j], tmp[i*N+j]);
 	  // second do vectorized copy into memory
           vector_store(gauge, parity * offset + x + (dir * M + i) * stride, vecTmp);
@@ -1566,13 +1566,13 @@ namespace quda {
           const int M = reconLen / N;
           real tmp[reconLen];
 
-#pragma unroll
+QUDA_UNROLL
           for (int i=0; i<M; i++) {
 	    // first do vectorized copy from memory into registers
             Vector vecTmp = vector_load<Vector>(
                 ghost[dir] + parity * faceVolumeCB[dir] * (M * N + hasPhase), i * faceVolumeCB[dir] + x);
             // second do copy converting into register type
-#pragma unroll
+QUDA_UNROLL
             for (int j = 0; j < N; j++) copy(tmp[i * N + j], reinterpret_cast<Float *>(&vecTmp)[j]);
           }
           real phase = 0.;
@@ -1599,11 +1599,11 @@ namespace quda {
           real tmp[reconLen];
           reconstruct.Pack(tmp, v);
 
-#pragma unroll
+QUDA_UNROLL
           for (int i=0; i<M; i++) {
 	    Vector vecTmp;
 	    // first do copy converting into storage type
-#pragma unroll
+QUDA_UNROLL
 	    for (int j=0; j<N; j++) copy(reinterpret_cast<Float*>(&vecTmp)[j], tmp[i*N+j]);
 	    // second do vectorized copy into memory
 	    vector_store(ghost[dir]+parity*faceVolumeCB[dir]*(M*N + hasPhase), i*faceVolumeCB[dir]+x, vecTmp);
@@ -1655,13 +1655,13 @@ namespace quda {
         const int M = reconLen / N;
         real tmp[reconLen];
 
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<M; i++) {
 	  // first do vectorized copy from memory
 	  Vector vecTmp = vector_load<Vector>(ghost[dim] + ((dir*2+parity)*geometry+g)*R[dim]*faceVolumeCB[dim]*(M*N + hasPhase),
 					      +i*R[dim]*faceVolumeCB[dim]+buff_idx);
 	  // second do copy converting into register type
-#pragma unroll
+QUDA_UNROLL
 	  for (int j=0; j<N; j++) copy(tmp[i*N+j], reinterpret_cast<Float*>(&vecTmp)[j]);
 	}
         real phase = 0.;
@@ -1681,11 +1681,11 @@ namespace quda {
         real tmp[reconLen];
         reconstruct.Pack(tmp, v);
 
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<M; i++) {
 	    Vector vecTmp;
 	    // first do copy converting into storage type
-#pragma unroll
+QUDA_UNROLL
 	    for (int j=0; j<N; j++) copy(reinterpret_cast<Float*>(&vecTmp)[j], tmp[i*N+j]);
 	    // second do vectorized copy to memory
 	    vector_store(ghost[dim] + ((dir*2+parity)*geometry+g)*R[dim]*faceVolumeCB[dim]*(M*N + hasPhase),
diff --git a/include/gauge_fix_ovr_hit_devf.cuh b/include/gauge_fix_ovr_hit_devf.cuh
index 209ef41567..7611b5a17c 100644
--- a/include/gauge_fix_ovr_hit_devf.cuh
+++ b/include/gauge_fix_ovr_hit_devf.cuh
@@ -175,7 +175,7 @@ namespace quda {
       if (mu == 0) {
         Float a0, a1, a2, a3;
         a0 = 0.0; a1 = 0.0; a2 = 0.0; a3 = 0.0;
-      #pragma unroll
+      QUDA_UNROLL
         for (int i = 0; i < gauge_dir; i++) {
           a0 += elems[tid + i * blockSize] + elems[tid + (i + 4) * blockSize];
           a1 += elems[tid + i * blockSize + blockSize * 8] + elems[tid + (i + 4) * blockSize + blockSize * 8];
@@ -252,7 +252,7 @@ namespace quda {
         elems[tid + blockSize * 3] = -(link(p,p).y - link(q,q).y);
       }
 
-#pragma unroll
+QUDA_UNROLL
       for (int mu_ = 1; mu_ < gauge_dir; mu_++) {
         cache.sync();
         if (mu_ == mu) {
@@ -263,7 +263,7 @@ namespace quda {
         }
       }
 
-#pragma unroll
+QUDA_UNROLL
       for (int mu_ = 4; mu_ < 4 + gauge_dir; mu_++) {
         cache.sync();
         if (mu_ == mu) {
@@ -435,7 +435,7 @@ namespace quda {
       if (mu == 0) {
         Float a0, a1, a2, a3;
         a0 = 0.0; a1 = 0.0; a2 = 0.0; a3 = 0.0;
-      #pragma unroll
+      QUDA_UNROLL
         for ( int i = 0; i < gauge_dir; i++ ) {
           a0 += elems[tid + i * blockSize];
           a1 += elems[tid + i * blockSize + blockSize * 4];
@@ -508,7 +508,7 @@ namespace quda {
         elems[tid + blockSize * 3] = (link1(p,p).y - link1(q,q).y) - (link(p,p).y - link(q,q).y);
       }
 
-#pragma unroll
+QUDA_UNROLL
       for (int mu_ = 1; mu_ < gauge_dir; mu_++) {
         cache.sync();
         if (mu_ == mu) {
diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index 2db11afbc6..f43ffdb38b 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -17,7 +17,7 @@ namespace quda {
   template <typename I, typename J, typename K>
   __device__ __host__ inline int linkIndexShift(const I &x, const J &dx, const K &X) {
     int y[4];
-#pragma unroll
+QUDA_UNROLL
     for ( int i = 0; i < 4; i++ ) y[i] = (x[i] + dx[i] + X[i]) % X[i];
     int idx = (((y[3] * X[2] + y[2]) * X[1] + y[1]) * X[0] + y[0]) >> 1;
     return idx;
@@ -34,7 +34,7 @@ namespace quda {
    */
   template <typename I, typename J, typename K, typename L>
   __device__ __host__ inline int linkIndexShift(I &y, const J &x, const K &dx, const L &X) {
-#pragma unroll
+QUDA_UNROLL
     for ( int i = 0; i < 4; i++ ) y[i] = (x[i] + dx[i] + X[i]) % X[i];
     int idx = (((y[3] * X[2] + y[2]) * X[1] + y[1]) * X[0] + y[0]) >> 1;
     return idx;
@@ -81,7 +81,7 @@ namespace quda {
   __device__ __host__ inline int linkIndexDn(const Coord &x, const I X[4], const int mu)
   {
     int y[4];
-#pragma unroll
+QUDA_UNROLL
     for ( int i = 0; i < 4; i++ ) y[i] = x[i];
     y[mu] = (y[mu] + n + X[mu]) % X[mu];
     int idx = (((y[3] * X[2] + y[2]) * X[1] + y[1]) * X[0] + y[0]) >> 1;
@@ -101,7 +101,7 @@ namespace quda {
   __device__ __host__ inline auto linkIndexHop(const Coord &x, const I X[4], const int mu, int nFace)
   {
     int y[4];
-#pragma unroll
+QUDA_UNROLL
     for ( int i = 0; i < 4; i++ ) y[i] = x[i];
     switch (mu) {
     case 0: y[0] = (y[0] + nFace + X[0]) % X[0]; break;
@@ -149,7 +149,7 @@ namespace quda {
   template <typename I>
   __device__ __host__ inline int linkNormalIndexP1(const int x[], const I X[4], const int mu) {
     int y[4];
-#pragma unroll
+QUDA_UNROLL
     for ( int i = 0; i < 4; i++ ) y[i] = x[i];
     y[mu] = (y[mu] + 1 + X[mu]) % X[mu];
     int idx = ((y[3] * X[2] + y[2]) * X[1] + y[1]) * X[0] + y[0];
@@ -292,7 +292,7 @@ namespace quda {
     x[2] = (zb - x[3] * X[2]);
     int x1odd = (x[1] + x[2] + x[3] + parity) & 1;
     x[0] = (2 * cb_index + x1odd  - za * X[0]);
-#pragma unroll
+QUDA_UNROLL
     for (int d=0; d<4; d++) x[d] += R[d];
     return;
   }
diff --git a/include/kernels/blas_core.cuh b/include/kernels/blas_core.cuh
index 58b0a92e9a..55e26d1724 100644
--- a/include/kernels/blas_core.cuh
+++ b/include/kernels/blas_core.cuh
@@ -99,7 +99,7 @@ namespace quda
       axpbyz_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &v) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) v[i] = a * x[i] + b * y[i];
       }                                  // use v not z to ensure same precision as y
       constexpr int flops() const { return 3; }   //! flops per element
@@ -115,7 +115,7 @@ namespace quda
       ax_(const real &a, const real &, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) x[i] *= a;
       }
       constexpr int flops() const { return 1; }   //! flops per element
@@ -131,7 +131,7 @@ namespace quda
       caxpy_(const complex<real> &a, const complex<real> &, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) y[i] = cmac(a, x[i], y[i]);
       }
       constexpr int flops() const { return 4; }   //! flops per element
@@ -164,7 +164,7 @@ namespace quda
       caxpby_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) _caxpby(a, x[i], b, y[i]);
       }
       constexpr int flops() const { return 7; }   //! flops per element
@@ -179,7 +179,7 @@ namespace quda
       caxpbypczw_(const complex<real> &a, const complex<real> &b, const complex<real> &c) : a(a), b(b), c(c) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &w, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           w[i] = y[i];
           _caxpby(a, x[i], b, w[i]);
@@ -201,7 +201,7 @@ namespace quda
       axpyBzpcx_(const real &a, const real &b, const real &c) : a(a), b(b), c(c) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] += a * x[i];
           x[i] = b * z[i] + c * x[i];
@@ -221,7 +221,7 @@ namespace quda
       axpyZpbx_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] += a * x[i];
           x[i] = z[i] + b * x[i];
@@ -241,7 +241,7 @@ namespace quda
       caxpyBzpx_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           x[i] = cmac(b, z[i], x[i]);
@@ -261,7 +261,7 @@ namespace quda
       caxpyBxpz_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           z[i] = cmac(b, x[i], z[i]);
@@ -281,7 +281,7 @@ namespace quda
       caxpbypzYmbw_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &w, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           z[i] = cmac(a, x[i], z[i]);
           z[i] = cmac(b, y[i], z[i]);
@@ -302,7 +302,7 @@ namespace quda
       cabxpyAx_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a.real()), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           x[i] *= a;
           y[i] = cmac(b, x[i], y[i]);
@@ -323,7 +323,7 @@ namespace quda
       caxpyxmaz_(const complex<real> &a, const complex<real> &, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           x[i] = cmac(-a, z[i], x[i]);
@@ -363,7 +363,7 @@ namespace quda
 
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           x[i] = cmac(-a, z[i], x[i]);
@@ -387,7 +387,7 @@ namespace quda
       tripleCGUpdate_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &w, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] += a * w[i];
           z[i] -= a * x[i];
diff --git a/include/kernels/block_orthogonalize.cuh b/include/kernels/block_orthogonalize.cuh
index 56e86dddb3..1ad46f8525 100644
--- a/include/kernels/block_orthogonalize.cuh
+++ b/include/kernels/block_orthogonalize.cuh
@@ -111,17 +111,17 @@ namespace quda {
 
     __device__ __host__ inline void load(ColorSpinor<real, nColor, spinBlock> &v, int parity, int x_cb, int chirality, int i)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int s = 0; s < spinBlock; s++)
-#pragma unroll
+QUDA_UNROLL
         for (int c = 0; c < nColor; c++) v(s, c) = arg.V(parity, x_cb, chirality * spinBlock + s, c, i);
     }
 
     __device__ __host__ inline void save(int parity, int x_cb, int chirality, int i, const ColorSpinor<real, nColor, spinBlock> &v)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int s = 0; s < spinBlock; s++)
-#pragma unroll
+QUDA_UNROLL
         for (int c = 0; c < nColor; c++) arg.V(parity, x_cb, chirality * spinBlock + s, c, i) = v(s, c);
     }
 
@@ -135,7 +135,7 @@ namespace quda {
       int x_offset_cb[n_sites_per_thread];
       int x_cb[n_sites_per_thread];
 
-#pragma unroll
+QUDA_UNROLL
       for (int tx = 0; tx < n_sites_per_thread; tx++) {
         int x_fine_offset_tx = x_fine_offset * n_sites_per_thread + tx;
         // all threads with x_fine_offset greater than aggregate_size_cb are second parity
@@ -157,19 +157,19 @@ namespace quda {
 
           ColorSpinor<real, nColor, spinBlock> v[mVec][n_sites_per_thread];
 
-#pragma unroll
+QUDA_UNROLL
           for (int tx = 0; tx < n_sites_per_thread; tx++) {
             if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
             if (n == 0) { // load from B on first Gram-Schmidt, otherwise V.
               if (chirality == 0) {
-#pragma unroll
+QUDA_UNROLL
                 for (int m = 0; m < mVec; m++) arg.B[j+m].template load<spinBlock>(v[m][tx].data, parity[tx], x_cb[tx], 0);
               } else {
-#pragma unroll
+QUDA_UNROLL
                 for (int m = 0; m < mVec; m++) arg.B[j+m].template load<spinBlock>(v[m][tx].data, parity[tx], x_cb[tx], 1);
               }
             } else {
-#pragma unroll
+QUDA_UNROLL
               for (int m = 0; m < mVec; m++) load(v[m][tx], parity[tx], x_cb[tx], chirality, j + m);
             }
           }
@@ -178,45 +178,45 @@ namespace quda {
             ColorSpinor<real, nColor, spinBlock> vi[n_sites_per_thread];
 
             dot_t dot;
-#pragma unroll
+QUDA_UNROLL
             for (int tx = 0; tx < n_sites_per_thread; tx++) {
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
               load(vi[tx], parity[tx], x_cb[tx], chirality, i);
 
-#pragma unroll
+QUDA_UNROLL
               for (int m = 0; m < mVec; m++) dot[m] += innerProduct(vi[tx], v[m][tx]);
             }
 
             dot = dot_reducer.AllSum(dot);
 
             // subtract the blocks to orthogonalise
-#pragma unroll
+QUDA_UNROLL
             for (int tx = 0; tx < n_sites_per_thread; tx++) {
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
-#pragma unroll
+QUDA_UNROLL
               for (int m = 0; m < mVec; m++) caxpy(-complex<real>(dot[m].real(), dot[m].imag()), vi[tx], v[m][tx]);
             }
           } // i
 
           // now orthogonalize over the block diagonal and normalize each entry
-#pragma unroll
+QUDA_UNROLL
           for (int m = 0; m < mVec; m++) {
 
             dot_t dot;
-#pragma unroll
+QUDA_UNROLL
             for (int tx = 0; tx < n_sites_per_thread; tx++) {
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
-#pragma unroll
+QUDA_UNROLL
               for (int i = 0; i < m; i++) dot[i] += innerProduct(v[i][tx], v[m][tx]);
             }
             
             dot = dot_reducer.AllSum(dot);
             
             sum_t nrm = 0.0;
-#pragma unroll
+QUDA_UNROLL
             for (int tx = 0; tx < n_sites_per_thread; tx++) {
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
-#pragma unroll
+QUDA_UNROLL
               for (int i = 0; i < m; i++) caxpy(-complex<real>(dot[i].real(), dot[i].imag()), v[i][tx], v[m][tx]); // subtract the blocks to orthogonalise
               nrm += norm2(v[m][tx]);
             }
@@ -224,17 +224,17 @@ namespace quda {
             nrm = norm_reducer.AllSum(nrm);
             auto nrm_inv = nrm > 0.0 ? quda::rsqrt(nrm) : 0.0;
 
-#pragma unroll
+QUDA_UNROLL
             for (int tx = 0; tx < n_sites_per_thread; tx++) {
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
               v[m][tx] *= nrm_inv;
             }
           }
 
-#pragma unroll
+QUDA_UNROLL
           for (int tx = 0; tx < n_sites_per_thread; tx++) {
             if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
-#pragma unroll
+QUDA_UNROLL
             for (int m = 0; m < mVec; m++) save(parity[tx], x_cb[tx], chirality, j + m, v[m][tx]);
           }
         } // j
diff --git a/include/kernels/clover_compute.cuh b/include/kernels/clover_compute.cuh
index 67f5560845..7677b04236 100644
--- a/include/kernels/clover_compute.cuh
+++ b/include/kernels/clover_compute.cuh
@@ -72,7 +72,7 @@ namespace quda {
 
       // Load the field-strength tensor from global memory
       Link F[6];
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<6; ++i) F[i] = arg.f(i, x_cb, parity);
 
       Complex I(0.0,1.0);
@@ -84,12 +84,12 @@ namespace quda {
       block2[1] = arg.coeff*(F[1]-F[4] - I*(F[2]+F[3])); // 126 floating-point ops
 
       // This uses lots of unnecessary memory
-#pragma unroll
+QUDA_UNROLL
       for (int ch=0; ch<2; ++ch) {
         HMatrix<real,N> A;
         // c = 0(1) => positive(negative) chiral block
         // Compute real diagonal elements
-#pragma unroll
+QUDA_UNROLL
         for (int i=0; i<N/2; ++i) {
           A(i+0,i+0) = 1.0 - block1[ch](i,i).real();
           A(i+3,i+3) = 1.0 + block1[ch](i,i).real();
diff --git a/include/kernels/clover_outer_product.cuh b/include/kernels/clover_outer_product.cuh
index a915521e54..4acf4ca6b8 100644
--- a/include/kernels/clover_outer_product.cuh
+++ b/include/kernels/clover_outer_product.cuh
@@ -63,7 +63,7 @@ namespace quda {
       Spinor A = arg.inA(x_cb, 0);
       Spinor C = arg.inC(x_cb, 0);
 
-#pragma unroll
+QUDA_UNROLL
       for (int dim=0; dim<4; ++dim) {
         int shift[4] = {0, 0, 0, 0};
         shift[dim] = 1;
diff --git a/include/kernels/clover_sigma_outer_product.cuh b/include/kernels/clover_sigma_outer_product.cuh
index 91d2de8ef5..1ae9f015b4 100644
--- a/include/kernels/clover_sigma_outer_product.cuh
+++ b/include/kernels/clover_sigma_outer_product.cuh
@@ -47,7 +47,7 @@ namespace quda
     using Link = Matrix<complex<typename Arg::real>, Arg::nColor>;
     Link result;
 
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < Arg::nvector; i++) {
       const Spinor A = arg.inA[i](x_cb, parity);
       const Spinor B = arg.inB[i](x_cb, parity);
diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 8fecb83189..93816c606f 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -265,11 +265,11 @@ namespace quda {
 
       int ghost_idx = ghostFaceIndex<1>(coord, arg.x_size, arg.dim, nFace);
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of gauge field
         auto U = make_tile_A<complex, false>(tile);
         U.load(arg.U, arg.dim, parity, x_cb, i0, k);
-#pragma unroll
+QUDA_UNROLL
         for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
           auto W = make_tile_B<complex, true>(tile);
           W.loadCS(Wacc, arg.dim, 1, (parity+1)&1, ghost_idx, s, k, j0);
@@ -281,11 +281,11 @@ namespace quda {
 
       int y_cb = linkIndexHop(coord, arg.x_size, arg.dim, nFace);
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of gauge field
         auto U = make_tile_A<complex, false>(tile);
         U.load(arg.U, arg.dim, parity, x_cb, i0, k);
-#pragma unroll
+QUDA_UNROLL
         for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
           auto W = make_tile_B<complex, false>(tile);
           W.loadCS(Wacc, 0, 0, (parity+1)&1, y_cb, s, k, j0);
@@ -295,7 +295,7 @@ namespace quda {
     }
 
     real uv_max = static_cast<real>(0.0);
-#pragma unroll
+QUDA_UNROLL
     for (int s = 0; s < uvSpin; s++) {
       IF_CONSTEXPR (Arg::compute_max) {
         uv_max = fmax(UV[s].abs_max(), uv_max);
@@ -339,13 +339,13 @@ namespace quda {
 
     if (arg.dir == QUDA_IN_PLACE) {
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of coarse clover field
-#pragma unroll
+QUDA_UNROLL
         for (int s_col = 0; s_col < Arg::fineSpin; s_col++) {
           auto W = make_tile_B<complex, false>(tile);
           W.loadCS(Wacc, 0, 0, parity, x_cb, s_col, k, j0);
-#pragma unroll
+QUDA_UNROLL
           for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
             auto C = make_tile_A<complex, false>(tile);
             C.load(arg.C, 0, parity, x_cb, s, s_col, i0, k);
@@ -358,13 +358,13 @@ namespace quda {
 
       int ghost_idx = ghostFaceIndex<1>(coord, arg.x_size, arg.dim, nFace);
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of gauge field
-#pragma unroll
+QUDA_UNROLL
         for (int s_col=0; s_col<Arg::fineSpin; s_col++) {
           auto W = make_tile_B<complex, true>(tile);
           W.loadCS(Wacc, arg.dim, 1, (parity+1)&1, ghost_idx, s_col, k, j0);
-#pragma unroll
+QUDA_UNROLL
           for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
             // on coarse lattice, if forwards then use forwards links
             auto U = make_tile_A<complex, false>(tile);
@@ -379,13 +379,13 @@ namespace quda {
 
       int y_cb = linkIndexHop(coord, arg.x_size, arg.dim, nFace);
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of gauge field
-#pragma unroll
+QUDA_UNROLL
         for (int s_col = 0; s_col < Arg::fineSpin; s_col++) {
           auto W = make_tile_B<complex, false>(tile);
           W.loadCS(Wacc, 0, 0, (parity+1)&1, y_cb, s_col, k, j0);
-#pragma unroll
+QUDA_UNROLL
           for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
             // on coarse lattice, if forwards then use forwards links
             auto U = make_tile_A<complex, false>(tile);
@@ -398,7 +398,7 @@ namespace quda {
     }
 
     real uv_max = static_cast<real>(0.0);
-#pragma unroll
+QUDA_UNROLL
     for (int s = 0; s < uvSpin; s++) {
       IF_CONSTEXPR (Arg::compute_max) {
         uv_max = fmax(UV[s].abs_max(), uv_max);
@@ -472,11 +472,11 @@ namespace quda {
       constexpr int N = Arg::fineSpin * Arg::fineColor / 2;
       HMatrix<real, N> A;
 
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < N; i++) {
         int s_i = 2 * ch + i / Arg::fineColor;
         int c_i = i % Arg::fineColor;
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j <= i; j++) {
           int s_j = 2 * ch + j / Arg::fineColor;
           int c_j = j % Arg::fineColor;
@@ -502,16 +502,16 @@ namespace quda {
 #endif
 
       if (!Arg::compute_max) {
-#pragma unroll
+QUDA_UNROLL
         for (int s = 0; s < Arg::fineSpin / 2; s++) {
-#pragma unroll
+QUDA_UNROLL
           for (int ic = 0; ic < Arg::fineColor; ic++) { arg.AV(parity, x_cb, 2 * ch + s, ic, ic_c) = AV(s, ic); }
         }
       } else {
         real max = static_cast<real>(0.0);
-#pragma unroll
+QUDA_UNROLL
         for (int s = 0; s < Arg::fineSpin / 2; s++) {
-#pragma unroll
+QUDA_UNROLL
           for (int ic = 0; ic < Arg::fineColor; ic++) {
             auto abs_max = fmax(abs(AV(s, ic).real()), abs(AV(s, ic).imag()));
             max = fmax(abs_max, max);
@@ -543,17 +543,17 @@ namespace quda {
       complex<real> fp(1./(1.+arg.mu*arg.mu),-arg.mu/(1.+arg.mu*arg.mu));
       complex<real> fm(1./(1.+arg.mu*arg.mu),+arg.mu/(1.+arg.mu*arg.mu));
 
-#pragma unroll
+QUDA_UNROLL
       for (int s = 0; s < Arg::fineSpin/2; s++) {
-#pragma unroll
+QUDA_UNROLL
         for (int c = 0; c < Arg::fineColor; c++) {
           arg.AV(parity,x_cb,s,c,v) = arg.V(parity,x_cb,s,c,v) * fp;
         }
       }
 
-#pragma unroll
+QUDA_UNROLL
       for (int s = Arg::fineSpin/2; s < Arg::fineSpin; s++) {
-#pragma unroll
+QUDA_UNROLL
         for (int c = 0; c < Arg::fineColor; c++) {
           arg.AV(parity,x_cb,s,c,v) = arg.V(parity,x_cb,s,c,v) * fm;
         }
@@ -589,11 +589,11 @@ namespace quda {
       constexpr int N = Arg::fineSpin * Arg::fineColor / 2;
       HMatrix<real, N> A;
 
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < N; i++) {
         int s_i = 2 * ch + i / Arg::fineColor;
         int c_i = i % Arg::fineColor;
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j <= i; j++) {
           int s_j = 2 * ch + j / Arg::fineColor;
           int c_j = j % Arg::fineColor;
@@ -606,9 +606,9 @@ namespace quda {
 
       ColorSpinor<real, Arg::fineColor, Arg::fineSpin / 2> V;
 
-#pragma unroll
+QUDA_UNROLL
       for (int s = 0; s < Arg::fineSpin / 2; s++) {
-#pragma unroll
+QUDA_UNROLL
         for (int c = 0; c < Arg::fineColor; c++) {
           V(s, c) = arg.V(parity, x_cb, 2 * ch + s, c, ic_c);
         }
@@ -624,11 +624,11 @@ namespace quda {
       if (!dynamic_clover_inverse()) {
         // load in the clover inverse matrix
         HMatrix<real, N> Ainv;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < N; i++) {
           int s_i = 2 * ch + i / Arg::fineColor;
           int c_i = i % Arg::fineColor;
-#pragma unroll
+QUDA_UNROLL
           for (int j = 0; j <= i; j++) {
             int s_j = 2 * ch + j / Arg::fineColor;
             int c_j = j % Arg::fineColor;
@@ -638,16 +638,16 @@ namespace quda {
         auto AV = Ainv * UV;
 
         if (!Arg::compute_max) {
-#pragma unroll
+QUDA_UNROLL
           for (int s = 0; s < Arg::fineSpin / 2; s++)
-#pragma unroll
+QUDA_UNROLL
             for (int c = 0; c < Arg::fineColor; c++)
               arg.AV(parity, x_cb, 2 * ch + s, c, ic_c) = AV(s, c);
         } else {
           real max = static_cast<real>(0.0);
-#pragma unroll
+QUDA_UNROLL
           for (int s = 0; s < Arg::fineSpin / 2; s++) {
-#pragma unroll
+QUDA_UNROLL
             for (int c = 0; c < Arg::fineColor; c++) {
               auto abs_max = fmax(abs(AV(s, c).real()), abs(AV(s, c).imag()));
               max = fmax(abs_max, max);
@@ -664,16 +664,16 @@ namespace quda {
         const auto AV = cholesky.backward(cholesky.forward(UV));
 
         if (!Arg::compute_max) {
-#pragma unroll
+QUDA_UNROLL
           for (int s = 0; s < Arg::fineSpin / 2; s++)
-#pragma unroll
+QUDA_UNROLL
             for (int c = 0; c < Arg::fineColor; c++)
               arg.AV(parity, x_cb, 2 * ch + s, c, ic_c) = AV(s, c);
         } else {
           real max = static_cast<real>(0.0);
-#pragma unroll
+QUDA_UNROLL
           for (int s = 0; s < Arg::fineSpin / 2; s++) {
-#pragma unroll
+QUDA_UNROLL
             for (int c = 0; c < Arg::fineColor; c++) {
               auto abs_max = fmax(abs(AV(s, c).real()), abs(AV(s, c).imag()));
               max = fmax(abs_max, max);
@@ -731,7 +731,7 @@ namespace quda {
     using TileType = typename Arg::vuvTileType;
     auto &tile = arg.vuvTile;
 
-#pragma unroll
+QUDA_UNROLL
     for (int s = 0; s < Arg::fineSpin; s++) { // Loop over fine spin
 
       //Spin part of the color matrix.  Will always consist
@@ -749,7 +749,7 @@ namespace quda {
       const int s_col = gamma.getcol(s);
       const int s_c_col = 1 - s_c_row; // always off-diagonal relative to row coord
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < TileType::k; k += TileType::K) { // Sum over fine color
 
         if (arg.dir == QUDA_BACKWARDS) {
@@ -764,9 +764,9 @@ namespace quda {
 
           //Off-diagonal Spin (backward link / positive projector applied)
           auto gammaV = make_tile_A<complex, false>(tile);
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < TileType::K; i++)
-#pragma unroll
+QUDA_UNROLL
             for (int j = 0; j < TileType::M; j++)
               gammaV(j, i) = gamma.apply(s, conj(V(i, j)));
 
@@ -785,9 +785,9 @@ namespace quda {
           //Off-diagonal Spin (forward link / negative projector applied)
           auto gammaAV = make_tile_A<complex, false>(tile);
 
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < TileType::K; i++)
-#pragma unroll
+QUDA_UNROLL
             for (int j = 0; j < TileType::M; j++)
               gammaAV(j, i) = -gamma.apply(s, conj(AV(i, j)));
 
@@ -898,13 +898,13 @@ namespace quda {
     using TileType = typename Arg::vuvTileType;
     auto &tile = arg.vuvTile;
 
-#pragma unroll
+QUDA_UNROLL
     for (int k = 0; k < TileType::k; k += TileType::K) { // Sum over fine color
-#pragma unroll
+QUDA_UNROLL
       for (int s = 0; s < Arg::fineSpin; s++) {
         auto AV = make_tile_At<complex, false>(tile);
         AV.loadCS(arg.AV, 0, 0, parity, x_cb, s, k, i0);
-#pragma unroll
+QUDA_UNROLL
         for (int s_col = 0; s_col < Arg::fineSpin; s_col++) { // which chiral block
           auto UV = make_tile_B<complex, false>(tile);
           UV.loadCS(arg.UV, 0, 0, parity, x_cb, s_col*Arg::fineSpin+s, k, j0);
@@ -964,9 +964,9 @@ namespace quda {
       int i_block0 = (threadIdx.y / (arg.parity_flip ? 1 : 2)) * TileType::M;
       int j_block0 = threadIdx.z * TileType::N;
 
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < TileType::M; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < TileType::N; j++) {
           if (tx < Arg::coarseSpin*Arg::coarseSpin) {
             if (pack.dir != QUDA_IN_PLACE) Y[i_block0+i][j_block0+j][x_][s_row][s_col] = 0;
@@ -977,24 +977,24 @@ namespace quda {
 
       __syncthreads();
 
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < TileType::M; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < TileType::N; j++) {
 
           if (pack.dir == QUDA_IN_PLACE || isDiagonal) {
-#pragma unroll
+QUDA_UNROLL
             for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-#pragma unroll
+QUDA_UNROLL
               for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
                 atomic_helper<real, storeType>(&X[i_block0+i][j_block0+j][x_][s_row][s_col],
                                                arg.X_atomic, vuv[s_row*Arg::coarseSpin+s_col](i,j));
               }
             }
           } else {
-#pragma unroll
+QUDA_UNROLL
             for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-#pragma unroll
+QUDA_UNROLL
               for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
                 atomic_helper<real, storeType>(&Y[i_block0+i][j_block0+j][x_][s_row][s_col],
                                                arg.Y_atomic, vuv[s_row*Arg::coarseSpin+s_col](i,j));
@@ -1008,9 +1008,9 @@ namespace quda {
 
       if (tx < Arg::coarseSpin*Arg::coarseSpin && (parity == 0 || arg.parity_flip == 1) ) {
 
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < TileType::M; i++) {
-#pragma unroll
+QUDA_UNROLL
           for (int j = 0; j < TileType::N; j++) {
             if (pack.dir == QUDA_IN_PLACE) {
               // same as dir == QUDA_FORWARDS
@@ -1065,25 +1065,25 @@ namespace quda {
 
     if (arg.dir == QUDA_IN_PLACE) {
       // same as dir == QUDA_FORWARDS
-#pragma unroll
+QUDA_UNROLL
       for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-#pragma unroll
+QUDA_UNROLL
         for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < TileType::M; i++)
-#pragma unroll
+QUDA_UNROLL
             for (int j = 0; j < TileType::N; j++)
               arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,s_row,s_col,i0+i,j0+j,vuv[s_row*Arg::coarseSpin+s_col](i,j));
         }
       }
     } else if (!isDiagonal) {
-#pragma unroll
+QUDA_UNROLL
       for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-#pragma unroll
+QUDA_UNROLL
         for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < TileType::M; i++)
-#pragma unroll
+QUDA_UNROLL
             for (int j = 0; j < TileType::N; j++)
               arg.Y_atomic.atomicAdd(dim_index,coarse_parity,coarse_x_cb,s_row,s_col,i0+i,j0+j,vuv[s_row*Arg::coarseSpin+s_col](i,j));
         }
@@ -1091,25 +1091,25 @@ namespace quda {
     } else {
 
       if (arg.dir == QUDA_BACKWARDS) {
-#pragma unroll
+QUDA_UNROLL
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-#pragma unroll
+QUDA_UNROLL
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < TileType::M; i++)
-#pragma unroll
+QUDA_UNROLL
               for (int j = 0; j < TileType::N; j++)
                 arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,s_col,s_row,j0+j,i0+i,conj(vuv[s_row*Arg::coarseSpin+s_col](i,j)));
           }
         }
       } else {
-#pragma unroll
+QUDA_UNROLL
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-#pragma unroll
+QUDA_UNROLL
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < TileType::M; i++)
-#pragma unroll
+QUDA_UNROLL
               for (int j = 0; j < TileType::N; j++)
                 arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,s_row,s_col,i0+i,j0+j,vuv[s_row*Arg::coarseSpin+s_col](i,j));
           }
@@ -1117,15 +1117,15 @@ namespace quda {
       }
 
       if (!arg.bidirectional) {
-#pragma unroll
+QUDA_UNROLL
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-#pragma unroll
+QUDA_UNROLL
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
             if (s_row != s_col) vuv[s_row * Arg::coarseSpin + s_col] *= static_cast<real>(-1.0);
             if (Arg::fineSpin != 1 || s_row != s_col) {
-#pragma unroll
+QUDA_UNROLL
               for (int i = 0; i < TileType::M; i++)
-#pragma unroll
+QUDA_UNROLL
                 for (int j = 0; j < TileType::N; j++)
                   arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,s_row,s_col,i0+i,j0+j,vuv[s_row*Arg::coarseSpin+s_col](i,j));
             }
@@ -1165,7 +1165,7 @@ namespace quda {
     multiplyVUV(vuv, arg, parity, x_cb, i0, j0);
 
     if (isDiagonal && !isFromCoarseClover) {
-#pragma unroll
+QUDA_UNROLL
       for (int s2=0; s2<Arg::coarseSpin*Arg::coarseSpin; s2++) vuv[s2] *= -arg.kappa;
     }
 
@@ -1296,17 +1296,17 @@ namespace quda {
       for (int i = 0; i < Arg::coarseSpin * Arg::coarseSpin; i++) X[i] = 0.0;
 
       // If Nspin = 4, then the clover term has structure C_{\mu\nu} = \gamma_{\mu\nu}C^{\mu\nu}
-#pragma unroll
+QUDA_UNROLL
       for (int s = 0; s < Arg::fineSpin; s++) { // Loop over fine spin row
         const int s_c = arg.spin_map(s,parity);
         // On the fine lattice, the clover field is chirally blocked, so loop over rows/columns
         // in the same chiral block.
-#pragma unroll
+QUDA_UNROLL
         for (int s_col = s_c * arg.spin_bs; s_col < (s_c+1) * arg.spin_bs; s_col++) { // Loop over fine spin column
-#pragma unroll
+QUDA_UNROLL
           for (int ic = 0; ic < Arg::fineColor; ic++) { // Sum over fine color row
             complex<real> CV = 0.0;
-#pragma unroll
+QUDA_UNROLL
             for (int jc = 0; jc < Arg::fineColor; jc++) {  // Sum over fine color column
               CV = cmac(arg.C(0, parity, x_cb, s, s_col, ic, jc), arg.V(parity, x_cb, s_col, jc, c_col), CV);
             } // Fine color column
@@ -1315,9 +1315,9 @@ namespace quda {
         }  // Fine spin column
       } // Fine spin
 
-#pragma unroll
+QUDA_UNROLL
       for (int si = 0; si < Arg::coarseSpin; si++) {
-#pragma unroll
+QUDA_UNROLL
         for (int sj = 0; sj < Arg::coarseSpin; sj++) {
           arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,si,sj, c_row, c_col, X[si*Arg::coarseSpin+sj]);
         }
@@ -1346,11 +1346,11 @@ namespace quda {
       int parity = parity_c_col / Arg::coarseColor;
       int c_col = parity_c_col % Arg::coarseColor;
 
-#pragma unroll
+QUDA_UNROLL
       for (int d=0; d<4; d++) {
-#pragma unroll
+QUDA_UNROLL
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { //Spin row
-#pragma unroll
+QUDA_UNROLL
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { //Spin column
             if (s_row == s_col && Arg::coarseSpin != 1)
               arg.Y(d+4, parity, x_cb, s_row, s_col, c_row, c_col) = arg.Y(d, parity, x_cb, s_row, s_col, c_row, c_col);
@@ -1463,9 +1463,9 @@ namespace quda {
         int d_in = arg.dim_index % in.geometry;
         int d_out = arg.dim_index % arg.Y.geometry;
 
-#pragma unroll
+QUDA_UNROLL
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { //Spin row
-#pragma unroll
+QUDA_UNROLL
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { //Spin column
             complex<real> M = in(d_in,parity,x_cb,s_row,s_col,c_row,c_col);
             arg.Y(d_out,parity,x_cb,s_row,s_col,c_row,c_col) = M;
@@ -1476,9 +1476,9 @@ namespace quda {
         int d_in = arg.dim_index % in.geometry;
         int d_out = arg.dim_index % arg.X.geometry;
 
-#pragma unroll
+QUDA_UNROLL
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { //Spin row
-#pragma unroll
+QUDA_UNROLL
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { //Spin column
             complex<real> M = in(d_in,parity,x_cb,s_row,s_col,c_row,c_col);
             arg.X(d_out,parity,x_cb,s_row,s_col,c_row,c_col) = M;
@@ -1508,9 +1508,9 @@ namespace quda {
       int c_col = parity_c_col % Arg::coarseColor; // color col index
       int parity = parity_c_col / Arg::coarseColor;
 
-#pragma unroll
+QUDA_UNROLL
       for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { //Spin row
-#pragma unroll
+QUDA_UNROLL
         for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { //Spin column
           complex<real> M = arg.Y(arg.dim_index,parity,x_cb,s_row,s_col,c_row,c_col);
           arg.Y(arg.dim_index,parity,x_cb,s_row,s_col,c_row,c_col) = arg.rescale*M;
diff --git a/include/kernels/coarse_op_preconditioned.cuh b/include/kernels/coarse_op_preconditioned.cuh
index 32a8dda770..dca015603f 100644
--- a/include/kernels/coarse_op_preconditioned.cuh
+++ b/include/kernels/coarse_op_preconditioned.cuh
@@ -59,7 +59,7 @@ namespace quda {
 
       auto yHat = make_tile_C<complex,true>(arg.tile);
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < Arg::yhatTileType::k; k += Arg::yhatTileType::K) {
         auto Y = make_tile_A<complex, true>(arg.tile);
         Y.load(arg.Y, d, 1-parity, ghost_idx, i0, k);
@@ -81,7 +81,7 @@ namespace quda {
 
       auto yHat = make_tile_C<complex,false>(arg.tile);
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < Arg::yhatTileType::k; k += Arg::yhatTileType::K) {
         auto Y = make_tile_A<complex, false>(arg.tile);
         Y.load(arg.Y, d, 1-parity, back_idx, i0, k);
@@ -101,7 +101,7 @@ namespace quda {
     { // now do the forwards links X^{-1} * Y^{-\mu}
       auto yHat = make_tile_C<complex, false>(arg.tile);
 
-#pragma unroll
+QUDA_UNROLL
       for (int k = 0; k < Arg::yhatTileType::k; k += Arg::yhatTileType::K) {
         auto X = make_tile_A<complex, false>(arg.tile);
         X.load(arg.Xinv, 0, parity, x_cb, i0, k);
diff --git a/include/kernels/color_spinor_pack.cuh b/include/kernels/color_spinor_pack.cuh
index 9e45f97d05..ce9450893b 100644
--- a/include/kernels/color_spinor_pack.cuh
+++ b/include/kernels/color_spinor_pack.cuh
@@ -127,7 +127,7 @@ namespace quda {
       cache.save(thread_max);
       cache.sync();
       real this_site_max = static_cast<real>(0);
-#pragma unroll
+QUDA_UNROLL
       for (int sc = 0; sc < color_spin_threads; sc++) {
         auto sc_max = cache.load_y(sc);
         this_site_max = this_site_max > sc_max ? this_site_max : sc_max;
@@ -154,10 +154,10 @@ namespace quda {
     real thread_max_r = 0.0;
     real thread_max_i = 0.0;
 
-#pragma unroll
+QUDA_UNROLL
     for (int spin_local=0; spin_local<Ms; spin_local++) {
       int s = spin_block + spin_local;
-#pragma unroll
+QUDA_UNROLL
       for (int color_local=0; color_local<Mc; color_local++) {
         int c = color_block + color_local;
         complex<real> z = arg.field(spinor_parity, x_cb, s, c);
@@ -228,10 +228,10 @@ namespace quda {
       int x_cb = indexFromFaceIndex(dim, dir, ghost_idx, parity, arg);
       auto max = compute_site_max<Arg>(arg, x_cb, spinor_parity, spin_block, color_block);
 
-#pragma unroll
+QUDA_UNROLL
       for (int spin_local=0; spin_local<Ms; spin_local++) {
         int s = spin_block + spin_local;
-#pragma unroll
+QUDA_UNROLL
         for (int color_local=0; color_local<Mc; color_local++) {
           int c = color_block + color_local;
           arg.field.Ghost(dim, dir, spinor_parity, ghost_idx, s, c, 0, max) = arg.field(spinor_parity, x_cb, s, c);
diff --git a/include/kernels/contraction.cuh b/include/kernels/contraction.cuh
index b1f6e7f052..7aa5ef8d25 100644
--- a/include/kernels/contraction.cuh
+++ b/include/kernels/contraction.cuh
@@ -50,9 +50,9 @@ namespace quda
       Vector y = arg.y(x_cb, parity);
 
       Matrix<complex<real>, nSpin> A;
-#pragma unroll
+QUDA_UNROLL
       for (int mu = 0; mu < nSpin; mu++) {
-#pragma unroll
+QUDA_UNROLL
         for (int nu = 0; nu < nSpin; nu++) {
           // Color inner product: <\phi(x)_{\mu} | \phi(y)_{\nu}>
           // The Bra is conjugated
diff --git a/include/kernels/copy_clover.cuh b/include/kernels/copy_clover.cuh
index 1e73d707f4..ed224ba876 100644
--- a/include/kernels/copy_clover.cuh
+++ b/include/kernels/copy_clover.cuh
@@ -34,7 +34,7 @@ namespace quda {
       typename mapper<typename Arg::store_out_t>::type out[Arg::length];
       typename mapper<typename Arg::store_in_t>::type in[Arg::length];
       arg.in.load(in, x_cb, parity);
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<Arg::length; i++) out[i] = in[i];
       arg.out.save(out, x_cb, parity);
     }
diff --git a/include/kernels/copy_field_offset.cuh b/include/kernels/copy_field_offset.cuh
index b75fd07320..c87f62bb9a 100644
--- a/include/kernels/copy_field_offset.cuh
+++ b/include/kernels/copy_field_offset.cuh
@@ -117,7 +117,7 @@ namespace quda
   copy_field(int out, int in, int parity, const Arg &arg)
   {
     using Element = typename Arg::Element;
-#pragma unroll
+QUDA_UNROLL
     for (int d = 0; d < 4; d++) {
       Element element = arg.in(d, in, parity);
       arg.out(d, out, parity) = element;
@@ -159,14 +159,14 @@ namespace quda
           // we are collecting so x_cb is the index for the input.
           idx_in = x_cb + arg.volume_4d_cb_in * s;
           getCoords5CB(coordinate, idx_in, arg.dim_in, arg.X0h_in, parity, QUDA_5D_PC);
-#pragma unroll
+QUDA_UNROLL
           for (int d = 0; d < 4; d++) { coordinate[d] += arg.offset[d]; }
           idx_out = linkIndex(coordinate, arg.dim_out) + arg.volume_4d_cb_out * coordinate[4];
         } else {
           // we are dispersing so x_cb is the index for the output.
           idx_out = x_cb + arg.volume_4d_cb_out * s;
           getCoords5CB(coordinate, idx_out, arg.dim_out, arg.X0h_out, parity, QUDA_5D_PC);
-#pragma unroll
+QUDA_UNROLL
           for (int d = 0; d < 4; d++) { coordinate[d] += arg.offset[d]; }
           idx_in = linkIndex(coordinate, arg.dim_in) + arg.volume_4d_cb_in * coordinate[4];
         }
diff --git a/include/kernels/dslash_clover_helper.cuh b/include/kernels/dslash_clover_helper.cuh
index e1b3bae568..ff12416b5d 100644
--- a/include/kernels/dslash_clover_helper.cuh
+++ b/include/kernels/dslash_clover_helper.cuh
@@ -87,7 +87,7 @@ namespace quda {
 
       in.toRel(); // change to chiral basis here
 
-#pragma unroll
+QUDA_UNROLL
       for (int chirality=0; chirality<2; chirality++) {
         HMatrix<real, N> A = arg.clover(x_cb, clover_parity, chirality);
         half_fermion chi = in.chiral_project(chirality);
@@ -129,7 +129,7 @@ namespace quda {
 
       in.toRel(); // change to chiral basis here
 
-#pragma unroll
+QUDA_UNROLL
       for (int chirality=0; chirality<2; chirality++) {
         // factor of 2 comes from clover normalization we need to correct for
         const complex<real> j(0.0, chirality == 0 ? static_cast<real>(0.5) : -static_cast<real>(0.5));
diff --git a/include/kernels/dslash_coarse.cuh b/include/kernels/dslash_coarse.cuh
index 63d77e36ce..2d6837345b 100644
--- a/include/kernels/dslash_coarse.cuh
+++ b/include/kernels/dslash_coarse.cuh
@@ -123,7 +123,7 @@ namespace quda {
     if (!thread_dir || target::is_host()) {
 
       //Forward gather - compute fwd offset for spinor fetch
-#pragma unroll
+QUDA_UNROLL
       for(int d = thread_dim; d < Arg::nDim; d += Arg::dim_stride) // loop over dimension
       {
 	const int fwd_idx = linkIndexP1(coord, arg.dim, d);
@@ -132,13 +132,13 @@ namespace quda {
 	  if (doHalo<Arg::type>()) {
             int ghost_idx = ghostFaceIndex<1, 5>(coord, arg.dim, d, arg.nFace);
 
-#pragma unroll
+QUDA_UNROLL
 	    for(int color_local = 0; color_local < Mc; color_local++) { //Color row
 	      int c_row = color_block + color_local; // global color index
 	      int row = s_row * Arg::nColor + c_row;
-#pragma unroll
+QUDA_UNROLL
 	      for(int s_col = 0; s_col < Arg::nSpin; s_col++) { //Spin column
-#pragma unroll
+QUDA_UNROLL
 		for(int c_col = 0; c_col < Arg::nColor; c_col += Arg::color_stride) { //Color column
 		  int col = s_col * Arg::nColor + c_col + color_offset;
 		  if (!Arg::dagger)
@@ -152,13 +152,13 @@ namespace quda {
 	    }
 	  }
 	} else if (doBulk<Arg::type>()) {
-#pragma unroll
+QUDA_UNROLL
 	  for(int color_local = 0; color_local < Mc; color_local++) { //Color row
 	    int c_row = color_block + color_local; // global color index
 	    int row = s_row * Arg::nColor + c_row;
-#pragma unroll
+QUDA_UNROLL
 	    for(int s_col = 0; s_col < Arg::nSpin; s_col++) { //Spin column
-#pragma unroll
+QUDA_UNROLL
 	      for(int c_col = 0; c_col < Arg::nColor; c_col += Arg::color_stride) { //Color column
 		int col = s_col * Arg::nColor + c_col + color_offset;
 		if (!Arg::dagger)
@@ -178,7 +178,7 @@ namespace quda {
     if (thread_dir || target::is_host()) {
 
       //Backward gather - compute back offset for spinor and gauge fetch
-#pragma unroll
+QUDA_UNROLL
       for(int d = thread_dim; d < Arg::nDim; d += Arg::dim_stride)
 	{
 	const int back_idx = linkIndexM1(coord, arg.dim, d);
@@ -186,13 +186,13 @@ namespace quda {
 	if ( arg.commDim[d] && (coord[d] - arg.nFace < 0) ) {
 	  if (doHalo<Arg::type>()) {
             const int ghost_idx = ghostFaceIndex<0, 5>(coord, arg.dim, d, arg.nFace);
-#pragma unroll
+QUDA_UNROLL
 	    for (int color_local=0; color_local<Mc; color_local++) {
 	      int c_row = color_block + color_local;
 	      int row = s_row * Arg::nColor + c_row;
-#pragma unroll
+QUDA_UNROLL
 	      for (int s_col=0; s_col < Arg::nSpin; s_col++)
-#pragma unroll
+QUDA_UNROLL
 		for (int c_col=0; c_col < Arg::nColor; c_col += Arg::color_stride) {
 		  int col = s_col * Arg::nColor + c_col + color_offset;
 		  if (!Arg::dagger)
@@ -205,13 +205,13 @@ namespace quda {
 	    }
 	  }
 	} else if (doBulk<Arg::type>()) {
-#pragma unroll
+QUDA_UNROLL
 	  for(int color_local = 0; color_local < Mc; color_local++) {
 	    int c_row = color_block + color_local;
 	    int row = s_row * Arg::nColor + c_row;
-#pragma unroll
+QUDA_UNROLL
 	    for(int s_col = 0; s_col < Arg::nSpin; s_col++)
-#pragma unroll
+QUDA_UNROLL
 	      for(int c_col = 0; c_col < Arg::nColor; c_col += Arg::color_stride) {
 		int col = s_col * Arg::nColor + c_col + color_offset;
 		if (!Arg::dagger)
@@ -244,13 +244,13 @@ namespace quda {
     const int spinor_parity = (arg.nParity == 2) ? parity : 0;
 
     // M is number of colors per thread
-#pragma unroll
+QUDA_UNROLL
     for(int color_local = 0; color_local < Mc; color_local++) {//Color out
       int c = color_block + color_local; // global color index
       int row = s * Arg::nColor + c;
-#pragma unroll
+QUDA_UNROLL
       for (int s_col = 0; s_col < Arg::nSpin; s_col++) //Spin in
-#pragma unroll
+QUDA_UNROLL
 	for (int c_col = 0; c_col < Arg::nColor; c_col += Arg::color_stride) { //Color in
 	  //Factor of kappa and diagonal addition now incorporated in X
 	  int col = s_col * Arg::nColor + c_col + color_offset;
@@ -281,14 +281,14 @@ namespace quda {
 
       if (dir == 0 && dim == 0) {
         // full split over dimension and direction
-#pragma unroll
+QUDA_UNROLL
         for (int d=1; d < Arg::dim_stride; d++) { // get remaining forward gathers (if any)
           // 4-way 1,2,3  (stride = 4)
           // 2-way 1      (stride = 2)
           out += cache.load_z(target::thread_idx().z + d * 2 + 0);
         }
 
-#pragma unroll
+QUDA_UNROLL
         for (int d=0; d < Arg::dim_stride; d++) { // get all backward gathers
           out += cache.load_z(target::thread_idx().z + d * 2 + 1);
         }
@@ -348,7 +348,7 @@ namespace quda {
         // reduce down to the first group of column-split threads
         out = warp_combine<Arg::color_stride>(out);
 
-#pragma unroll
+QUDA_UNROLL
         for (int color_local=0; color_local<Mc; color_local++) {
           int c = color_block + color_local; // global color index
           if (color_offset == 0) {
diff --git a/include/kernels/dslash_mdw_fused.cuh b/include/kernels/dslash_mdw_fused.cuh
index 8885fba82e..e493c0e73c 100644
--- a/include/kernels/dslash_mdw_fused.cuh
+++ b/include/kernels/dslash_mdw_fused.cuh
@@ -166,7 +166,7 @@ namespace quda {
     __device__ inline bool is_halo_4d(const int coordinate[4], const int dim[4], const int halo_shift[4])
     {
       bool ret = false;
-#pragma unroll
+QUDA_UNROLL
       for (int d = 0; d < 4; d++) {
         ret = ret or (coordinate[d] >= dim[d] - halo_shift[d] or coordinate[d] < halo_shift[d]);
       }
@@ -179,14 +179,14 @@ namespace quda {
       int back_x[4];
       int back_dim[4];
 
-#pragma unroll
+QUDA_UNROLL
       for (int d = 0; d < 4; d++) {
         back_x[d] = comm[d] ? x[d] - pad : x[d];
         back_dim[d] = comm[d] ? dim[d] - pad * 2 : dim[d];
       }
 
       bool is_center = true;
-#pragma unroll
+QUDA_UNROLL
       for (int d = 0; d < 4; d++) { is_center = is_center && (back_x[d] >= 0 && back_x[d] < back_dim[d]); }
 
       if (is_center) {
@@ -210,7 +210,7 @@ namespace quda {
 
       const int index_4d_cb = index_4d_cb_from_coordinate_4d(coordinate, arg.dim);
 
-#pragma unroll
+QUDA_UNROLL
       for (int d = 0; d < 4; d++) // loop over dimension
       {
         int x[4] = {coordinate[0], coordinate[1], coordinate[2], coordinate[3]};
@@ -261,7 +261,7 @@ namespace quda {
       int aux[4];
       aux[0] = shrunk_index * 2;
 
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < 3; i++) { aux[i + 1] = aux[i] / shrunk_dim[i]; }
 
       coordinate[0] = aux[0] - aux[1] * shrunk_dim[0];
@@ -274,7 +274,7 @@ namespace quda {
         += (shift[0] + shift[1] + shift[2] + shift[3] + parity + coordinate[3] + coordinate[2] + coordinate[1]) & 1;
 
 // Now go back to the extended volume.
-#pragma unroll
+QUDA_UNROLL
       for (int d = 0; d < 4; d++) { coordinate[d] += shift[d]; }
     }
 
@@ -355,7 +355,7 @@ namespace quda {
         mma::MmaOperandA op_a[Arg::reload ? 1 : tk_dim];
         mma::MmaOperandA op_a_aux[Arg::reload ? 1 : tk_dim];
         if (!Arg::reload) { // the data in registers can be resued.
-#pragma unroll
+QUDA_UNROLL
           for (int tile_k = 0; tile_k < tk_dim; tile_k++) { op_a[tile_k].template load<M_sm>(sm_a, tile_k, warp_m, wrm); }
         }
 
@@ -365,7 +365,7 @@ namespace quda {
             construct_matrix_a_m5inv<M_sm, true, Arg>(arg, sm_a); // dagger = true
             __syncthreads();
 
-#pragma unroll
+QUDA_UNROLL
             for (int tile_k = 0; tile_k < tk_dim; tile_k++) {
               op_a_aux[tile_k].template load<M_sm>(sm_a, tile_k, warp_m, wrm);
             }
diff --git a/include/kernels/dslash_staggered.cuh b/include/kernels/dslash_staggered.cuh
index 8f772165bf..ef7428d827 100644
--- a/include/kernels/dslash_staggered.cuh
+++ b/include/kernels/dslash_staggered.cuh
@@ -92,7 +92,7 @@ namespace quda
     typedef Matrix<complex<real>, Arg::nColor> Link;
     const int their_spinor_parity = (arg.nParity == 2) ? 1 - parity : 0;
 
-#pragma unroll
+QUDA_UNROLL
     for (int d = 0; d < 4; d++) { // loop over dimension
 
       // standard - forward direction
diff --git a/include/kernels/dslash_twisted_clover_preconditioned.cuh b/include/kernels/dslash_twisted_clover_preconditioned.cuh
index 2572e1f936..79946b1936 100644
--- a/include/kernels/dslash_twisted_clover_preconditioned.cuh
+++ b/include/kernels/dslash_twisted_clover_preconditioned.cuh
@@ -78,7 +78,7 @@ namespace quda
 
         Vector tmp;
 
-#pragma unroll
+QUDA_UNROLL
         for (int chirality = 0; chirality < 2; chirality++) {
 
           const complex<real> b(0.0, chirality == 0 ? static_cast<real>(arg.b) : -static_cast<real>(arg.b));
diff --git a/include/kernels/dslash_twisted_mass_preconditioned.cuh b/include/kernels/dslash_twisted_mass_preconditioned.cuh
index 496e12276d..912ff95987 100644
--- a/include/kernels/dslash_twisted_mass_preconditioned.cuh
+++ b/include/kernels/dslash_twisted_mass_preconditioned.cuh
@@ -55,7 +55,7 @@ namespace quda
     typedef Matrix<complex<real>, Arg::nColor> Link;
     const int their_spinor_parity = nParity == 2 ? 1 - parity : 0;
 
-#pragma unroll
+QUDA_UNROLL
     for (int d = 0; d < Arg::nDim; d++) { // loop over dimension
       {                              // Forward gather - compute fwd offset for vector fetch
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index 0b56afc7ad..001d87746b 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -77,7 +77,7 @@ namespace quda
     // parity for gauge field - include residual parity from 5-d => 4-d checkerboarding
     const int gauge_parity = (Arg::nDim == 5 ? (coord.x_cb / arg.dc.volume_4d_cb + parity) % 2 : parity);
 
-#pragma unroll
+QUDA_UNROLL
     for (int d = 0; d < 4; d++) { // loop over dimension - 4 and not nDim since this is used for DWF as well
       {                           // Forward gather - compute fwd offset for vector fetch
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
diff --git a/include/kernels/dslash_wilson_clover.cuh b/include/kernels/dslash_wilson_clover.cuh
index ffc8b880e8..a51a473375 100644
--- a/include/kernels/dslash_wilson_clover.cuh
+++ b/include/kernels/dslash_wilson_clover.cuh
@@ -68,7 +68,7 @@ namespace quda
 
         Vector tmp;
 
-#pragma unroll
+QUDA_UNROLL
         for (int chirality = 0; chirality < 2; chirality++) {
           constexpr int n = Arg::nColor * Arg::nSpin / 2;
           HMatrix<real, n> A = arg.A(coord.x_cb, parity, chirality);
diff --git a/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh b/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
index e6923bf707..a906f6886e 100644
--- a/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
+++ b/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
@@ -68,7 +68,7 @@ namespace quda
 
         Vector tmp;
 
-#pragma unroll
+QUDA_UNROLL
         for (int chirality = 0; chirality < 2; chirality++) {
           constexpr int n = Arg::nColor * Arg::nSpin / 2;
           HMatrix<real, n> A = arg.A(coord.x_cb, parity, chirality);
diff --git a/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh b/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
index e28bd33ce4..60af0fde19 100644
--- a/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
@@ -82,7 +82,7 @@ namespace quda
 
           Vector tmp;
 
-#pragma unroll
+QUDA_UNROLL
           for (int chirality = 0; chirality < 2; chirality++) {
 
             HMatrix<real, Arg::nColor * Arg::nSpin / 2> A_inv = arg.A_inv(coord.x_cb, parity, chirality);
@@ -110,7 +110,7 @@ namespace quda
           Vector x = arg.x(coord.x_cb, my_spinor_parity);
           x.toRel();
           Vector tmp;
-#pragma unroll
+QUDA_UNROLL
           for (int chirality = 0; chirality < 2; chirality++) {
 
             HMatrix<real, Arg::nColor *Arg::nSpin / 2> A = arg.A(coord.x_cb, parity, chirality);
diff --git a/include/kernels/dslash_wilson_clover_preconditioned.cuh b/include/kernels/dslash_wilson_clover_preconditioned.cuh
index 45ad58edfb..a778628bcb 100644
--- a/include/kernels/dslash_wilson_clover_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_preconditioned.cuh
@@ -73,7 +73,7 @@ namespace quda
 
         Vector tmp;
 
-#pragma unroll
+QUDA_UNROLL
         for (int chirality = 0; chirality < 2; chirality++) {
 
           HMatrix<real, Arg::nColor * Arg::nSpin / 2> A = arg.A(coord.x_cb, parity, chirality);
diff --git a/include/kernels/gauge_det_trace.cuh b/include/kernels/gauge_det_trace.cuh
index c409b7b2c4..026c15065d 100644
--- a/include/kernels/gauge_det_trace.cuh
+++ b/include/kernels/gauge_det_trace.cuh
@@ -43,17 +43,17 @@ namespace quda {
     __device__ __host__ inline reduce_t operator()(reduce_t &value, int x_cb, int parity)
     {
       int X[4];
-#pragma unroll
+QUDA_UNROLL
       for(int dr=0; dr<4; ++dr) X[dr] = arg.X[dr];
 
       int x[4];
       getCoords(x, x_cb, X, parity);
-#pragma unroll
+QUDA_UNROLL
       for(int dr=0; dr<4; ++dr) {
         x[dr] += arg.border[dr];
         X[dr] += 2*arg.border[dr];
       }
-#pragma unroll
+QUDA_UNROLL
       for (int mu = 0; mu < 4; mu++) {
         Matrix<complex<typename Arg::real>, Arg::nColor> U = arg.u(mu, linkIndex(x, X), parity);
         auto local = Arg::type == compute_type::determinant ? getDeterminant(U) : getTrace(U);
diff --git a/include/kernels/gauge_fix_fft.cuh b/include/kernels/gauge_fix_fft.cuh
index 3680491e65..cae267c20f 100644
--- a/include/kernels/gauge_fix_fft.cuh
+++ b/include/kernels/gauge_fix_fft.cuh
@@ -237,29 +237,29 @@ namespace quda {
     Float t1 = 0.0;
     //first normalize first row
     //sum of squares of row
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) t1 += norm(U(0,c));
     t1 = (Float)1.0 / sqrt(t1);
     //14
     //used to normalize row
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) U(0,c) *= t1;
     //6
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) t2 += conj(U(0,c)) * U(1,c);
     //24
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) U(1,c) -= t2 * U(0,c);
     //24
     //normalize second row
     //sum of squares of row
     t1 = 0.0;
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) t1 += norm(U(1,c));
     t1 = (Float)1.0 / sqrt(t1);
     //14
     //used to normalize row
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) U(1, c) *= t1;
     //6
     //Reconstruct last row
diff --git a/include/kernels/gauge_fix_ovr.cuh b/include/kernels/gauge_fix_ovr.cuh
index 25bbdb86f9..aacffeb5d6 100644
--- a/include/kernels/gauge_fix_ovr.cuh
+++ b/include/kernels/gauge_fix_ovr.cuh
@@ -56,12 +56,12 @@ namespace quda {
       using Link = Matrix<complex<typename Arg::real>, 3>;
 
       int X[4];
-#pragma unroll
+QUDA_UNROLL
       for (int dr = 0; dr < 4; dr++) X[dr] = arg.X[dr];
 
       int x[4];
       getCoords(x, x_cb, X, parity);
-#pragma unroll
+QUDA_UNROLL
       for (int dr = 0; dr < 4; ++dr) {
         x[dr] += arg.border[dr];
         X[dr] += 2 * arg.border[dr];
@@ -139,7 +139,7 @@ namespace quda {
       int parity = arg.parity;
 
       int X[4], x[4];
-#pragma unroll
+QUDA_UNROLL
       for (int dr = 0; dr < 4; dr++) X[dr] = arg.X[dr];
 
       if (!Arg::halo) {
@@ -154,7 +154,7 @@ namespace quda {
         x[0] = idx % X[0];
       }
 
-#pragma unroll
+QUDA_UNROLL
       for (int dr = 0; dr < 4; dr++) {
         x[dr] += arg.border[dr];
         X[dr] += 2 * arg.border[dr];
diff --git a/include/kernels/gauge_heatbath.cuh b/include/kernels/gauge_heatbath.cuh
index a366a99c73..522d79f335 100644
--- a/include/kernels/gauge_heatbath.cuh
+++ b/include/kernels/gauge_heatbath.cuh
@@ -592,12 +592,12 @@ namespace quda {
       auto parity = arg.parity;
 
       int X[4];
-#pragma unroll
+QUDA_UNROLL
       for ( int dr = 0; dr < 4; ++dr ) X[dr] = arg.X[dr];
 
       int x[4];
       getCoords(x, x_cb, X, parity);
-#pragma unroll
+QUDA_UNROLL
       for ( int dr = 0; dr < 4; ++dr ) {
         x[dr] += arg.border[dr];
         X[dr] += 2 * arg.border[dr];
diff --git a/include/kernels/gauge_plaq.cuh b/include/kernels/gauge_plaq.cuh
index d4afb02041..00c44d7ea8 100644
--- a/include/kernels/gauge_plaq.cuh
+++ b/include/kernels/gauge_plaq.cuh
@@ -69,12 +69,12 @@ namespace quda {
 
       int x[4];
       getCoords(x, x_cb, arg.X, parity);
-#pragma unroll
+QUDA_UNROLL
       for (int dr=0; dr<4; ++dr) x[dr] += arg.border[dr]; // extended grid coordinates
 
-#pragma unroll
+QUDA_UNROLL
       for (int mu = 0; mu < 3; mu++) {
-#pragma unroll
+QUDA_UNROLL
         for (int nu = 0; nu < 3; nu++) {
           if (nu >= mu + 1) plaq[0] += plaquette(arg, x, parity, mu, nu);
         }
diff --git a/include/kernels/gauge_qcharge.cuh b/include/kernels/gauge_qcharge.cuh
index b05f40aab6..d019179a46 100644
--- a/include/kernels/gauge_qcharge.cuh
+++ b/include/kernels/gauge_qcharge.cuh
@@ -55,7 +55,7 @@ namespace quda
       // first compute the field energy
       Link iden;
       setIdentity(&iden);
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<6; i++) {
         // Make traceless
         auto tmp = F[i] - n_inv * getTrace(F[i]) * iden;
@@ -69,7 +69,7 @@ namespace quda
       double Q_idx = 0.0;
       double Qi[3] = {0.0,0.0,0.0};
       // unroll computation
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<3; i++) {
         Qi[i] = getTrace(F[i] * F[5 - i]).real();
       }
diff --git a/include/kernels/gauge_utils.cuh b/include/kernels/gauge_utils.cuh
index 411597a255..ea75444565 100644
--- a/include/kernels/gauge_utils.cuh
+++ b/include/kernels/gauge_utils.cuh
@@ -22,7 +22,7 @@ namespace quda
   {
     setZero(&staple);
     int dx[4] = { };
-#pragma unroll
+QUDA_UNROLL
     for (int mu = 0; mu < 4 ; mu++) {
       // Identify directions orthogonal to the link and
       // ignore the dir_ignore direction (usually the temporal dim
diff --git a/include/kernels/hisq_paths_force.cuh b/include/kernels/hisq_paths_force.cuh
index 893c7ea587..907a84877a 100644
--- a/include/kernels/hisq_paths_force.cuh
+++ b/include/kernels/hisq_paths_force.cuh
@@ -204,7 +204,7 @@ namespace quda {
       {
         int x[4];
         getCoords(x, x_cb, arg.X, parity);
-#pragma unroll
+QUDA_UNROLL
         for (int d=0; d<4; d++) x[d] += arg.border[d];
         int e_cb = linkIndex(x,arg.E);
 
@@ -611,7 +611,7 @@ namespace quda {
         for (int d=0; d<4; d++) x[d] += arg.border[d];
         int e_cb = linkIndex(x,arg.E);
 
-#pragma unroll
+QUDA_UNROLL
         for (int sig=0; sig<4; ++sig) {
           Link Uw = arg.link(sig, e_cb, parity);
           Link Ox = arg.oProd(sig, e_cb, parity);
@@ -672,7 +672,7 @@ namespace quda {
          */
 
         // compute the force for forward long links
-#pragma unroll
+QUDA_UNROLL
         for (int sig=0; sig<4; sig++) {
           int point_c = e_cb;
 
diff --git a/include/kernels/laplace.cuh b/include/kernels/laplace.cuh
index da24bf77b7..81fdcbab24 100644
--- a/include/kernels/laplace.cuh
+++ b/include/kernels/laplace.cuh
@@ -81,7 +81,7 @@ namespace quda
     typedef Matrix<complex<real>, Arg::nColor> Link;
     const int their_spinor_parity = (arg.nParity == 2) ? 1 - parity : 0;
 
-#pragma unroll
+QUDA_UNROLL
     for (int d = 0; d < Arg::nDim; d++) { // loop over dimension
       if (d != dir) {
         {
diff --git a/include/kernels/momentum.cuh b/include/kernels/momentum.cuh
index 581b64b94c..805f75dab2 100644
--- a/include/kernels/momentum.cuh
+++ b/include/kernels/momentum.cuh
@@ -126,7 +126,7 @@ namespace quda {
       for (int d=0; d<4; d++) x[d] += arg.border[d];
       int e_cb = linkIndex(x, arg.E);
 
-#pragma unroll
+QUDA_UNROLL
       for (int d=0; d<4; d++) {
         Matrix<complex<typename Arg::Float>, Arg::nColor> m = arg.mom(d, x_cb, parity);
         Matrix<complex<typename Arg::Float>, Arg::nColor> f = arg.force(d, e_cb, parity);
diff --git a/include/kernels/multi_blas_core.cuh b/include/kernels/multi_blas_core.cuh
index 653a4fdb60..77bd13e73f 100644
--- a/include/kernels/multi_blas_core.cuh
+++ b/include/kernels/multi_blas_core.cuh
@@ -100,7 +100,7 @@ namespace quda
           w = ::quda::zero<complex<typename Arg::real>, Arg::n/2>();
         }
 
-#pragma unroll
+QUDA_UNROLL
         for (int l_ = 0; l_ < Arg::NXZ; l_ += warp_split) {
           const int l = l_ + l_idx;
           if (l < Arg::NXZ || warp_split == 1) {
@@ -147,7 +147,7 @@ namespace quda
 
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &, int i, int j) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int k = 0; k < x.size(); k++) y[k] += a(j, i) * x[k];
       }
 
@@ -169,7 +169,7 @@ namespace quda
 
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &, int i, int j) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int k = 0; k < x.size(); k++) y[k] = cmac(a(j, i), x[k], y[k]);
       }
 
@@ -191,7 +191,7 @@ namespace quda
 
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &w, int i, int j) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int k = 0; k < x.size(); k++) {
           if (j == 0) w[k] = y[k];
           w[k] = cmac(a(j, i), x[k], w[k]);
@@ -221,7 +221,7 @@ namespace quda
 
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &w, int i, int) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int k = 0; k < x.size(); k++) {
           y[k] += a[i] * w[k];
           w[k] = b[i] * x[k] + c[i] * w[k];
@@ -257,7 +257,7 @@ namespace quda
       // i loops over NYW, j loops over NXZ
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &w, int, int j) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int k = 0; k < x.size(); k++) {
           y[k] = cmac(a[j], x[k], y[k]);
           w[k] = cmac(b[j], x[k], w[k]);
diff --git a/include/kernels/multi_reduce_core.cuh b/include/kernels/multi_reduce_core.cuh
index 05f172c9b6..e20c54e6cc 100644
--- a/include/kernels/multi_reduce_core.cuh
+++ b/include/kernels/multi_reduce_core.cuh
@@ -97,7 +97,7 @@ namespace quda
         // Each NYW owns its own thread.
         // The NXZ's are all in the same thread block,
         // so they can share the same memory.
-#pragma unroll
+QUDA_UNROLL
         for (int l = 0; l < Arg::NXZ; l++) {
           if (arg.f.read.X) arg.X[l].load(x, i, parity);
           if (arg.f.read.Z) arg.Z[l].load(z, i, parity);
@@ -150,7 +150,7 @@ namespace quda
 
       template <typename T> __device__ __host__ inline void operator()(reduce_t &sum, T &x, T &y, T &, T &, int, int) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int k=0; k < x.size(); k++) dot_<reduce_t, real>(sum, x[k], y[k]);
       }
 
@@ -181,7 +181,7 @@ namespace quda
 
       template <typename T> __device__ __host__ inline void operator()(reduce_t &sum, T &x, T &y, T &, T &, int, int) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int k=0; k < x.size(); k++) cdot_<reduce_t, real>(sum, x[k], y[k]);
       }
 
@@ -199,7 +199,7 @@ namespace quda
 
       template <typename T> __device__ __host__ inline void operator()(reduce_t &sum, T &x, T &y, T &, T &w, int i, int j) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int k = 0; k < x.size(); k++) {
           cdot_<reduce_t, real>(sum, x[k], y[k]);
           if (i == j) w[k] = y[k];
diff --git a/include/kernels/pgauge_init.cuh b/include/kernels/pgauge_init.cuh
index 583c567d6f..46525be8ee 100644
--- a/include/kernels/pgauge_init.cuh
+++ b/include/kernels/pgauge_init.cuh
@@ -73,29 +73,29 @@ namespace quda {
     Float t1 = 0.0;
     //first normalize first row
     //sum of squares of row
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) t1 += norm(U(0,c));
     t1 = (Float)1.0 / sqrt(t1);
     //14
     //used to normalize row
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) U(0,c) *= t1;
     //6
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) t2 += conj(U(0,c)) * U(1,c);
     //24
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) U(1,c) -= t2 * U(0,c);
     //24
     //normalize second row
     //sum of squares of row
     t1 = 0.0;
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) t1 += norm(U(1,c));
     t1 = (Float)1.0 / sqrt(t1);
     //14
     //used to normalize row
-#pragma unroll
+QUDA_UNROLL
     for ( int c = 0; c < 3; c++ ) U(1, c) *= t1;
     //6
     //Reconstruct lat row
diff --git a/include/kernels/prolongator.cuh b/include/kernels/prolongator.cuh
index 6843ceef84..91209e1bcb 100644
--- a/include/kernels/prolongator.cuh
+++ b/include/kernels/prolongator.cuh
@@ -48,9 +48,9 @@ namespace quda {
     int parity_coarse = (x_coarse >= arg.in.VolumeCB()) ? 1 : 0;
     int x_coarse_cb = x_coarse - parity_coarse*arg.in.VolumeCB();
 
-#pragma unroll
+QUDA_UNROLL
     for (int s=0; s<Arg::fineSpin; s++) {
-#pragma unroll
+QUDA_UNROLL
       for (int c=0; c<Arg::coarseColor; c++) {
         out[s*Arg::coarseColor+c] = arg.in(parity_coarse, x_coarse_cb, arg.spin_map(s,parity), c);
       }
@@ -70,25 +70,25 @@ namespace quda {
 
     constexpr int color_unroll = 2;
 
-#pragma unroll
+QUDA_UNROLL
     for (int s=0; s<Arg::fineSpin; s++) {
-#pragma unroll
+QUDA_UNROLL
       for (int fine_color_local = 0; fine_color_local < fine_color_per_thread; fine_color_local++) {
         int i = fine_color_block + fine_color_local; // global fine color index
 
         complex<typename Arg::real> partial[color_unroll];
-#pragma unroll
+QUDA_UNROLL
         for (int k=0; k<color_unroll; k++) partial[k] = 0.0;
 
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<Arg::coarseColor; j+=color_unroll) {
           // V is a ColorMatrixField with internal dimensions Ns * Nc * Nvec
-#pragma unroll
+QUDA_UNROLL
           for (int k=0; k<color_unroll; k++)
             partial[k] = cmac(arg.V(v_parity, x_cb, s, i, j + k), in[s * Arg::coarseColor + j + k], partial[k]);
         }
 
-#pragma unroll
+QUDA_UNROLL
         for (int k=1; k<color_unroll; k++) partial[0] += partial[k];
         arg.out(spinor_parity, x_cb, s, i) = partial[0];
       }
diff --git a/include/kernels/reduce_core.cuh b/include/kernels/reduce_core.cuh
index 6eaa5ec35b..dc91dd360b 100644
--- a/include/kernels/reduce_core.cuh
+++ b/include/kernels/reduce_core.cuh
@@ -127,7 +127,7 @@ namespace quda
       Norm1(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i=0; i < x.size(); i++) sum += norm1_<reduce_t, real>(x[i]);
       }
       constexpr int flops() const { return 2; }   //! flops per element
@@ -149,7 +149,7 @@ namespace quda
       Norm2(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) norm2_<reduce_t, real>(sum, x[i]);
       }
       constexpr int flops() const { return 2; }   //! flops per element
@@ -172,7 +172,7 @@ namespace quda
       Dot(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) dot_<reduce_t, real>(sum, x[i], y[i]);
       }
       constexpr int flops() const { return 2; }   //! flops per element
@@ -191,7 +191,7 @@ namespace quda
       axpbyzNorm2(const real &a, const real &b) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           z[i] = a * x[i] + b * y[i];
           norm2_<reduce_t, real>(sum, z[i]);
@@ -212,7 +212,7 @@ namespace quda
       AxpyReDot(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] += a * x[i];
           dot_<reduce_t, real>(sum, x[i], y[i]);
@@ -233,7 +233,7 @@ namespace quda
       caxpyNorm2(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           norm2_<reduce_t, real>(sum, y[i]);
@@ -256,7 +256,7 @@ namespace quda
       caxpyxmaznormx(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           x[i] = cmac(-a, z[i], x[i]);
@@ -281,7 +281,7 @@ namespace quda
       cabxpyzaxnorm(const complex<real> &a, const complex<real> &b) : a(a.real()), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           x[i] *= a;
           z[i] = cmac(b, x[i], y[i]);
@@ -312,7 +312,7 @@ namespace quda
       Cdot(const complex<real> &, const complex<real> &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) cdot_<reduce_t, real>(sum, x[i], y[i]);
       }
       constexpr int flops() const { return 4; }   //! flops per element
@@ -332,7 +332,7 @@ namespace quda
       caxpydotzy(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           cdot_<reduce_t, real>(sum, z[i], y[i]);
@@ -375,7 +375,7 @@ namespace quda
       CdotNormA(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) cdotNormA_<reduce_t>(sum, x[i], y[i]);
       }
       constexpr int flops() const { return 6; }   //! flops per element
@@ -395,7 +395,7 @@ namespace quda
       caxpbypzYmbwcDotProductUYNormY_(const complex<real> &a, const complex<real> &b) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &v) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           y[i] = cmac(b, z[i], y[i]);
@@ -421,7 +421,7 @@ namespace quda
       axpyCGNorm2(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           auto y_new = y[i] + a * x[i];
           norm2_<real_reduce_t, real>(sum.x, y_new);
@@ -458,7 +458,7 @@ namespace quda
 
       template <typename T> __device__ __host__ void operator()(reduce_t &, T &x, T &y, T &, T &, T &)
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           norm2_<real_reduce_t, real>(aux.x, x[i]);
           norm2_<real_reduce_t, real>(aux.y, y[i]);
@@ -503,7 +503,7 @@ namespace quda
 
       template <typename T> __device__ __host__ void operator()(reduce_t &, T &x, T &y, T &z, T &, T &)
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           norm2_<real_reduce_t, real>(aux.x, x[i] + y[i]);
           norm2_<real_reduce_t, real>(aux.y, z[i]);
@@ -536,7 +536,7 @@ namespace quda
       tripleCGReduction_(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           norm2_<real_reduce_t, real>(sum.x, x[i]);
           norm2_<real_reduce_t, real>(sum.y, y[i]);
@@ -561,7 +561,7 @@ namespace quda
       quadrupleCGReduction_(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           norm2_<real_reduce_t, real>(sum.x, x[i]);
           norm2_<real_reduce_t, real>(sum.y, y[i]);
@@ -588,7 +588,7 @@ namespace quda
       quadrupleCG3InitNorm_(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &v) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           z[i] = x[i];
           w[i] = y[i];
@@ -619,7 +619,7 @@ namespace quda
       quadrupleCG3UpdateNorm_(const real &a, const real &b) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &v) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           auto tmpx = x[i];
           auto tmpy = y[i];
diff --git a/include/kernels/restrictor.cuh b/include/kernels/restrictor.cuh
index 132d8689c8..e304e35f37 100644
--- a/include/kernels/restrictor.cuh
+++ b/include/kernels/restrictor.cuh
@@ -72,33 +72,33 @@ namespace quda {
     const int spinor_parity = (arg.nParity == 2) ? parity : 0;
     const int v_parity = (arg.V.Nparity() == 2) ? parity : 0;
 
-#pragma unroll
+QUDA_UNROLL
     for (int s=0; s<Arg::fineSpin; s++)
-#pragma unroll
+QUDA_UNROLL
       for (int coarse_color_local=0; coarse_color_local<coarse_color_per_thread; coarse_color_local++) {
 	out[s*coarse_color_per_thread+coarse_color_local] = 0.0;
       }
 
-#pragma unroll
+QUDA_UNROLL
     for (int coarse_color_local=0; coarse_color_local<coarse_color_per_thread; coarse_color_local++) {
       int i = coarse_color_block + coarse_color_local;
-#pragma unroll
+QUDA_UNROLL
       for (int s=0; s<Arg::fineSpin; s++) {
 
 	constexpr int color_unroll = Arg::fineColor == 3 ? 3 : 2;
 
 	complex<typename Arg::real> partial[color_unroll];
-#pragma unroll
+QUDA_UNROLL
 	for (int k=0; k<color_unroll; k++) partial[k] = 0.0;
 
-#pragma unroll
+QUDA_UNROLL
 	for (int j=0; j<Arg::fineColor; j+=color_unroll) {
-#pragma unroll
+QUDA_UNROLL
 	  for (int k=0; k<color_unroll; k++)
 	    partial[k] = cmac(conj(arg.V(v_parity, x_cb, s, j+k, i)), arg.in(spinor_parity, x_cb, s, j+k), partial[k]);
 	}
 
-#pragma unroll
+QUDA_UNROLL
 	for (int k=0; k<color_unroll; k++) out[s*coarse_color_per_thread + coarse_color_local] += partial[k];
       }
     }
@@ -137,9 +137,9 @@ namespace quda {
         rotateCoarseColor(tmp, arg, parity, x_fine_cb, coarse_color_block);
 
         // perform any local spin coarsening
-#pragma unroll
+QUDA_UNROLL
         for (int s=0; s<Arg::fineSpin; s++) {
-#pragma unroll
+QUDA_UNROLL
           for (int v=0; v<coarse_color_per_thread; v++) {
             reduced[arg.spin_map(s,parity)*coarse_color_per_thread+v] += tmp[s*coarse_color_per_thread+v];
           }
@@ -152,9 +152,9 @@ namespace quda {
         const int parity_coarse = x_coarse >= arg.out.VolumeCB() ? 1 : 0;
         const int x_coarse_cb = x_coarse - parity_coarse*arg.out.VolumeCB();
 
-#pragma unroll
+QUDA_UNROLL
         for (int s = 0; s < Arg::coarseSpin; s++) {
-#pragma unroll
+QUDA_UNROLL
           for (int coarse_color_local=0; coarse_color_local<coarse_color_per_thread; coarse_color_local++) {
             int v = coarse_color_thread * coarse_color_per_thread + coarse_color_local;
             arg.out(parity_coarse, x_coarse_cb, s, v) = reduced[s*coarse_color_per_thread+coarse_color_local];
diff --git a/include/kernels/staggered_coarse_op_kernel.cuh b/include/kernels/staggered_coarse_op_kernel.cuh
index 548d1e6f0d..6749d5426e 100644
--- a/include/kernels/staggered_coarse_op_kernel.cuh
+++ b/include/kernels/staggered_coarse_op_kernel.cuh
@@ -90,7 +90,7 @@ namespace quda {
       int hyperCorner = 4*(coord[3]%2)+2*(coord[2]%2)+(coord[1]%2);
       int c_row = 8*ic_f + hyperCorner;
 
-#pragma unroll
+QUDA_UNROLL
       for (int mu=0; mu < nDim; mu++) {
         // The coarse color column depends on my fine hypercube+mu corner.
         coord[mu]++; // linkIndexP1, it's fine if this wraps because we're modding by 2.
@@ -122,9 +122,9 @@ namespace quda {
 
       // lastly, add staggered mass term to diagonal
       if (ic_f == 0 && jc_f == 0 && x_cb < arg.coarseVolumeCB) {
-#pragma unroll
+QUDA_UNROLL
         for (int s = 0; s < Arg::coarseSpin; s++) {
-#pragma unroll
+QUDA_UNROLL
           for (int c = 0; c < Arg::coarseColor; c++) {
             arg.X(0,parity,x_cb,s,s,c,c) = static_cast<real>(2.0) * complex<real>(arg.mass, 0.0); // staggered conventions. No need to +=
           } //Color
diff --git a/include/kernels/staggered_kd_apply_xinv_kernel.cuh b/include/kernels/staggered_kd_apply_xinv_kernel.cuh
index 0f7961116b..e435f5bd63 100644
--- a/include/kernels/staggered_kd_apply_xinv_kernel.cuh
+++ b/include/kernels/staggered_kd_apply_xinv_kernel.cuh
@@ -163,30 +163,30 @@ namespace quda {
       //////////////////////
 
       // Zero the shared memory buffer, which is 48 components
-#pragma unroll
+QUDA_UNROLL
       for (unsigned int coarse_row = 0; coarse_row < Arg::coarseDof; coarse_row += Arg::blockSizeKD) {
         out_buffer[Arg::paddedSpinorSizeKD * Arg::fineColor * mid_idx + fast_idx + coarse_row] = { 0, 0 };
       }
 
-#pragma unroll
+QUDA_UNROLL
       for (int tile_row = 0; tile_row < Arg::numTiles; tile_row++) {
-#pragma unroll
+QUDA_UNROLL
         for (int tile_col = 0; tile_col < Arg::numTiles; tile_col++) {
           // load Xinv
           if (Arg::dagger) {
-#pragma unroll
+QUDA_UNROLL
             for (int col = 0; col < Arg::xinvColTileSize; col++) {
               xinv_buffer[fast_idx * Arg::xinvPaddedColTileSize + col] = conj(arg.xInv(0, parity_coarse_xinv, x_coarse_xinv_cb, 0, 0, Arg::xinvColTileSize * tile_col + col, Arg::xinvRowTileSize * tile_row + fast_idx));
             }
           } else {
-#pragma unroll
+QUDA_UNROLL
             for (int row = 0; row < Arg::xinvColTileSize; row++) {
               xinv_buffer[row * Arg::xinvPaddedColTileSize + fast_idx] = arg.xInv(0, parity_coarse_xinv, x_coarse_xinv_cb, 0, 0, Arg::xinvRowTileSize * tile_row + row, Arg::xinvColTileSize * tile_col + fast_idx);
             }
           }
 
           // do the tile multiplication
-#pragma unroll
+QUDA_UNROLL
           for (int row = 0; row < Arg::xinvRowTileSize; row++) {
             const complex xinv_elem = xinv_buffer[row * Arg::xinvPaddedColTileSize + fast_idx];
             const complex cs_component = in_buffer[Arg::paddedSpinorSizeKD * Arg::fineColor * mid_idx + Arg::xinvColTileSize * tile_col + fast_idx];
@@ -208,7 +208,7 @@ namespace quda {
 
       Vector out;
 
-#pragma unroll
+QUDA_UNROLL
       for (int c_f = 0; c_f < Arg::fineColor; c_f++) {
         out(0,c_f) = static_cast<complex_spinor>(out_buffer[buffer_index + 8 * c_f]);
       }
diff --git a/include/kernels/staggered_outer_product.cuh b/include/kernels/staggered_outer_product.cuh
index e0a333652d..534e9f3d90 100644
--- a/include/kernels/staggered_outer_product.cuh
+++ b/include/kernels/staggered_outer_product.cuh
@@ -61,7 +61,7 @@ namespace quda {
       matrix result;
       const vector x = arg.inA(x_cb, 0);
 
-#pragma unroll
+QUDA_UNROLL
       for (int dim=0; dim<4; ++dim) {
         int shift[4] = {0,0,0,0};
         shift[dim] = 1;
diff --git a/include/linalg.cuh b/include/linalg.cuh
index 9bca67012c..41faa87c28 100644
--- a/include/linalg.cuh
+++ b/include/linalg.cuh
@@ -49,12 +49,12 @@ namespace quda {
       __device__ __host__ inline Cholesky(const Mat<T,N> &A) {
 	const Mat<T,N> &L = L_;
 
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<N; i++) {
-#pragma unroll
+QUDA_UNROLL
 	  for (int j=0; j<N; j++) if (j<i+1) {
 	    complex<T> s = 0;
-#pragma unroll
+QUDA_UNROLL
 	    for (int k=0; k<N; k++) {
 	      if (k==0) {
 		s.x  = L(i,k).real()*L(j,k).real();
@@ -98,10 +98,10 @@ namespace quda {
       __device__ __host__ inline Vector forward(const Vector &b) {
 	const Mat<T,N> &L = L_;
 	Vector x;
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<N; i++) {
 	  x(i) = b(i);
-#pragma unroll
+QUDA_UNROLL
 	  for (int j=0; j<N; j++) if (j<i) {
 	    x(i).x -= L(i,j).real()*x(j).real();
 	    x(i).x += L(i,j).imag()*x(j).imag();
@@ -124,10 +124,10 @@ namespace quda {
       __device__ __host__ inline Vector backward(const Vector &b) {
 	const Mat<T,N> &L = L_;
 	Vector x;
-#pragma unroll
+QUDA_UNROLL
 	for (int i=N-1; i>=0; i--) {
 	  x(i) = b(i);
-#pragma unroll
+QUDA_UNROLL
 	  for (int j=0; j<N; j++) if (j>=i+1) {
 	    x(i).x -= L(i,j).real()*x(j).real();
 	    x(i).x += L(i,j).imag()*x(j).imag();
@@ -149,17 +149,17 @@ namespace quda {
 	Mat<T,N> Ainv;
 	ColorSpinor<T,1,N> v;
 
-#pragma unroll
+QUDA_UNROLL
 	for (int k=0;k<N;k++) {
 
 	  // forward substitute
 	  if (!fast) v(k) = complex<T>(static_cast<T>(1.0)/L(k,k).real());
 	  else v(k) = L(k,k).real();
 
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<N; i++) if (i>k) {
 	    v(i) = complex<T>(0.0);
-#pragma unroll
+QUDA_UNROLL
 	    for (int j=0; j<N; j++) if (j>=k && j<i) {
 	      v(i).x -= L(i,j).real() * v(j).real();
 	      v(i).x += L(i,j).imag() * v(j).imag();
@@ -174,9 +174,9 @@ namespace quda {
 	  if (!fast) v(N-1) *= static_cast<T>(1.0) / L(N-1,N-1);
 	  else v(N-1) *= L(N-1,N-1);
 
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=N-2; i>=0; i--) if (i>=k) {
-#pragma unroll
+QUDA_UNROLL
 	    for (int j=0; j<N; j++) if (j>i) {
 	      v(i).x -= L(i,j).real() * v(j).real();
 	      v(i).x += L(i,j).imag() * v(j).imag();
@@ -190,7 +190,7 @@ namespace quda {
 	  // Overwrite column k
 	  Ainv(k,k) = v(k);
 
-#pragma unroll
+QUDA_UNROLL
 	  for(int i=0;i<N;i++) if (i>k) Ainv(i,k) = v(i);
 	}
 
diff --git a/include/matrix_tile.cuh b/include/matrix_tile.cuh
index b16e18b352..1cf9d0195d 100644
--- a/include/matrix_tile.cuh
+++ b/include/matrix_tile.cuh
@@ -20,9 +20,9 @@ namespace quda {
 
     template <typename T_> inline __device__ __host__ void operator*=(const T_ &a)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<n; j++) {
           tile[i*n+j] *= a;
         }
@@ -37,11 +37,11 @@ namespace quda {
     template <int k, bool ghost_a, bool ghost_b>
     inline __device__ __host__ void mma_nn(const MatrixTile<T, m, k, ghost_a> &a, const MatrixTile<T, k, n, ghost_b> &b)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < n; j++) {
-#pragma unroll
+QUDA_UNROLL
           for (int l = 0; l < k; l++) {
             tile[i*n+j] = cmac(a.tile[i*k+l], b.tile[l*n+j], tile[i*n+j]);
           }
@@ -57,11 +57,11 @@ namespace quda {
     template <int k, bool ghost_a, bool ghost_b>
     inline __device__ __host__ void mma_nt(const MatrixTile<T, m, k, ghost_a> &a, const MatrixTile<T, n, k, ghost_b> &b)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < n; j++) {
-#pragma unroll
+QUDA_UNROLL
           for (int l = 0; l < k; l++) {
             tile[i*n+j] = cmac(a.tile[i*k+l], conj(b.tile[j*k+l]), tile[i*n+j]);
           }
@@ -77,11 +77,11 @@ namespace quda {
     template <int k, bool ghost_a, bool ghost_b>
     inline __device__ __host__ void mma_tn(const MatrixTile<T, k, m, ghost_a> &a, const MatrixTile<T, k, n, ghost_b> &b)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < n; j++) {
-#pragma unroll
+QUDA_UNROLL
           for (int l = 0; l < k; l++) {
             tile[i*n+j] = cmac(conj(a.tile[l*m+i]), b.tile[l*n+j], tile[i*n+j]);
           }
@@ -94,9 +94,9 @@ namespace quda {
      */
     template <typename Accessor> inline __device__ __host__ std::enable_if_t<Accessor::supports_ghost_zone, void> load(const Accessor &a, int d, int parity, int x_cb, int i0, int j0)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<n; j++) {
           if (ghost) // ideally `if constexpr`, if we had this we could avoid this whole `supports_ghost_zone` thing.
             tile[i*n+j] = a.Ghost(d, parity, x_cb, i0 + i, j0 + j);
@@ -112,9 +112,9 @@ namespace quda {
     template <typename Accessor> inline __device__ __host__ std::enable_if_t<!Accessor::supports_ghost_zone, void> load(const Accessor &a, int d, int parity, int x_cb, int i0, int j0)
     {
       static_assert(!ghost, "Cannot use ghost MatrixTile with field that doesn't support ghost zones");
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<n; j++) {
           tile[i*n+j] = a(d, parity, x_cb, i0 + i, j0 + j);
         }
@@ -123,9 +123,9 @@ namespace quda {
 
     template <typename Accessor> inline __device__ __host__ std::enable_if_t<Accessor::supports_ghost_zone, void> load(const Accessor &a, int d, int parity, int x_cb, int si, int sj, int i0, int j0)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<n; j++) {
           if (ghost)
             tile[i*n+j] = a.Ghost(d, parity, x_cb, si, sj, i0 + i, j0 + j);
@@ -138,9 +138,9 @@ namespace quda {
     template <typename Accessor> inline __device__ __host__ std::enable_if_t<!Accessor::supports_ghost_zone, void> load(const Accessor &a, int d, int parity, int x_cb, int si, int sj, int i0, int j0)
     {
       static_assert(!ghost, "Cannot use ghost MatrixTile with field that doesn't support ghost zones");
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<n; j++) {
           tile[i*n+j] = a(d, parity, x_cb, si, sj, i0 + i, j0 + j);
         }
@@ -149,9 +149,9 @@ namespace quda {
 
     template <typename Accessor> inline __device__ __host__ void save(Accessor &a, int d, int parity, int x_cb, int i0, int j0)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < n; j++) {
           if (ghost)
             a.Ghost(d, parity, x_cb, i0 + i, j0 + j) = tile[i*n+j];
@@ -163,9 +163,9 @@ namespace quda {
 
     template <typename Accessor> inline __device__ __host__ void save(Accessor &a, int d, int parity, int x_cb, int si, int sj, int i0, int j0)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < n; j++) {
           if (ghost)
             a.Ghost(d, parity, x_cb, si, sj, i0 + i, j0 + j) = tile[i*n+j];
@@ -177,9 +177,9 @@ namespace quda {
 
     template <typename Accessor> inline __device__ __host__ void loadCS(const Accessor &a, int d, int dir, int parity, int x_cb, int s, int i0, int j0)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<n; j++) {
           if (ghost)
             tile[i*n+j] = a.Ghost(d, dir, parity, x_cb, s, i0 + i, j0 + j);
@@ -191,9 +191,9 @@ namespace quda {
 
     template <typename Accessor> inline __device__ __host__ void saveCS(Accessor &a, int d, int dir, int parity, int x_cb, int s, int i0, int j0)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<n; j++) {
           if (ghost)
             a.Ghost(d, dir, parity, x_cb, s, i0 + i, j0 + j) = tile[i*n+j];
@@ -206,18 +206,18 @@ namespace quda {
     inline __device__ __host__ real abs_max()
     {
       real maxTile[m*n];
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < n; j++) {
           maxTile[i*n+j] = fmax(fabs(tile[i*n+j].real()), fabs(tile[i*n+j].imag()));
         }
       }
 
       real max = 0.0;
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < m; i++) {
-#pragma unroll
+QUDA_UNROLL
         for (int j = 0; j < n; j++) {
           max = fmax(max, maxTile[i*n+j]);
         }
diff --git a/include/quda_api.h b/include/quda_api.h
index 82c90d0093..6fff4e4628 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -1,6 +1,11 @@
 #pragma once
 
 #ifdef QUDA_BACKEND_OMPTARGET
+
+#ifndef QUDA_UNROLL
+#define QUDA_UNROLL
+#endif
+
 // OMPTARGET SPECIFIC workarounds
 #define __host__
 #define __device__
@@ -98,6 +103,10 @@ extern LaunchParam launch_param;
 
 #endif // QUDA_BACKEND_OMPTARGET
 
+#ifndef QUDA_UNROLL
+#define QUDA_UNROLL _Pragma("unroll")
+#endif
+
 #include <quda_define.h>
 #include <string>
 #include <enum_quda.h>
diff --git a/include/quda_matrix.h b/include/quda_matrix.h
index afc5c364ae..4e614c223d 100644
--- a/include/quda_matrix.h
+++ b/include/quda_matrix.h
@@ -82,13 +82,13 @@ namespace quda {
 
         template <class U> __device__ __host__ inline Matrix(const Matrix<U, N> &a)
         {
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < N * N; i++) data[i] = a.data[i];
         }
 
         __device__ __host__ inline Matrix(const T data_[])
         {
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<N*N; i++) data[i] = data_[i];
         }
 
@@ -116,7 +116,7 @@ namespace quda {
 
 	template<class U>
 	  __device__ __host__ inline void operator=(const Matrix<U,N> & b) {
-#pragma unroll
+QUDA_UNROLL
 	  for (int i=0; i<N*N; i++) data[i] = b.data[i];
 	}
 
@@ -139,10 +139,10 @@ namespace quda {
         */
         __device__ __host__ inline real L1() {
           real l1 = 0;
-#pragma unroll
+QUDA_UNROLL
           for (int j=0; j<N; j++) {
             real col_sum = 0;
-#pragma unroll
+QUDA_UNROLL
             for (int i=0; i<N; i++) {
               col_sum += abs(data[i*N + j]);
             }
@@ -158,9 +158,9 @@ namespace quda {
         */
         __device__ __host__ inline real L2() {
           real l2 = 0;
-#pragma unroll
+QUDA_UNROLL
           for (int j=0; j<N; j++) {
-#pragma unroll
+QUDA_UNROLL
             for (int i=0; i<N; i++) {
               l2 += norm(data[i*N + j]);
             }
@@ -175,10 +175,10 @@ namespace quda {
         */
         __device__ __host__ inline real Linf() {
           real linf = 0;
-#pragma unroll
+QUDA_UNROLL
           for (int i=0; i<N; i++) {
             real row_sum = 0;
-#pragma unroll
+QUDA_UNROLL
             for (int j=0; j<N; j++) {
               row_sum += abs(data[i*N + j]);
             }
@@ -206,12 +206,12 @@ namespace quda {
         {
           const auto identity = conj(*this) * *this;
 
-#pragma unroll
+QUDA_UNROLL
           for (int i=0; i<N; ++i){
             if( fabs(identity(i,i).real() - 1.0) > max_error ||
                 fabs(identity(i,i).imag()) > max_error) return false;
 
-#pragma unroll
+QUDA_UNROLL
             for (int j=i+1; j<N; ++j){
               if( fabs(identity(i,j).real()) > max_error ||
                   fabs(identity(i,j).imag()) > max_error ||
@@ -222,9 +222,9 @@ namespace quda {
             }
           }
 
-#pragma unroll
+QUDA_UNROLL
           for (int i=0; i<N; i++) {
-#pragma unroll
+QUDA_UNROLL
             for (int j=0; j<N; j++) {
               if (is_nan((*this)(i,j).real()) ||
                   is_nan((*this)(i,j).imag())) return false;
@@ -305,7 +305,7 @@ namespace quda {
       constexpr int size() const { return N * N; }
 
       __device__ __host__ inline HMatrix() {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < N * N; i++) data[i] = (T)0.0;
       }
 
@@ -315,7 +315,7 @@ namespace quda {
       HMatrix& operator=(HMatrix<T,N> &&) = default;
 
       __device__ __host__ inline HMatrix(const T data_[]) {
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<N*N; i++) data[i] = data_[i];
       }
 
@@ -336,7 +336,7 @@ namespace quda {
 
       template<class U>
 	__device__ __host__ inline void operator=(const HMatrix<U,N> & b) {
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<N*N; i++) data[i] = b.data[i];
       }
 
@@ -353,15 +353,15 @@ namespace quda {
       __device__ __host__ inline HMatrix<T,N> square() const {
 	HMatrix<T,N> result;
 	complex<T> tmp;
-#pragma unroll
+QUDA_UNROLL
 	for (int i=0; i<N; i++) {
-#pragma unroll
+QUDA_UNROLL
 	  for (int k=0; k<N; k++) if (i<=k) { // else compiler can't handle triangular unroll
             tmp.real(             (*this)(i,0).real() * (*this)(0,k).real());
 	    tmp.real(tmp.real() - (*this)(i,0).imag() * (*this)(0,k).imag());
             tmp.imag(             (*this)(i,0).real() * (*this)(0,k).imag());
 	    tmp.imag(tmp.imag() + (*this)(i,0).imag() * (*this)(0,k).real());
-#pragma unroll
+QUDA_UNROLL
 	    for (int j=1; j<N; j++) {
               tmp.real(tmp.real() + (*this)(i,j).real() * (*this)(j,k).real());
               tmp.real(tmp.real() - (*this)(i,j).imag() * (*this)(j,k).imag());
@@ -382,7 +382,7 @@ namespace quda {
       {
         HMatrix<T, N> result;
         T max = static_cast<T>(0.0);
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < N * N; i++) max = (abs(data[i]) > max ? abs(data[i]) : max);
         return max;
       }
@@ -402,9 +402,9 @@ namespace quda {
 
   template<class T,int N>
     __device__ __host__ Matrix<T,N>::Matrix(const HMatrix<typename RealType<T>::type,N> &a) {
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<N; i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int j=0; j<N; j++) {
 	(*this)(i,j) = a(i,j);
       }
@@ -433,7 +433,7 @@ namespace quda {
     __device__ __host__ inline Mat<T,N> operator+(const Mat<T,N> & a, const Mat<T,N> & b)
     {
       Mat<T,N> result;
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<a.size(); i++) result.data[i] = a.data[i] + b.data[i];
       return result;
     }
@@ -442,7 +442,7 @@ namespace quda {
   template< template<typename,int> class Mat, class T, int N>
     __device__ __host__ inline Mat<T,N> operator+=(Mat<T,N> & a, const Mat<T,N> & b)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<a.size(); i++) a.data[i] += b.data[i];
       return a;
     }
@@ -450,7 +450,7 @@ namespace quda {
   template< template<typename,int> class Mat, class T, int N>
     __device__ __host__ inline Mat<T,N> operator+=(Mat<T,N> & a, const T & b)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<a.rows(); i++) a(i,i) += b;
       return a;
     }
@@ -458,7 +458,7 @@ namespace quda {
   template< template<typename,int> class Mat, class T, int N>
     __device__ __host__ inline Mat<T,N> operator-=(Mat<T,N> & a, const Mat<T,N> & b)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<a.size(); i++) a.data[i] -= b.data[i];
       return a;
     }
@@ -467,7 +467,7 @@ namespace quda {
     __device__ __host__ inline Mat<T,N> operator-(const Mat<T,N> & a, const Mat<T,N> & b)
     {
       Mat<T,N> result;
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<a.size(); i++) result.data[i] = a.data[i] - b.data[i];
       return result;
     }
@@ -475,7 +475,7 @@ namespace quda {
   template< template<typename,int> class Mat, class T, int N, class S>
     __device__ __host__ inline Mat<T,N> operator*(const S & scalar, const Mat<T,N> & a){
       Mat<T,N> result;
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<a.size(); ++i) result.data[i] = scalar*a.data[i];
       return result;
     }
@@ -494,7 +494,7 @@ namespace quda {
   template< template<typename,int> class Mat, class T, int N>
     __device__ __host__ inline Mat<T,N> operator-(const Mat<T,N> & a){
       Mat<T,N> result;
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<a.size(); ++i) result.data[i] = -a.data[i];
       return result;
     }
@@ -507,12 +507,12 @@ namespace quda {
     __device__ __host__ inline Mat<T,N> operator*(const Mat<T,N> &a, const Mat<T,N> &b)
     {
       Mat<T,N> result;
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; i++) {
-#pragma unroll
+QUDA_UNROLL
 	for (int k=0; k<N; k++) {
 	  result(i,k) = a(i,0) * b(0,k);
-#pragma unroll
+QUDA_UNROLL
 	  for (int j=1; j<N; j++) {
 	    result(i,k) += a(i,j) * b(j,k);
 	  }
@@ -528,15 +528,15 @@ namespace quda {
     __device__ __host__ inline Matrix<complex<T>,N> operator*(const Matrix<complex<T>,N> &a, const Matrix<complex<T>,N> &b)
     {
       Matrix<complex<T>,N> result;
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; i++) {
-#pragma unroll
+QUDA_UNROLL
 	for (int k=0; k<N; k++) {
           result(i,k).real(                     a(i,0).real() * b(0,k).real());
           result(i,k).real(result(i,k).real() - a(i,0).imag() * b(0,k).imag());
           result(i,k).imag(                     a(i,0).real() * b(0,k).imag());
           result(i,k).imag(result(i,k).imag() + a(i,0).imag() * b(0,k).real());
-#pragma unroll
+QUDA_UNROLL
 	  for (int j=1; j<N; j++) {
 	    result(i,k).real(result(i,k).real() + a(i,j).real() * b(j,k).real());
 	    result(i,k).real(result(i,k).real() - a(i,j).imag() * b(j,k).imag());
@@ -563,12 +563,12 @@ namespace quda {
                                                                                      const Matrix<U, N> &b)
   {
     Matrix<typename PromoteTypeId<T, U>::type, N> result;
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; i++) {
-#pragma unroll
+QUDA_UNROLL
 	for (int k=0; k<N; k++) {
 	  result(i,k) = a(i,0) * b(0,k);
-#pragma unroll
+QUDA_UNROLL
 	  for (int j=1; j<N; j++) {
 	    result(i,k) += a(i,j) * b(j,k);
 	  }
@@ -594,9 +594,9 @@ namespace quda {
     __device__ __host__ inline
     Matrix<T,N> conj(const Matrix<T,N> & other){
       Matrix<T,N> result;
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<N; ++j){
           result(i,j) = conj( other(j,i) );
         }
@@ -651,10 +651,10 @@ namespace quda {
     __device__ __host__ inline
     void setIdentity(Matrix<T,N>* m){
 
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
         (*m)(i,i) = 1;
-#pragma unroll
+QUDA_UNROLL
         for (int j=i+1; j<N; ++j){
           (*m)(i,j) = (*m)(j,i) = 0;
         }
@@ -666,10 +666,10 @@ namespace quda {
     __device__ __host__ inline
     void setIdentity(Matrix<float2,N>* m){
 
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
         (*m)(i,i) = make_float2(1,0);
-#pragma unroll
+QUDA_UNROLL
         for (int j=i+1; j<N; ++j){
           (*m)(i,j) = (*m)(j,i) = make_float2(0.,0.);
         }
@@ -681,10 +681,10 @@ namespace quda {
     __device__ __host__ inline
     void setIdentity(Matrix<double2,N>* m){
 
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
         (*m)(i,i) = make_double2(1,0);
-#pragma unroll
+QUDA_UNROLL
         for (int j=i+1; j<N; ++j){
           (*m)(i,j) = (*m)(j,i) = make_double2(0.,0.);
         }
@@ -697,9 +697,9 @@ namespace quda {
     __device__ __host__ inline
     void setZero(Matrix<T,N>* m){
 
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<N; ++j){
           (*m)(i,j) = 0;
         }
@@ -711,9 +711,9 @@ namespace quda {
     __device__ __host__ inline
     void setZero(Matrix<float2,N>* m){
 
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<N; ++j){
           (*m)(i,j) = make_float2(0.,0.);
         }
@@ -725,9 +725,9 @@ namespace quda {
     __device__ __host__ inline
     void setZero(Matrix<double2,N>* m){
 
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<N; ++j){
           (*m)(i,j) = make_double2(0.,0.);
         }
@@ -743,9 +743,9 @@ namespace quda {
 
     // second make it traceless
     real imag_trace = 0.0;
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<N; i++) imag_trace += am(i,i).y;
-#pragma unroll
+QUDA_UNROLL
     for (int i=0; i<N; i++) {
       am(i,i).y -= imag_trace/N;
     }
@@ -760,9 +760,9 @@ namespace quda {
 
     // second make it traceless
     real imag_trace = 0.0;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < N; i++) imag_trace += am(i, i).y;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < N; i++) { am(i, i).y -= imag_trace / N; }
     // third scale out anti hermitian part
     Complex i_2(0.0, 0.5);
@@ -800,7 +800,7 @@ namespace quda {
     __device__  __host__ inline
     void copyColumn(const Matrix<T,N>& m, int c, Array<T,N>* a)
     {
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
         (*a)[i] = m(i,c); // c is the column index
       }
@@ -810,9 +810,9 @@ namespace quda {
   // Need some print utilities
   template<class T, int N>
     std::ostream & operator << (std::ostream & os, const Matrix<T,N> & m){
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<N; ++i){
-#pragma unroll
+QUDA_UNROLL
         for (int j=0; j<N; ++j){
           os << m(i,j) << " ";
         }
@@ -923,9 +923,9 @@ namespace quda {
       int j=0;
 
       //error = ||U^dagger U - I||_L2
-#pragma unroll
+QUDA_UNROLL
       for (i=0; i<3; ++i)
-#pragma unroll
+QUDA_UNROLL
 	for (j=0; j<3; ++j)
 	  if(i==j) {
 	    temp = identity_comp(i,j);
diff --git a/include/reduce_helper.h b/include/reduce_helper.h
index 8f02e7fc25..f237bd16aa 100644
--- a/include/reduce_helper.h
+++ b/include/reduce_helper.h
@@ -241,7 +241,7 @@ namespace quda
             constexpr size_t n = sizeof(T) / sizeof(atomic_t);
             atomic_t sum_tmp[n];
             memcpy(sum_tmp, &sum, sizeof(sum));
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < n; i++) {
               // catch the case where the computed value is equal to the init_value
               sum_tmp[i] = sum_tmp[i] == init_value<atomic_t>() ? terminate_value<atomic_t>() : sum_tmp[i];
diff --git a/include/shared_memory_cache_helper.cuh b/include/shared_memory_cache_helper.cuh
index 59c3a928b0..55b21a002d 100644
--- a/include/shared_memory_cache_helper.cuh
+++ b/include/shared_memory_cache_helper.cuh
@@ -112,7 +112,7 @@ namespace quda
       atom_t tmp[n_element];
       memcpy(tmp, (void*)&a, sizeof(T));
       int j = (z * block.y + y) * block.x + x;
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < n_element; i++) cache<dynamic>()[i * stride + j] = tmp[i];
     }
 
@@ -120,7 +120,7 @@ namespace quda
     {
       atom_t tmp[n_element];
       int j = (z * block.y + y) * block.x + x;
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < n_element; i++) tmp[i] = cache<dynamic>()[i * stride + j];
       T a;
       memcpy((void*)&a, tmp, sizeof(T));
diff --git a/include/su3_project.cuh b/include/su3_project.cuh
index 0a725ff89a..58e67a2f39 100644
--- a/include/su3_project.cuh
+++ b/include/su3_project.cuh
@@ -24,9 +24,9 @@ namespace quda {
   __host__ __device__ inline bool checkUnitary(const Matrix &inv, const Matrix &in, const Float tol)
   {
     // first check U - U^{-1} = 0
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < in.rows(); i++) {
-#pragma unroll
+QUDA_UNROLL
       for (int j = 0; j < in.rows(); j++) {
         if (fabs(in(i,j).real() - inv(j,i).real()) > tol ||
             fabs(in(i,j).imag() + inv(j,i).imag()) > tol) return false;
@@ -36,12 +36,12 @@ namespace quda {
     // now check 1 - U^dag * U = 0
     // this check is more expensive so delay until we have passed first check
     const Matrix identity = conj(in)*in;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < in.rows(); i++) {
       if (fabs(identity(i,i).real() - static_cast<Float>(1.0)) > tol ||
           fabs(identity(i,i).imag()) > tol)
         return false;
-#pragma unroll
+QUDA_UNROLL
       for (int j = 0; j < in.rows(); j++) {
         if (i>j) { // off-diagonal identity check
         if (fabs(identity(i,j).real()) > tol || fabs(identity(i,j).imag()) > tol ||
diff --git a/include/targets/generic/aos.h b/include/targets/generic/aos.h
index c3e50c9d65..393adbc61c 100644
--- a/include/targets/generic/aos.h
+++ b/include/targets/generic/aos.h
@@ -11,7 +11,7 @@ namespace quda {
    */
   template <typename T, int n> __host__ __device__ void block_load(T out[n], const T *in)
   {
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < n; i++) out[i] = in[i];
   }
 
@@ -24,7 +24,7 @@ namespace quda {
    */
   template <typename T, int n> __host__ __device__ void block_store(T *out, const T in[n])
   {
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < n; i++) out[i] = in[i];
   }
 
diff --git a/include/targets/omptarget/cub/block/block_reduce.cuh b/include/targets/omptarget/cub/block/block_reduce.cuh
index 30876a4424..3c94a40910 100644
--- a/include/targets/omptarget/cub/block/block_reduce.cuh
+++ b/include/targets/omptarget/cub/block/block_reduce.cuh
@@ -33,7 +33,7 @@ struct WarpReduce
   {}
   inline T Sum(T input, int valid_items = warp_threads)  // FIXME warp_num_valid < warp_threads
   {
-    #pragma unroll
+    QUDA_UNROLL
     for(int i=0;i<warp_steps;++i){
       temp_storage.reduce[lane_id] = input;
       const int offset = 1 << i;
@@ -78,7 +78,7 @@ struct BlockReduce
     if (lane_id == 0) temp_storage.warp_aggregates[warp_id] = warp_aggregate;
     #pragma omp barrier
     if (tid == 0) {
-#pragma unroll
+QUDA_UNROLL
       for(int i=1;i<warps;++i) warp_aggregate += temp_storage.warp_aggregates[i];
     }
     return warp_aggregate;
diff --git a/include/targets/omptarget/warp_collective.h b/include/targets/omptarget/warp_collective.h
index cf3f592b74..6d20a25527 100644
--- a/include/targets/omptarget/warp_collective.h
+++ b/include/targets/omptarget/warp_collective.h
@@ -5,10 +5,10 @@ namespace quda {
 #ifdef __CUDA_ARCH__
     constexpr int warp_size = device::warp_size();
     if (warp_split > 1) {
-#pragma unroll
+QUDA_UNROLL
       for (int i = 0; i < x.size(); i++) {
         // reduce down to the first group of column-split threads
-#pragma unroll
+QUDA_UNROLL
         for (int offset = warp_size / 2; offset >= warp_size / warp_split; offset /= 2) {
           // TODO - add support for non-converged warps
           x[i].real(x[i].real() + __shfl_down_sync(device::warp_converged_mask(), x[i].real(), offset));

From a9819a5a621287bb257d4506aa9959995e005797 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 4 Oct 2021 21:52:14 +0000
Subject: [PATCH 090/198] omp: use omp atomic capture compare properly

---
 include/atomic.cuh | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/include/atomic.cuh b/include/atomic.cuh
index b99dcd2f06..fe97bbbfe0 100644
--- a/include/atomic.cuh
+++ b/include/atomic.cuh
@@ -52,17 +52,45 @@ static inline double atomicAdd(double *x, double v)
 }
 static inline unsigned int atomicCAS(unsigned int*x, unsigned int c, unsigned int v)
 {
+#if 0
+  unsigned int old = *x, assumed;
+  do{
+    assumed = old;
+    if(assumed==c){
+      #pragma omp atomic capture
+      {old=*x; *x=v;}
+    }else{
+      old=*x;
+    }
+  }while(assumed!=old);
+  return old;
+#else
   unsigned int z;
-#pragma omp atomic capture
-  {z=*x; *x=z==c?v:z;}
+  #pragma omp atomic capture compare
+  {z=*x; if(*x==c){*x=v;}}
   return z;
+#endif
 }
 static inline uint32_t atomicMax(uint32_t *x, uint32_t v)
 {
+#if 0
+  unsigned int old = *x, assumed;
+  do{
+    assumed = old;
+    if(assumed<v){
+      #pragma omp atomic capture
+      {old=*x; *x=v;}
+    }else{
+      old=*x;
+    }
+  }while(assumed!=old);
+  return old;
+#else
   uint32_t z;
-#pragma omp atomic capture
-  {z=*x; *x=z<v?v:z;}
+  #pragma omp atomic capture compare
+  {z=*x; if(*x<v){*x=v;}}
   return z;
+#endif
 }
 #endif
 

From 77c0be14c032c1df64e6b2028236ad7be2dcd07f Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 5 Oct 2021 03:03:36 +0000
Subject: [PATCH 091/198] omp: fix last merge

---
 include/reduce_helper.h            | 4 ++--
 lib/targets/omptarget/quda_api.cpp | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/reduce_helper.h b/include/reduce_helper.h
index 5c8c8b54d2..4d7f338441 100644
--- a/include/reduce_helper.h
+++ b/include/reduce_helper.h
@@ -356,9 +356,9 @@ QUDA_UNROLL
         arg.result_d[idx] = sum;
         arg.count[idx] = 0; // set to zero for next time
       }
-#endif
-    }
     }
 #endif
+  }
+#endif
 
 } // namespace quda
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 2215db5b2c..3bc34bcee2 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -519,7 +519,7 @@ namespace quda
   float qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
                               const char *func, const char *file, const char *line)
   {
-    return const_cast<float>(reinterpret_cast<QudaEvent *>(stop.event)->time - reinterpret_cast<QudaEvent *>(start.event)->time);
+    return static_cast<float>(reinterpret_cast<QudaEvent *>(stop.event)->time - reinterpret_cast<QudaEvent *>(start.event)->time);
   }
 
   void qudaEventDestroy_(qudaEvent_t &event, const char *func, const char *file, const char *line)

From 5fd11fde9262298832bcfd1c834dfc287cb952d9 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 6 Oct 2021 15:56:09 +0000
Subject: [PATCH 092/198] omp: replace extra pragma unroll

---
 include/kernels/coarse_op_kernel.cuh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 307af8274c..f60371ed16 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -489,9 +489,9 @@ QUDA_UNROLL
       }
 
       ColorSpinor<real, Arg::fineColor, Arg::fineSpin / 2> V;
-#pragma unroll
+QUDA_UNROLL
       for (int s = 0; s < Arg::fineSpin / 2; s++) {
-#pragma unroll
+QUDA_UNROLL
         for (int c = 0; c < Arg::fineColor; c++) { V(s, c) = arg.V(parity, x_cb, 2 * ch + s, c, ic_c); }
       }
 

From f3676eef2de2036ae47898ce8b2825b79bb7b5f0 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 29 Oct 2021 18:45:39 +0000
Subject: [PATCH 093/198] omp: define threadIdx for DW m5 dslash

---
 include/kernels/dslash_domain_wall_m5.cuh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/kernels/dslash_domain_wall_m5.cuh b/include/kernels/dslash_domain_wall_m5.cuh
index 3e140bce91..65226f4674 100644
--- a/include/kernels/dslash_domain_wall_m5.cuh
+++ b/include/kernels/dslash_domain_wall_m5.cuh
@@ -210,6 +210,7 @@ namespace quda
   template <bool sync, bool dagger, bool shared, class Vector, class Arg, Dslash5Type type = Arg::type>
   __device__ __host__ inline Vector d5(const Arg &arg, const Vector &in, int parity, int x_cb, int s)
   {
+    QUDA_RT_CONSTS;
 
     using real = typename Arg::real;
     constexpr bool is_variable = true;

From 65f79dff7a1404194a6350832b1d18b0e331967a Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 8 Dec 2021 14:49:10 +0000
Subject: [PATCH 094/198] omp: fix last merge and clean up target files

---
 include/float_vector.h                        |  13 +
 include/quda_api.h                            | 103 ----
 include/reducer.h                             |   1 +
 include/targets/generic/load_store.h          |   2 +
 include/targets/omptarget/atomic_helper.h     |  55 ++
 .../omptarget/block_reduction_kernel.h        |  88 ++-
 include/targets/omptarget/quda_api.h          | 103 ++++
 include/targets/omptarget/quda_cuda_api.h     |  21 -
 include/targets/omptarget/reduce_helper.h     | 159 ++++++
 include/targets/omptarget/reduction_kernel.h  | 523 ++++++++----------
 include/targets/omptarget/target_device.h     |   2 +-
 lib/targets/omptarget/comm_target.cpp         |   1 -
 lib/targets/omptarget/device.cpp              | 350 +++---------
 13 files changed, 699 insertions(+), 722 deletions(-)
 create mode 100644 include/targets/omptarget/atomic_helper.h
 create mode 100644 include/targets/omptarget/quda_api.h
 delete mode 100644 include/targets/omptarget/quda_cuda_api.h
 create mode 100644 include/targets/omptarget/reduce_helper.h

diff --git a/include/float_vector.h b/include/float_vector.h
index 2eaceaa8a4..3d2e2842f5 100644
--- a/include/float_vector.h
+++ b/include/float_vector.h
@@ -65,6 +65,19 @@ namespace quda {
     return v;
   }
 
+  template <> __device__ __host__ inline array<double,1> zero<array<double,1>>() { return zero<double,1>(); }
+  template <> __device__ __host__ inline array<double,2> zero<array<double,2>>() { return zero<double,2>(); }
+  template <> __device__ __host__ inline array<double,3> zero<array<double,3>>() { return zero<double,3>(); }
+  template <> __device__ __host__ inline array<double,4> zero<array<double,4>>() { return zero<double,4>(); }
+  template <> __device__ __host__ inline array<double,8> zero<array<double,8>>() { return zero<double,8>(); }
+  template <> __device__ __host__ inline array<double,16> zero<array<double,16>>() { return zero<double,16>(); }
+  template <> __device__ __host__ inline array<double2,1> zero<array<double2,1>>() { return zero<double2,1>(); }
+  template <> __device__ __host__ inline array<double2,2> zero<array<double2,2>>() { return zero<double2,2>(); }
+  template <> __device__ __host__ inline array<double2,3> zero<array<double2,3>>() { return zero<double2,3>(); }
+  template <> __device__ __host__ inline array<double2,4> zero<array<double2,4>>() { return zero<double2,4>(); }
+  template <> __device__ __host__ inline array<double2,8> zero<array<double2,8>>() { return zero<double2,8>(); }
+  template <> __device__ __host__ inline array<double2,16> zero<array<double2,16>>() { return zero<double2,16>(); }
+
   template <typename T> struct RealType {
   };
   template <> struct RealType<double> {
diff --git a/include/quda_api.h b/include/quda_api.h
index 7f9062fe5d..6cf0848283 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -1,108 +1,5 @@
 #pragma once
 
-#ifdef QUDA_BACKEND_OMPTARGET
-
-#ifndef QUDA_UNROLL
-#define QUDA_UNROLL
-#endif
-
-// OMPTARGET SPECIFIC workarounds
-#define __host__
-#define __device__
-#define __global__
-#define __constant__ static
-#define __launch_bounds__(...)
-#define __syncthreads() _Pragma("omp barrier")
-#define __forceinline__ inline __attribute__((always_inline))
-
-#define CUDA_SUCCESS QUDA_SUCCESS
-
-#define QUDA_RT_CONSTS \
-  const dim3\
-    blockDim=launch_param.block,\
-    gridDim=launch_param.grid,\
-    threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y)),\
-    blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y))
-
-#include <functional>
-#include <iostream>
-#include <omp.h>
-
-struct dim3 {unsigned int x,y,z;
-  constexpr dim3():x(1u),y(1u),z(1u){}
-  constexpr dim3(unsigned int x):x(x),y(1u),z(1u){}
-  constexpr dim3(unsigned int x,unsigned int y):x(x),y(y),z(1u){}
-  constexpr dim3(unsigned int x,unsigned int y,unsigned int z):x(x),y(y),z(z){}
-};
-struct int2 {int x,y;};
-struct int4 {int x,y,z,w;};
-static inline int4 make_int4(int x,int y,int z,int w){return int4{x,y,z,w};}
-struct uint2 {unsigned int x,y;};
-static inline uint2 make_uint2(unsigned int x,unsigned int y){return uint2{x,y};}
-struct uint3 {unsigned int x,y,z;};
-static inline uint3 make_uint3(unsigned int x,unsigned int y,unsigned int z){return uint3{x,y,z};}
-struct char2 {char x,y;};
-static inline char2 make_char2(char x,char y){return char2{x,y};}
-struct char3 {char x,y,z;};
-struct char4 {char x,y,z,w;};
-struct short2 {short x,y;};
-static inline short2 make_short2(short x,short y){return short2{x,y};}
-struct short3 {short x,y,z;};
-struct short4 {short x,y,z,w;};
-struct float2 {float x,y;};
-static inline float2 make_float2(float x,float y){return float2{x,y};}
-struct float3 {float x,y,z;};
-static inline float3 make_float3(float x,float y,float z){return float3{x,y,z};}
-struct float4 {float x,y,z,w;};
-static inline float4 make_float4(float x,float y,float z,float w){return float4{x,y,z,w};}
-struct double2 {double x,y;};
-static inline double2 make_double2(double x,double y){return double2{x,y};}
-struct double3 {double x,y,z;};
-static inline double3 make_double3(double x,double y,double z){return double3{x,y,z};}
-struct double4 {double x,y,z,w;};
-static inline double4 make_double4(double x,double y,double z,double w){return double4{x,y,z,w};}
-
-template <typename... Fmt>
-static inline int
-ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f, Fmt... fmt)
-{
-  if(0==omp_get_team_num()&&0==omp_get_thread_num()){
-    printf("OMP WIP: ");
-    printf(fmt...);
-    printf(" %s:%ld %s\n", file, line, func);
-  }
-  f();
-  return 0;
-}
-template <typename... Fmt>
-static inline int
-ompwip_(const char * const file, const size_t line, const char * const func, const char * const msg, Fmt... fmt)
-{
-  return ompwip_(file,line,func,[](){},msg,fmt...);
-}
-static inline int
-ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f=[](){})
-{
-  return ompwip_(file,line,func,f,"");
-}
-#define ompwip(...) ompwip_(__FILE__,__LINE__,__PRETTY_FUNCTION__,##__VA_ARGS__)
-
-using cudaStream_t = int;  // device.h:/cudaStream_t
-using CUresult = int;  // ../lib/coarse_op.cuh:/CUresult
-
-struct SharedCache{
-  int *addr;
-};
-extern SharedCache shared_cache;
-
-struct LaunchParam{
-  dim3 block;
-  dim3 grid;
-};
-extern LaunchParam launch_param;
-
-#endif // QUDA_BACKEND_OMPTARGET
-
 #ifndef QUDA_UNROLL
 #define QUDA_UNROLL _Pragma("unroll")
 #endif
diff --git a/include/reducer.h b/include/reducer.h
index ab94e7ad9f..f98d47f921 100644
--- a/include/reducer.h
+++ b/include/reducer.h
@@ -3,6 +3,7 @@
 #include "complex_quda.h"
 #include "quda_constants.h"
 #include "quda_api.h"
+#include "float_vector.h"
 
 /**
    @file reducer.h
diff --git a/include/targets/generic/load_store.h b/include/targets/generic/load_store.h
index cc34dd1d55..3a513b648d 100644
--- a/include/targets/generic/load_store.h
+++ b/include/targets/generic/load_store.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <target_device.h>
+
 namespace quda
 {
 
diff --git a/include/targets/omptarget/atomic_helper.h b/include/targets/omptarget/atomic_helper.h
new file mode 100644
index 0000000000..7da42a09a8
--- /dev/null
+++ b/include/targets/omptarget/atomic_helper.h
@@ -0,0 +1,55 @@
+#pragma once
+
+#include <array.h>
+
+/**
+   @file atomic_helper.h
+
+   @section Provides definitions of atomic functions that are used in QUDA.
+ */
+
+namespace quda
+{
+
+  /**
+     @brief atomic_fetch_add function performs similarly as atomic_ref::fetch_add
+     @param[in,out] addr The memory address of the variable we are
+     updating atomically
+     @param[in] val The value we summing to the value at addr
+  */
+  template <typename T> __device__ __host__ inline void atomic_fetch_add(T *addr, T val)
+  {
+#pragma omp atomic update
+      *addr += val;
+  }
+
+  template <typename T> __device__ __host__ inline void atomic_fetch_add(complex<T> *addr, complex<T> val)
+  {
+    atomic_fetch_add(reinterpret_cast<T *>(addr) + 0, val.real());
+    atomic_fetch_add(reinterpret_cast<T *>(addr) + 1, val.imag());
+  }
+
+  template <typename T, int n> __device__ __host__ inline void atomic_fetch_add(array<T, n> *addr, array<T, n> val)
+  {
+    for (int i = 0; i < n; i++) atomic_fetch_add(&(*addr)[i], val[i]);
+  }
+
+  /**
+     @brief atomic_fetch_max function that does an atomic max.
+     @param[in,out] addr The memory address of the variable we are
+     updating atomically
+     @param[in] val The value we are comparing against.  Must be
+     positive valued else result is undefined.
+  */
+  __device__ __host__ inline void atomic_fetch_abs_max(float *addr, float val)
+  {
+#pragma omp atomic compare
+    if(*addr<val){*addr=val;}
+  }
+  __device__ __host__ inline void atomic_fetch_abs_max(double *addr, double val)
+  {
+#pragma omp atomic compare
+    if(*addr<val){*addr=val;}
+  }
+
+} // namespace quda
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 0c01ceb219..697a9783a8 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -13,6 +13,22 @@ namespace quda {
      determines if we are swizzling and a parameter "swizzle_factor"
      which is the effective matrix dimension that we are tranposing in
      this mapping.
+
+     Specifically, the thread block id is remapped by
+     transposing its coordinates: if the original order can be
+     parameterized by
+
+     blockIdx.x = j * swizzle + i,
+
+     then the new order is
+
+     block_idx = i * (gridDim.x / swizzle) + j
+
+     We need to factor out any remainder and leave this in original
+     ordering.
+
+     @param arg Kernel argument struct
+     @return Swizzled block index
    */
   template <typename Arg> __device__ constexpr int virtual_block_idx(const Arg &arg)
   {
@@ -22,7 +38,6 @@ namespace quda {
       // the portion of the grid that is exactly divisible by the number of SMs
       const auto gridp = gridDim.x - gridDim.x % arg.swizzle_factor;
 
-      block_idx = blockIdx.x;
       if (block_idx < gridp) {
         // this is the portion of the block that we are going to transpose
         const int i = blockIdx.x % arg.swizzle_factor;
@@ -40,6 +55,9 @@ namespace quda {
      creates and curries in the block size.  This allows the block
      size to be set statically at launch time in the actual argument
      class that is passed to the kernel.
+
+     @tparam block_size x-dimension block-size
+     @param[in] arg Kernel argument
    */
   template <unsigned int block_size_, typename Arg_> struct BlockKernelArg : Arg_ {
     using Arg = Arg_;
@@ -48,28 +66,49 @@ namespace quda {
   };
 
   /**
-     @brief Generic block kernel.  Here, we split the block and thread
-     indices in the x and y dimension and pass these indices
-     separately to the transform functor.  The x thread dimension is
-     templated, e.g., for efficient reductions, and typically the y
-     thread dimension is a trivial vectorizable dimension.
+     @brief BlockKernel2D_impl is the implementation of the Generic
+     block kernel.  Here, we split the block (CTA) and thread indices
+     in the x and y dimension and pass these indices separately to the
+     transform functor.  The x thread dimension is templated
+     (Arg::block_size), e.g., for efficient reductions, and typically
+     the y thread dimension is a trivial vectorizable dimension.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @param[in] arg Kernel argument
   */
-  template <template <typename> class Transformer, typename Arg>
+  template <template <typename> class Functor, typename Arg>
   __forceinline__ __device__ void BlockKernel2D_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
     const dim3 block_idx(virtual_block_idx(arg), blockIdx.y, 0);
     const dim3 thread_idx(threadIdx.x, threadIdx.y, 0);
-    auto j = blockDim.y*blockIdx.y + threadIdx.y;
+    auto j = blockDim.y * blockIdx.y + threadIdx.y;
     if (j >= arg.threads.y) return;
 
-    Transformer<Arg> t(arg);
+    Functor<Arg> t(arg);
     t(block_idx, thread_idx);
   }
 
-  template <template <typename> class Transformer, typename Arg, bool grid_stride = false>
-    __launch_bounds__(Arg::launch_bounds || Arg::block_size > 512 ? Arg::block_size : 0)
-    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
+  /**
+     @brief BlockKernel2D is the entry point of the generic block
+     kernel.  This is the specialization where the kernel argument
+     struct is passed by value directly to the kernel.  The kernel
+     type will impose launch bounds if requested (Arg::launch_bounds)
+     or if a block_size > 512 is required.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension).  Not supported at present.
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  __launch_bounds__(Arg::launch_bounds || Arg::block_size > 512 ?
+                      Arg::block_size :
+                      0) __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
@@ -92,15 +131,30 @@ namespace quda {
       //          "omp reports: teams %d threads %d\n",
       //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
       //          omp_get_num_teams(), omp_get_num_threads());
-      BlockKernel2D_impl<Transformer, Arg>(arg);
+      BlockKernel2D_impl<Functor, Arg>(arg);
     }
     }
     omp_target_free(dparg, omp_get_default_device());
   }
 
-  template <template <typename> class Transformer, typename Arg, bool grid_stride = false>
-    __launch_bounds__(Arg::launch_bounds || Arg::block_size > 512 ? Arg::block_size : 0)
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D()
+  /**
+     @brief BlockKernel2D is the entry point of the generic block
+     kernel.  This is the specialization where the kernel argument
+     struct is copied to the device prior to kernel launch.  The kernel
+     type will impose launch bounds if requested (Arg::launch_bounds)
+     or if a block_size > 512 is required.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension).  Not supported at present.
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  __launch_bounds__(Arg::launch_bounds || Arg::block_size > 512 ?
+                      Arg::block_size :
+                      0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D()
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
@@ -117,7 +171,7 @@ namespace quda {
       //          "omp reports: teams %d threads %d\n",
       //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
       //          omp_get_num_teams(), omp_get_num_threads());
-      BlockKernel2D_impl<Transformer, Arg>(device::get_arg<Arg>());
+      BlockKernel2D_impl<Functor, Arg>(device::get_arg<Arg>());
     }
     }
   }
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
new file mode 100644
index 0000000000..0d712521bb
--- /dev/null
+++ b/include/targets/omptarget/quda_api.h
@@ -0,0 +1,103 @@
+#pragma once
+
+#ifndef QUDA_UNROLL
+#define QUDA_UNROLL
+#endif
+
+// OMPTARGET SPECIFIC workarounds
+#define __host__
+#define __device__
+#define __global__
+#define __constant__ static
+#define __launch_bounds__(...)
+#define __syncthreads() _Pragma("omp barrier")
+#define __threadfence() _Pragma("omp barrier")
+#define __forceinline__ inline __attribute__((always_inline))
+
+#define CUDA_SUCCESS QUDA_SUCCESS
+
+#define QUDA_RT_CONSTS \
+  const dim3\
+    blockDim=launch_param.block,\
+    gridDim=launch_param.grid,\
+    threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y)),\
+    blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y))
+
+#include <functional>
+#include <iostream>
+#include <omp.h>
+
+struct dim3 {unsigned int x,y,z;
+  constexpr dim3():x(1u),y(1u),z(1u){}
+  constexpr dim3(unsigned int x):x(x),y(1u),z(1u){}
+  constexpr dim3(unsigned int x,unsigned int y):x(x),y(y),z(1u){}
+  constexpr dim3(unsigned int x,unsigned int y,unsigned int z):x(x),y(y),z(z){}
+};
+struct int2 {int x,y;};
+struct int4 {int x,y,z,w;};
+static inline int4 make_int4(int x,int y,int z,int w){return int4{x,y,z,w};}
+struct uint2 {unsigned int x,y;};
+static inline uint2 make_uint2(unsigned int x,unsigned int y){return uint2{x,y};}
+struct uint3 {unsigned int x,y,z;};
+static inline uint3 make_uint3(unsigned int x,unsigned int y,unsigned int z){return uint3{x,y,z};}
+struct char2 {char x,y;};
+static inline char2 make_char2(char x,char y){return char2{x,y};}
+struct char3 {char x,y,z;};
+struct char4 {char x,y,z,w;};
+struct short2 {short x,y;};
+static inline short2 make_short2(short x,short y){return short2{x,y};}
+struct short3 {short x,y,z;};
+struct short4 {short x,y,z,w;};
+struct float2 {float x,y;};
+static inline float2 make_float2(float x,float y){return float2{x,y};}
+struct float3 {float x,y,z;};
+static inline float3 make_float3(float x,float y,float z){return float3{x,y,z};}
+struct float4 {float x,y,z,w;};
+static inline float4 make_float4(float x,float y,float z,float w){return float4{x,y,z,w};}
+struct double2 {double x,y;};
+static inline double2 make_double2(double x,double y){return double2{x,y};}
+struct double3 {double x,y,z;};
+static inline double3 make_double3(double x,double y,double z){return double3{x,y,z};}
+struct double4 {double x,y,z,w;};
+static inline double4 make_double4(double x,double y,double z,double w){return double4{x,y,z,w};}
+
+template <typename... Fmt>
+static inline int
+ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f, Fmt... fmt)
+{
+  if(0==omp_get_team_num()&&0==omp_get_thread_num()){
+    printf("OMP WIP: ");
+    printf(fmt...);
+    printf(" %s:%ld %s\n", file, line, func);
+  }
+  f();
+  return 0;
+}
+template <typename... Fmt>
+static inline int
+ompwip_(const char * const file, const size_t line, const char * const func, const char * const msg, Fmt... fmt)
+{
+  return ompwip_(file,line,func,[](){},msg,fmt...);
+}
+static inline int
+ompwip_(const char * const file, const size_t line, const char * const func, std::function<void(void)>f=[](){})
+{
+  return ompwip_(file,line,func,f,"");
+}
+#define ompwip(...) ompwip_(__FILE__,__LINE__,__PRETTY_FUNCTION__,##__VA_ARGS__)
+
+using cudaStream_t = int;  // device.h:/cudaStream_t
+using CUresult = int;  // ../lib/coarse_op.cuh:/CUresult
+
+struct SharedCache{
+  int *addr;
+};
+extern SharedCache shared_cache;
+
+struct LaunchParam{
+  dim3 block;
+  dim3 grid;
+};
+extern LaunchParam launch_param;
+
+#include "../../quda_api.h"
diff --git a/include/targets/omptarget/quda_cuda_api.h b/include/targets/omptarget/quda_cuda_api.h
deleted file mode 100644
index b84d0ba949..0000000000
--- a/include/targets/omptarget/quda_cuda_api.h
+++ /dev/null
@@ -1,21 +0,0 @@
-using cudaError_t = int;
-
-namespace quda {
-
-  /**
-     @file quda_cuda_api.h
-     @brief Header file that declares some functions that will be called from within the CUDA target
-  */
-
-  namespace cuda {
-
-    void set_runtime_error(cudaError_t error, const char *api_func, const char *func, const char *file, const char *line,
-                           bool allow_error = false);
-
-    // defined in quda_api.cpp
-    void set_driver_error(CUresult error, const char *api_func, const char *func, const char *file, const char *line,
-                          bool allow_error = false);
-
-
-  }
-}
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
new file mode 100644
index 0000000000..833b01ed95
--- /dev/null
+++ b/include/targets/omptarget/reduce_helper.h
@@ -0,0 +1,159 @@
+#pragma once
+
+#include <quda_internal.h>
+#include <target_device.h>
+#include <block_reduce_helper.h>
+#include <kernel_helper.h>
+
+#ifdef QUAD_SUM
+using device_reduce_t = doubledouble;
+#else
+using device_reduce_t = double;
+#endif
+
+using count_t = unsigned int;
+
+namespace quda
+{
+
+/*
+  // OMP TARGET TODO: due to limitation of openmp implementation, we inline reduce manually.
+  // declaration of reduce function
+  template <int block_size_x, int block_size_y = 1, typename Reducer, typename Arg, typename T>
+  __device__ __forceinline__ void reduce(Arg &arg, const Reducer &r, const T &in, const int idx = 0);
+*/
+
+  /**
+     @brief ReduceArg is the argument type that all kernel arguments
+     shoud inherit from if the kernel is to utilize global reductions.
+     @tparam T the type that will be reduced
+     @tparam use_kernel_arg Whether the kernel will source the
+     parameter struct as an explicit kernel argument or from constant
+     memory
+   */
+  template <typename T, bool use_kernel_arg = true> struct ReduceArg : kernel_param<use_kernel_arg> {
+
+    // OMP TARGET TODO: due to limitation of openmp implementation, we inline reduce manually.
+    // We also remove private restrictions below.
+    // template <int, int, typename Reducer, typename Arg, typename I>
+    // friend __device__ void reduce(Arg &, const Reducer &, const I &, const int);
+    qudaError_t launch_error; /** only do complete if no launch error to avoid hang */
+
+    const int n_reduce; /** number of reductions of length n_item */
+    T *partial; /** device buffer */
+    T *result_d; /** device-mapped host buffer */
+    T *result_h; /** host buffer */
+    count_t *count; /** count array that is used to track the number of completed thread blocks at a given batch index */
+
+    /**
+       @brief Constructor for ReduceArg
+       @param[in] threads The number threads partaking in the kernel
+       @param[in] n_reduce The number of reductions
+    */
+    ReduceArg(dim3 threads, int n_reduce = 1, bool = false) :
+      kernel_param<use_kernel_arg>(threads),
+      launch_error(QUDA_ERROR_UNINITIALIZED),
+      n_reduce(n_reduce),
+      partial(static_cast<decltype(partial)>(reducer::get_device_buffer())),
+      result_d(static_cast<decltype(result_d)>(reducer::get_mapped_buffer())),
+      result_h(static_cast<decltype(result_h)>(reducer::get_host_buffer())),
+      count {reducer::get_count<count_t>()}
+    {
+      // check reduction buffers are large enough if requested
+      auto max_reduce_blocks = 2 * device::processor_count();
+      auto reduce_size = max_reduce_blocks * n_reduce * sizeof(*partial);
+      if (reduce_size > reducer::buffer_size())
+        errorQuda("Requested reduction requires a larger buffer %lu than allocated %lu", reduce_size,
+                  reducer::buffer_size());
+
+      if (commAsyncReduction()) result_d = partial;
+    }
+
+    /**
+       @brief Finalize the reduction, returning the computed reduction
+       into result.  The generic path posts an event after the kernel
+       and then polls on completion of the event.
+       @param[out] result The reduction result is copied here
+       @param[in] stream The stream on which we the reduction is being done
+     */
+    template <typename host_t, typename device_t = host_t>
+    void complete(std::vector<host_t> &result, const qudaStream_t stream = device::get_default_stream())
+    {
+      if (launch_error == QUDA_ERROR) return; // kernel launch failed so return
+      if (launch_error == QUDA_ERROR_UNINITIALIZED) errorQuda("No reduction kernel appears to have been launched");
+      auto event = reducer::get_event();
+      qudaEventRecord(event, stream);
+      while (!qudaEventQuery(event)) { }
+
+      // copy back result element by element and convert if necessary to host reduce type
+      // unit size here may differ from system_atomic_t size, e.g., if doing double-double
+      const int n_element = n_reduce * sizeof(T) / sizeof(device_t);
+      if (result.size() != (unsigned)n_element)
+        errorQuda("result vector length %lu does not match n_reduce %d", result.size(), n_element);
+      // OMP TARGET TODO: try shared memory for result_d/h
+      omp_target_memcpy(result_h, result_d, n_element*sizeof(device_t), 0, 0, omp_get_initial_device(), omp_get_default_device());
+      for (int i = 0; i < n_element; i++) result[i] = reinterpret_cast<device_t *>(result_h)[i];
+    }
+  };
+
+  /**
+     @brief Generic reduction function that reduces block-distributed
+     data "in" per thread to a single value.  This is the generic
+     variant which require explicit host-device synchronization to
+     signal the completion of the reduction to the host.
+
+     @param arg The kernel argument, this must derive from ReduceArg
+     @param r Instance of the reducer to be used in this reduction
+     @param in The input per-thread data to be reduced
+     @param idx In the case of multiple reductions, idx identifies
+     which reduction this thread block corresponds to.  Typically idx
+     will be constant along constant block_idx().y and block_idx().z.
+
+     OMP TARGET: should be run only as the main thread of each team.
+     here, we need the block reduction value, `aggregate`.
+     OMP TARGET TODO: not supported for now.  We manually inline this.
+
+  template <int block_size_x, int block_size_y, typename Reducer, typename Arg, typename T>
+  __device__ __forceinline__ void reduce(Arg &arg, const Reducer &r, const T &aggregate, const int idx)
+  {
+    printf("reduce::aggregate team %d %g\n", omp_get_team_num(), *(double*)(&aggregate));
+    bool isLastBlockDone;
+
+    arg.partial[idx * target::grid_dim().x + target::block_idx().x] = aggregate;
+    __threadfence(); // flush result
+
+    // increment global block counter
+    // auto value = atomicInc(&arg.count[idx], gridDim.x);
+    unsigned int value = 0;
+    unsigned int *c = &arg.count[idx];
+    #pragma omp atomic capture
+    { value = *c; *c = *c + 1; }
+
+    // determine if last block
+    isLastBlockDone = (value == (target::grid_dim().x - 1));
+
+    __syncthreads();
+
+    // finish the reduction if last block
+#pragma omp declare reduction(OMPReduce_ : typename Reducer::reduce_t : omp_out=Reducer::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Reducer::init_omp())
+    if (isLastBlockDone) {
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      T sum = arg.init();
+      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+      {
+        auto i = target::thread_idx().y * block_size_x + target::thread_idx().x;
+        while (i < target::grid_dim().x) {
+          sum = r(sum, const_cast<T &>(static_cast<volatile T *>(arg.partial)[idx * target::grid_dim().x + i]));
+          i += block_size_x * block_size_y;
+        }
+      }
+
+      // write out the final reduced value
+      printf("reduce::sum %g\n", *(double*)(&sum));
+      arg.result_d[idx] = sum;
+      arg.count[idx] = 0; // set to zero for next time
+    }
+  }
+  */
+
+} // namespace quda
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index c9d72d0794..94d46b78a3 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -10,6 +10,10 @@ namespace quda {
      creates and curries in the block size.  This allows the block
      size to be set statically at launch time in the actual argument
      class that is passed to the kernel.
+
+     @tparam block_size_x x-dimension block-size
+     @tparam block_size_y y-dimension block-size
+     @tparam Arg Kernel argument struct
   */
   template <int block_size_x_, int block_size_y_, typename Arg_> struct ReduceKernelArg : Arg_ {
     using Arg = Arg_;
@@ -18,44 +22,19 @@ namespace quda {
     ReduceKernelArg(const Arg &arg) : Arg(arg) { }
   };
 
-  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
-  __forceinline__ __device__ void Reduction2D_impl(const Arg &arg)
-  {
-    const dim3
-      blockDim=launch_param.block,
-      gridDim=launch_param.grid,
-      blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
-
-    using reduce_t = typename Transformer<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
-    reduce_t value = arg.init();
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
-    {
-      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-      //          "omp reports: teams %d threads %d\n",
-      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-      //          omp_get_num_teams(), omp_get_num_threads());
-      dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-
-      Transformer<Arg> t(arg);
-
-      auto idx = threadIdx.x + blockIdx.x * blockDim.x;
-      auto j = threadIdx.y;
-
-      while (idx < arg.threads.x) {
-        value = t(value, idx, j);
-        if (grid_stride) idx += blockDim.x * gridDim.x; else break;
-      }
-    }
-
-    // perform final inter-block reduction and write out result
-    Transformer<Arg> t(arg);
-    reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value);
-  }
-
-  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  /**
+     @brief Reduction2D is the entry point of the generic 2-d
+     reduction kernel.  This is the specialization where the kernel
+     argument struct is passed by value directly to the kernel.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
@@ -71,14 +50,15 @@ namespace quda {
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
       shared_cache.addr = cache;
-      // Reduction2D_impl<Transformer, Arg, grid_stride>(*dparg);
+      // OMP TARGET TODO: issues with reduction, not performed if not inlined.
+      // Reduction2D_impl<Functor, Arg, grid_stride>(*dparg);
       const dim3
         blockDim=launch_param.block,
         gridDim=launch_param.grid,
         blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
 
-      using reduce_t = typename Transformer<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+      using reduce_t = typename Functor<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Functor<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Functor<Arg>::init_omp())
       reduce_t value = dparg->init();
       #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
       {
@@ -89,74 +69,77 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
-        Transformer<Arg> t(*dparg);
+        Functor<Arg> t(*dparg);
 
         auto idx = threadIdx.x + blockIdx.x * blockDim.x;
         auto j = threadIdx.y;
 
-        // if(blockIdx.x==0&&threadIdx.x==0) printf("team %d: thread %d: in value: %g\n", omp_get_team_num(), omp_get_thread_num(), *reinterpret_cast<double *>(&value));
         while (idx < dparg->threads.x) {
           value = t(value, idx, j);
-          // if(blockIdx.x==0&&threadIdx.x==0) printf("team %d: thread %d: idx %d: value: %g\n", omp_get_team_num(), omp_get_thread_num(), idx, *reinterpret_cast<double *>(&value));
-          if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+          if (grid_stride)
+            idx += blockDim.x * gridDim.x;
+          else
+            break;
         }
       }
-      // perform final inter-block reduction and write out result
-      Transformer<Arg> t(*dparg);
-      // reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value);
-      // ../../reduce_helper.h:/reduce
-      {
-        const auto idx = 0;
-        // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
-        bool isLastBlockDone;
-        // if (threadIdx.x == 0 && threadIdx.y == 0)
-        { // This is the main thread per team
-          // if(blockIdx.x==0) printf("team %d: value: %g\n", omp_get_team_num(), *reinterpret_cast<double *>(&value));
-          dparg->partial[idx * gridDim.x + blockIdx.x] = value;
-
-          // increment global block counter
-          // auto value = atomicInc(&arg.count[idx], gridDim.x);
-          unsigned int cvalue = 0;
-          unsigned int *c = &dparg->count[idx];
-          #pragma omp atomic capture
-          { cvalue = *c; *c = *c + 1; }
-          // { cvalue = dparg->count[idx]; dparg->count[idx] = ((dparg->count[idx] >= gridDim.x) ? 0 : (dparg->count[idx]+1)); }
-
-          // determine if last block
-          isLastBlockDone = (cvalue == (gridDim.x - 1));
-        }
-        // finish the reduction if last block
-        if (isLastBlockDone) {
-          reduce_t sum = dparg->init();
-          #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-          {
-            dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-            auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
-            while (i < gridDim.x) {
-              sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-              i += Arg::block_size_x * Arg::block_size_y;
-            }
-          }
 
-          // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
+      // perform final inter-block reduction and write out result
+      Functor<Arg> t(*dparg);
 
-          // printf("lastBlock: team %d: idx: %d value: ", omp_get_team_num(), idx);
-          // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
-          // printf("\n");
-          // write out the final reduced value
-          // if (threadIdx.y * Arg::block_size_x + threadIdx.x == 0)
-          { // This is the main thread per team
-            dparg->result_d[idx] = sum;
-            dparg->count[idx] = 0; // set to zero for next time
+      // OMP TARGET TODO: fully inline everything for now
+      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value);
+      int idx = 0;
+      bool isLastBlockDone;
+
+      dparg->partial[idx * gridDim.x + blockIdx.x] = value;
+      // __threadfence(); // flush result
+
+      // increment global block counter
+      unsigned int cvalue = 0;
+      unsigned int *c = &dparg->count[idx];
+      #pragma omp atomic capture
+      { cvalue = *c; *c = *c + 1; }
+
+      // determine if last block
+      isLastBlockDone = (cvalue == (gridDim.x - 1));
+
+      // __syncthreads();
+
+      // finish the reduction if last block
+      if (isLastBlockDone) {
+        reduce_t sum = dparg->init();
+        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+        {
+          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+          auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
+          while (i < gridDim.x) {
+            sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
+            i += Arg::block_size_x * Arg::block_size_y;
           }
         }
+
+        // write out the final reduced value
+        dparg->result_d[idx] = sum;
+        dparg->count[idx] = 0; // set to zero for next time
       }
     }
     omp_target_free(dparg, omp_get_default_device());
   }
 
-  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D()
+  /**
+     @brief Reduction2D is the entry point of the generic 2-d
+     reduction kernel.  This is the specialization where the kernel
+     argument struct is copied to the device prior to kernel launch.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D()
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
@@ -165,15 +148,15 @@ namespace quda {
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
       shared_cache.addr = cache;
-      // Reduction2D_impl<Transformer, Arg, grid_stride>(device::get_arg<Arg>());
+      // Reduction2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       Arg *dparg = &device::get_arg<Arg>();
       const dim3
         blockDim=launch_param.block,
         gridDim=launch_param.grid,
         blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
 
-      using reduce_t = typename Transformer<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+      using reduce_t = typename Functor<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Functor<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Functor<Arg>::init_omp())
       reduce_t value = dparg->init();
       #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
       {
@@ -184,112 +167,75 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
-        Transformer<Arg> t(*dparg);
+        Functor<Arg> t(*dparg);
 
         auto idx = threadIdx.x + blockIdx.x * blockDim.x;
         auto j = threadIdx.y;
 
         while (idx < dparg->threads.x) {
           value = t(value, idx, j);
-          if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+          if (grid_stride)
+            idx += blockDim.x * gridDim.x;
+          else
+            break;
         }
       }
-      // perform final inter-block reduction and write out result
-      Transformer<Arg> t(*dparg);
-      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value);
-      // ../../reduce_helper.h:/reduce
-      {
-        const auto idx = 0;
-        // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
-        bool isLastBlockDone;
-        // if (threadIdx.x == 0 && threadIdx.y == 0)
-        { // This is the main thread per team
-          dparg->partial[idx * gridDim.x + blockIdx.x] = value;
-
-          // increment global block counter
-          // auto value = atomicInc(&arg.count[idx], gridDim.x);
-          unsigned int cvalue = 0;
-          unsigned int *c = &dparg->count[idx];
-          #pragma omp atomic capture
-          { cvalue = *c; *c = *c + 1; }
-          // { cvalue = dparg->count[idx]; dparg->count[idx] = ((dparg->count[idx] >= gridDim.x) ? 0 : (dparg->count[idx]+1)); }
-
-          // determine if last block
-          isLastBlockDone = (cvalue == (gridDim.x - 1));
-        }
-        // finish the reduction if last block
-        if (isLastBlockDone) {
-          reduce_t sum = dparg->init();
-          #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-          {
-            dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-            auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
-            while (i < gridDim.x) {
-              sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-              i += Arg::block_size_x * Arg::block_size_y;
-            }
-          }
 
-          // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
+      // perform final inter-block reduction and write out result
+      Functor<Arg> t(*dparg);
 
-          // printf("lastBlock: team %d: idx: %d value: ", omp_get_team_num(), idx);
-          // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
-          // printf("\n");
-          // write out the final reduced value
-          // if (threadIdx.y * Arg::block_size_x + threadIdx.x == 0)
-          { // This is the main thread per team
-            dparg->result_d[idx] = sum;
-            dparg->count[idx] = 0; // set to zero for next time
+      // OMP TARGET TODO: fully inline everything for now
+      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value);
+      int idx = 0;
+      bool isLastBlockDone;
+
+      dparg->partial[idx * gridDim.x + blockIdx.x] = value;
+      // __threadfence(); // flush result
+
+      // increment global block counter
+      unsigned int cvalue = 0;
+      unsigned int *c = &dparg->count[idx];
+      #pragma omp atomic capture
+      { cvalue = *c; *c = *c + 1; }
+
+      // determine if last block
+      isLastBlockDone = (cvalue == (gridDim.x - 1));
+
+      // __syncthreads();
+
+      // finish the reduction if last block
+      if (isLastBlockDone) {
+        reduce_t sum = dparg->init();
+        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+        {
+          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+          auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
+          while (i < gridDim.x) {
+            sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
+            i += Arg::block_size_x * Arg::block_size_y;
           }
         }
-      }
-    }
-  }
 
-
-  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
-  __forceinline__ __device__ void MultiReduction_impl(const Arg &arg)
-  {
-    const dim3
-      blockDim=launch_param.block,
-      gridDim=launch_param.grid,
-      blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
-
-    using reduce_t = typename Transformer<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
-    reduce_t value = arg.init();
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
-    {
-      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-      //          "omp reports: teams %d threads %d\n",
-      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-      //          omp_get_num_teams(), omp_get_num_threads());
-      dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-
-      Transformer<Arg> t(arg);
-
-      auto idx = threadIdx.x + blockIdx.x * blockDim.x;
-      auto j = threadIdx.y + blockIdx.y * blockDim.y;
-      auto k = threadIdx.z;
-
-      if (j < arg.threads.y){
-        while (idx < arg.threads.x) {
-          value = t(value, idx, j, k);
-          if (grid_stride) idx += blockDim.x * gridDim.x; else break;
-        }
+        // write out the final reduced value
+        dparg->result_d[idx] = sum;
+        dparg->count[idx] = 0; // set to zero for next time
       }
     }
-    // perform final inter-block reduction and write out result
-    Transformer<Arg> t(arg);
-    auto j = blockIdx.y * blockDim.y;
-    reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value, j);
   }
 
-  /** Workaround compiler limitations.  We have to inline parallel regions with target teams. **/
-
-  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  /**
+     @brief MultiReduction is the entry point of the generic
+     multi-reduction kernel.  This is the specialization where the
+     kernel argument struct is passed by value directly to the kernel.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
@@ -305,14 +251,14 @@ namespace quda {
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
       shared_cache.addr = cache;
-      // MultiReduction_impl<Transformer, Arg, grid_stride>(*dparg);
       const dim3
         blockDim=launch_param.block,
         gridDim=launch_param.grid,
         blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
 
-      using reduce_t = typename Transformer<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+      using reduce_t = typename Functor<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Functor<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Functor<Arg>::init_omp())
       reduce_t value = dparg->init();
       #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
       {
@@ -323,76 +269,79 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
-        Transformer<Arg> t(*dparg);
+        Functor<Arg> t(*dparg);
 
         auto idx = threadIdx.x + blockIdx.x * blockDim.x;
         auto j = threadIdx.y + blockIdx.y * blockDim.y;
         auto k = threadIdx.z;
 
-        if (j < dparg->threads.y){
+        if (j < dparg->threads.y) {
           while (idx < dparg->threads.x) {
             value = t(value, idx, j, k);
-            if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+            if (grid_stride)
+              idx += blockDim.x * gridDim.x;
+            else
+              break;
           }
         }
       }
+
       // perform final inter-block reduction and write out result
-      Transformer<Arg> t(*dparg);
+      Functor<Arg> t(*dparg);
       auto j = blockIdx.y * blockDim.y;
-      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value, j);
-      // ../../reduce_helper.h:/reduce
-      {
-        const auto idx = j;
-        // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
-        bool isLastBlockDone;
-        // if (threadIdx.x == 0 && threadIdx.y == 0)
-        { // This is the main thread per team
-          // if(blockIdx.x==0) printf("team %d: value: %g\n", omp_get_team_num(), *reinterpret_cast<double *>(&value));
-          dparg->partial[idx * gridDim.x + blockIdx.x] = value;
-
-          // increment global block counter
-          // auto value = atomicInc(&arg.count[idx], gridDim.x);
-          unsigned int cvalue = 0;
-          unsigned int *c = &dparg->count[idx];
-          #pragma omp atomic capture
-          { cvalue = *c; *c = *c + 1; }
-          // { cvalue = dparg->count[idx]; dparg->count[idx] = ((dparg->count[idx] >= gridDim.x) ? 0 : (dparg->count[idx]+1)); }
-
-          // determine if last block
-          isLastBlockDone = (cvalue == (gridDim.x - 1));
-        }
-        // finish the reduction if last block
-        if (isLastBlockDone) {
-          reduce_t sum = dparg->init();
-          #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-          {
-            dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-            auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
-            while (i < gridDim.x) {
-              sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-              i += Arg::block_size_x * Arg::block_size_y;
-            }
-          }
-
-          // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
-
-          // printf("lastBlock: team %d: idx: %d value: ", omp_get_team_num(), idx);
-          // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
-          // printf("\n");
-          // write out the final reduced value
-          // if (threadIdx.y * Arg::block_size_x + threadIdx.x == 0)
-          { // This is the main thread per team
-            dparg->result_d[idx] = sum;
-            dparg->count[idx] = 0; // set to zero for next time
+      // reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value, j);
+      int idx = j;
+      bool isLastBlockDone;
+
+      dparg->partial[idx * gridDim.x + blockIdx.x] = value;
+      // __threadfence(); // flush result
+
+      // increment global block counter
+      unsigned int cvalue = 0;
+      unsigned int *c = &dparg->count[idx];
+      #pragma omp atomic capture
+      { cvalue = *c; *c = *c + 1; }
+
+      // determine if last block
+      isLastBlockDone = (cvalue == (gridDim.x - 1));
+
+      // __syncthreads();
+
+      // finish the reduction if last block
+      if (isLastBlockDone) {
+        reduce_t sum = dparg->init();
+        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+        {
+          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+          auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
+          while (i < gridDim.x) {
+            sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
+            i += Arg::block_size_x * Arg::block_size_y;
           }
         }
+
+        // write out the final reduced value
+        dparg->result_d[idx] = sum;
+        dparg->count[idx] = 0; // set to zero for next time
       }
     }
     omp_target_free(dparg, omp_get_default_device());
   }
 
-  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction()
+  /**
+     @brief MultiReduction is the entry point of the generic
+     multi-reduction kernel.  This is the specialization where the
+     kernel argument struct is passed by value directly to the kernel.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction()
   {
     const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
     const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
@@ -401,15 +350,15 @@ namespace quda {
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
       shared_cache.addr = cache;
-      // MultiReduction_impl<Transformer, Arg, grid_stride>(device::get_arg<Arg>());
-      Arg *dparg = &device::get_arg<Arg>();
+      Arg *dparg = device::get_arg<Arg>();
       const dim3
         blockDim=launch_param.block,
         gridDim=launch_param.grid,
         blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
+      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
 
-      using reduce_t = typename Transformer<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Transformer<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Transformer<Arg>::init_omp())
+      using reduce_t = typename Functor<Arg>::reduce_t;
+#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Functor<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Functor<Arg>::init_omp())
       reduce_t value = dparg->init();
       #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
       {
@@ -420,70 +369,62 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
 
-        Transformer<Arg> t(*dparg);
+        Functor<Arg> t(*dparg);
 
         auto idx = threadIdx.x + blockIdx.x * blockDim.x;
         auto j = threadIdx.y + blockIdx.y * blockDim.y;
         auto k = threadIdx.z;
 
-        if (j < dparg->threads.y){
+        if (j < dparg->threads.y) {
           while (idx < dparg->threads.x) {
             value = t(value, idx, j, k);
-            if (grid_stride) idx += blockDim.x * gridDim.x; else break;
+            if (grid_stride)
+              idx += blockDim.x * gridDim.x;
+            else
+              break;
           }
         }
       }
+
       // perform final inter-block reduction and write out result
-      Transformer<Arg> t(*dparg);
+      Functor<Arg> t(*dparg);
       auto j = blockIdx.y * blockDim.y;
-      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value, j);
-      // ../../reduce_helper.h:/reduce
-      {
-        const auto idx = j;
-        // In OpenMP, this runs in the main thread of each team, and `in` is already the block reduction value.
-        bool isLastBlockDone;
-        // if (threadIdx.x == 0 && threadIdx.y == 0)
-        { // This is the main thread per team
-          dparg->partial[idx * gridDim.x + blockIdx.x] = value;
-
-          // increment global block counter
-          // auto value = atomicInc(&arg.count[idx], gridDim.x);
-          unsigned int cvalue = 0;
-          unsigned int *c = &dparg->count[idx];
-          #pragma omp atomic capture
-          { cvalue = *c; *c = *c + 1; }
-          // { cvalue = dparg->count[idx]; dparg->count[idx] = ((dparg->count[idx] >= gridDim.x) ? 0 : (dparg->count[idx]+1)); }
-
-          // determine if last block
-          isLastBlockDone = (cvalue == (gridDim.x - 1));
-        }
-        // finish the reduction if last block
-        if (isLastBlockDone) {
-          reduce_t sum = dparg->init();
-          #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-          {
-            dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-            auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
-            while (i < gridDim.x) {
-              sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-              i += Arg::block_size_x * Arg::block_size_y;
-            }
-          }
-
-          // sum = (Reducer::do_sum ? BlockReduce(cub_tmp).Sum(sum) : BlockReduce(cub_tmp).Reduce(sum, r));
-
-          // printf("lastBlock: team %d: idx: %d value: ", omp_get_team_num(), idx);
-          // for(int l=0;l<sizeof(sum)/sizeof(double);++l) printf(" %.16g", reinterpret_cast<double *>(&sum)[l]);
-          // printf("\n");
-          // write out the final reduced value
-          // if (threadIdx.y * Arg::block_size_x + threadIdx.x == 0)
-          { // This is the main thread per team
-            dparg->result_d[idx] = sum;
-            dparg->count[idx] = 0; // set to zero for next time
+      // reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value, j);
+      int idx = j;
+      bool isLastBlockDone;
+
+      dparg->partial[idx * gridDim.x + blockIdx.x] = value;
+      // __threadfence(); // flush result
+
+      // increment global block counter
+      unsigned int cvalue = 0;
+      unsigned int *c = &dparg->count[idx];
+      #pragma omp atomic capture
+      { cvalue = *c; *c = *c + 1; }
+
+      // determine if last block
+      isLastBlockDone = (cvalue == (gridDim.x - 1));
+
+      // __syncthreads();
+
+      // finish the reduction if last block
+      if (isLastBlockDone) {
+        reduce_t sum = dparg->init();
+        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
+        {
+          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
+          auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
+          while (i < gridDim.x) {
+            sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
+            i += Arg::block_size_x * Arg::block_size_y;
           }
         }
+
+        // write out the final reduced value
+        dparg->result_d[idx] = sum;
+        dparg->count[idx] = 0; // set to zero for next time
       }
     }
   }
 
-}
+} // namespace quda
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 2675bae7f2..86644e8eff 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -159,7 +159,7 @@ namespace quda {
        @brief Use a compile time fixed size for the shared local memory,
        until we can find a way to set it dynamically.
     */
-    constexpr unsigned int max_shared_memory_size() { return 48*1024; /* Some amount below maximum */ }
+    constexpr unsigned int max_shared_memory_size() { return 32*1024; /* Some amount below maximum */ }
 
     /**
        @brief Helper function that returns true if we are to pass the
diff --git a/lib/targets/omptarget/comm_target.cpp b/lib/targets/omptarget/comm_target.cpp
index ecb6837d38..313250b0c6 100644
--- a/lib/targets/omptarget/comm_target.cpp
+++ b/lib/targets/omptarget/comm_target.cpp
@@ -1,6 +1,5 @@
 #include <comm_quda.h>
 #include <quda_api.h>
-#include <quda_cuda_api.h>
 #include <algorithm>
 #include <shmem_helper.cuh>
 
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index d7c892ed81..1c169f07e2 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -1,111 +1,49 @@
 #include <omp.h>
 #include <util_quda.h>
 #include <quda_internal.h>
-#include <quda_cuda_api.h>
 #include <target_device.h>
 
-#ifdef QUDA_NVML
-#include <nvml.h>
-#endif
-
-#ifdef NUMA_NVML
-#include <numa_affinity.h>
-#endif
+// OMP TARGET TODO: most of what follows are guess work.
+// We also don't support streams.
 
 static char ompdevname[] = "OpenMP Target Device";
 static char omphostname[] = "OpenMP Host Device";
-struct cudaDeviceProp{
+struct DeviceProp{
   char*name;
-  int major,minor;
-  size_t sharedMemPerBlock;
-  unsigned int maxThreadsPerBlock;
-  unsigned int maxThreadsPerMultiProcessor;
-  unsigned int maxThreadsDim[3];
-  unsigned int maxGridSize[3];
-  unsigned int multiProcessorCount;
-  unsigned int warpSize;
-  unsigned long totalGlobalMem;
-  unsigned long totalConstMem;
-  unsigned long memPitch;
-  unsigned int regsPerBlock;
-  int unifiedAddressing;
-  int deviceOverlap;
+  int id;
+  unsigned int num_procs;
+  unsigned int max_teams;
+  unsigned int max_threads;
 };
 
-static int cudaDriverGetVersion(int*v){*v=0;return 0;}
-static int cudaRuntimeGetVersion(int*v){*v=0;return 0;}
-static int cudaGetDeviceCount(int*c){*c=omp_get_num_devices();return 0;}
-static int cudaGetDeviceProperties(cudaDeviceProp*p,int dev)
+static void getDeviceProperties(DeviceProp*p,int dev)
 {
-  /* FIXME totally fake numbers */
-  if(0<omp_get_num_devices()){
+  int m = 0;
+
+  if(dev>=0 && dev<=omp_get_num_devices())
     p->name = ompdevname;
-    p->major = 7;
-    p->minor = 0;
-    p->sharedMemPerBlock = 0;
-    p->maxThreadsPerBlock = 1024;
-    p->maxThreadsPerMultiProcessor = 1024;
-    p->maxThreadsDim[0] = 1024;
-    p->maxThreadsDim[1] = 1024;
-    p->maxThreadsDim[2] = 64;
-    p->maxGridSize[0] = 65536;
-    p->maxGridSize[1] = 65536;
-    p->maxGridSize[2] = 65536;
-    p->multiProcessorCount = 16;
-  }else{
+  else
     p->name = omphostname;
-    p->major = 7;
-    p->minor = 0;
-    p->sharedMemPerBlock = 0;
-    p->maxThreadsPerBlock = 1024;
-    p->maxThreadsPerMultiProcessor = 1024;
-    p->maxThreadsDim[0] = 1024;
-    p->maxThreadsDim[1] = 1024;
-    p->maxThreadsDim[2] = 64;
-    p->maxGridSize[0] = 65536;
-    p->maxGridSize[1] = 65536;
-    p->maxGridSize[2] = 65536;
-    p->multiProcessorCount = 16;
-  }
-  p->warpSize = 8;
-  p->totalGlobalMem = 1u<34;
-  p->totalConstMem = 65536;
-  p->memPitch = 1u<31;
-  p->regsPerBlock = 32768;
-  p->unifiedAddressing = 1;
-  p->deviceOverlap = 0;
-
-  /* Now actually querying the device in some way. */
-  int m = 0;
-  #pragma omp target teams map(tofrom:m)
-  if(omp_get_team_num()==0)
-    m = omp_get_max_threads();
-  p->maxThreadsPerMultiProcessor = m;
-  p->maxThreadsPerBlock = m;
-  p->maxThreadsDim[0] = m;
-  p->maxThreadsDim[1] = m;
-  p->maxThreadsDim[2] = m;
-  #pragma omp target map(tofrom:m)
+
+  p->id = dev;
+  #pragma omp target device(dev) map(from:m)
   m = omp_get_num_procs();
-  p->multiProcessorCount = m/8;
-  return 0;
-}
+  p->num_procs = m;
 
-enum {cudaFuncCachePreferL1};
-enum {cudaStreamDefault};
-static int cudaSetDevice(int d){omp_set_default_device(d); return 0;}
-static int cudaDeviceSetCacheConfig(int a){ompwip();return 0;}
-static int cudaDeviceGetStreamPriorityRange(int*lo,int*hi){ompwip();lo=0;hi=0;return 0;}
-static int cudaStreamCreateWithPriority(cudaStream_t*s,int lo,int hi){ompwip();*s=0;return 0;}
-static int cudaStreamDestroy(cudaStream_t s){ompwip();return 0;}
-static int cudaDeviceReset(){ompwip();return 0;}
+/*
+  #pragma omp target device(dev) map(from:m)
+  m = omp_get_max_teams();
+  p->max_teams = m;
+*/
+  p->max_teams = 65536;
 
-static cudaDeviceProp deviceProp;
-static cudaStream_t *streams;
-static const int Nstream = 9;
+  #pragma omp target teams device(dev) map(from:m)
+  if(omp_get_team_num()==0)
+    m = omp_get_max_threads();
+  p->max_threads= m;
+}
 
-#define CHECK_CUDA_ERROR(func)                                          \
-  cuda::set_runtime_error(func, #func, __func__, __FILE__, __STRINGIFY__(__LINE__));
+static DeviceProp deviceProp;
 
 namespace quda
 {
@@ -115,251 +53,87 @@ namespace quda
 
     static bool initialized = false;
 
+    void print_device(DeviceProp dp)
+    {
+      int dev = dp.id;
+      printfQuda("%d - name:        %s\n", dev, dp.name);
+      printfQuda("%d - num_procs:   %d\n", dev, dp.num_procs);
+      printfQuda("%d - max_teams:   %d\n", dev, dp.max_teams);
+      printfQuda("%d - max_threads: %d\n", dev, dp.max_threads);
+    }
+
     void init(int dev)
     {
       if (initialized) return;
       initialized = true;
 
-      int driver_version;
-      CHECK_CUDA_ERROR(cudaDriverGetVersion(&driver_version));
-      printfQuda("CUDA Driver version = %d\n", driver_version);
-
-      int runtime_version;
-      CHECK_CUDA_ERROR(cudaRuntimeGetVersion(&runtime_version));
-      printfQuda("CUDA Runtime version = %d\n", runtime_version);
-
-#ifdef QUDA_NVML
-      nvmlReturn_t result = nvmlInit();
-      if (NVML_SUCCESS != result) errorQuda("NVML Init failed with error %d", result);
-      const int length = 80;
-      char graphics_version[length];
-      result = nvmlSystemGetDriverVersion(graphics_version, length);
-      if (NVML_SUCCESS != result) errorQuda("nvmlSystemGetDriverVersion failed with error %d", result);
-      printfQuda("Graphic driver version = %s\n", graphics_version);
-      result = nvmlShutdown();
-      if (NVML_SUCCESS != result) errorQuda("NVML Shutdown failed with error %d", result);
-#endif
-
-      if(get_device_count() == 0)  /* TODO: get CPU device properties */
-        cudaGetDeviceProperties(&deviceProp, -1);
-      for (int i = 0; i < get_device_count(); i++) {
-        CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, i));
-        if (getVerbosity() >= QUDA_SUMMARIZE) {
-          printfQuda("Found device %d: %s\n", i, deviceProp.name);
-        }
-      }
-
-/*
-      CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, dev));
-      if (deviceProp.major < 1) errorQuda("Device %d does not support CUDA", dev);
-
-      // Check GPU and QUDA build compatibiliy
-      // 4 cases:
-      // a) QUDA and GPU match: great
-      // b) QUDA built for higher compute capability: error
-      // c) QUDA built for lower major compute capability: warn if QUDA_ALLOW_JIT, else error
-      // d) QUDA built for same major compute capability but lower minor: warn
-
-      const int my_major = __COMPUTE_CAPABILITY__ / 100;
-      const int my_minor = (__COMPUTE_CAPABILITY__ - my_major * 100) / 10;
-      // b) QUDA was compiled for a higher compute capability
-      if (deviceProp.major * 100 + deviceProp.minor * 10 < __COMPUTE_CAPABILITY__)
-        errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. ** \n --- "
-                  "Please set the correct QUDA_GPU_ARCH when running cmake.\n",
-                  deviceProp.major, deviceProp.minor, my_major, my_minor);
-
-      // c) QUDA was compiled for a lower compute capability
-      if (deviceProp.major < my_major) {
-        char *allow_jit_env = getenv("QUDA_ALLOW_JIT");
-        if (allow_jit_env && strcmp(allow_jit_env, "1") == 0) {
-          if (getVerbosity() > QUDA_SILENT)
-            warningQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- "
-                        "Jitting the PTX since QUDA_ALLOW_JIT=1 was set. Note that this will take some time.\n",
-                        deviceProp.major, deviceProp.minor, my_major, my_minor);
-        } else {
-          errorQuda("** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n --- "
-                    "Please set the correct QUDA_GPU_ARCH when running cmake.\n If you want the PTX to be jitted for "
-                    "your current GPU arch please set the enviroment variable QUDA_ALLOW_JIT=1.",
-                    deviceProp.major, deviceProp.minor, my_major, my_minor);
-        }
-      }
-      // d) QUDA built for same major compute capability but lower minor
-      if (deviceProp.major == my_major and deviceProp.minor > my_minor) {
-        warningQuda(
-          "** Running on a device with compute capability %i.%i but QUDA was compiled for %i.%i. **\n -- This might "
-          "result in a lower performance. Please consider adjusting QUDA_GPU_ARCH when running cmake.\n",
-          deviceProp.major, deviceProp.minor, my_major, my_minor);
-      }
-*/
-
-      if (!deviceProp.unifiedAddressing) warningQuda("Device %d does not support unified addressing", dev);
+      getDeviceProperties(&deviceProp, dev);
 
-      if (getVerbosity() >= QUDA_SUMMARIZE) printfQuda("Using device %d: %s\n", dev, deviceProp.name);
-#ifndef USE_QDPJIT
-      CHECK_CUDA_ERROR(cudaSetDevice(dev));
-#endif
-
-#ifdef NUMA_NVML
-      char *enable_numa_env = getenv("QUDA_ENABLE_NUMA");
-      if (enable_numa_env && strcmp(enable_numa_env, "0") == 0) {
-        if (getVerbosity() > QUDA_SILENT) printfQuda("Disabling numa_affinity\n");
-      } else {
-        setNumaAffinityNVML(dev);
+      if (getVerbosity() >= QUDA_SUMMARIZE) {
+        printfQuda("Using device %d: %s\n", dev, deviceProp.name);
+        print_device(deviceProp);
       }
-#endif
-
-      CHECK_CUDA_ERROR(cudaDeviceSetCacheConfig(cudaFuncCachePreferL1));
-      //cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte);
-      // cudaGetDeviceProperties(&deviceProp, dev);
+      omp_set_default_device(dev);
     }
 
     int get_device_count()
     {
-      static int device_count = 0;
-      if (device_count == 0) {
-        CHECK_CUDA_ERROR(cudaGetDeviceCount(&device_count));
-        if (device_count == 0) warningQuda("No CUDA devices found");
+      static int device_count = -1;
+      if (device_count < 0) {
+        device_count = omp_get_num_devices();
+        if (device_count == 0)
+          warningQuda("No non-host devices found");
       }
       return device_count;
     }
 
     void print_device_properties()
     {
+      DeviceProp dp;
       for (int device = 0; device < get_device_count(); device++) {
-
-        // cudaDeviceProp deviceProp;
-        CHECK_CUDA_ERROR(cudaGetDeviceProperties(&deviceProp, device));
-        printfQuda("%d - name:                    %s\n", device, deviceProp.name);
-        printfQuda("%d - totalGlobalMem:          %lu bytes ( %.2f Gbytes)\n", device, deviceProp.totalGlobalMem,
-                   deviceProp.totalGlobalMem / (float)(1024 * 1024 * 1024));
-        printfQuda("%d - sharedMemPerBlock:       %lu bytes ( %.2f Kbytes)\n", device, deviceProp.sharedMemPerBlock,
-                   deviceProp.sharedMemPerBlock / (float)1024);
-        printfQuda("%d - regsPerBlock:            %d\n", device, deviceProp.regsPerBlock);
-        printfQuda("%d - warpSize:                %d\n", device, deviceProp.warpSize);
-        printfQuda("%d - memPitch:                %lu\n", device, deviceProp.memPitch);
-        printfQuda("%d - maxThreadsPerBlock:      %d\n", device, deviceProp.maxThreadsPerBlock);
-        printfQuda("%d - maxThreadsDim[0]:        %d\n", device, deviceProp.maxThreadsDim[0]);
-        printfQuda("%d - maxThreadsDim[1]:        %d\n", device, deviceProp.maxThreadsDim[1]);
-        printfQuda("%d - maxThreadsDim[2]:        %d\n", device, deviceProp.maxThreadsDim[2]);
-        printfQuda("%d - maxGridSize[0]:          %d\n", device, deviceProp.maxGridSize[0]);
-        printfQuda("%d - maxGridSize[1]:          %d\n", device, deviceProp.maxGridSize[1]);
-        printfQuda("%d - maxGridSize[2]:          %d\n", device, deviceProp.maxGridSize[2]);
-        printfQuda("%d - totalConstMem:           %lu bytes ( %.2f Kbytes)\n", device, deviceProp.totalConstMem,
-                   deviceProp.totalConstMem / (float)1024);
-        printfQuda("%d - compute capability:      %d.%d\n", device, deviceProp.major, deviceProp.minor);
-        printfQuda("%d - deviceOverlap            %s\n", device, (deviceProp.deviceOverlap ? "true" : "false"));
-        printfQuda("%d - multiProcessorCount      %d\n", device, deviceProp.multiProcessorCount);
-/*
-        printfQuda("%d - kernelExecTimeoutEnabled %s\n", device,
-                   (deviceProp.kernelExecTimeoutEnabled ? "true" : "false"));
-        printfQuda("%d - integrated               %s\n", device, (deviceProp.integrated ? "true" : "false"));
-        printfQuda("%d - canMapHostMemory         %s\n", device, (deviceProp.canMapHostMemory ? "true" : "false"));
-        switch (deviceProp.computeMode) {
-        case 0: printfQuda("%d - computeMode              0: cudaComputeModeDefault\n", device); break;
-        case 1: printfQuda("%d - computeMode              1: cudaComputeModeExclusive\n", device); break;
-        case 2: printfQuda("%d - computeMode              2: cudaComputeModeProhibited\n", device); break;
-        case 3: printfQuda("%d - computeMode              3: cudaComputeModeExclusiveProcess\n", device); break;
-        default: errorQuda("Unknown deviceProp.computeMode.");
-        }
-
-        printfQuda("%d - surfaceAlignment         %lu\n", device, deviceProp.surfaceAlignment);
-        printfQuda("%d - concurrentKernels        %s\n", device, (deviceProp.concurrentKernels ? "true" : "false"));
-        printfQuda("%d - ECCEnabled               %s\n", device, (deviceProp.ECCEnabled ? "true" : "false"));
-        printfQuda("%d - pciBusID                 %d\n", device, deviceProp.pciBusID);
-        printfQuda("%d - pciDeviceID              %d\n", device, deviceProp.pciDeviceID);
-        printfQuda("%d - pciDomainID              %d\n", device, deviceProp.pciDomainID);
-        printfQuda("%d - tccDriver                %s\n", device, (deviceProp.tccDriver ? "true" : "false"));
-        switch (deviceProp.asyncEngineCount) {
-        case 0: printfQuda("%d - asyncEngineCount         1: host -> device only\n", device); break;
-        case 1: printfQuda("%d - asyncEngineCount         2: host <-> device\n", device); break;
-        case 2: printfQuda("%d - asyncEngineCount         0: not supported\n", device); break;
-        default: errorQuda("Unknown deviceProp.asyncEngineCount.");
-        }
-        printfQuda("%d - unifiedAddressing        %s\n", device, (deviceProp.unifiedAddressing ? "true" : "false"));
-        printfQuda("%d - memoryClockRate          %d kilohertz\n", device, deviceProp.memoryClockRate);
-        printfQuda("%d - memoryBusWidth           %d bits\n", device, deviceProp.memoryBusWidth);
-        printfQuda("%d - l2CacheSize              %d bytes\n", device, deviceProp.l2CacheSize);
-*/
-        printfQuda("%d - maxThreadsPerMultiProcessor          %d\n\n", device, deviceProp.maxThreadsPerMultiProcessor);
-      }
-    }
-
-    void create_context()
-    {
-      streams = new cudaStream_t[Nstream];
-
-      int greatestPriority;
-      int leastPriority;
-      CHECK_CUDA_ERROR(cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority));
-      for (int i=0; i<Nstream-1; i++) {
-        CHECK_CUDA_ERROR(cudaStreamCreateWithPriority(&streams[i], cudaStreamDefault, greatestPriority));
+        getDeviceProperties(&dp, device);
+        print_device(dp);
       }
-      CHECK_CUDA_ERROR(cudaStreamCreateWithPriority(&streams[Nstream-1], cudaStreamDefault, leastPriority));
     }
 
-    void destroy()
-    {
-      if (streams) {
-        for (int i = 0; i < Nstream; i++) CHECK_CUDA_ERROR(cudaStreamDestroy(streams[i]));
-        delete []streams;
-        streams = nullptr;
-      }
-
-      char *device_reset_env = getenv("QUDA_DEVICE_RESET");
-      if (device_reset_env && strcmp(device_reset_env, "1") == 0) {
-        // end this CUDA context
-        CHECK_CUDA_ERROR(cudaDeviceReset());
-      }
-    }
+    void create_context() { }
 
-    cudaStream_t get_cuda_stream(const qudaStream_t &stream)
-    {
-      return streams[stream.idx];
-    }
+    void destroy() { }
 
     qudaStream_t get_stream(unsigned int i)
     {
-      if (i > Nstream) errorQuda("Invalid stream index %u", i);
-      qudaStream_t stream;
-      stream.idx = i;
-      return stream;
-      //return qudaStream_t(i);
-      // return streams[i];
+      return qudaStream_t{static_cast<int>(i)};
     }
 
     qudaStream_t get_default_stream()
     {
-      qudaStream_t stream;
-      stream.idx = Nstream - 1;
-      return stream;
-      //return qudaStream_t(Nstream - 1);
-      //return streams[Nstream - 1];
+      return qudaStream_t{0};
     }
 
     unsigned int get_default_stream_idx()
     {
-      return Nstream - 1;
+      return 0;
     }
 
     bool managed_memory_supported() { return false; }
 
     bool shared_memory_atomic_supported() { return false; }
 
-    size_t max_default_shared_memory() { return deviceProp.sharedMemPerBlock; }
+    size_t max_default_shared_memory() { return device::max_shared_memory_size(); }
 
     size_t max_dynamic_shared_memory() { return device::max_shared_memory_size(); }
 
-    unsigned int max_threads_per_block() { return deviceProp.maxThreadsPerBlock; }
+    unsigned int max_threads_per_block() { return deviceProp.max_threads; }
 
-    unsigned int max_threads_per_processor() { return deviceProp.maxThreadsPerMultiProcessor; }
+    unsigned int max_threads_per_processor() { return deviceProp.max_threads; }
 
-    unsigned int max_threads_per_block_dim(int i) { return deviceProp.maxThreadsDim[i]; }
+    unsigned int max_threads_per_block_dim(int i) { return deviceProp.max_threads; }
 
-    unsigned int max_grid_size(int i) { return deviceProp.maxGridSize[i]; }
+    unsigned int max_grid_size(int i) { return deviceProp.max_teams; }
 
-    unsigned int processor_count() { return deviceProp.multiProcessorCount; }
+    unsigned int processor_count() { return deviceProp.num_procs/8; }
 
-    unsigned int max_blocks_per_processor() { return 1; }
+    unsigned int max_blocks_per_processor() { return 32; }
 
     namespace profile
     {

From 5f91ffd1610f204bd4ade05c243c0e830265fcba Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 8 Dec 2021 21:15:12 +0000
Subject: [PATCH 095/198] omp: save unknown pointers in get_pointer_location

---
 lib/targets/omptarget/malloc.cpp | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index 8de065c49b..29e6c9a5bb 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -2,6 +2,7 @@
 #include <cstdio>
 #include <string>
 #include <map>
+#include <set>
 #include <unistd.h>   // for getpagesize()
 #include <execinfo.h> // for backtrace
 #include <quda_internal.h>
@@ -616,15 +617,21 @@ namespace quda
 
   QudaFieldLocation get_pointer_location(const void *ptr)
   {
-    if(alloc[DEVICE].count(const_cast<void*>(ptr)) || alloc[DEVICE_PINNED].count(const_cast<void*>(ptr))){
+    static std::set<void *> other_ptr;  // OMP TARGET TODO: unknown pointers, assume on CPU
+    void *p = const_cast<void*>(ptr);
+    QudaFieldLocation fl = QUDA_INVALID_FIELD_LOCATION;
+    if(alloc[DEVICE].count(p) || alloc[DEVICE_PINNED].count(p)){
       // ompwip("get_pointer_location %p returns QUDA_CUDA_FIELD_LOCATION",ptr);
-      return QUDA_CUDA_FIELD_LOCATION;
-    }else if(alloc[HOST].count(const_cast<void*>(ptr)) || alloc[PINNED].count(const_cast<void*>(ptr)) || alloc[MAPPED].count(const_cast<void*>(ptr))){
+      fl = QUDA_CUDA_FIELD_LOCATION;
+    }else if(alloc[HOST].count(p) || alloc[PINNED].count(p) || alloc[MAPPED].count(p) || other_ptr.count(p)){
       // ompwip("get_pointer_location %p returns QUDA_CPU_FIELD_LOCATION",ptr);
-      return QUDA_CPU_FIELD_LOCATION;
+      fl = QUDA_CPU_FIELD_LOCATION;
+    }else{
+      ompwip("WARNING: get_pointer_location assumes %p to be QUDA_CPU_FIELD_LOCATION",ptr);
+      other_ptr.insert(p);
+      fl = QUDA_CPU_FIELD_LOCATION;
     }
-    ompwip("WARNING: get_pointer_location assumes %p to be QUDA_CPU_FIELD_LOCATION",ptr);
-    return QUDA_CPU_FIELD_LOCATION;
+    return fl;
 /*
     CUpointer_attribute attribute[] = {CU_POINTER_ATTRIBUTE_MEMORY_TYPE};
     CUmemorytype mem_type;

From 77b2c0f246974d57d03dc19ae7765617380a5bb9 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 8 Dec 2021 22:35:34 +0000
Subject: [PATCH 096/198] omp: introduce defines for unknown device properties

---
 include/targets/omptarget/target_device.h | 18 +++++++++++++---
 lib/targets/omptarget/device.cpp          | 26 +++++++++++++++++++----
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 86644e8eff..904d4b3c95 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -1,5 +1,17 @@
 #pragma once
 
+#ifndef QUDA_WARP_SIZE
+#define QUDA_WARP_SIZE 16
+#endif
+
+#ifndef QUDA_MAX_BLOCK_SIZE
+#define QUDA_MAX_BLOCK_SIZE 1024
+#endif
+
+#ifndef QUDA_MAX_SHARED_MEMORY_SIZE
+#define QUDA_MAX_SHARED_MEMORY_SIZE 32*1024
+#endif
+
 namespace quda {
 
   namespace target {
@@ -88,7 +100,7 @@ namespace quda {
        @brief Helper function that returns the warp-size of the
        architecture we are running on.
     */
-    constexpr int warp_size() { return 16; }
+    constexpr int warp_size() { return QUDA_WARP_SIZE; }
 
     /**
        @brief Return the thread mask for a converged warp.
@@ -102,7 +114,7 @@ namespace quda {
     template <int block_size_y = 1, int block_size_z = 1>
       constexpr unsigned int max_block_size()
       {
-        return std::max(warp_size(), 512 / (block_size_y * block_size_z));
+        return QUDA_MAX_BLOCK_SIZE / (block_size_y * block_size_z);
       }
 
     /**
@@ -159,7 +171,7 @@ namespace quda {
        @brief Use a compile time fixed size for the shared local memory,
        until we can find a way to set it dynamically.
     */
-    constexpr unsigned int max_shared_memory_size() { return 32*1024; /* Some amount below maximum */ }
+    constexpr unsigned int max_shared_memory_size() { return QUDA_MAX_SHARED_MEMORY_SIZE; }
 
     /**
        @brief Helper function that returns true if we are to pass the
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 1c169f07e2..462d49af58 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -6,6 +6,24 @@
 // OMP TARGET TODO: most of what follows are guess work.
 // We also don't support streams.
 
+// until we have a way to query device properties correctly
+
+#ifndef QUDA_OMP_MAX_TEAMS
+#define QUDA_OMP_MAX_TEAMS 65536
+#endif
+
+#ifndef QUDA_MAX_THREADS_PER_PROCESSOR
+#define QUDA_MAX_THREADS_PER_PROCESSOR 2048
+#endif
+
+#ifndef QUDA_PROCESSOR_COUNT
+#define QUDA_PROCESSOR_COUNT 64
+#endif
+
+#ifndef QUDA_MAX_BLOCKS_PER_PROCESSOR
+#define QUDA_MAX_BLOCKS_PER_PROCESSOR 32
+#endif
+
 static char ompdevname[] = "OpenMP Target Device";
 static char omphostname[] = "OpenMP Host Device";
 struct DeviceProp{
@@ -35,7 +53,7 @@ static void getDeviceProperties(DeviceProp*p,int dev)
   m = omp_get_max_teams();
   p->max_teams = m;
 */
-  p->max_teams = 65536;
+  p->max_teams = QUDA_OMP_MAX_TEAMS;
 
   #pragma omp target teams device(dev) map(from:m)
   if(omp_get_team_num()==0)
@@ -125,15 +143,15 @@ namespace quda
 
     unsigned int max_threads_per_block() { return deviceProp.max_threads; }
 
-    unsigned int max_threads_per_processor() { return deviceProp.max_threads; }
+    unsigned int max_threads_per_processor() { return QUDA_MAX_THREADS_PER_PROCESSOR; }
 
     unsigned int max_threads_per_block_dim(int i) { return deviceProp.max_threads; }
 
     unsigned int max_grid_size(int i) { return deviceProp.max_teams; }
 
-    unsigned int processor_count() { return deviceProp.num_procs/8; }
+    unsigned int processor_count() { return QUDA_PROCESSOR_COUNT; }
 
-    unsigned int max_blocks_per_processor() { return 32; }
+    unsigned int max_blocks_per_processor() { return QUDA_MAX_BLOCKS_PER_PROCESSOR; }
 
     namespace profile
     {

From ce5f14bf17125b520c4175d4d26a8fa9c330f5bb Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 9 Dec 2021 17:27:34 +0000
Subject: [PATCH 097/198] omp: back out change to maxGridSize

---
 include/tune_quda.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/include/tune_quda.h b/include/tune_quda.h
index 8351b51c61..6f8b1ea04f 100644
--- a/include/tune_quda.h
+++ b/include/tune_quda.h
@@ -114,9 +114,6 @@ namespace quda {
       return device::max_threads_per_block() / (param.block.y * param.block.z);
     }
 
-#ifdef QUDA_BACKEND_OMPTARGET
-    virtual unsigned int maxGridSize() const { return device::processor_count(); }
-#else
     /**
        @brief Return the maximum grid size in the x dimension explored
        by the autotuner.  This defaults to twice the number of
@@ -125,7 +122,6 @@ namespace quda {
        will find this through increased block size.
      */
     virtual unsigned int maxGridSize() const { return 2 * device::processor_count(); }
-#endif
 
     /**
        @brief Return the minimum grid size in the x dimension explored

From f02478856bb56bf59bc40bedcd9d8bf42439370c Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 10 Dec 2021 20:49:27 +0000
Subject: [PATCH 098/198] replace remaining pragma unroll with QUDA_UNROLL

---
 include/clover_field_order.h      | 38 +++++++++++++++----------------
 include/float_vector.h            |  4 ++--
 include/kernels/clover_invert.cuh |  2 +-
 include/kernels/copy_clover.cuh   |  2 +-
 include/linalg.cuh                |  4 ++--
 5 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/include/clover_field_order.h b/include/clover_field_order.h
index f4b0a4f73e..2ad6dc5f91 100644
--- a/include/clover_field_order.h
+++ b/include/clover_field_order.h
@@ -141,13 +141,13 @@ namespace quda {
 
       template <typename T1, typename T2> __device__ __host__ inline void unpack(T1 &out, const T2 &in) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < compressed_block_size(); i++) out[unpack_idx(i)] = in[i];
 
           // first reconstruct second set of diagonal elements before we reconstruct the first set
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < 3; i++) out[i + 3] = diagonal - out[i];
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < 3; i++) out[i + 0] = diagonal + out[i];
 
         out[30] = -out[6];
@@ -160,10 +160,10 @@ namespace quda {
 
       template <typename T1, typename T2> __device__ __host__ inline void pack(T1 &out, const T2 &in) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < compressed_block_size(); i++) out[i] = in[unpack_idx(i)];
           // remove diagonal constant
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < 3; i++) out[i] -= diagonal;
         out[3] = 0.0; // intentionally zero this so that it can't contribute to the max element
       }
@@ -182,13 +182,13 @@ namespace quda {
 
       template <typename T1, typename T2> __device__ __host__ inline void unpack(T1 &out, const T2 &in) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < block; i++) out[i] = in[i];
       }
 
       template <typename T1, typename T2> __device__ __host__ inline void pack(T1 &out, const T2 &in) const
       {
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < block; i++) out[i] = in[i];
       }
     };
@@ -828,14 +828,14 @@ QUDA_UNROLL
 	  // factor of 0.5 comes from basis change
           Float v_[length];
           block_load<Float, length>(v_, &clover[parity * offset + x * length]);
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < length; i++) v[i] = 0.5 * v_[i];
         }
 
         __device__ __host__ inline void save(const RegType v[length], int x, int parity) const
         {
           Float v_[length];
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < length; i++) v_[i] = 2.0 * v[i];
           block_store<Float, length>(&clover[parity * offset + x * length], v_);
         }
@@ -871,16 +871,16 @@ QUDA_UNROLL
 
 	__device__ __host__ inline void load(RegType v[length], int x, int parity) const {
 	  // the factor of 0.5 comes from a basis change
-#pragma unroll
+QUDA_UNROLL
           for (int chirality = 0; chirality < 2; chirality++) {
             // set diagonal elements
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < 6; i++) {
               v[chirality*36 + i] = 0.5*diag[((i*2 + chirality)*2 + parity)*volumeCB + x];
             }
 
             // the off diagonal elements
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < 30; i++) {
               int z = i%2;
 	      int off = i/2;
@@ -893,16 +893,16 @@ QUDA_UNROLL
         __device__ __host__ inline void save(const RegType v[length], int x, int parity) const
         {
           // the factor of 2.0 comes from undoing the basis change
-#pragma unroll
+QUDA_UNROLL
           for (int chirality = 0; chirality < 2; chirality++) {
             // set diagonal elements
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < 6; i++) {
               diag[((i*2 + chirality)*2 + parity)*volumeCB + x] = 2.0*v[chirality*36 + i];
             }
 
             // the off diagonal elements
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < 30; i++) {
               int z = i%2;
 	      int off = i/2;
@@ -958,18 +958,18 @@ QUDA_UNROLL
 
           // flip the sign of the imaginary components
           int sign[36];
-#pragma unroll
+QUDA_UNROLL
           for (int i = 0; i < 6; i++) sign[i] = 1;
-#pragma unroll
+QUDA_UNROLL
           for (int i = 6; i < 36; i += 2) {
             if ( (i >= 10 && i<= 15) || (i >= 18 && i <= 29) )  { sign[i] = -1; sign[i+1] = -1; }
 	    else { sign[i] = 1; sign[i+1] = -1; }
           }
 
           const int M = length / 2;
-#pragma unroll
+QUDA_UNROLL
           for (int chirality = 0; chirality < 2; chirality++)
-#pragma unroll
+QUDA_UNROLL
             for (int i = 0; i < M; i++)
               v[chirality * M + i] = sign[i] * clover[parity][x * length + chirality * M + bq[i]];
         }
diff --git a/include/float_vector.h b/include/float_vector.h
index 3d2e2842f5..cf4a86ac6e 100644
--- a/include/float_vector.h
+++ b/include/float_vector.h
@@ -37,7 +37,7 @@ namespace quda {
   __device__ __host__ inline array<T, n> operator+(const array<T, n> &a, const array<T, n> &b)
   {
     array<T, n> c;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < n; i++) c[i] = a[i] + b[i];
     return c;
   }
@@ -60,7 +60,7 @@ namespace quda {
   template <typename T, int n> __device__ __host__ inline array<T, n> zero()
   {
     array<T, n> v;
-#pragma unroll
+QUDA_UNROLL
     for (int i = 0; i < n; i++) v[i] = zero<T>();
     return v;
   }
diff --git a/include/kernels/clover_invert.cuh b/include/kernels/clover_invert.cuh
index 788e17125c..12d2074fc5 100644
--- a/include/kernels/clover_invert.cuh
+++ b/include/kernels/clover_invert.cuh
@@ -52,7 +52,7 @@ namespace quda
       using Mat = HMatrix<real, N>;
       double trLogA = 0.0;
 
-#pragma unroll
+QUDA_UNROLL
       for (int ch = 0; ch < 2; ch++) {
         Mat A = arg.clover(x_cb, parity, ch);
         A *= static_cast<real>(2.0); // factor of two is inherent to QUDA clover storage
diff --git a/include/kernels/copy_clover.cuh b/include/kernels/copy_clover.cuh
index f102b47078..02cd67a0ed 100644
--- a/include/kernels/copy_clover.cuh
+++ b/include/kernels/copy_clover.cuh
@@ -73,7 +73,7 @@ QUDA_UNROLL
       typename mapper<typename Arg::store_out_t>::type out[length];
       typename mapper<typename Arg::store_in_t>::type in[length];
       arg.in.raw_load(in, x_cb, parity);
-#pragma unroll
+QUDA_UNROLL
       for (int i=0; i<length; i++) out[i] = in[i];
 
       arg.out.raw_save(out, x_cb, parity);
diff --git a/include/linalg.cuh b/include/linalg.cuh
index cdf15eefc0..f62eb2071a 100644
--- a/include/linalg.cuh
+++ b/include/linalg.cuh
@@ -157,14 +157,14 @@ QUDA_UNROLL
       {
         // copy source vector into factorization precision
 	ColorSpinor<T,1,N> b_;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < N; i++) b_(i) = b(i);
 
         auto x_ = backward(forward(b_));
 
         // copy solution vector into desired precision
         Vector x;
-#pragma unroll
+QUDA_UNROLL
         for (int i = 0; i < N; i++) x(i) = x_(i);
 
         return x;

From 3df91c9de04cfd9d9843293d12dc10125e1669a2 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 10 Dec 2021 21:51:51 +0000
Subject: [PATCH 099/198] omp: prepare for cmake (not cmake ready yet)

---
 CMakeLists.txt                         | 12 ++++++++++--
 include/blas_magma.h                   |  2 +-
 include/index_helper.cuh               |  2 +-
 include/kernels/coarse_op_kernel.cuh   |  2 +-
 include/kernels/momentum.cuh           |  2 +-
 include/kernels/transform_reduce.cuh   |  2 +-
 include/quda_arch.h                    |  4 ++++
 include/quda_define.h.in               |  3 ++-
 include/reducer.h                      |  6 +++---
 include/shared_memory_cache_helper.cuh |  4 ++--
 include/targets/omptarget/FFT_Plans.h  |  2 +-
 include/tune_quda.h                    |  2 +-
 lib/CMakeLists.txt                     |  4 ++++
 lib/gauge_fix_fft.cu                   |  2 +-
 lib/targets/omptarget/CMakeLists.txt   |  2 ++
 15 files changed, 35 insertions(+), 16 deletions(-)
 create mode 100644 lib/targets/omptarget/CMakeLists.txt

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bb4adce2cb..c4bbdf39b4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -56,11 +56,11 @@ else()
   set(DEFTARGET "CUDA")
 endif()
 
-set(VALID_TARGET_TYPES CUDA HIP)
+set(VALID_TARGET_TYPES CUDA HIP OMPTARGET)
 set(QUDA_TARGET_TYPE
     "${DEFTARGET}"
     CACHE STRING "Choose the type of target, options are: ${VALID_TARGET_TYPES}")
-set_property(CACHE QUDA_TARGET_TYPE PROPERTY STRINGS CUDA HIP)
+set_property(CACHE QUDA_TARGET_TYPE PROPERTY STRINGS CUDA HIP OMPTARGET)
 
 string(TOUPPER ${QUDA_TARGET_TYPE} CHECK_TARGET_TYPE)
 list(FIND VALID_TARGET_TYPES ${CHECK_TARGET_TYPE} TARGET_TYPE_VALID)
@@ -78,6 +78,14 @@ if( ${CHECK_TARGET_TYPE} STREQUAL "HIP")
   set(QUDA_TARGET_HIP ON)
   set(QUDA_TARGET_LIBRARY quda_hip_target)
 endif()
+
+if( ${CHECK_TARGET_TYPE} STREQUAL "OMPTARGET")
+  set(QUDA_TARGET_OMPTARGET ON)
+  set(QUDA_TARGET_LIBRARY quda_omptarget_target)
+  set(QUDA_DEFAULT_GPU_ARCH omp_gpu)
+  set(CMAKE_CXX_COMPILER "icpx")
+  set(CMAKE_CUDA_COMPILER "icpx")
+endif()
 #
 # PROJECT is QUDA
 #
diff --git a/include/blas_magma.h b/include/blas_magma.h
index d8afcfbc84..54b2563320 100644
--- a/include/blas_magma.h
+++ b/include/blas_magma.h
@@ -1,7 +1,7 @@
 #ifndef _BLAS_MAGMA_H
 #define _BLAS_MAGMA_H
 
-#ifndef QUDA_BACKEND_OMPTARGET
+#ifndef QUDA_TARGET_OMPTARGET
 #include <cuda.h>
 #include <cuda_runtime.h>
 #include <string>
diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index 1c084749bd..380fccce8b 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <enum_quda.h>
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
 #include <omp.h>
 #endif
 
diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 69c75a63d4..5156aa906c 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -948,7 +948,7 @@ QUDA_UNROLL
       using real = typename Arg::Float;
       using TileType = typename Arg::vuvTileType;
       const int dim_index = arg.dim_index % arg.Y_atomic.geometry;
-#ifndef QUDA_BACKEND_OMPTARGET
+#ifndef QUDA_TARGET_OMPTARGET
       __shared__ complex<storeType> X[Arg::max_color_height_per_block][Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
       __shared__ complex<storeType> Y[Arg::max_color_height_per_block][Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
 #else
diff --git a/include/kernels/momentum.cuh b/include/kernels/momentum.cuh
index af85786db5..f0d946d6c9 100644
--- a/include/kernels/momentum.cuh
+++ b/include/kernels/momentum.cuh
@@ -94,7 +94,7 @@ namespace quda {
 
     static constexpr bool do_sum = false;
 
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
     static reduce_t reduce_omp(const reduce_t &a, const reduce_t &b)
     {
       auto c = a;
diff --git a/include/kernels/transform_reduce.cuh b/include/kernels/transform_reduce.cuh
index cf67a208f2..f08856e5ea 100644
--- a/include/kernels/transform_reduce.cuh
+++ b/include/kernels/transform_reduce.cuh
@@ -45,7 +45,7 @@ namespace quda {
 
     static constexpr bool do_sum = Arg::reducer::do_sum;
 
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
     static reduce_t reduce_omp(reduce_t a, reduce_t b) { return Arg::reducer::reduce_omp(a, b); }
     static reduce_t init_omp() { return Arg::reducer::init_omp(); }
 #endif
diff --git a/include/quda_arch.h b/include/quda_arch.h
index feda28371c..20a2980415 100644
--- a/include/quda_arch.h
+++ b/include/quda_arch.h
@@ -12,3 +12,7 @@
 #elif defined(QUDA_TARGET_HIP)
 #include <hip/hip_runtime.h>
 #endif
+
+#elif defined(QUDA_TARGET_OMPTARGET)
+#include <omp.h>
+#endif
diff --git a/include/quda_define.h.in b/include/quda_define.h.in
index 7fd4fb021e..8042293c49 100644
--- a/include/quda_define.h.in
+++ b/include/quda_define.h.in
@@ -67,7 +67,8 @@
  */
 #cmakedefine QUDA_TARGET_CUDA
 #cmakedefine QUDA_TARGET_HIP
+#cmakedefine QUDA_TARGET_OMPTARGET
 
-#if !defined(QUDA_TARGET_CUDA) && !defined(QUDA_TARGET_HIP)
+#if !defined(QUDA_TARGET_CUDA) && !defined(QUDA_TARGET_HIP) && !defined(QUDA_TARGET_OMPTARGET)
 #error "No QUDA_TARGET selected"
 #endif
diff --git a/include/reducer.h b/include/reducer.h
index f98d47f921..be6d302a4e 100644
--- a/include/reducer.h
+++ b/include/reducer.h
@@ -66,7 +66,7 @@ namespace quda
   template <typename T> struct plus {
     static constexpr bool do_sum = true;
     __device__ __host__ inline T operator()(T a, T b) const { return a + b; }
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
     static inline T reduce_omp(T a, T b) { return a + b; }
     static inline T init_omp() { return ::quda::zero<T>(); }
 #endif
@@ -78,7 +78,7 @@ namespace quda
   template <typename T> struct maximum {
     static constexpr bool do_sum = false;
     __device__ __host__ inline T operator()(T a, T b) const { return a > b ? a : b; }
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
     static inline T reduce_omp(T a, T b) { return a > b ? a : b; }
     static inline T init_omp() { return ::quda::zero<T>(); }  // FIXME wrong for negative values.
 #endif
@@ -90,7 +90,7 @@ namespace quda
   template <typename T> struct minimum {
     static constexpr bool do_sum = false;
     __device__ __host__ inline T operator()(T a, T b) const { return a < b ? a : b; }
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
     static inline T reduce_omp(T a, T b) { return a < b ? a : b; }
     static inline T init_omp() { return ::quda::zero<T>(); }  // FIXME wrong for positive values.
 #endif
diff --git a/include/shared_memory_cache_helper.cuh b/include/shared_memory_cache_helper.cuh
index 175d4a1782..45e8678599 100644
--- a/include/shared_memory_cache_helper.cuh
+++ b/include/shared_memory_cache_helper.cuh
@@ -72,7 +72,7 @@ namespace quda
     template <typename dummy> struct cache_dynamic<true, dummy> {
       __device__ inline atom_t* operator()()
       {
-#ifndef QUDA_BACKEND_OMPTARGET
+#ifndef QUDA_TARGET_OMPTARGET
         extern __shared__ int cache_[];
         return reinterpret_cast<atom_t*>(cache_);
 #else
@@ -88,7 +88,7 @@ namespace quda
     template <typename dummy> struct cache_static<true, dummy> {
       __device__ inline atom_t* operator()()
       {
-#ifndef QUDA_BACKEND_OMPTARGET
+#ifndef QUDA_TARGET_OMPTARGET
         static __shared__ atom_t cache_[n_element * block_size_x * block_size_y * block_size_z];
         return reinterpret_cast<atom_t*>(cache_);
 #else
diff --git a/include/targets/omptarget/FFT_Plans.h b/include/targets/omptarget/FFT_Plans.h
index e53bfe1a5f..baed179446 100644
--- a/include/targets/omptarget/FFT_Plans.h
+++ b/include/targets/omptarget/FFT_Plans.h
@@ -13,7 +13,7 @@ using FFTPlanHandle = cufftHandle;
 
 #ifndef GPU_GAUGE_ALG
 */
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
 #define CUFFT_SAFE_CALL(call)
 
 inline void ApplyFFT(FFTPlanHandle &, float2 *, float2 *, int)
diff --git a/include/tune_quda.h b/include/tune_quda.h
index 6f8b1ea04f..e2962c97ce 100644
--- a/include/tune_quda.h
+++ b/include/tune_quda.h
@@ -74,7 +74,7 @@ namespace quda {
       if (tuneGridDim()) {
         const int step = gridStep();
         param.grid.x += step;
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
         /* Makes sure the runtime allows us to set the parameters. */
         bool failed = false;
         {
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 68d3578659..43aa66c6a2 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -272,6 +272,10 @@ if(${QUDA_TARGET_TYPE} STREQUAL "HIP")
   add_subdirectory(targets/hip)
   target_include_directories(quda PRIVATE ../include/targets/hip)
 endif()
+if(${QUDA_TARGET_TYPE} STREQUAL "OMPTARGET")
+  add_subdirectory(targets/omptarget)
+  target_include_directories(quda PRIVATE ../include/targets/omptarget)
+endif()
 
 add_subdirectory(targets/generic)
 target_include_directories(quda PRIVATE ../include/targets/generic)
diff --git a/lib/gauge_fix_fft.cu b/lib/gauge_fix_fft.cu
index 8587d91f3c..b4da419bd5 100644
--- a/lib/gauge_fix_fft.cu
+++ b/lib/gauge_fix_fft.cu
@@ -185,7 +185,7 @@ namespace quda {
   void gaugeFixingFFT(GaugeField& data, int Nsteps, int verbose_interval,
                       double alpha0, int autotune, double tolerance, int stopWtheta)
   {
-#ifdef QUDA_BACKEND_OMPTARGET
+#ifdef QUDA_TARGET_OMPTARGET
     ompwip("unimplemented");
 #else
     TimeProfile profileInternalGaugeFixFFT("InternalGaugeFixQudaFFT", false);
diff --git a/lib/targets/omptarget/CMakeLists.txt b/lib/targets/omptarget/CMakeLists.txt
new file mode 100644
index 0000000000..a81643e172
--- /dev/null
+++ b/lib/targets/omptarget/CMakeLists.txt
@@ -0,0 +1,2 @@
+# add target specific files / options
+target_sources(quda_cpp PRIVATE  blas_lapack_cublas.cpp comm_target.cpp device.cpp malloc.cpp quda_api.cpp)

From 319e1972ba3e2d178e64f84697a7accca1daf207 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 10 Dec 2021 22:06:21 +0000
Subject: [PATCH 100/198] omp: fix quda_arch.h

---
 include/quda_arch.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/include/quda_arch.h b/include/quda_arch.h
index 20a2980415..fe24d813b1 100644
--- a/include/quda_arch.h
+++ b/include/quda_arch.h
@@ -11,7 +11,6 @@
 
 #elif defined(QUDA_TARGET_HIP)
 #include <hip/hip_runtime.h>
-#endif
 
 #elif defined(QUDA_TARGET_OMPTARGET)
 #include <omp.h>

From c7b70c536999f136940e993cc6c3686a984d5645 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 10 Dec 2021 22:07:04 +0000
Subject: [PATCH 101/198] omp: remove extra omp.h includes

---
 include/index_helper.cuh         | 3 ---
 lib/targets/omptarget/device.cpp | 1 -
 2 files changed, 4 deletions(-)

diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index 380fccce8b..23679e2c05 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -1,9 +1,6 @@
 #pragma once
 
 #include <enum_quda.h>
-#ifdef QUDA_TARGET_OMPTARGET
-#include <omp.h>
-#endif
 
 namespace quda {
   /**
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 462d49af58..bd81a0d4b7 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -1,4 +1,3 @@
-#include <omp.h>
 #include <util_quda.h>
 #include <quda_internal.h>
 #include <target_device.h>

From 9bc64c140f61f43bf643ec6253ed224c2092666a Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 16 Dec 2021 18:04:57 +0000
Subject: [PATCH 102/198] omp: include math_helper in blas_helper

---
 include/blas_helper.cuh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index a150bb74f7..61090e53dc 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -5,6 +5,7 @@
 #include <convert.h>
 #include <float_vector.h>
 #include <array.h>
+#include <math_helper.cuh>
 
 //#define QUAD_SUM
 #ifdef QUAD_SUM

From 2c2c392d5741fdb7046407489cdb7f7d8cd5f204 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 11 Apr 2022 22:18:00 +0000
Subject: [PATCH 103/198] omp: update reducer with generic block reduce

---
 include/kernels/coarse_op_kernel.cuh          |   2 +-
 include/reducer.h                             |   4 +-
 include/shared_memory_cache_helper.cuh        |   4 +-
 .../targets/omptarget/block_reduce_helper.h   | 241 +++++++++
 .../omptarget/block_reduction_kernel.h        |  12 +-
 .../omptarget/cub/block/block_reduce.cuh      |  88 ----
 include/targets/omptarget/kernel.h            |  51 +-
 include/targets/omptarget/quda_api.h          |  32 +-
 include/targets/omptarget/reduce_helper.h     | 135 ++---
 include/targets/omptarget/reduction_kernel.h  | 418 ++++------------
 include/targets/omptarget/target_device.h     |   8 +-
 include/targets/omptarget/tunable_kernel.h    |  10 +-
 lib/targets/omptarget/comm_target.cpp         | 183 +------
 lib/targets/omptarget/malloc.cpp              |  39 +-
 lib/targets/omptarget/quda_api.cpp            | 464 ++++++------------
 15 files changed, 698 insertions(+), 993 deletions(-)
 create mode 100644 include/targets/omptarget/block_reduce_helper.h
 delete mode 100644 include/targets/omptarget/cub/block/block_reduce.cuh

diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 83a3c49235..19c8f235da 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -1373,7 +1373,7 @@ QUDA_UNROLL
       __shared__ complex<storeType> Y[Arg::max_color_height_per_block][Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
 #else
       typedef complex<storeType> TyX0[Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
-      TyX0 *X = reinterpret_cast<TyX0 *>(shared_cache.addr);
+      TyX0 *X = reinterpret_cast<TyX0 *>(target::omptarget::shared_cache.addr);
       TyX0 *Y = X+Arg::max_color_height_per_block;
 #endif
 
diff --git a/include/reducer.h b/include/reducer.h
index af930e9b7b..d52e7a7d47 100644
--- a/include/reducer.h
+++ b/include/reducer.h
@@ -91,7 +91,7 @@ namespace quda
     using reducer_t = maximum<T>;
     template <typename U> static inline void comm_reduce(std::vector<U> &a) { comm_allreduce_max(a); }
     __device__ __host__ static inline T init() { return low<T>::value(); }
-    __device__ __host__ static inline T apply(T a, T b) { return max(a, b); }
+    __device__ __host__ static inline T apply(T a, T b) { return std::max(a, b); }
     __device__ __host__ inline T operator()(T a, T b) const { return apply(a, b); }
   };
 
@@ -104,7 +104,7 @@ namespace quda
     using reducer_t = minimum<T>;
     template <typename U> static inline void comm_reduce(std::vector<U> &a) { comm_allreduce_min(a); }
     __device__ __host__ static inline T init() { return high<T>::value(); }
-    __device__ __host__ static inline T apply(T a, T b) { return min(a, b); }
+    __device__ __host__ static inline T apply(T a, T b) { return std::min(a, b); }
     __device__ __host__ inline T operator()(T a, T b) const { return apply(a, b); }
   };
 
diff --git a/include/shared_memory_cache_helper.cuh b/include/shared_memory_cache_helper.cuh
index 85c1d0000b..902d0f3cfd 100644
--- a/include/shared_memory_cache_helper.cuh
+++ b/include/shared_memory_cache_helper.cuh
@@ -76,7 +76,7 @@ namespace quda
         extern __shared__ int cache_[];
         return reinterpret_cast<atom_t*>(cache_);
 #else
-        return reinterpret_cast<atom_t*>(shared_cache.addr);
+        return reinterpret_cast<atom_t*>(target::omptarget::shared_cache.addr);
 #endif
       }
     };
@@ -92,7 +92,7 @@ namespace quda
         static __shared__ atom_t cache_[n_element * block_size_x * block_size_y * block_size_z];
         return reinterpret_cast<atom_t*>(cache_);
 #else
-        return reinterpret_cast<atom_t*>(shared_cache.addr);
+        return reinterpret_cast<atom_t*>(target::omptarget::shared_cache.addr);
 #endif
       }
     };
diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
new file mode 100644
index 0000000000..7c43094ebf
--- /dev/null
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -0,0 +1,241 @@
+#pragma once
+
+#include <target_device.h>
+#include <reducer.h>
+
+/**
+   @file block_reduce_helper.h
+
+   @section This files contains the CUDA device specialziations for
+   warp- and block-level reductions, using the CUB library
+ */
+
+// ensures we use shfl_sync and not shfl when compiling with clang
+#if defined(__clang__) && defined(__CUDA__)
+#define CUB_USE_COOPERATIVE_GROUPS
+#endif
+
+using namespace quda;
+
+namespace cub {
+
+  // Naive implementation of cub collectives in OpenMP target.
+  // FIXME: use team shared memory
+
+  enum BlockReduceAlgorithm {BLOCK_REDUCE_WARP_REDUCTIONS};
+
+  constexpr int get_warp_steps(int s)
+  {
+    // log2(s)
+    return s==1?0:s==2?1:s==4?2:s==8?3:s==16?4:s==32?5:s==64?6:-1;
+  }
+
+// https://github.com/NVlabs/cub/blob/1.8.0/cub/warp/specializations/warp_reduce_smem.cuh
+template <typename T, int warp_threads, int ptx_arch = __COMPUTE_CAPABILITY__>
+struct WarpReduce
+{
+  static constexpr int warp_steps = get_warp_steps(warp_threads);
+  static_assert(warp_steps>=0, "unsupported warp_threads");
+  static constexpr int half_warp_threads = 1 << (warp_steps-1);
+  static constexpr int warp_smem_elments = warp_threads + half_warp_threads;
+  struct TempStorage
+  {
+    T reduce[warp_smem_elments];
+  };
+  TempStorage &temp_storage;
+  int tid,lane_id;
+  inline WarpReduce(TempStorage &temp_storage) :
+    temp_storage(temp_storage),
+    tid(omp_get_thread_num()),
+    lane_id(tid%warp_threads)
+  {}
+  template <typename ReductionOp>
+  inline T Reduce(T input, int valid_items, ReductionOp reduction_op)
+  {
+    QUDA_UNROLL
+    for(int i=0;i<warp_steps;++i){
+      temp_storage.reduce[lane_id] = input;
+      const int offset = 1 << i;
+      // this is bad, but we don't have warp barrier
+      #pragma omp barrier
+      if((lane_id + offset) < valid_items)
+        input = reduction_op(input, temp_storage.reduce[lane_id+offset]);
+    }
+    return input;
+  }
+  inline T Sum(T input, int valid_items = warp_threads)  // FIXME warp_num_valid < warp_threads
+  {
+    QUDA_UNROLL
+    for(int i=0;i<warp_steps;++i){
+      temp_storage.reduce[lane_id] = input;
+      const int offset = 1 << i;
+      // this is bad, but we don't have warp barrier
+      #pragma omp barrier
+      if((lane_id + offset) < valid_items)
+        input += temp_storage.reduce[lane_id+offset];
+    }
+    return input;
+  }
+};
+
+template <typename T, int warp_threads, int ptx_arch = __COMPUTE_CAPABILITY__>
+struct WarpScan
+{
+  struct TempStorage
+  {
+    T v;
+  };
+  TempStorage &temp_storage;
+  int lane_id;
+  inline WarpScan(TempStorage &temp_storage) :
+    temp_storage(temp_storage),
+    lane_id(omp_get_thread_num()%warp_threads)
+  {}
+  inline T Broadcast(T input, unsigned int src_lane)
+  {
+    if (lane_id == src_lane)
+      temp_storage.v = input;
+    #pragma omp barrier
+    return temp_storage.v;
+  }
+};
+
+// https://github.com/NVlabs/cub/blob/1.8.0/cub/block/specializations/block_reduce_warp_reductions.cuh
+template <typename T, int block_dim_x, BlockReduceAlgorithm alg = BLOCK_REDUCE_WARP_REDUCTIONS, int block_dim_y = 1, int block_dim_z = 1, int ptx_arch = __COMPUTE_CAPABILITY__>
+struct BlockReduce
+{
+  static constexpr int block_threads = block_dim_x*block_dim_y*block_dim_z;
+  static constexpr int warp_threads = device::warp_size();
+  static constexpr int warps = (block_threads + warp_threads - 1) / warp_threads;
+  static constexpr int logical_warp_size = std::min(block_threads, warp_threads);
+  static constexpr bool even_warp_multiple = block_threads%logical_warp_size == 0;
+  typedef WarpReduce<T, logical_warp_size, ptx_arch> WarpReduce_;
+  struct TempStorage
+  {
+    typename WarpReduce_::TempStorage warp_reduce[warps];
+    T warp_aggregates[warps];
+  };
+  TempStorage &temp_storage;
+  int tid,lane_id,warp_id;
+  inline BlockReduce(TempStorage &temp_storage) :
+    temp_storage(temp_storage),
+    tid(omp_get_thread_num()),
+    lane_id(tid%logical_warp_size),
+    warp_id(tid/logical_warp_size)
+  {}
+  template <typename ReductionOp>
+  inline T Reduce(T input, ReductionOp reduction_op)
+  {
+    int warp_offset = warp_id * logical_warp_size;
+    int warp_num_valid = (even_warp_multiple || (warp_offset + logical_warp_size <= block_threads)) ? logical_warp_size : block_threads - warp_offset;
+    T warp_aggregate = WarpReduce_(temp_storage.warp_reduce[warp_id]).Reduce(input, warp_num_valid, reduction_op);
+
+    if (lane_id == 0) temp_storage.warp_aggregates[warp_id] = warp_aggregate;
+    #pragma omp barrier
+    if (tid == 0) {
+QUDA_UNROLL
+      for(int i=1;i<warps;++i) warp_aggregate = reduction_op(warp_aggregate, temp_storage.warp_aggregates[i]);
+    }
+    return warp_aggregate;
+  }
+  inline T Sum(T input)
+  {
+    return Reduce(input, plus<T>());
+  }
+};
+
+}
+
+namespace quda
+{
+
+  // pre-declaration of warp_reduce that we wish to specialize
+  template <bool> struct warp_reduce;
+
+  /**
+     @brief CUDA specialization of warp_reduce, utilizing cub::WarpReduce
+  */
+  template <> struct warp_reduce<true> {
+
+    /**
+       @brief Perform a warp-wide reduction
+       @param[in] value_ thread-local value to be reduced
+       @param[in] all Whether we want all threads to have visibility
+       to the result (all = true) or just the first thread in the
+       warp (all = false)
+       @param[in] r The reduction operation we want to apply
+       @return The warp-wide reduced value
+     */
+    template <typename T, typename reducer_t, typename param_t>
+    __device__ inline T operator()(const T &value_, bool all, const reducer_t &r, const param_t &)
+    {
+      using warp_reduce_t = cub::WarpReduce<T, param_t::width, __COMPUTE_CAPABILITY__>;
+      using tempStorage = typename warp_reduce_t::TempStorage;
+      static_assert(sizeof(tempStorage) <= sizeof(target::omptarget::shared_cache.addr[0])*1024, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, 1024 is used below
+      tempStorage *dummy_storage = (tempStorage*)&target::omptarget::shared_cache.addr[1024];
+      warp_reduce_t warp_reduce(*dummy_storage);
+      T value = reducer_t::do_sum ? warp_reduce.Sum(value_) : warp_reduce.Reduce(value_, r);
+
+      if (all) {
+        using warp_scan_t = cub::WarpScan<T, param_t::width, __COMPUTE_CAPABILITY__>;
+        using tempStorage = typename warp_scan_t::TempStorage;
+        static_assert(sizeof(tempStorage) <= sizeof(target::omptarget::shared_cache.addr[0])*1024, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, 1024 is arbitrary
+        tempStorage *dummy_storage = (tempStorage*)&target::omptarget::shared_cache.addr[2048];
+        // typename warp_scan_t::TempStorage dummy_storage;
+        warp_scan_t warp_scan(*dummy_storage);
+        value = warp_scan.Broadcast(value, 0);
+      }
+
+      return value;
+    }
+  };
+
+  // pre-declaration of block_reduce that we wish to specialize
+  template <bool> struct block_reduce;
+
+  /**
+     @brief CUDA specialization of block_reduce, utilizing cub::BlockReduce
+  */
+  template <> struct block_reduce<true> {
+
+    /**
+       @brief Perform a block-wide reduction
+       @param[in] value_ thread-local value to be reduced
+       @param[in] async Whether this reduction will be performed
+       asynchronously with respect to the calling threads
+       @param[in] batch The batch index of the reduction
+       @param[in] all Whether we want all threads to have visibility
+       to the result (all = true) or just the first thread in the
+       block (all = false)
+       @param[in] r The reduction operation we want to apply
+       @return The block-wide reduced value
+     */
+    template <typename T, typename reducer_t, typename param_t>
+    __device__ inline T operator()(const T &value_, bool async, int batch, bool all, const reducer_t &r,
+                                   const param_t &)
+    {
+      using block_reduce_t = cub::BlockReduce<T, param_t::block_size_x, cub::BLOCK_REDUCE_WARP_REDUCTIONS,
+                                              param_t::block_size_y, param_t::block_size_z, __COMPUTE_CAPABILITY__>;
+      using tempStorage = typename block_reduce_t::TempStorage;
+      tempStorage *storage = (tempStorage*)&target::omptarget::shared_cache.addr[1024];  // FIXME arbitrary
+      // typename block_reduce_t::TempStorage storage[param_t::batch_size];
+      block_reduce_t block_reduce(storage[batch]);
+      if (!async){
+        #pragma omp barrier
+        // __syncthreads(); // only synchronize if we are not pipelining
+      }
+      T value = reducer_t::do_sum ? block_reduce.Sum(value_) : block_reduce.Reduce(value_, r);
+      QUDA_RT_CONSTS;
+      if (all) {
+        T &value_shared = reinterpret_cast<T &>(storage[batch]);
+        if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) value_shared = value;
+        #pragma omp barrier
+        value = value_shared;
+      }
+      return value;
+    }
+  };
+
+} // namespace quda
+
+#include "../generic/block_reduce_helper.h"
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 697a9783a8..e31a3d98d1 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -111,8 +111,8 @@ namespace quda {
                       0) __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // const int tx = arg.threads.x;
     // const int ty = arg.threads.y;
     // const int tz = arg.threads.z;
@@ -123,7 +123,7 @@ namespace quda {
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      shared_cache.addr = cache;
+      target::omptarget::shared_cache.addr = cache;
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -157,13 +157,13 @@ namespace quda {
                       0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D()
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      shared_cache.addr = cache;
+      target::omptarget::shared_cache.addr = cache;
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
diff --git a/include/targets/omptarget/cub/block/block_reduce.cuh b/include/targets/omptarget/cub/block/block_reduce.cuh
deleted file mode 100644
index 3c94a40910..0000000000
--- a/include/targets/omptarget/cub/block/block_reduce.cuh
+++ /dev/null
@@ -1,88 +0,0 @@
-#include <target_device.h>
-
-namespace cub {
-
-  // Naive implementation of cub collectives in OpenMP target.
-
-  enum BlockReduceAlgorithm {BLOCK_REDUCE_WARP_REDUCTIONS};
-
-  constexpr int get_warp_steps(int s)
-  {
-    // log2(s)
-    return s==1?0:s==2?1:s==4?2:s==8?3:s==16?4:s==32?5:s==64?6:-1;
-  }
-
-// https://github.com/NVlabs/cub/blob/1.8.0/cub/warp/specializations/warp_reduce_smem.cuh
-template <typename T, int warp_threads, int ptx_arch = __COMPUTE_CAPABILITY__>
-struct WarpReduce
-{
-  static constexpr int warp_steps = get_warp_steps(warp_threads);
-  static_assert(warp_steps>=0, "unsupported warp_threads");
-  static constexpr int half_warp_threads = 1 << (warp_steps-1);
-  static constexpr int warp_smem_elments = warp_threads + half_warp_threads;
-  struct TempStorage
-  {
-    T reduce[warp_smem_elments];
-  };
-  TempStorage &temp_storage;
-  int tid,lane_id;
-  inline WarpReduce(TempStorage &temp_storage) :
-    temp_storage(temp_storage),
-    tid(omp_get_thread_num()),
-    lane_id(tid%warp_threads)
-  {}
-  inline T Sum(T input, int valid_items = warp_threads)  // FIXME warp_num_valid < warp_threads
-  {
-    QUDA_UNROLL
-    for(int i=0;i<warp_steps;++i){
-      temp_storage.reduce[lane_id] = input;
-      const int offset = 1 << i;
-      // this is bad, but we don't have warp barrier
-      #pragma omp barrier
-      if((lane_id + offset) < valid_items)
-        input += temp_storage.reduce[lane_id+offset];
-    }
-    return input;
-  }
-};
-
-// https://github.com/NVlabs/cub/blob/1.8.0/cub/block/specializations/block_reduce_warp_reductions.cuh
-template <typename T, int block_dim_x, BlockReduceAlgorithm alg = BLOCK_REDUCE_WARP_REDUCTIONS, int block_dim_y = 1, int block_dim_z = 1, int ptx_arch = __COMPUTE_CAPABILITY__>
-struct BlockReduce
-{
-  static constexpr int block_threads = block_dim_x*block_dim_y*block_dim_z;
-  static constexpr int warp_threads = device::warp_size();
-  static constexpr int warps = (block_threads + warp_threads - 1) / warp_threads;
-  static constexpr int logical_warp_size = std::min(block_threads, warp_threads);
-  static constexpr bool even_warp_multiple = block_threads%logical_warp_size == 0;
-  typedef WarpReduce<T, logical_warp_size, ptx_arch> WarpReduce_;
-  struct TempStorage
-  {
-    typename WarpReduce_::TempStorage warp_reduce[warps];
-    T warp_aggregates[warps];
-  };
-  TempStorage &temp_storage;
-  int tid,lane_id,warp_id;
-  inline BlockReduce(TempStorage &temp_storage) :
-    temp_storage(temp_storage),
-    tid(omp_get_thread_num()),
-    lane_id(tid%logical_warp_size),
-    warp_id(tid/logical_warp_size)
-  {}
-  inline T Sum(T input)
-  {
-    int warp_offset = warp_id * logical_warp_size;
-    int warp_num_valid = (even_warp_multiple || (warp_offset + logical_warp_size <= block_threads)) ? logical_warp_size : block_threads - warp_offset;
-    T warp_aggregate = WarpReduce_(temp_storage.warp_reduce[warp_id]).Sum(input, warp_num_valid);
-
-    if (lane_id == 0) temp_storage.warp_aggregates[warp_id] = warp_aggregate;
-    #pragma omp barrier
-    if (tid == 0) {
-QUDA_UNROLL
-      for(int i=1;i<warps;++i) warp_aggregate += temp_storage.warp_aggregates[i];
-    }
-    return warp_aggregate;
-  }
-};
-
-}
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 9912eb4f59..6debc1b793 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -22,19 +22,19 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
     __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg)
     {
-      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-      const int tx = arg.threads.x;
-      const int ty = arg.threads.y;
-      const int tz = arg.threads.z;
-      // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
       Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
       // printf("dparg %p\n", dparg);
       omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+      // const int tx = arg.threads.x;
+      // const int ty = arg.threads.y;
+      // const int tz = arg.threads.z;
+      // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
       {
         int cache[device::max_shared_memory_size()/sizeof(int)];
-        shared_cache.addr = cache;
+        target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -47,19 +47,20 @@ namespace quda {
         Kernel1D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
       }
+      // printf("Kernel1D: exited\n");
       omp_target_free(dparg, omp_get_default_device());
     }
 
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D()
     {
-      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
       // printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
       {
         int cache[device::max_shared_memory_size()/sizeof(int)];
-        shared_cache.addr = cache;
+        target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -92,8 +93,8 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
     __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg)
     {
-      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
       const int tx = arg.threads.x;
       const int ty = arg.threads.y;
       const int tz = arg.threads.z;
@@ -104,7 +105,7 @@ namespace quda {
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
       {
         int cache[device::max_shared_memory_size()/sizeof(int)];
-        shared_cache.addr = cache;
+        target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -123,13 +124,13 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D()
     {
-      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
       // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
       {
         int cache[device::max_shared_memory_size()/sizeof(int)];
-        shared_cache.addr = cache;
+        target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -164,8 +165,8 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
     __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg)
     {
-      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
       const int tx = arg.threads.x;
       const int ty = arg.threads.y;
       const int tz = arg.threads.z;
@@ -176,7 +177,7 @@ namespace quda {
       #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
       {
         int cache[device::max_shared_memory_size()/sizeof(int)];
-        shared_cache.addr = cache;
+        target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -195,13 +196,13 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
     __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D()
     {
-      const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
       // printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
       #pragma omp target teams num_teams(gd) thread_limit(ld)
       {
         int cache[device::max_shared_memory_size()/sizeof(int)];
-        shared_cache.addr = cache;
+        target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -217,8 +218,8 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
-    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     const int tx = arg.threads.x;
     const int ty = arg.threads.y;
     const int tz = arg.threads.z;
@@ -229,7 +230,7 @@ namespace quda {
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
       {
         int cache[device::max_shared_memory_size()/sizeof(int)];
-        shared_cache.addr = cache;
+        target::omptarget::shared_cache.addr = cache;
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index 0d712521bb..90a58deadc 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -18,10 +18,10 @@
 
 #define QUDA_RT_CONSTS \
   const dim3\
-    blockDim=launch_param.block,\
-    gridDim=launch_param.grid,\
-    threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y)),\
-    blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y))
+    blockDim=target::omptarget::launch_param.block,\
+    gridDim=target::omptarget::launch_param.grid,\
+    threadIdx(omp_get_thread_num()%target::omptarget::launch_param.block.x, (omp_get_thread_num()/target::omptarget::launch_param.block.x)%target::omptarget::launch_param.block.y, omp_get_thread_num()/(target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y)),\
+    blockIdx(omp_get_team_num()%target::omptarget::launch_param.grid.x, (omp_get_team_num()/target::omptarget::launch_param.grid.x)%target::omptarget::launch_param.grid.y, omp_get_team_num()/(target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y))
 
 #include <functional>
 #include <iostream>
@@ -89,15 +89,21 @@ ompwip_(const char * const file, const size_t line, const char * const func, std
 using cudaStream_t = int;  // device.h:/cudaStream_t
 using CUresult = int;  // ../lib/coarse_op.cuh:/CUresult
 
-struct SharedCache{
-  int *addr;
-};
-extern SharedCache shared_cache;
+namespace quda {
+  namespace target {
+    namespace omptarget {
+      struct SharedCache{
+        int *addr;
+      };
+      extern SharedCache shared_cache;
 
-struct LaunchParam{
-  dim3 block;
-  dim3 grid;
-};
-extern LaunchParam launch_param;
+      struct LaunchParam{
+        dim3 block;
+        dim3 grid;
+      };
+      extern LaunchParam launch_param;
+    }
+  }
+}
 
 #include "../../quda_api.h"
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 833b01ed95..0db6836748 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -16,12 +16,9 @@ using count_t = unsigned int;
 namespace quda
 {
 
-/*
-  // OMP TARGET TODO: due to limitation of openmp implementation, we inline reduce manually.
   // declaration of reduce function
   template <int block_size_x, int block_size_y = 1, typename Reducer, typename Arg, typename T>
-  __device__ __forceinline__ void reduce(Arg &arg, const Reducer &r, const T &in, const int idx = 0);
-*/
+  __device__ inline void reduce(Arg &arg, const Reducer &r, const T &in, const int idx = 0);
 
   /**
      @brief ReduceArg is the argument type that all kernel arguments
@@ -32,43 +29,57 @@ namespace quda
      memory
    */
   template <typename T, bool use_kernel_arg = true> struct ReduceArg : kernel_param<use_kernel_arg> {
+    using reduce_t = T;
 
-    // OMP TARGET TODO: due to limitation of openmp implementation, we inline reduce manually.
-    // We also remove private restrictions below.
-    // template <int, int, typename Reducer, typename Arg, typename I>
-    // friend __device__ void reduce(Arg &, const Reducer &, const I &, const int);
+    template <int, int, typename Reducer, typename Arg, typename I>
+    friend __device__ void reduce(Arg &, const Reducer &, const I &, const int);
     qudaError_t launch_error; /** only do complete if no launch error to avoid hang */
+    static constexpr unsigned int max_n_batch_block
+      = 1; /** by default reductions do not support batching withing the block */
 
+  private:
     const int n_reduce; /** number of reductions of length n_item */
     T *partial; /** device buffer */
     T *result_d; /** device-mapped host buffer */
     T *result_h; /** host buffer */
     count_t *count; /** count array that is used to track the number of completed thread blocks at a given batch index */
+    T *device_output_async_buffer = nullptr; // Optional device output buffer for the reduction result
 
+  public:
     /**
        @brief Constructor for ReduceArg
        @param[in] threads The number threads partaking in the kernel
        @param[in] n_reduce The number of reductions
     */
     ReduceArg(dim3 threads, int n_reduce = 1, bool = false) :
-      kernel_param<use_kernel_arg>(threads),
-      launch_error(QUDA_ERROR_UNINITIALIZED),
-      n_reduce(n_reduce),
-      partial(static_cast<decltype(partial)>(reducer::get_device_buffer())),
-      result_d(static_cast<decltype(result_d)>(reducer::get_mapped_buffer())),
-      result_h(static_cast<decltype(result_h)>(reducer::get_host_buffer())),
-      count {reducer::get_count<count_t>()}
+      kernel_param<use_kernel_arg>(threads), launch_error(QUDA_ERROR_UNINITIALIZED), n_reduce(n_reduce)
     {
-      // check reduction buffers are large enough if requested
-      auto max_reduce_blocks = 2 * device::processor_count();
-      auto reduce_size = max_reduce_blocks * n_reduce * sizeof(*partial);
-      if (reduce_size > reducer::buffer_size())
-        errorQuda("Requested reduction requires a larger buffer %lu than allocated %lu", reduce_size,
-                  reducer::buffer_size());
+      reducer::init(n_reduce, sizeof(*partial));
+      // these buffers may be allocated in init, so we can't set the local copies until now
+      partial = static_cast<decltype(partial)>(reducer::get_device_buffer());
+      result_d = static_cast<decltype(result_d)>(reducer::get_mapped_buffer());
+      result_h = static_cast<decltype(result_h)>(reducer::get_host_buffer());
+      count = reducer::get_count<count_t>();
 
       if (commAsyncReduction()) result_d = partial;
     }
 
+    /**
+      @brief Set device_output_async_buffer
+    */
+    void set_output_async_buffer(T *ptr)
+    {
+      if (!commAsyncReduction()) {
+        errorQuda("When setting the asynchronous buffer the commAsyncReduction option must be set.");
+      }
+      device_output_async_buffer = ptr;
+    }
+
+    /**
+      @brief Get device_output_async_buffer
+    */
+    __device__ __host__ T *get_output_async_buffer() const { return device_output_async_buffer; }
+
     /**
        @brief Finalize the reduction, returning the computed reduction
        into result.  The generic path posts an event after the kernel
@@ -90,8 +101,6 @@ namespace quda
       const int n_element = n_reduce * sizeof(T) / sizeof(device_t);
       if (result.size() != (unsigned)n_element)
         errorQuda("result vector length %lu does not match n_reduce %d", result.size(), n_element);
-      // OMP TARGET TODO: try shared memory for result_d/h
-      omp_target_memcpy(result_h, result_d, n_element*sizeof(device_t), 0, 0, omp_get_initial_device(), omp_get_default_device());
       for (int i = 0; i < n_element; i++) result[i] = reinterpret_cast<device_t *>(result_h)[i];
     }
   };
@@ -108,52 +117,60 @@ namespace quda
      @param idx In the case of multiple reductions, idx identifies
      which reduction this thread block corresponds to.  Typically idx
      will be constant along constant block_idx().y and block_idx().z.
-
-     OMP TARGET: should be run only as the main thread of each team.
-     here, we need the block reduction value, `aggregate`.
-     OMP TARGET TODO: not supported for now.  We manually inline this.
-
+  */
   template <int block_size_x, int block_size_y, typename Reducer, typename Arg, typename T>
-  __device__ __forceinline__ void reduce(Arg &arg, const Reducer &r, const T &aggregate, const int idx)
+  __device__ inline void reduce(Arg &arg, const Reducer &r, const T &in, const int idx)
   {
-    printf("reduce::aggregate team %d %g\n", omp_get_team_num(), *(double*)(&aggregate));
-    bool isLastBlockDone;
-
-    arg.partial[idx * target::grid_dim().x + target::block_idx().x] = aggregate;
-    __threadfence(); // flush result
-
-    // increment global block counter
-    // auto value = atomicInc(&arg.count[idx], gridDim.x);
-    unsigned int value = 0;
-    unsigned int *c = &arg.count[idx];
-    #pragma omp atomic capture
-    { value = *c; *c = *c + 1; }
-
-    // determine if last block
-    isLastBlockDone = (value == (target::grid_dim().x - 1));
+    constexpr auto n_batch_block
+      = std::min(Arg::max_n_batch_block, device::max_block_size() / (block_size_x * block_size_y));
+    using BlockReduce = BlockReduce<T, block_size_x, block_size_y, n_batch_block, true>;
+    // bool isLastBlockDone[n_batch_block];
+    static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::shared_cache.addr[0])*1024, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 1024 is used in block_reduce_helper.h:/dummy_storage/
+    bool *isLastBlockDone = (bool*)target::omptarget::shared_cache.addr;
+
+    T aggregate = BlockReduce(target::thread_idx().z).Reduce(in, r);
+    // printf("team %d thread %d  r %g  aggregate %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&in), *(double*)(&aggregate));
+
+    if (target::thread_idx().x == 0 && target::thread_idx().y == 0) {
+      arg.partial[idx * target::grid_dim().x + target::block_idx().x] = aggregate;
+      __threadfence(); // flush result
+
+      // increment global block counter
+      // auto value = atomicInc(&arg.count[idx], target::grid_dim().x);
+      unsigned int value = 0u;
+      unsigned int *c = &arg.count[idx];
+      #pragma omp atomic capture
+      { value = *c; *c = *c + 1; }
+
+      // determine if last block
+      isLastBlockDone[target::thread_idx().z] = (value == (target::grid_dim().x - 1));
+    }
 
     __syncthreads();
 
     // finish the reduction if last block
-#pragma omp declare reduction(OMPReduce_ : typename Reducer::reduce_t : omp_out=Reducer::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Reducer::init_omp())
-    if (isLastBlockDone) {
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-      T sum = arg.init();
-      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-      {
-        auto i = target::thread_idx().y * block_size_x + target::thread_idx().x;
-        while (i < target::grid_dim().x) {
-          sum = r(sum, const_cast<T &>(static_cast<volatile T *>(arg.partial)[idx * target::grid_dim().x + i]));
-          i += block_size_x * block_size_y;
-        }
+    if (isLastBlockDone[target::thread_idx().z]) {
+      auto i = target::thread_idx().y * block_size_x + target::thread_idx().x;
+      T sum = r.init();
+      while (i < target::grid_dim().x) {
+        sum = r(sum, const_cast<T &>(static_cast<volatile T *>(arg.partial)[idx * target::grid_dim().x + i]));
+        // printf("team %d thread %d  sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
+        i += block_size_x * block_size_y;
       }
 
+      sum = BlockReduce(target::thread_idx().z).Reduce(sum, r);
+
       // write out the final reduced value
-      printf("reduce::sum %g\n", *(double*)(&sum));
-      arg.result_d[idx] = sum;
-      arg.count[idx] = 0; // set to zero for next time
+      if (target::thread_idx().y * block_size_x + target::thread_idx().x == 0) {
+        // printf("team %d thread %d  final sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
+        if (arg.get_output_async_buffer()) {
+          arg.get_output_async_buffer()[idx] = sum;
+        } else {
+          arg.result_d[idx] = sum;
+        }
+        arg.count[idx] = 0; // set to zero for next time
+      }
     }
   }
-  */
 
 } // namespace quda
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 94d46b78a3..d95310833c 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -3,7 +3,8 @@
 #include <target_device.h>
 #include <reduce_helper.h>
 
-namespace quda {
+namespace quda
+{
 
   /**
      @brief This class is derived from the arg class that the functor
@@ -22,6 +23,44 @@ namespace quda {
     ReduceKernelArg(const Arg &arg) : Arg(arg) { }
   };
 
+  /**
+     @brief Reduction2D_impl is the implementation of the generic 2-d
+     reduction kernel.  Functors that utilize this kernel have two
+     parallelization dimensions.  The y thread dimenion is constrained
+     to remain inside the thread block and this dimension is
+     contracted in the reduction.
+
+     @tparam Transformer Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Transformer, typename Arg, bool grid_stride = true>
+  __forceinline__ __device__ void Reduction2D_impl(const Arg &arg)
+  {
+    QUDA_RT_CONSTS;
+    using reduce_t = typename Transformer<Arg>::reduce_t;
+    Transformer<Arg> t(arg);
+
+    auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+    auto j = threadIdx.y;
+
+    reduce_t value = t.init();
+
+    while (idx < arg.threads.x) {
+      value = t(value, idx, j);
+      if (grid_stride)
+        idx += blockDim.x * gridDim.x;
+      else
+        break;
+    }
+
+    // perform final inter-block reduction and write out result
+    reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value);
+  }
+
   /**
      @brief Reduction2D is the entry point of the generic 2-d
      reduction kernel.  This is the specialization where the kernel
@@ -37,91 +76,20 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
   {
-    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    // const int tx = arg.threads.x;
-    // const int ty = arg.threads.y;
-    // const int tz = arg.threads.z;
-    // printf("Reduction2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      shared_cache.addr = cache;
-      // OMP TARGET TODO: issues with reduction, not performed if not inlined.
-      // Reduction2D_impl<Functor, Arg, grid_stride>(*dparg);
-      const dim3
-        blockDim=launch_param.block,
-        gridDim=launch_param.grid,
-        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
-
-      using reduce_t = typename Functor<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Functor<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Functor<Arg>::init_omp())
-      reduce_t value = dparg->init();
-      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-
-        Functor<Arg> t(*dparg);
-
-        auto idx = threadIdx.x + blockIdx.x * blockDim.x;
-        auto j = threadIdx.y;
-
-        while (idx < dparg->threads.x) {
-          value = t(value, idx, j);
-          if (grid_stride)
-            idx += blockDim.x * gridDim.x;
-          else
-            break;
-        }
-      }
-
-      // perform final inter-block reduction and write out result
-      Functor<Arg> t(*dparg);
-
-      // OMP TARGET TODO: fully inline everything for now
-      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value);
-      int idx = 0;
-      bool isLastBlockDone;
-
-      dparg->partial[idx * gridDim.x + blockIdx.x] = value;
-      // __threadfence(); // flush result
-
-      // increment global block counter
-      unsigned int cvalue = 0;
-      unsigned int *c = &dparg->count[idx];
-      #pragma omp atomic capture
-      { cvalue = *c; *c = *c + 1; }
-
-      // determine if last block
-      isLastBlockDone = (cvalue == (gridDim.x - 1));
-
-      // __syncthreads();
-
-      // finish the reduction if last block
-      if (isLastBlockDone) {
-        reduce_t sum = dparg->init();
-        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-        {
-          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-          auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
-          while (i < gridDim.x) {
-            sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-            i += Arg::block_size_x * Arg::block_size_y;
-          }
-        }
-
-        // write out the final reduced value
-        dparg->result_d[idx] = sum;
-        dparg->count[idx] = 0; // set to zero for next time
-      }
+      target::omptarget::shared_cache.addr = cache;
+    #pragma omp parallel num_threads(ld)
+    {
+      char buffer[sizeof(Arg)];
+      memcpy(buffer, (void *)dparg, sizeof(Arg));
+      Reduction2D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
+    }
     }
     omp_target_free(dparg, omp_get_default_device());
   }
@@ -141,86 +109,59 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D()
   {
-    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    // printf("Reduction2D: launch parameter: gd %d ld %d\n", gd, ld);
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      shared_cache.addr = cache;
-      // Reduction2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
-      Arg *dparg = &device::get_arg<Arg>();
-      const dim3
-        blockDim=launch_param.block,
-        gridDim=launch_param.grid,
-        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
-
-      using reduce_t = typename Functor<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Functor<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Functor<Arg>::init_omp())
-      reduce_t value = dparg->init();
-      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-
-        Functor<Arg> t(*dparg);
-
-        auto idx = threadIdx.x + blockIdx.x * blockDim.x;
-        auto j = threadIdx.y;
-
-        while (idx < dparg->threads.x) {
-          value = t(value, idx, j);
-          if (grid_stride)
-            idx += blockDim.x * gridDim.x;
-          else
-            break;
-        }
-      }
-
-      // perform final inter-block reduction and write out result
-      Functor<Arg> t(*dparg);
-
-      // OMP TARGET TODO: fully inline everything for now
-      // reduce<Arg::block_size_x, Arg::block_size_y>(*dparg, t, value);
-      int idx = 0;
-      bool isLastBlockDone;
+      target::omptarget::shared_cache.addr = cache;
+    #pragma omp parallel num_threads(ld)
+    {
+      Reduction2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+    }
+    }
+  }
 
-      dparg->partial[idx * gridDim.x + blockIdx.x] = value;
-      // __threadfence(); // flush result
+  /**
+     @brief MultiReduction_impl is the implementation of the generic
+     multi-reduction kernel.  Functors that utilize this kernel have
+     three parallelization dimensions.  The y thread dimension is
+     constrained to remain inside the thread block and this dimension
+     is contracted in the reduction.  The z thread dimension is a
+     batch dimension that is not contracted in the reduction.
 
-      // increment global block counter
-      unsigned int cvalue = 0;
-      unsigned int *c = &dparg->count[idx];
-      #pragma omp atomic capture
-      { cvalue = *c; *c = *c + 1; }
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
+  __forceinline__ __device__ void MultiReduction_impl(const Arg &arg)
+  {
+    QUDA_RT_CONSTS;
+    using reduce_t = typename Functor<Arg>::reduce_t;
+    Functor<Arg> t(arg);
 
-      // determine if last block
-      isLastBlockDone = (cvalue == (gridDim.x - 1));
+    auto idx = threadIdx.x + blockIdx.x * blockDim.x;
+    auto k = threadIdx.y;
+    auto j = threadIdx.z + blockIdx.z * blockDim.z;
 
-      // __syncthreads();
+    if (j >= arg.threads.z) return;
 
-      // finish the reduction if last block
-      if (isLastBlockDone) {
-        reduce_t sum = dparg->init();
-        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-        {
-          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-          auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
-          while (i < gridDim.x) {
-            sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-            i += Arg::block_size_x * Arg::block_size_y;
-          }
-        }
+    reduce_t value = t.init();
 
-        // write out the final reduced value
-        dparg->result_d[idx] = sum;
-        dparg->count[idx] = 0; // set to zero for next time
-      }
+    while (idx < arg.threads.x) {
+      value = t(value, idx, k, j);
+      if (grid_stride)
+        idx += blockDim.x * gridDim.x;
+      else
+        break;
     }
+
+    // perform final inter-block reduction and write out result
+    reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value, j);
   }
 
   /**
@@ -238,92 +179,20 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
   {
-    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    // const int tx = arg.threads.x;
-    // const int ty = arg.threads.y;
-    // const int tz = arg.threads.z;
-    // printf("MultiReduction: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      shared_cache.addr = cache;
-      const dim3
-        blockDim=launch_param.block,
-        gridDim=launch_param.grid,
-        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-
-      using reduce_t = typename Functor<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Functor<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Functor<Arg>::init_omp())
-      reduce_t value = dparg->init();
-      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-
-        Functor<Arg> t(*dparg);
-
-        auto idx = threadIdx.x + blockIdx.x * blockDim.x;
-        auto j = threadIdx.y + blockIdx.y * blockDim.y;
-        auto k = threadIdx.z;
-
-        if (j < dparg->threads.y) {
-          while (idx < dparg->threads.x) {
-            value = t(value, idx, j, k);
-            if (grid_stride)
-              idx += blockDim.x * gridDim.x;
-            else
-              break;
-          }
-        }
-      }
-
-      // perform final inter-block reduction and write out result
-      Functor<Arg> t(*dparg);
-      auto j = blockIdx.y * blockDim.y;
-      // reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value, j);
-      int idx = j;
-      bool isLastBlockDone;
-
-      dparg->partial[idx * gridDim.x + blockIdx.x] = value;
-      // __threadfence(); // flush result
-
-      // increment global block counter
-      unsigned int cvalue = 0;
-      unsigned int *c = &dparg->count[idx];
-      #pragma omp atomic capture
-      { cvalue = *c; *c = *c + 1; }
-
-      // determine if last block
-      isLastBlockDone = (cvalue == (gridDim.x - 1));
-
-      // __syncthreads();
-
-      // finish the reduction if last block
-      if (isLastBlockDone) {
-        reduce_t sum = dparg->init();
-        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-        {
-          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-          auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
-          while (i < gridDim.x) {
-            sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-            i += Arg::block_size_x * Arg::block_size_y;
-          }
-        }
-
-        // write out the final reduced value
-        dparg->result_d[idx] = sum;
-        dparg->count[idx] = 0; // set to zero for next time
-      }
+      target::omptarget::shared_cache.addr = cache;
+    #pragma omp parallel num_threads(ld)
+    {
+      char buffer[sizeof(Arg)];
+      memcpy(buffer, (void *)dparg, sizeof(Arg));
+      MultiReduction_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
+    }
     }
     omp_target_free(dparg, omp_get_default_device());
   }
@@ -343,87 +212,16 @@ namespace quda {
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction()
   {
-    const int gd = launch_param.grid.x*launch_param.grid.y*launch_param.grid.z;
-    const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-    // printf("MultiReduction: launch parameter: gd %d ld %d\n", gd, ld);
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      shared_cache.addr = cache;
-      Arg *dparg = device::get_arg<Arg>();
-      const dim3
-        blockDim=launch_param.block,
-        gridDim=launch_param.grid,
-        blockIdx(omp_get_team_num()%launch_param.grid.x, (omp_get_team_num()/launch_param.grid.x)%launch_param.grid.y, omp_get_team_num()/(launch_param.grid.x*launch_param.grid.y));
-      const int ld = launch_param.block.x*launch_param.block.y*launch_param.block.z;
-
-      using reduce_t = typename Functor<Arg>::reduce_t;
-#pragma omp declare reduction(OMPReduce_ : reduce_t : omp_out=Functor<Arg>::reduce_omp(omp_out,omp_in)) initializer(omp_priv=Functor<Arg>::init_omp())
-      reduce_t value = dparg->init();
-      #pragma omp parallel num_threads(ld) reduction(OMPReduce_:value)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-
-        Functor<Arg> t(*dparg);
-
-        auto idx = threadIdx.x + blockIdx.x * blockDim.x;
-        auto j = threadIdx.y + blockIdx.y * blockDim.y;
-        auto k = threadIdx.z;
-
-        if (j < dparg->threads.y) {
-          while (idx < dparg->threads.x) {
-            value = t(value, idx, j, k);
-            if (grid_stride)
-              idx += blockDim.x * gridDim.x;
-            else
-              break;
-          }
-        }
-      }
-
-      // perform final inter-block reduction and write out result
-      Functor<Arg> t(*dparg);
-      auto j = blockIdx.y * blockDim.y;
-      // reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value, j);
-      int idx = j;
-      bool isLastBlockDone;
-
-      dparg->partial[idx * gridDim.x + blockIdx.x] = value;
-      // __threadfence(); // flush result
-
-      // increment global block counter
-      unsigned int cvalue = 0;
-      unsigned int *c = &dparg->count[idx];
-      #pragma omp atomic capture
-      { cvalue = *c; *c = *c + 1; }
-
-      // determine if last block
-      isLastBlockDone = (cvalue == (gridDim.x - 1));
-
-      // __syncthreads();
-
-      // finish the reduction if last block
-      if (isLastBlockDone) {
-        reduce_t sum = dparg->init();
-        #pragma omp parallel num_threads(ld) reduction(OMPReduce_:sum)
-        {
-          dim3 threadIdx(omp_get_thread_num()%launch_param.block.x, (omp_get_thread_num()/launch_param.block.x)%launch_param.block.y, omp_get_thread_num()/(launch_param.block.x*launch_param.block.y));
-          auto i = threadIdx.y * Arg::block_size_x + threadIdx.x;
-          while (i < gridDim.x) {
-            sum = t(sum, const_cast<reduce_t &>(static_cast<volatile reduce_t *>(dparg->partial)[idx * gridDim.x + i]));
-            i += Arg::block_size_x * Arg::block_size_y;
-          }
-        }
-
-        // write out the final reduced value
-        dparg->result_d[idx] = sum;
-        dparg->count[idx] = 0; // set to zero for next time
-      }
+      target::omptarget::shared_cache.addr = cache;
+    #pragma omp parallel num_threads(ld)
+    {
+      MultiReduction_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+    }
     }
   }
 
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 904d4b3c95..0d64dcf629 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -57,7 +57,7 @@ namespace quda {
     */
     __device__ __host__ inline dim3 block_dim()
     {
-      return launch_param.block;
+      return target::omptarget::launch_param.block;
     }
 
     /**
@@ -67,7 +67,7 @@ namespace quda {
     */
     __device__ __host__ inline dim3 grid_dim()
     {
-      return launch_param.grid;
+      return target::omptarget::launch_param.grid;
     }
 
     /**
@@ -78,7 +78,7 @@ namespace quda {
     __device__ __host__ inline dim3 block_idx()
     {
       const int n = omp_get_team_num();
-      return dim3(n%launch_param.grid.x, (n/launch_param.grid.x)%launch_param.grid.y, n/(launch_param.grid.x*launch_param.grid.y));
+      return dim3(n%target::omptarget::launch_param.grid.x, (n/target::omptarget::launch_param.grid.x)%target::omptarget::launch_param.grid.y, n/(target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y));
     }
 
     /**
@@ -89,7 +89,7 @@ namespace quda {
     __device__ __host__ inline dim3 thread_idx()
     {
       const int n = omp_get_thread_num();
-      return dim3(n%launch_param.block.x, (n/launch_param.block.x)%launch_param.block.y, n/(launch_param.block.x*launch_param.block.y));
+      return dim3(n%target::omptarget::launch_param.block.x, (n/target::omptarget::launch_param.block.x)%target::omptarget::launch_param.block.y, n/(target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y));
     }
 
   }
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index ef4183cf60..b664fc5063 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -11,13 +11,17 @@
 
 namespace quda {
 
-  // defined in ../../../lib/targets/omptarget/quda_api.cpp:/qudaSetupLaunchParameter
-  void qudaSetupLaunchParameter(const TuneParam &);
+  namespace target {
+    namespace omptarget {
+      // defined in ../../../lib/targets/omptarget/quda_api.cpp:/qudaSetupLaunchParameter
+      void qudaSetupLaunchParameter(const TuneParam &);
+    }
+  }
 
   template <typename... Arg>
   qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, qudaStream_t stream, const Arg &... arg)
   {
-    qudaSetupLaunchParameter(tp);
+    target::omptarget::qudaSetupLaunchParameter(tp);
     reinterpret_cast<void(*)(Arg...)>(const_cast<void*>(func))(arg...);
     return QUDA_SUCCESS;
   }
diff --git a/lib/targets/omptarget/comm_target.cpp b/lib/targets/omptarget/comm_target.cpp
index 313250b0c6..fd43158016 100644
--- a/lib/targets/omptarget/comm_target.cpp
+++ b/lib/targets/omptarget/comm_target.cpp
@@ -3,175 +3,38 @@
 #include <algorithm>
 #include <shmem_helper.cuh>
 
-#define CHECK_CUDA_ERROR(func)                                          \
-  quda::cuda::set_runtime_error(func, #func, __func__, __FILE__, __STRINGIFY__(__LINE__));
-
-bool comm_peer2peer_possible(int local_gpuid, int neighbor_gpuid)
-{
-  ompwip("comm_peer2peer_possible -> false");
-  return false;
-/*
-  int canAccessPeer[2];
-  CHECK_CUDA_ERROR(cudaDeviceCanAccessPeer(&canAccessPeer[0], local_gpuid, neighbor_gpuid));
-  CHECK_CUDA_ERROR(cudaDeviceCanAccessPeer(&canAccessPeer[1], neighbor_gpuid, local_gpuid));
-
-  // require symmetric peer-to-peer access to enable peer-to-peer
-  return canAccessPeer[0] && canAccessPeer[1];
-*/
-}
-
-int comm_peer2peer_performance(int local_gpuid, int neighbor_gpuid)
+namespace quda
 {
-  ompwip("comm_peer2peer_performance -> 0");
-  return 0;
-/*
-  int accessRank[2] = { };
-  if (comm_peer2peer_possible(local_gpuid, neighbor_gpuid)) {
-    CHECK_CUDA_ERROR(cudaDeviceGetP2PAttribute(&accessRank[0], cudaDevP2PAttrPerformanceRank, local_gpuid, neighbor_gpuid));
-    CHECK_CUDA_ERROR(cudaDeviceGetP2PAttribute(&accessRank[1], cudaDevP2PAttrPerformanceRank, neighbor_gpuid, local_gpuid));
+  bool comm_peer2peer_possible(int local_gpuid, int neighbor_gpuid)
+  {
+    ompwip("comm_peer2peer_possible -> false");
+    return false;
   }
 
-  // return the slowest direction of access (lower is faster)
-  return std::max(accessRank[0], accessRank[1]);
-*/
-}
-
-void comm_create_neighbor_memory(void *remote[QUDA_MAX_DIM][2], void *local)
-{
-  ompwip("unimplemented");
-/*
-  // handles for obtained ghost pointers
-  cudaIpcMemHandle_t remote_handle[2][QUDA_MAX_DIM];
-
-  for (int dim=0; dim<4; ++dim) {
-    if (comm_dim(dim)==1) continue;
-    for (int dir=0; dir<2; ++dir) {
-      MsgHandle* sendHandle = nullptr;
-      MsgHandle* receiveHandle = nullptr;
-      int disp = (dir == 1) ? +1 : -1;
-
-      // first set up receive
-      if (comm_peer2peer_enabled(1-dir,dim)) {
-        receiveHandle = comm_declare_receive_relative(&remote_handle[1-dir][dim],
-                                                      dim, -disp, sizeof(remote_handle));
-      }
-      // now send
-      cudaIpcMemHandle_t local_handle;
-      if (comm_peer2peer_enabled(dir,dim)) {
-        CHECK_CUDA_ERROR(cudaIpcGetMemHandle(&local_handle, local));
-        sendHandle = comm_declare_send_relative(&local_handle, dim, disp, sizeof(local_handle));
-      }
-      if (receiveHandle) comm_start(receiveHandle);
-      if (sendHandle) comm_start(sendHandle);
-
-      if (receiveHandle) comm_wait(receiveHandle);
-      if (sendHandle) comm_wait(sendHandle);
-
-      if (sendHandle) comm_free(sendHandle);
-      if (receiveHandle) comm_free(receiveHandle);
-    }
+  int comm_peer2peer_performance(int local_gpuid, int neighbor_gpuid)
+  {
+    ompwip("comm_peer2peer_performance -> 0");
+    return 0;
   }
 
-  // open the remote memory handles and set the send ghost pointers
-  for (int dim=0; dim<4; ++dim) {
-    if (comm_dim(dim)==1) continue;
-    // even if comm_dim(2) == 2, we might not have p2p enabled in both directions, so check this
-    const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
-    for (int dir = 0; dir < num_dir; dir++) {
-      remote[dim][dir] = nullptr;
-      if (!comm_peer2peer_enabled(dir, dim)) continue;
-      CHECK_CUDA_ERROR(cudaIpcOpenMemHandle(&remote[dim][dir], remote_handle[dir][dim], cudaIpcMemLazyEnablePeerAccess));
-    }
-    if (num_dir == 1) remote[dim][1] = remote[dim][0];
+  void comm_create_neighbor_memory(void *remote[QUDA_MAX_DIM][2], void *local)
+  {
+    ompwip("unimplemented");
   }
-*/
-}
-
-void comm_destroy_neighbor_memory(void *remote[QUDA_MAX_DIM][2])
-{
-  ompwip("unimplemented");
-/*
-  for (int dim=0; dim<4; ++dim) {
 
-    if (comm_dim(dim)==1) continue;
-    const int num_dir = (comm_dim(dim) == 2 && comm_peer2peer_enabled(0,dim) && comm_peer2peer_enabled(1,dim)) ? 1 : 2;
-
-    if (comm_peer2peer_enabled(1,dim)) {
-      // only close this handle if it doesn't alias the back ghost
-      if (num_dir == 2 && remote[dim][1]) CHECK_CUDA_ERROR(cudaIpcCloseMemHandle(remote[dim][1]));
-    }
-
-    if (comm_peer2peer_enabled(0,dim)) {
-      if (remote[dim][0]) CHECK_CUDA_ERROR(cudaIpcCloseMemHandle(remote[dim][0]));
-    }
-  } // iterate over dim
-*/
-}
-
-void comm_create_neighbor_event(qudaEvent_t remote[2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
-{
-  ompwip("unimplemented");
-/*
-  // handles for obtained events
-  cudaIpcEventHandle_t ipcRemoteEventHandle[2][QUDA_MAX_DIM];
-
-  for (int dim=0; dim<4; ++dim) {
-    if (comm_dim(dim)==1) continue;
-    for (int dir=0; dir<2; ++dir) {
-      MsgHandle* sendHandle = nullptr;
-      MsgHandle* receiveHandle = nullptr;
-      int disp = (dir == 1) ? +1 : -1;
-
-      // first set up receive
-      if (comm_peer2peer_enabled(1-dir,dim)) {
-        receiveHandle = comm_declare_receive_relative(&ipcRemoteEventHandle[1-dir][dim], dim, -disp,
-                                                      sizeof(ipcRemoteEventHandle[1-dir][dim]));
-      }
-
-      cudaIpcEventHandle_t handle;
-
-      // now send
-      if (comm_peer2peer_enabled(dir,dim)) {
-        cudaEvent_t event;
-        CHECK_CUDA_ERROR(cudaEventCreate(&event, cudaEventDisableTiming | cudaEventInterprocess));
-        local[dir][dim].event = reinterpret_cast<void*>(event);
-        CHECK_CUDA_ERROR(cudaIpcGetEventHandle(&handle, event));
-        sendHandle = comm_declare_send_relative(&handle, dim, disp, sizeof(handle));
-      }
-
-      if (receiveHandle) comm_start(receiveHandle);
-      if (sendHandle) comm_start(sendHandle);
-
-      if (receiveHandle) comm_wait(receiveHandle);
-      if (sendHandle) comm_wait(sendHandle);
+  void comm_destroy_neighbor_memory(void *remote[QUDA_MAX_DIM][2])
+  {
+    ompwip("unimplemented");
+  }
 
-      if (sendHandle) comm_free(sendHandle);
-      if (receiveHandle) comm_free(receiveHandle);
-    }
+  void comm_create_neighbor_event(qudaEvent_t remote[2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
+  {
+    ompwip("unimplemented");
   }
 
-  for (int dim=0; dim<4; ++dim) {
-    if (comm_dim(dim)==1) continue;
-    for (int dir=0; dir<2; ++dir) {
-      if (!comm_peer2peer_enabled(dir,dim)) continue;
-      cudaEvent_t event;
-      CHECK_CUDA_ERROR(cudaIpcOpenEventHandle(&event, ipcRemoteEventHandle[dir][dim]));
-      remote[dir][dim].event = reinterpret_cast<void*>(event);
-    }
+  void comm_destroy_neighbor_event(qudaEvent_t [2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
+  {
+    ompwip("unimplemented");
   }
-*/
-}
 
-void comm_destroy_neighbor_event(qudaEvent_t [2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
-{
-  ompwip("unimplemented");
-/*
-  for (int dim=0; dim<4; ++dim) {
-    if (comm_dim(dim)==1) continue;
-    for (int dir = 0; dir <2; dir++) {
-      cudaEvent_t &event = reinterpret_cast<cudaEvent_t&>(local[dir][dim].event);
-      if (comm_peer2peer_enabled(dir, dim)) CHECK_CUDA_ERROR(cudaEventDestroy(event));
-    } 
-  } // iterate over dim
-*/
-}
+} // namespace quda
diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index 29e6c9a5bb..bb68096537 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -18,6 +18,8 @@
 #include "backward.hpp"
 #endif
 
+// #pragma omp requires unified_shared_memory
+
 namespace quda
 {
 
@@ -139,9 +141,12 @@ namespace quda
     alloc[type].erase(ptr);
   }
 
+#ifdef OMPTARGET_MAPPED_USE_ASSOCIATE_PTR
   namespace target {
     static std::map<const void*,void*> omp_mapped_ptr;  // host -> device
   }
+#endif
+
   /**
    * Under CUDA 4.0, cudaHostRegister seems to require that both the
    * beginning and end of the buffer be aligned on page boundaries.
@@ -336,16 +341,7 @@ namespace quda
   {
     MemAlloc a(func, file, line);
 
-#if 0
-    void *ptr;
-    static int page_size = 2*getpagesize();
-    a.base_size = ((size + page_size - 1) / page_size) * page_size; // round up to the nearest multiple of page_size
-    a.size = size;
-    cudaError_t err = cudaHostAlloc(&ptr, a.base_size, cudaHostAllocMapped | cudaHostAllocPortable);
-    if (err != cudaSuccess) {
-      errorQuda("cudaHostAlloc failed of size %zu (%s:%d in %s())\n", size, file, line, func); }
-    }
-#else
+#ifdef OMPTARGET_MAPPED_USE_ASSOCIATE_PTR
     void *ptr = aligned_malloc(a, size);
     print_trace();
     if(0<omp_get_num_devices()){
@@ -361,6 +357,15 @@ namespace quda
       warningQuda("%s:%d %s() mapped malloc without a device", file, line, func);
       target::omp_mapped_ptr[ptr] = ptr;
     }
+#else
+    a.size = a.base_size = size;
+    void *ptr;
+    if(0<omp_get_num_devices()){
+      ptr = omp_target_alloc_shared(size, omp_get_default_device());   // FIXME non-portable
+    }else{
+      warningQuda("%s:%d %s() mapped malloc without a device", file, line, func);
+      ptr = aligned_malloc(a, size);
+    }
 #endif
     track_malloc(MAPPED, a, ptr);
 #ifdef HOST_DEBUG
@@ -525,13 +530,18 @@ namespace quda
       free(ptr);
     } else if (alloc[MAPPED].count(ptr)) {
 #ifdef HOST_ALLOC
+#ifdef OMPTARGET_MAPPED_USE_ASSOCIATE_PTR
       ompwip("ERROR: HOST_ALLOC alloc[MAPPED].count(ptr) untested code path: %p",ptr); print_trace();
       free(ptr);
+#else
+      omp_target_free(ptr, omp_get_default_device());
+#endif
 /*
       cudaError_t err = cudaFreeHost(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to free host memory (%s:%d in %s())\n", file, line, func); }
 */
 #else
+#ifdef OMPTARGET_MAPPED_USE_ASSOCIATE_PTR
       // ompwip("!HOST_ALLOC alloc[MAPPED].count(ptr) free: %p",ptr);
       void *dp = target::omp_mapped_ptr[ptr];
       if(dp!=ptr){
@@ -540,6 +550,9 @@ namespace quda
         omp_target_free(dp, omp_get_default_device());
       }
       free(ptr);
+#else
+      omp_target_free(ptr, omp_get_default_device());
+#endif
 /*
       cudaError_t err = cudaHostUnregister(ptr);
       if (err != cudaSuccess) {
@@ -657,10 +670,14 @@ namespace quda
 
   void *get_mapped_device_pointer_(const char *func, const char *file, int line, const void *host)
   {
+#ifdef OMPTARGET_MAPPED_USE_ASSOCIATE_PTR
     print_trace();
     auto dp = target::omp_mapped_ptr[host];
     ompwip("WARNING: get_mapped_device_pointer_ host: %p  device: %p  make sure to copy to host before reading from host",host,dp);
     return dp;
+#else
+    return (void*)host;
+#endif
 /*
     void *device;
     auto error = cudaHostGetDevicePointer(&device, const_cast<void *>(host), 0);
@@ -835,7 +852,6 @@ namespace quda
 
     void flush_pinned()
     {
-      ompwip("ERROR: untested pool::flush_pinned");
       if (pinned_memory_pool) {
         for (auto it : pinnedCache) { host_free(it.second); }
         pinnedCache.clear();
@@ -844,7 +860,6 @@ namespace quda
 
     void flush_device()
     {
-      ompwip("ERROR: untested pool:flush_device");
       if (device_memory_pool) {
         for (auto it : deviceCache) { device_free(it.second); }
         deviceCache.clear();
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 3bc34bcee2..6882eb6453 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -5,197 +5,11 @@
 #include <timer.h>
 #include <device.h>
 
-using cudaError_t = int;
-enum {cudaSuccess};
-enum cudaMemcpyKind{cudaMemcpyHostToHost, cudaMemcpyHostToDevice, cudaMemcpyDeviceToHost, cudaMemcpyDeviceToDevice, cudaMemcpyDefault};
-
-static char FIXME[]="OMP FIXME";
-
-#define cudaGetErrorString(a) "OMP FIXME"
-#define cudaGetLastError()
-#define cuGetErrorName(a,b) ompwip([&](){(*(b))=FIXME;})
-
-#if 0
-
-#define cudaMemcpy(a,b,c,d) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld",a,b,c)
-#define cudaMemcpyAsync(a,b,c,d,e) ompwip([&](){ompwipMemcpy(a,(void*)b,c,d);},"memcpy %p <- %p %ld",a,b,c)
-#define cudaMemset(a,b,c) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld",a,b,c)
-#define cudaMemsetAsync(a,b,c,d) ompwip([&](){ompwipMemset(a,b,c);},"memset %p %d %ld",a,b,c)
-#define cudaMemset2D(a,b,c,d,e) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld",a,b,c,d,e)
-#define cudaMemset2DAsync(a,b,c,d,e,f) ompwip([&](){ompwipMemset2D(a,b,c,d,e);},"memset %p(%ld) %d %ld %ld",a,b,c,d,e)
-
-#else
-
-#define cudaMemcpy(a,b,c,d) ompwipMemcpy(a,(void*)b,c,d)
-#define cudaMemcpyAsync(a,b,c,d,e) ompwipMemcpy(a,(void*)b,c,d)
-#define cudaMemset(a,b,c) ompwipMemset(a,b,c)
-#define cudaMemsetAsync(a,b,c,d) ompwipMemset(a,b,c)
-#define cudaMemset2D(a,b,c,d,e) ompwipMemset2D(a,b,c,d,e)
-#define cudaMemset2DAsync(a,b,c,d,e,f) ompwipMemset2D(a,b,c,d,e)
-
-#endif
-
-static inline int
-ompwipMemset(void *p, unsigned char b, std::size_t s)
-{
-  unsigned char *c = reinterpret_cast<unsigned char *>(p);
-#pragma omp target teams distribute parallel for simd is_device_ptr(c)
-  for(std::size_t i=0;i<s;++i) c[i] = b;
-  return 0;
-}
-
-static inline int
-ompwipMemset2D(void *p, size_t pitch, unsigned char b, size_t w, size_t h)
-{
-  unsigned char *c = reinterpret_cast<unsigned char *>(p);
-#pragma omp target teams distribute parallel for simd is_device_ptr(c) collapse(2)
-  for(std::size_t i=0;i<h;++i)
-    for(std::size_t j=0;j<w;++j)
-      c[j+i*pitch] = b;
-  return 0;
-}
-
-#if 1
-#define printmem(d,m,h)
-#else
-static inline void
-printmem(void *d, std::size_t m, int host)
-{
-  constexpr size_t loc[] = {0, 1, 2, 3, 5, 7, 11, 13};
-  constexpr size_t nloc = sizeof(loc)/sizeof(loc[0]);
-  unsigned char buf[nloc*8];
-  unsigned char *p = (unsigned char *)d;
-
-  if(host){
-    for(size_t i=0; i<nloc; ++i){
-      if(i*8>=m) break;
-      for(size_t j=0; j<8; ++j)
-        buf[j+i*8] = p[j+loc[i]*8];
-    }
-    printf("host   mem:");
-  }else{
-    #pragma omp target map(tofrom:buf[:8*nloc]) map(to:loc[:nloc]) is_device_ptr(p)
-    {
-    for(size_t i=0; i<nloc; ++i){
-      if(i*8>=m) break;
-      for(size_t j=0; j<8; ++j)
-        buf[j+i*8] = p[j+loc[i]*8];
-    }
-    }
-    printf("target mem:");
-  }
-  for(size_t i=0; i<nloc; ++i){
-    printf(" ");
-    for(size_t j=0; j<8; ++j)
-      printf("%02X", buf[j+i*8]);
-  }
-  printf("\n");
-  fflush(0);
-}
-#endif
-
-static inline int
-ompwipMemcpy(void *d, void *s, std::size_t c, cudaMemcpyKind k)
-{
-  int r = 0;  // return value from omp_target_memcpy, note that no return value is reserved for memcpy.
-  switch(k){
-  case cudaMemcpyHostToHost:
-    printmem(s,c,1);
-    memcpy(d,s,c);
-    printmem(d,c,1);
-    break;
-  case cudaMemcpyHostToDevice:
-    printmem(s,c,1);
-    if(0<omp_get_num_devices()){
-      r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
-      printmem(d,c,0);
-    }else{
-      warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy");
-      memcpy(d,s,c);
-      printmem(d,c,1);
-    }
-    break;
-  case cudaMemcpyDeviceToHost:
-    if(0<omp_get_num_devices()){
-      printmem(s,c,0);
-      r = omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
-    }else{
-      warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy");
-      printmem(s,c,1);
-      memcpy(d,s,c);
-    }
-    printmem(d,c,1);
-    break;
-  case cudaMemcpyDeviceToDevice:
-    printmem(s,c,0);
-    r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
-    printmem(d,c,0);
-    break;
-  case cudaMemcpyDefault:
-    if(0<omp_get_num_devices()){
-      if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(d)){
-        if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
-          // ompwip("cudaMemcpyDefault calling device to device");
-          printmem(s,c,0);
-          r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
-          printmem(d,c,0);
-        }else{
-          // ompwip("cudaMemcpyDefault calling host to device");
-          printmem(s,c,1);
-          r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
-          printmem(d,c,0);
-        }
-      }else{
-        if(QUDA_CUDA_FIELD_LOCATION==get_pointer_location(s)){
-          // ompwip("cudaMemcpyDefault calling device to host");
-          printmem(s,c,0);
-          r = omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
-          printmem(d,c,1);
-        }else{
-          // ompwip("cudaMemcpyDefault calling host to host");
-          printmem(s,c,1);
-          memcpy(d,s,c);
-          printmem(d,c,1);
-        }
-      }
-    }else{
-      warningQuda("cudaMemcpyDefault without a device, calling memcpy");
-      printmem(s,c,1);
-      memcpy(d,s,c);
-      printmem(d,c,1);
-    }
-    break;
-  default: errorQuda("Unsupported cudaMemcpyType %d", k);
-  }
-  return r;
-}
-
-#define cudaGetSymbolAddress(a,b) ompwip("WARNING: unimplemented cudaGetSymbolAddress")
-
-// if this macro is defined then we profile the CUDA API calls
-//#define API_PROFILE
-
-#ifdef API_PROFILE
-#define PROFILE(f, idx)                                                                                                \
-  apiTimer.TPSTART(idx);                                                                                               \
-  f;                                                                                                                   \
-  apiTimer.TPSTOP(idx);
-#else
-#define PROFILE(f, idx) f;
-#endif
-
-#pragma omp declare target
-SharedCache shared_cache;
-#pragma omp end declare target
-
-LaunchParam launch_param;
-#pragma omp declare target to(launch_param)
-
 namespace quda
 {
 
   static qudaError_t last_error = QUDA_SUCCESS;
-  static std::string last_error_str("CUDA_SUCCESS");
+  static std::string last_error_str("OMPTARGET_SUCCESS");
 
   qudaError_t qudaGetLastError()
   {
@@ -207,68 +21,129 @@ namespace quda
   std::string qudaGetLastErrorString()
   {
     auto rtn = last_error_str;
-    last_error_str = QUDA_SUCCESS;
+    last_error_str = "OMPTARGET_SUCCESS";
     return rtn;
   }
 
-  void qudaSetupLaunchParameter(const TuneParam &tp)
-  {
-    launch_param.grid = tp.grid;
-    launch_param.block = tp.block;
-    #pragma omp target update to(launch_param)
-  }
+  namespace target {
+    namespace omptarget {
+      #pragma omp declare target
+      SharedCache shared_cache;
+      #pragma omp end declare target
 
-  namespace cuda {
+      LaunchParam launch_param;
+      #pragma omp declare target to(launch_param)
 
-    void set_runtime_error(cudaError_t error, const char *api_func, const char *func, const char *file, const char *line,
-                           bool allow_error = false)
-    {
-      if (error == cudaSuccess) return;
-      last_error = error == cudaSuccess ? QUDA_SUCCESS : QUDA_ERROR;
-      last_error_str = cudaGetErrorString(error);
-      if (!allow_error) errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, cudaGetErrorString(error), file, line, func);
-      else cudaGetLastError(); // clear the error state
-    }
+      void qudaSetupLaunchParameter(const TuneParam &tp)
+      {
+        launch_param.grid = tp.grid;
+        launch_param.block = tp.block;
+        #pragma omp target update to(launch_param)
+      }
 
-    void set_driver_error(CUresult error, const char *api_func, const char *func, const char *file, const char *line,
-                          bool allow_error = false)
-    {
-      if (error == CUDA_SUCCESS) return;
-      last_error = error == CUDA_SUCCESS ? QUDA_SUCCESS : QUDA_ERROR;
-      const char *str;
-      cuGetErrorName(error, &str);
-      last_error_str = str;
-      if (!allow_error) errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, str, file, line, func);
-      else cudaGetLastError(); // clear the error state
-    }
+      static inline int
+      ompMemset(void *p, unsigned char b, std::size_t s)
+      {
+        unsigned char *c = reinterpret_cast<unsigned char *>(p);
+      #pragma omp target teams distribute parallel for simd is_device_ptr(c)
+        for(std::size_t i=0;i<s;++i) c[i] = b;
+        return 0;
+      }
 
-  }
+      static inline int
+      ompMemsetAsync(void *p, unsigned char b, std::size_t s, qudaStream_t stream)
+      {
+        return ompMemset(p, b, s);
+      }
 
-  using namespace cuda;
+      static inline int
+      ompMemset2D(void *p, size_t pitch, unsigned char b, size_t w, size_t h)
+      {
+        unsigned char *c = reinterpret_cast<unsigned char *>(p);
+      #pragma omp target teams distribute parallel for simd is_device_ptr(c) collapse(2)
+        for(std::size_t i=0;i<h;++i)
+          for(std::size_t j=0;j<w;++j)
+            c[j+i*pitch] = b;
+        return 0;
+      }
 
-  // Agnostic way to return a cuda API flag
-  namespace {
-    inline
-    cudaMemcpyKind qudaMemcpyKindToAPI( const qudaMemcpyKind& k)
-    {
-      switch(k) {
-      case qudaMemcpyHostToHost : return cudaMemcpyHostToHost;
-      case qudaMemcpyHostToDevice : return cudaMemcpyHostToDevice;
-      case qudaMemcpyDeviceToHost : return cudaMemcpyDeviceToHost;
-      case qudaMemcpyDeviceToDevice : return cudaMemcpyDeviceToDevice;
-      case qudaMemcpyDefault : return cudaMemcpyDefault;
-      default:
-	errorQuda(" unknown value for qudaMemcpyKind %d", static_cast<int>(k));
-	return cudaMemcpyDefault; // keep warnings away
+      static inline int
+      ompMemset2DAsync(void *p, size_t pitch, unsigned char b, size_t w, size_t h, qudaStream_t stream)
+      {
+        return ompMemset2D(p, pitch, b, w, h);
+      }
+
+      static inline int
+      ompMemcpy(void *d, void *s, std::size_t c, qudaMemcpyKind k)
+      {
+        int r = 0;  // return value from omp_target_memcpy, note that no return value is reserved for memcpy.
+        switch(k){
+        case qudaMemcpyHostToHost:
+          memcpy(d,s,c);
+          break;
+        case qudaMemcpyHostToDevice:
+          if(0<omp_get_num_devices()){
+            r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+          }else{
+            warningQuda("cudaMemcpyHostToDevice without a device, calling memcpy");
+            memcpy(d,s,c);
+          }
+          break;
+        case qudaMemcpyDeviceToHost:
+          if(0<omp_get_num_devices()){
+            r = omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
+          }else{
+            warningQuda("cudaMemcpyDeviceToHost without a device, calling memcpy");
+            memcpy(d,s,c);
+          }
+          break;
+        case qudaMemcpyDeviceToDevice:
+          r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
+          break;
+        case qudaMemcpyDefault:
+          if(0<omp_get_num_devices()){
+            if(QUDA_CUDA_FIELD_LOCATION==quda::get_pointer_location(d)){
+              if(QUDA_CUDA_FIELD_LOCATION==quda::get_pointer_location(s)){
+                r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_default_device());
+              }else{
+                r = omp_target_memcpy(d,s,c,0,0,omp_get_default_device(),omp_get_initial_device());
+              }
+            }else{
+              if(QUDA_CUDA_FIELD_LOCATION==quda::get_pointer_location(s)){
+                r = omp_target_memcpy(d,s,c,0,0,omp_get_initial_device(),omp_get_default_device());
+              }else{
+                memcpy(d,s,c);
+              }
+            }
+          }else{
+            warningQuda("cudaMemcpyDefault without a device, calling memcpy");
+            memcpy(d,s,c);
+          }
+          break;
+        default: errorQuda("Unsupported qudaMemcpyType %d", k);
+        }
+        return r;
+      }
+
+      static inline int
+      ompMemcpyAsync(void *d, void *s, std::size_t c, qudaMemcpyKind k, qudaStream_t stream)
+      {
+        return ompMemcpy(d, s, c, k);
+      }
+
+      void set_runtime_error(int error, const char *api_func, const char *func, const char *file,
+                             const char *line, bool allow_error = false)
+      {
+        if (error == 0) return;
+        last_error = error == 0 ? QUDA_SUCCESS : QUDA_ERROR;
+        last_error_str = "OMPTARGET_ERROR";
+        if (!allow_error)
+          errorQuda("%s returned %s\n (%s:%s in %s())\n", api_func, last_error_str.c_str(), file, line, func);
       }
     }
-  }
+  } // namespace target
 
-#ifdef USE_DRIVER_API
-  static TimeProfile apiTimer("CUDA API calls (driver)");
-#else
-  static TimeProfile apiTimer("CUDA API calls (runtime)");
-#endif
+  using namespace target::omptarget;
 
   class QudaMem : public Tunable
   {
@@ -277,7 +152,7 @@ namespace quda
     const size_t count;
     const int value;
     const bool copy;
-    const cudaMemcpyKind kind;
+    const qudaMemcpyKind kind;
     const bool async;
     const char *name;
     const bool active_tuning;
@@ -289,7 +164,7 @@ namespace quda
     unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
 
   public:
-    inline QudaMem(void *dst, const void *src, size_t count, cudaMemcpyKind kind, const qudaStream_t &stream,
+    inline QudaMem(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const qudaStream_t &stream,
                    bool async, const char *func, const char *file, const char *line) :
       dst(dst),
       src(src),
@@ -305,21 +180,21 @@ namespace quda
     {
       if (!async) {
         switch (kind) {
-        case cudaMemcpyDeviceToHost: name = "cudaMemcpyDeviceToHost"; break;
-        case cudaMemcpyHostToDevice: name = "cudaMemcpyHostToDevice"; break;
-        case cudaMemcpyHostToHost: name = "cudaMemcpyHostToHost"; break;
-        case cudaMemcpyDeviceToDevice: name = "cudaMemcpyDeviceToDevice"; break;
-        case cudaMemcpyDefault: name = "cudaMemcpyDefault"; break;
-        default: errorQuda("Unsupported cudaMemcpyKind %d", kind);
+        case qudaMemcpyDeviceToHost: name = "qudaMemcpyDeviceToHost"; break;
+        case qudaMemcpyHostToDevice: name = "qudaMemcpyHostToDevice"; break;
+        case qudaMemcpyHostToHost: name = "qudaMemcpyHostToHost"; break;
+        case qudaMemcpyDeviceToDevice: name = "qudaMemcpyDeviceToDevice"; break;
+        case qudaMemcpyDefault: name = "qudaMemcpyDefault"; break;
+        default: errorQuda("Unsupported qudaMemcpyKind %d", kind);
         }
       } else {
         switch (kind) {
-        case cudaMemcpyDeviceToHost: name = "cudaMemcpyAsyncDeviceToHost"; break;
-        case cudaMemcpyHostToDevice: name = "cudaMemcpyAsyncHostToDevice"; break;
-        case cudaMemcpyHostToHost: name = "cudaMemcpyAsyncHostToHost"; break;
-        case cudaMemcpyDeviceToDevice: name = "cudaMemcpyAsyncDeviceToDevice"; break;
-        case cudaMemcpyDefault: name = "cudaMemcpyAsyncDefault"; break;
-        default: errorQuda("Unsupported cudaMemcpyKind %d", kind);
+        case qudaMemcpyDeviceToHost: name = "qudaMemcpyAsyncDeviceToHost"; break;
+        case qudaMemcpyHostToDevice: name = "qudaMemcpyAsyncHostToDevice"; break;
+        case qudaMemcpyHostToHost: name = "qudaMemcpyAsyncHostToHost"; break;
+        case qudaMemcpyDeviceToDevice: name = "qudaMemcpyAsyncDeviceToDevice"; break;
+        case qudaMemcpyDefault: name = "qudaMemcpyAsyncDefault"; break;
+        default: errorQuda("Unsupported qudaMemcpyKind %d", kind);
         }
       }
       strcpy(aux, func);
@@ -338,11 +213,11 @@ namespace quda
       count(count),
       value(value),
       copy(false),
-      kind(cudaMemcpyDefault),
+      kind(qudaMemcpyDefault),
       async(async),
       active_tuning(activeTuning())
     {
-      name = !async ? "cudaMemset" : "cudaMemsetAsync";
+      name = !async ? "qudaMemset" : "qudaMemsetAsync";
       strcpy(aux, func);
       strcat(aux, ",");
       strcat(aux, file);
@@ -358,28 +233,16 @@ namespace quda
 
       if (copy) {
         if (async) {
-#ifdef API_PROFILE
-          QudaProfileType type = QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC;
-          switch (kind) {
-          case cudaMemcpyDeviceToHost: type = QUDA_PROFILE_MEMCPY_D2H_ASYNC; break;
-          case cudaMemcpyHostToDevice: type = QUDA_PROFILE_MEMCPY_H2D_ASYNC; break;
-          case cudaMemcpyDeviceToDevice: type = QUDA_PROFILE_MEMCPY_D2D_ASYNC; break;
-          case cudaMemcpyDefault: type = QUDA_PROFILE_MEMCPY_DEFAULT_ASYNC; break;
-          default: errorQuda("Unsupported cudaMemcpyTypeAsync %d", kind);
-          }
-#endif
-          cudaError_t error;
-          PROFILE(error = cudaMemcpyAsync(dst, src, count, kind, device::get_cuda_stream(stream)), type);
-          set_runtime_error(error, "cudaMemcpyAsync", func, file, line, active_tuning);
+          auto error = ompMemcpyAsync(dst, (void*)src, count, kind, stream);
+          set_runtime_error(error, "qudaMemcpyAsync", func, file, line, active_tuning);
         } else {
-          cudaError_t error = cudaMemcpy(dst, src, count, kind);
-          set_runtime_error(error, "cudaMemcpy", func, file, line, active_tuning);
+          auto error = ompMemcpy(dst, (void*)src, count, kind);
+          set_runtime_error(error, "qudaMemcpy", func, file, line, active_tuning);
         }
       } else {
-        cudaError_t error = async ?
-          cudaMemsetAsync(dst, value, count, device::get_cuda_stream(stream)) :
-          cudaMemset(dst, value, count);
-        set_runtime_error(error, "cudaMemset", func, file, line, active_tuning);
+        auto error
+          = async ? ompMemsetAsync(dst, value, count, stream) : ompMemset(dst, value, count);
+        set_runtime_error(error, "qudaMemset", func, file, line, active_tuning);
       }
     }
 
@@ -394,14 +257,14 @@ namespace quda
     }
 
     long long flops() const { return 0; }
-    long long bytes() const { return kind == cudaMemcpyDeviceToDevice ? 2 * count : count; }
+    long long bytes() const { return kind == qudaMemcpyDeviceToDevice ? 2 * count : count; }
   };
 
-  void qudaMemcpy_(void *dst, const void *src, size_t count, qudaMemcpyKind kind,
-                   const char *func, const char *file, const char *line)
+  void qudaMemcpy_(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const char *func, const char *file,
+                   const char *line)
   {
     if (count == 0) return;
-    QudaMem copy(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_default_stream(), false, func, file, line);
+    QudaMem copy(dst, src, count, kind, device::get_default_stream(), false, func, file, line);
   }
 
   void qudaMemcpyAsync_(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const qudaStream_t &stream,
@@ -410,20 +273,18 @@ namespace quda
     if (count == 0) return;
 
     if (kind == qudaMemcpyDeviceToDevice) {
-      QudaMem copy(dst, src, count, qudaMemcpyKindToAPI(kind), stream, true, func, file, line);
+      QudaMem copy(dst, src, count, kind, stream, true, func, file, line);
     } else {
-      PROFILE(cudaMemcpyAsync(dst, src, count, qudaMemcpyKindToAPI(kind), device::get_cuda_stream(stream)),
-              kind == qudaMemcpyDeviceToHost ? QUDA_PROFILE_MEMCPY_D2H_ASYNC : QUDA_PROFILE_MEMCPY_H2D_ASYNC);
+      ompMemcpyAsync(dst, (void*)src, count, kind, stream);
     }
   }
 
-  void qudaMemcpyP2PAsync_(void *dst, const void *src, size_t count, const qudaStream_t &stream,
-                           const char *func, const char *file, const char *line)
+  void qudaMemcpyP2PAsync_(void *dst, const void *src, size_t count, const qudaStream_t &stream, const char *func,
+                           const char *file, const char *line)
   {
     if (count == 0) return;
-    auto error = cudaMemcpyAsync(dst, src, count, cudaMemcpyDeviceToDevice, device::get_cuda_stream(stream));
-    if (error != cudaSuccess)
-      errorQuda("cudaMemcpyAsync returned %s\n (%s:%s in %s())\n", cudaGetErrorString(error), file, line, func);
+    auto error = ompMemcpyAsync(dst, (void*)src, count, qudaMemcpyDeviceToDevice, stream);
+    set_runtime_error(error, "cudaMemcpyAsync", func, file, line);
   }
 
   void qudaMemset_(void *ptr, int value, size_t count, const char *func, const char *file, const char *line)
@@ -442,14 +303,14 @@ namespace quda
   void qudaMemset2D_(void *ptr, size_t pitch, int value, size_t width, size_t height, const char *func,
                      const char *file, const char *line)
   {
-    cudaError_t error = cudaMemset2D(ptr, pitch, value, width, height);
+    auto error = ompMemset2D(ptr, pitch, value, width, height);
     set_runtime_error(error, __func__, func, file, line);
   }
 
   void qudaMemset2DAsync_(void *ptr, size_t pitch, int value, size_t width, size_t height, const qudaStream_t &stream,
                           const char *func, const char *file, const char *line)
   {
-    cudaError_t error = cudaMemset2DAsync(ptr, pitch, value, width, height, device::get_cuda_stream(stream));
+    auto error = ompMemset2DAsync(ptr, pitch, value, width, height, stream);
     set_runtime_error(error, __func__, func, file, line);
   }
 
@@ -457,18 +318,6 @@ namespace quda
                              const char *func, const char *file, const char *line)
   {
     ompwip("doing nothing");
-/*
-    int dev_id = 0;
-    if (mem_space == QUDA_CUDA_FIELD_LOCATION)
-      dev_id = comm_gpuid();
-    else if (mem_space == QUDA_CPU_FIELD_LOCATION)
-      dev_id = cudaCpuDeviceId;
-    else
-      errorQuda("Invalid QudaFieldLocation.");
-
-    cudaError_t error = cudaMemPrefetchAsync(ptr, count, dev_id, device::get_cuda_stream(stream));
-    set_runtime_error(error, __func__, func, file, line);
-*/
   }
 
   constexpr int max_quda_event = 16;
@@ -516,8 +365,8 @@ namespace quda
     return quda_event;
   }
 
-  float qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop,
-                              const char *func, const char *file, const char *line)
+  float qudaEventElapsedTime_(const qudaEvent_t &start, const qudaEvent_t &stop, const char *func, const char *file,
+                              const char *line)
   {
     return static_cast<float>(reinterpret_cast<QudaEvent *>(stop.event)->time - reinterpret_cast<QudaEvent *>(start.event)->time);
   }
@@ -533,29 +382,28 @@ namespace quda
 
   void qudaEventSynchronize_(const qudaEvent_t &quda_event, const char *func, const char *file, const char *line)
   {
+    // ompwip("unimplemented");
   }
 
   void qudaStreamSynchronize_(const qudaStream_t &stream, const char *func, const char *file, const char *line)
   {
+    // ompwip("unimplemented");
   }
 
   void qudaDeviceSynchronize_(const char *func, const char *file, const char *line)
   {
+    // ompwip("unimplemented");
   }
 
-  void* qudaGetSymbolAddress_(const char *symbol, const char *func, const char *file, const char *line)
+  void *qudaGetSymbolAddress_(const char *symbol, const char *func, const char *file, const char *line)
   {
-    void *ptr;
-    cudaError_t error = cudaGetSymbolAddress(&ptr, symbol);
-    set_runtime_error(error, __func__, func, file, line);
-    return ptr;
+    ompwip("unimplemented");
+    return nullptr;
   }
 
   void printAPIProfile()
   {
-#ifdef API_PROFILE
-    apiTimer.Print();
-#endif
+    ompwip("unimplemented");
   }
 
 } // namespace quda

From ca04701f12b13e7f40dc069f280930b30bffe1f1 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 14 Apr 2022 17:55:05 +0000
Subject: [PATCH 104/198] omp: fix get_pointer_location with a hack

---
 lib/targets/omptarget/malloc.cpp   | 25 ++++++++++++++++++++-----
 lib/targets/omptarget/quda_api.cpp |  1 +
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index bb68096537..a25e0d5610 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -119,6 +119,7 @@ namespace quda
 
   static void track_malloc(const AllocType &type, const MemAlloc &a, void *ptr)
   {
+    // ompwip("track malloc %d %p %d\n", ptr, type, a.base_size);
     total_bytes[type] += a.base_size;
     if (total_bytes[type] > max_total_bytes[type]) { max_total_bytes[type] = total_bytes[type]; }
     if (type != DEVICE && type != DEVICE_PINNED && type != SHMEM) {
@@ -631,18 +632,32 @@ namespace quda
   QudaFieldLocation get_pointer_location(const void *ptr)
   {
     static std::set<void *> other_ptr;  // OMP TARGET TODO: unknown pointers, assume on CPU
+    static std::set<void *> other_target_ptr;  // OMP TARGET TODO: unknown pointers, assume on TARGET
     void *p = const_cast<void*>(ptr);
     QudaFieldLocation fl = QUDA_INVALID_FIELD_LOCATION;
-    if(alloc[DEVICE].count(p) || alloc[DEVICE_PINNED].count(p)){
+    if(alloc[DEVICE].count(p) || alloc[DEVICE_PINNED].count(p) || other_target_ptr.count(p)){
       // ompwip("get_pointer_location %p returns QUDA_CUDA_FIELD_LOCATION",ptr);
       fl = QUDA_CUDA_FIELD_LOCATION;
-    }else if(alloc[HOST].count(p) || alloc[PINNED].count(p) || alloc[MAPPED].count(p) || other_ptr.count(p)){
+    }else if(alloc[HOST].count(p) || alloc[PINNED].count(p) || other_ptr.count(p)){
       // ompwip("get_pointer_location %p returns QUDA_CPU_FIELD_LOCATION",ptr);
       fl = QUDA_CPU_FIELD_LOCATION;
-    }else{
-      ompwip("WARNING: get_pointer_location assumes %p to be QUDA_CPU_FIELD_LOCATION",ptr);
-      other_ptr.insert(p);
+    }else if(alloc[MAPPED].count(p)){
+#ifdef OMPTARGET_MAPPED_USE_ASSOCIATE_PTR
       fl = QUDA_CPU_FIELD_LOCATION;
+#else
+      fl = QUDA_CUDA_FIELD_LOCATION;
+#endif
+    }else{
+      // OMP TARGET ARCHITECTURE SPECIFIC HACK
+      if(((uintptr_t)ptr)>>48){
+        ompwip("WARNING: get_pointer_location assumes %p to be QUDA_CUDA_FIELD_LOCATION",ptr);
+        other_target_ptr.insert(p);
+        fl = QUDA_CUDA_FIELD_LOCATION;
+      }else{
+        ompwip("WARNING: get_pointer_location assumes %p to be QUDA_CPU_FIELD_LOCATION",ptr);
+        other_ptr.insert(p);
+        fl = QUDA_CPU_FIELD_LOCATION;
+      }
     }
     return fl;
 /*
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 6882eb6453..ed5d23f1c4 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -76,6 +76,7 @@ namespace quda
       static inline int
       ompMemcpy(void *d, void *s, std::size_t c, qudaMemcpyKind k)
       {
+        // ompwip("memcpy 0x%p <- 0x%p %d %d\n", d, s, c, k);
         int r = 0;  // return value from omp_target_memcpy, note that no return value is reserved for memcpy.
         switch(k){
         case qudaMemcpyHostToHost:

From fc6982616fda9c065ac1f63852385eb4938f3367 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 15 Apr 2022 21:19:16 +0000
Subject: [PATCH 105/198] omp: update from cuda (mostly formatting)

---
 .../targets/omptarget/block_reduce_helper.h   |   5 +-
 .../omptarget/block_reduction_kernel.h        |  19 +-
 .../targets/omptarget/constant_kernel_arg.h   |  25 +-
 include/targets/omptarget/kernel.h            | 312 +++++++++++++-----
 include/targets/omptarget/reduce_helper.h     |   7 +-
 include/targets/omptarget/warp_collective.h   |  47 ++-
 6 files changed, 288 insertions(+), 127 deletions(-)

diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index 7c43094ebf..6b430d7594 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -217,7 +217,8 @@ namespace quda
       using block_reduce_t = cub::BlockReduce<T, param_t::block_size_x, cub::BLOCK_REDUCE_WARP_REDUCTIONS,
                                               param_t::block_size_y, param_t::block_size_z, __COMPUTE_CAPABILITY__>;
       using tempStorage = typename block_reduce_t::TempStorage;
-      tempStorage *storage = (tempStorage*)&target::omptarget::shared_cache.addr[1024];  // FIXME arbitrary
+      static_assert(param_t::batch_size*sizeof(tempStorage) <= sizeof(target::omptarget::shared_cache.addr[0])*(8192+4096+2048), "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, do we have that many?
+      tempStorage *storage = (tempStorage*)&target::omptarget::shared_cache.addr[128];  // FIXME arbitrary
       // typename block_reduce_t::TempStorage storage[param_t::batch_size];
       block_reduce_t block_reduce(storage[batch]);
       if (!async){
@@ -225,8 +226,8 @@ namespace quda
         // __syncthreads(); // only synchronize if we are not pipelining
       }
       T value = reducer_t::do_sum ? block_reduce.Sum(value_) : block_reduce.Reduce(value_, r);
-      QUDA_RT_CONSTS;
       if (all) {
+        QUDA_RT_CONSTS;
         T &value_shared = reinterpret_cast<T &>(storage[batch]);
         if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) value_shared = value;
         #pragma omp barrier
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index e31a3d98d1..6277d39232 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -1,9 +1,11 @@
 #pragma once
 
 #include <target_device.h>
-#include <reduce_helper.h>
+#include <kernel_helper.h>
+#include <block_reduce_helper.h>
 
-namespace quda {
+namespace quda
+{
 
   /**
      @brief This helper function swizzles the block index through
@@ -68,10 +70,9 @@ namespace quda {
   /**
      @brief BlockKernel2D_impl is the implementation of the Generic
      block kernel.  Here, we split the block (CTA) and thread indices
-     in the x and y dimension and pass these indices separately to the
-     transform functor.  The x thread dimension is templated
-     (Arg::block_size), e.g., for efficient reductions, and typically
-     the y thread dimension is a trivial vectorizable dimension.
+     and pass them separately to the transform functor.  The x thread
+     dimension is templated (Arg::block_size), e.g., for efficient
+     reductions.
 
      @tparam Functor Kernel functor that defines the kernel
      @tparam Arg Kernel argument struct that set any required meta
@@ -82,10 +83,12 @@ namespace quda {
   __forceinline__ __device__ void BlockKernel2D_impl(const Arg &arg)
   {
     QUDA_RT_CONSTS;
-    const dim3 block_idx(virtual_block_idx(arg), blockIdx.y, 0);
-    const dim3 thread_idx(threadIdx.x, threadIdx.y, 0);
+    const dim3 block_idx(virtual_block_idx(arg), blockIdx.y, blockIdx.z);
+    const dim3 thread_idx(threadIdx.x, threadIdx.y, threadIdx.z);
     auto j = blockDim.y * blockIdx.y + threadIdx.y;
+    auto k = blockDim.z * blockIdx.z + threadIdx.z;
     if (j >= arg.threads.y) return;
+    if (k >= arg.threads.z) return;
 
     Functor<Arg> t(arg);
     t(block_idx, thread_idx);
diff --git a/include/targets/omptarget/constant_kernel_arg.h b/include/targets/omptarget/constant_kernel_arg.h
index 3e33c89f89..59bb3ee328 100644
--- a/include/targets/omptarget/constant_kernel_arg.h
+++ b/include/targets/omptarget/constant_kernel_arg.h
@@ -14,14 +14,16 @@
    instantiation correctly.
  */
 
-namespace quda {
+namespace quda
+{
+
+  namespace device
+  {
 
-  namespace device {
-    
     /**
        @brief The __constant__ buffer used for kernel parameters
     */
-#if defined(__CUDACC_RDC__) && !defined(QUDA_CONSTANT_DEFINE) // what is the equivalent macro for nvc++?
+#if defined(__CUDACC_RDC__) && !defined(QUDA_CONSTANT_DEFINE)
     // rdc is enabled when NVSHMEM is enabled, so we need to make the
     // buffer as extern and define it in one place only
     extern __constant__ char buffer[max_constant_size()];
@@ -35,15 +37,20 @@ namespace quda {
        @brief Helper function that returns kernel argument from
        __constant__ memory.
      */
-    template <typename Arg>
-      constexpr std::enable_if_t<!use_kernel_arg<Arg>(), Arg&> get_arg() { return reinterpret_cast<Arg&>(buffer); }
+    template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), Arg &> get_arg()
+    {
+      return reinterpret_cast<Arg &>(buffer);
+    }
 
     /**
        @brief Helper function that returns a pointer to the
        __constant__ memory buffer.
      */
-    template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), void*> get_constant_buffer() { return omp_get_mapped_ptr(buffer, omp_get_default_device()); }
+    template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), void *> get_constant_buffer()
+    {
+      return omp_get_mapped_ptr(buffer, omp_get_default_device());
+    }
 
-  }
+  } // namespace device
 
-}
+} // namespace quda
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 6debc1b793..de4ca6e037 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -3,8 +3,21 @@
 #include <target_device.h>
 #include <kernel_helper.h>
 
-namespace quda {
+namespace quda
+{
 
+  /**
+     @brief Kernel1D_impl is the implementation of the generic 1-d
+     kernel.  Functors that utilize this kernel have a
+     single parallelization dimension.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread.
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __forceinline__ __device__ void Kernel1D_impl(const Arg &arg)
   {
@@ -15,26 +28,41 @@ namespace quda {
 
     while (i < arg.threads.x) {
       f(i);
-      if (grid_stride) i += gridDim.x * blockDim.x; else break;
+      if (grid_stride)
+        i += gridDim.x * blockDim.x;
+      else
+        break;
     }
   }
 
+  /**
+     @brief Kernel1D is the entry point of the generic 1-d kernel.
+     This is the specialization where the kernel argument struct is
+     passed by value directly to the kernel.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread.
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg)
+  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg)
+  {
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // printf("dparg %p\n", dparg);
+    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    // const int tx = arg.threads.x;
+    // const int ty = arg.threads.y;
+    // const int tz = arg.threads.z;
+    // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-      // printf("dparg %p\n", dparg);
-      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-      // const int tx = arg.threads.x;
-      // const int ty = arg.threads.y;
-      // const int tz = arg.threads.z;
-      // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-      #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
-      {
-        int cache[device::max_shared_memory_size()/sizeof(int)];
-        target::omptarget::shared_cache.addr = cache;
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -46,21 +74,33 @@ namespace quda {
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel1D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
-      }
-      // printf("Kernel1D: exited\n");
-      omp_target_free(dparg, omp_get_default_device());
     }
+    // printf("Kernel1D: exited\n");
+    omp_target_free(dparg, omp_get_default_device());
+  }
 
+  /**
+     @brief Kernel1D is the entry point of the generic 1-d kernel.
+     This is the specialization where the kernel argument struct is
+     copied to the device prior to kernel launch.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread.
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D()
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D()
+  {
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    // printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
+    #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-      // printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
-      #pragma omp target teams num_teams(gd) thread_limit(ld)
-      {
-        int cache[device::max_shared_memory_size()/sizeof(int)];
-        target::omptarget::shared_cache.addr = cache;
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -70,10 +110,21 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
-      }
     }
+  }
 
+  /**
+     @brief Kernel2D_impl is the implementation of the generic 2-d
+     kernel.  Functors that utilize this kernel have two
+     parallelization dimensions.
 
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __forceinline__ __device__ void Kernel2D_impl(const Arg &arg)
   {
@@ -86,26 +137,41 @@ namespace quda {
 
     while (i < arg.threads.x) {
       f(i, j);
-      if (grid_stride) i += gridDim.x * blockDim.x; else break;
+      if (grid_stride)
+        i += gridDim.x * blockDim.x;
+      else
+        break;
     }
   }
 
+  /**
+     @brief Kernel2D is the entry point of the generic 2-d kernel.
+     This is the specialization where the kernel argument struct is
+     passed by value directly to the kernel.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg)
+  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg)
+  {
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    const int tx = arg.threads.x;
+    const int ty = arg.threads.y;
+    const int tz = arg.threads.z;
+    // printf("Kernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // printf("dparg %p\n", dparg);
+    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-      const int tx = arg.threads.x;
-      const int ty = arg.threads.y;
-      const int tz = arg.threads.z;
-      // printf("Kernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-      // printf("dparg %p\n", dparg);
-      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-      #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
-      {
-        int cache[device::max_shared_memory_size()/sizeof(int)];
-        target::omptarget::shared_cache.addr = cache;
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -117,18 +183,30 @@ namespace quda {
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel2D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
-      }
-      omp_target_free(dparg, omp_get_default_device());
     }
+    omp_target_free(dparg, omp_get_default_device());
+  }
+
+  /**
+     @brief Kernel2D is the entry point of the generic 2-d kernel.
+     This is the specialization where the kernel argument struct is
+     copied to the device prior to kernel launch.
 
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D()
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D()
+  {
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+    #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-      // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
-      #pragma omp target teams num_teams(gd) thread_limit(ld)
-      {
         int cache[device::max_shared_memory_size()/sizeof(int)];
         target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
@@ -140,10 +218,21 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
-      }
     }
+  }
 
+  /**
+     @brief Kernel3D_impl is the implementation of the generic 3-d
+     kernel.  Functors that utilize this kernel have three
+     parallelization dimensions.
 
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __forceinline__ __device__ void Kernel3D_impl(const Arg &arg)
   {
@@ -158,26 +247,41 @@ namespace quda {
 
     while (i < arg.threads.x) {
       f(i, j, k);
-      if (grid_stride) i += gridDim.x * blockDim.x; else break;
+      if (grid_stride)
+        i += gridDim.x * blockDim.x;
+      else
+        break;
     }
   }
 
+  /**
+     @brief Kernel3D is the entry point of the generic 3-d kernel.
+     This is the specialization where the kernel argument struct is
+     passed by value directly to the kernel.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg)
+  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg)
+  {
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    const int tx = arg.threads.x;
+    const int ty = arg.threads.y;
+    const int tz = arg.threads.z;
+    // printf("Kernel3D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // printf("dparg %p\n", dparg);
+    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-      const int tx = arg.threads.x;
-      const int ty = arg.threads.y;
-      const int tz = arg.threads.z;
-      // printf("Kernel3D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-      Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-      // printf("dparg %p\n", dparg);
-      omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-      #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
-      {
-        int cache[device::max_shared_memory_size()/sizeof(int)];
-        target::omptarget::shared_cache.addr = cache;
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -189,20 +293,32 @@ namespace quda {
         memcpy(buffer, (void *)dparg, sizeof(Arg));
         Kernel3D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
       }
-      }
-      omp_target_free(dparg, omp_get_default_device());
     }
+    omp_target_free(dparg, omp_get_default_device());
+  }
 
+  /**
+     @brief Kernel3D is the entry point of the generic 3-d kernel.
+     This is the specialization where the kernel argument struct is
+     passed by value directly to the kernel.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam grid_stride Whether the kernel does multiple computations
+     per thread (in the x dimension)
+     @param[in] arg Kernel argument
+   */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-    __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D()
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D()
+  {
+    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
+    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    // printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
+    #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-      const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-      // printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
-      #pragma omp target teams num_teams(gd) thread_limit(ld)
-      {
-        int cache[device::max_shared_memory_size()/sizeof(int)];
-        target::omptarget::shared_cache.addr = cache;
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      target::omptarget::shared_cache.addr = cache;
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -212,10 +328,24 @@ namespace quda {
         //          omp_get_num_teams(), omp_get_num_threads());
         Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
       }
-      }
     }
+  }
 
-  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
+  /**
+     @brief raw_kernel is used for CUDA-specific kernels where we want
+     to avoid using the generic framework.  For these kernels, we
+     delegate the parallelism and bounds checking for the kernel
+     functor.
+
+     @tparam Functor Kernel functor that defines the kernel
+     @tparam Arg Kernel argument struct that set any required meta
+     data for the kernel
+     @tparam dummy unused template parameter, present to allow us to
+     utilize the generic launching framework
+
+     @param[in] arg Kernel argument
+   */
+  template <template <typename> class Functor, typename Arg, bool dummy = false>
   __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
@@ -228,19 +358,19 @@ namespace quda {
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
-      {
-        int cache[device::max_shared_memory_size()/sizeof(int)];
-        target::omptarget::shared_cache.addr = cache;
-    #pragma omp parallel num_threads(ld)
     {
-      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-      //          "omp reports: teams %d threads %d\n",
-      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-      //          omp_get_num_teams(), omp_get_num_threads());
-      Functor<Arg> f(*dparg);
-      f();
-    }
+      int cache[device::max_shared_memory_size()/sizeof(int)];
+      target::omptarget::shared_cache.addr = cache;
+      #pragma omp parallel num_threads(ld)
+      {
+        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
+        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
+        //          "omp reports: teams %d threads %d\n",
+        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
+        //          omp_get_num_teams(), omp_get_num_threads());
+        Functor<Arg> f(*dparg);
+        f();
+      }
     }
     omp_target_free(dparg, omp_get_default_device());
   }
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 0db6836748..514b962d0b 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -125,15 +125,17 @@ namespace quda
       = std::min(Arg::max_n_batch_block, device::max_block_size() / (block_size_x * block_size_y));
     using BlockReduce = BlockReduce<T, block_size_x, block_size_y, n_batch_block, true>;
     // bool isLastBlockDone[n_batch_block];
-    static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::shared_cache.addr[0])*1024, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 1024 is used in block_reduce_helper.h:/dummy_storage/
+    static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::shared_cache.addr[0])*128, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 128 is used in block_reduce_helper.h:/tempStorage/
     bool *isLastBlockDone = (bool*)target::omptarget::shared_cache.addr;
+    // printf("team %d thread %d isLastBlockDone %p\n", omp_get_team_num(), omp_get_thread_num(), isLastBlockDone);
 
     T aggregate = BlockReduce(target::thread_idx().z).Reduce(in, r);
     // printf("team %d thread %d  r %g  aggregate %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&in), *(double*)(&aggregate));
 
     if (target::thread_idx().x == 0 && target::thread_idx().y == 0) {
       arg.partial[idx * target::grid_dim().x + target::block_idx().x] = aggregate;
-      __threadfence(); // flush result
+      // __threadfence(); // flush result
+      #pragma omp flush
 
       // increment global block counter
       // auto value = atomicInc(&arg.count[idx], target::grid_dim().x);
@@ -150,6 +152,7 @@ namespace quda
 
     // finish the reduction if last block
     if (isLastBlockDone[target::thread_idx().z]) {
+      #pragma omp flush
       auto i = target::thread_idx().y * block_size_x + target::thread_idx().x;
       T sum = r.init();
       while (i < target::grid_dim().x) {
diff --git a/include/targets/omptarget/warp_collective.h b/include/targets/omptarget/warp_collective.h
index 6d20a25527..8f4c203e6e 100644
--- a/include/targets/omptarget/warp_collective.h
+++ b/include/targets/omptarget/warp_collective.h
@@ -1,23 +1,40 @@
-namespace quda {
+#pragma once
 
-  template <int warp_split, typename T> __device__ __host__ inline T warp_combine(T &x)
-  {
-#ifdef __CUDA_ARCH__
-    constexpr int warp_size = device::warp_size();
-    if (warp_split > 1) {
+#include <target_device.h>
+
+namespace quda
+{
+
+  template <bool is_device> struct warp_combine_impl {
+    template <typename T> T operator()(T &x, int) { return x; }
+  };
+
+  template <> struct warp_combine_impl<true> {
+    template <typename T> __device__ inline T operator()(T &x, int warp_split)
+    {
+#if 0
+      // OMP FIXME
+      constexpr int warp_size = device::warp_size();
+      if (warp_split > 1) {
 QUDA_UNROLL
-      for (int i = 0; i < x.size(); i++) {
-        // reduce down to the first group of column-split threads
+        for (int i = 0; i < x.size(); i++) {
+          // reduce down to the first group of column-split threads
 QUDA_UNROLL
-        for (int offset = warp_size / 2; offset >= warp_size / warp_split; offset /= 2) {
-          // TODO - add support for non-converged warps
-          x[i].real(x[i].real() + __shfl_down_sync(device::warp_converged_mask(), x[i].real(), offset));
-          x[i].imag(x[i].imag() + __shfl_down_sync(device::warp_converged_mask(), x[i].imag(), offset));
+          for (int offset = warp_size / 2; offset >= warp_size / warp_split; offset /= 2) {
+            // TODO - add support for non-converged warps
+            x[i].real(x[i].real() + __shfl_down_sync(device::warp_converged_mask(), x[i].real(), offset));
+            x[i].imag(x[i].imag() + __shfl_down_sync(device::warp_converged_mask(), x[i].imag(), offset));
+          }
         }
       }
-    }
 #endif
-    return x;
+      return x;
+    }
+  };
+
+  template <int warp_split, typename T> __device__ __host__ inline T warp_combine(T &x)
+  {
+    return target::dispatch<warp_combine_impl>(x, warp_split);
   }
 
-}
+} // namespace quda

From 3efc505cfbd46c8b9a950a77083cd61b94693492 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Sat, 16 Apr 2022 05:46:26 +0000
Subject: [PATCH 106/198] omp: workaround issues with omp barrier

omp barrier requires all threads in the team to participate.
This commit makes sure that happens with
1. reduce() calls BlockReduce in all threads in the team with last block done;
2. qudaLaunchKernel() rejects threads parameters that could result in threads have no work.
---
 include/targets/omptarget/reduce_helper.h  | 11 +++++++++-
 include/targets/omptarget/tunable_kernel.h | 25 +++++++++++++++++++---
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 514b962d0b..8e7546778d 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -125,9 +125,12 @@ namespace quda
       = std::min(Arg::max_n_batch_block, device::max_block_size() / (block_size_x * block_size_y));
     using BlockReduce = BlockReduce<T, block_size_x, block_size_y, n_batch_block, true>;
     // bool isLastBlockDone[n_batch_block];
-    static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::shared_cache.addr[0])*128, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 128 is used in block_reduce_helper.h:/tempStorage/
+    static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::shared_cache.addr[0])*64, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 128 is used in block_reduce_helper.h:/tempStorage/
     bool *isLastBlockDone = (bool*)target::omptarget::shared_cache.addr;
+    bool *hasLastBlockDone = (bool*)&target::omptarget::shared_cache.addr[64];
     // printf("team %d thread %d isLastBlockDone %p\n", omp_get_team_num(), omp_get_thread_num(), isLastBlockDone);
+    if (target::thread_idx().x == 0 && target::thread_idx().y == 0 && target::thread_idx().z == 0)
+      *hasLastBlockDone = 0;
 
     T aggregate = BlockReduce(target::thread_idx().z).Reduce(in, r);
     // printf("team %d thread %d  r %g  aggregate %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&in), *(double*)(&aggregate));
@@ -146,11 +149,14 @@ namespace quda
 
       // determine if last block
       isLastBlockDone[target::thread_idx().z] = (value == (target::grid_dim().x - 1));
+      if(value == (target::grid_dim().x - 1))
+        *hasLastBlockDone = 1;
     }
 
     __syncthreads();
 
     // finish the reduction if last block
+    if (*hasLastBlockDone) {
     if (isLastBlockDone[target::thread_idx().z]) {
       #pragma omp flush
       auto i = target::thread_idx().y * block_size_x + target::thread_idx().x;
@@ -173,7 +179,10 @@ namespace quda
         }
         arg.count[idx] = 0; // set to zero for next time
       }
+    }else{
+      BlockReduce(target::thread_idx().z).Reduce(r.init(), r);
     }
+    }  // hasLastBlockDone
   }
 
 } // namespace quda
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index b664fc5063..a2be594283 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -15,15 +15,34 @@ namespace quda {
     namespace omptarget {
       // defined in ../../../lib/targets/omptarget/quda_api.cpp:/qudaSetupLaunchParameter
       void qudaSetupLaunchParameter(const TuneParam &);
+      void set_runtime_error(int error, const char *api_func, const char *func, const char *file,
+                             const char *line, bool allow_error = false);
     }
   }
 
+  template <typename Arg>
+  static bool acceptThreads(const TuneParam &tp, const Arg &arg)
+  {
+    bool r = arg.threads.x<tp.block.x ||    // trivial cases where arg.threads.x == 1 or few
+       arg.threads.y<tp.block.y ||
+       arg.threads.z<tp.block.z ||
+      (arg.threads.x%(tp.grid.x*tp.block.x)==0 &&
+       arg.threads.y%(tp.grid.y*tp.block.y)==0 &&
+       arg.threads.z%(tp.grid.z*tp.block.z)==0);
+    if(!r && getVerbosity() >= QUDA_VERBOSE)
+      ompwip("WARNING: rejecting threads setup arg %d %d %d tp (%d %d %d)x(%d %d %d)",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
+    return r;
+  }
   template <typename... Arg>
   qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, qudaStream_t stream, const Arg &... arg)
   {
-    target::omptarget::qudaSetupLaunchParameter(tp);
-    reinterpret_cast<void(*)(Arg...)>(const_cast<void*>(func))(arg...);
-    return QUDA_SUCCESS;
+    if(acceptThreads(tp, arg...)){
+      target::omptarget::qudaSetupLaunchParameter(tp);
+      reinterpret_cast<void(*)(Arg...)>(const_cast<void*>(func))(arg...);
+      return QUDA_SUCCESS;
+    }else
+      target::omptarget::set_runtime_error(QUDA_ERROR, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
+      return QUDA_ERROR;
   }
 
   class TunableKernel : public Tunable

From 4ccd102500c77df5dd082660dc6ad333955e8190 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Sat, 16 Apr 2022 17:48:44 +0000
Subject: [PATCH 107/198] omp: reorder reduce() so all threads call BlockReduce
 at a single place

---
 .../targets/omptarget/block_reduce_helper.h   | 33 +++++++++++++++++
 include/targets/omptarget/reduce_helper.h     | 35 +++++++++----------
 2 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index 6b430d7594..7c108d632c 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -214,6 +214,7 @@ namespace quda
     __device__ inline T operator()(const T &value_, bool async, int batch, bool all, const reducer_t &r,
                                    const param_t &)
     {
+#if 1
       using block_reduce_t = cub::BlockReduce<T, param_t::block_size_x, cub::BLOCK_REDUCE_WARP_REDUCTIONS,
                                               param_t::block_size_y, param_t::block_size_z, __COMPUTE_CAPABILITY__>;
       using tempStorage = typename block_reduce_t::TempStorage;
@@ -233,6 +234,38 @@ namespace quda
         #pragma omp barrier
         value = value_shared;
       }
+#else
+      // this block of code only works for a single batch the size of the thread block
+      // test with QUDA/BlasTest.verify/reDotProduct_block_double_double --nsrc 2 --msrc 2
+      constexpr int nthr = param_t::block_size_x*param_t::block_size_y*param_t::block_size_z;
+      int tid = omp_get_thread_num();
+      static_assert(nthr*sizeof(T) <= sizeof(target::omptarget::shared_cache.addr[0])*8192, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, do we have that many?
+      T *storage = (T*)&target::omptarget::shared_cache.addr[128];  // FIXME arbitrary
+      storage[tid] = value_;
+      int offset = 1;
+      constexpr int bs = param_t::batch_size;
+      static_assert(nthr%bs==0, "Block size not divisible by batch_size");
+      constexpr int subblock = nthr/bs;
+      do{
+        #pragma omp for collapse(2)
+        for(int b=0;b<nthr;b+=subblock)
+          for(int i=0;i<subblock-offset;i+=2*offset)
+            storage[b+i] = r(storage[b+i], storage[b+i+offset]);
+        offset *= 2;
+      }while(offset<subblock);
+/*
+      int b = subblock*batch;
+      do{
+        #pragma omp for
+        for(int i=0;i<subblock-offset;i+=2*offset)
+          storage[b+i] = r(storage[b+i], storage[b+i+offset]);
+        offset *= 2;
+      }while(offset<subblock);
+*/
+      T value = storage[tid];
+      if(all)
+        value = storage[subblock*(tid/subblock)];
+#endif
       return value;
     }
   };
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 8e7546778d..3cb285a8ba 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -138,7 +138,6 @@ namespace quda
     if (target::thread_idx().x == 0 && target::thread_idx().y == 0) {
       arg.partial[idx * target::grid_dim().x + target::block_idx().x] = aggregate;
       // __threadfence(); // flush result
-      #pragma omp flush
 
       // increment global block counter
       // auto value = atomicInc(&arg.count[idx], target::grid_dim().x);
@@ -157,31 +156,31 @@ namespace quda
 
     // finish the reduction if last block
     if (*hasLastBlockDone) {
-    if (isLastBlockDone[target::thread_idx().z]) {
-      #pragma omp flush
-      auto i = target::thread_idx().y * block_size_x + target::thread_idx().x;
+      bool thisSubBlock = isLastBlockDone[target::thread_idx().z];
       T sum = r.init();
-      while (i < target::grid_dim().x) {
-        sum = r(sum, const_cast<T &>(static_cast<volatile T *>(arg.partial)[idx * target::grid_dim().x + i]));
-        // printf("team %d thread %d  sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
-        i += block_size_x * block_size_y;
+      if (thisSubBlock) {
+        auto i = target::thread_idx().y * block_size_x + target::thread_idx().x;
+        while (i < target::grid_dim().x) {
+          sum = r(sum, const_cast<T &>(static_cast<volatile T *>(arg.partial)[idx * target::grid_dim().x + i]));
+          // printf("team %d thread %d  sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
+          i += block_size_x * block_size_y;
+        }
       }
 
       sum = BlockReduce(target::thread_idx().z).Reduce(sum, r);
 
       // write out the final reduced value
-      if (target::thread_idx().y * block_size_x + target::thread_idx().x == 0) {
-        // printf("team %d thread %d  final sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
-        if (arg.get_output_async_buffer()) {
-          arg.get_output_async_buffer()[idx] = sum;
-        } else {
-          arg.result_d[idx] = sum;
+      if (thisSubBlock) {
+        if (target::thread_idx().y * block_size_x + target::thread_idx().x == 0) {
+          // printf("team %d thread %d  final sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
+          if (arg.get_output_async_buffer()) {
+            arg.get_output_async_buffer()[idx] = sum;
+          } else {
+            arg.result_d[idx] = sum;
+          }
+          arg.count[idx] = 0; // set to zero for next time
         }
-        arg.count[idx] = 0; // set to zero for next time
       }
-    }else{
-      BlockReduce(target::thread_idx().z).Reduce(r.init(), r);
-    }
     }  // hasLastBlockDone
   }
 

From b11cb1c84c2c6d860e4925a598ce659218576ba5 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 19 Apr 2022 17:33:20 +0000
Subject: [PATCH 108/198] omp: use explicit memcpy for arg in block reduction

---
 include/targets/omptarget/block_reduction_kernel.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 6277d39232..0ddb3a9195 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -134,7 +134,9 @@ namespace quda
       //          "omp reports: teams %d threads %d\n",
       //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
       //          omp_get_num_teams(), omp_get_num_threads());
-      BlockKernel2D_impl<Functor, Arg>(arg);
+      char buffer[sizeof(Arg)];
+      memcpy(buffer, (void *)dparg, sizeof(Arg));
+      BlockKernel2D_impl<Functor, Arg>(*(Arg *)buffer);
     }
     }
     omp_target_free(dparg, omp_get_default_device());

From f942ec2fa8cd4f66c0dfe712ffaeac79e88e3b99 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 19 Apr 2022 22:12:35 +0000
Subject: [PATCH 109/198] omp: cmake works

---
 include/quda_api.h                           | 15 +++-
 lib/CMakeLists.txt                           |  3 +
 lib/targets/omptarget/target_omptarget.cmake | 78 ++++++++++++++++++++
 3 files changed, 92 insertions(+), 4 deletions(-)
 create mode 100644 lib/targets/omptarget/target_omptarget.cmake

diff --git a/include/quda_api.h b/include/quda_api.h
index 6cf0848283..9382fe80d6 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -1,13 +1,20 @@
 #pragma once
 
-#ifndef QUDA_UNROLL
-#define QUDA_UNROLL _Pragma("unroll")
-#endif
-
 #include <quda_define.h>
 #include <string>
 #include <enum_quda.h>
 
+/* We have to overwrite some cuda-ism here even for public interface,
+   other wise we can't compile tests.
+ */
+#ifdef QUDA_TARGET_OMPTARGET
+#include "targets/omptarget/quda_api.h"
+#endif
+
+#ifndef QUDA_UNROLL
+#define QUDA_UNROLL _Pragma("unroll")
+#endif
+
 /**
    @file quda_api.h
 
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 42bb03f6f2..bf8525875d 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -192,6 +192,9 @@ endif()
 if(${QUDA_TARGET_TYPE} STREQUAL "HIP")
 	include(targets/hip/target_hip.cmake)
 endif()
+if(${QUDA_TARGET_TYPE} STREQUAL "OMPTARGET")
+	include(targets/omptarget/target_omptarget.cmake)
+endif()
 
 # make one library
 target_sources(quda PRIVATE $<TARGET_OBJECTS:quda_cpp> $<$<TARGET_EXISTS:quda_pack>:$<TARGET_OBJECTS:quda_pack>>
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
new file mode 100644
index 0000000000..9b794c808e
--- /dev/null
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -0,0 +1,78 @@
+# ######################################################################################################################
+# OMPTARGET specific part of CMakeLists
+set(QUDA_TARGET_OMPTARGET ON)
+
+if(DEFINED ENV{QUDA_GPU_ARCH})
+  set(QUDA_DEFAULT_GPU_ARCH $ENV{QUDA_GPU_ARCH})
+else()
+  set(QUDA_DEFAULT_GPU_ARCH xehp)
+endif()
+
+set(QUDA_GPU_ARCH
+    ${QUDA_DEFAULT_GPU_ARCH}
+    CACHE STRING "set the GPU architecture (xehp)")
+set_property(CACHE QUDA_GPU_ARCH PROPERTY STRINGS xehp)
+
+message(STATUS "Building for GPU Architectures: ${QUDA_GPU_ARCH}")
+
+set(QUDA_WARP_SIZE 16 CACHE STRING "OpenMP target logical warp size")
+set_property(CACHE QUDA_WARP_SIZE PROPERTY STRINGS 8 16 32)
+target_compile_definitions(quda PUBLIC QUDA_WARP_SIZE=${QUDA_WARP_SIZE})
+message(STATUS "Using logical warp size: ${QUDA_WARP_SIZE}")
+
+set(QUDA_MAX_BLOCK_SIZE 1024 CACHE STRING "OpenMP target maximum team size (number of threads per team)")
+set_property(CACHE QUDA_MAX_BLOCK_SIZE PROPERTY STRINGS 256 512 768 1024)
+target_compile_definitions(quda PUBLIC QUDA_MAX_BLOCK_SIZE=${QUDA_MAX_BLOCK_SIZE})
+message(STATUS "Using maximum team size: ${QUDA_MAX_BLOCK_SIZE}")
+
+set(QUDA_MAX_SHARED_MEMORY_SIZE 32768 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
+set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 16384 32768 49152)
+target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
+message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")
+
+# ######################################################################################################################
+# define omptarget flags
+set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51)
+message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
+
+# QUDA_HASH for tunecache
+set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${QUDA_GPU_ARCH},cxx_version=${CMAKE_CXX_COMPILER_VERSION})
+set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:${QUDA_GPU_ARCH}")
+
+# ######################################################################################################################
+# omptarget specific compile options
+
+target_include_directories(quda PRIVATE
+  ${CMAKE_SOURCE_DIR}/include/targets/omptarget)
+
+# We need to overwrite some cuda-ism
+target_include_directories(quda PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include/targets/omptarget>
+  $<INSTALL_INTERFACE:include/targets/omptarget>)
+target_compile_options(
+  quda
+  PRIVATE -Wall
+          -Wextra
+          -Wno-unknown-pragmas
+          -Wno-unused-result
+          $<$<CONFIG:STRICT>:-Werror
+          -Wno-error=pass-failed>
+          $<$<CONFIG:SANITIZE>:-fsanitize=address
+          -fsanitize=undefined>)
+
+target_compile_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS})
+
+target_link_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS} -Xopenmp-target-backend "-device ${QUDA_GPU_ARCH}")
+
+set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES LANGUAGE CXX)
+set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES COMPILE_FLAGS "-x c++")
+
+# target_link_libraries(quda PUBLIC hip::hiprand roc::rocrand hip::hipcub roc::rocprim_hip)
+# target_link_libraries(quda PUBLIC roc::hipblas roc::rocblas)
+
+# if(QUDA_GAUGE_ALG)
+#   target_include_directories(quda PUBLIC ${ROCM_PATH}/hipfft/include)
+#   target_link_libraries(quda PUBLIC hip::hipfft)
+# endif(QUDA_GAUGE_ALG)
+
+add_subdirectory(targets/omptarget)

From 80cc4a589554253f0ee538bd9ab404a45084b188 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 20 Apr 2022 21:15:22 +0000
Subject: [PATCH 110/198] omp: use our own implementation of MRG32k3a

---
 include/externals/rocrand/README              |  26 -
 include/externals/rocrand/rocrand_common.h    | 150 ---
 include/externals/rocrand/rocrand_mrg32k3a.h  | 506 -----------
 .../rocrand/rocrand_mrg32k3a_precomputed.h    | 859 ------------------
 include/targets/omptarget/mrg32k3a.h          | 206 +++++
 include/targets/omptarget/random_helper.h     |  86 +-
 6 files changed, 253 insertions(+), 1580 deletions(-)
 delete mode 100644 include/externals/rocrand/README
 delete mode 100644 include/externals/rocrand/rocrand_common.h
 delete mode 100644 include/externals/rocrand/rocrand_mrg32k3a.h
 delete mode 100644 include/externals/rocrand/rocrand_mrg32k3a_precomputed.h
 create mode 100644 include/targets/omptarget/mrg32k3a.h

diff --git a/include/externals/rocrand/README b/include/externals/rocrand/README
deleted file mode 100644
index 7e4a0c8676..0000000000
--- a/include/externals/rocrand/README
+++ /dev/null
@@ -1,26 +0,0 @@
-This directory contains a standalone rocRAND implementation of mrg32k3a, imported from
-https://github.com/ROCmSoftwarePlatform/rocRAND/commit/56b64021e145e65d3f6bee0fef6ceae15df289b4
-with modifications by Xiao-Yong Jin.
-
-LICENSE
-
-Copyright (c) 2021, QUDA Developers
-Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/include/externals/rocrand/rocrand_common.h b/include/externals/rocrand/rocrand_common.h
deleted file mode 100644
index a6ee5e68fd..0000000000
--- a/include/externals/rocrand/rocrand_common.h
+++ /dev/null
@@ -1,150 +0,0 @@
-// Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-
-#ifndef ROCRAND_COMMON_H_
-#define ROCRAND_COMMON_H_
-
-#define ROCRAND_2POW16_INV (1.5258789e-05f)
-#define ROCRAND_2POW16_INV_2PI (1.5258789e-05f * 6.2831855f)
-#define ROCRAND_2POW32_INV (2.3283064e-10f)
-#define ROCRAND_2POW32_INV_DOUBLE (2.3283064365386963e-10)
-#define ROCRAND_2POW32_INV_2PI (2.3283064e-10f * 6.2831855f)
-#define ROCRAND_2POW53_INV_DOUBLE (1.1102230246251565e-16)
-#define ROCRAND_PI  (3.1415926f)
-#define ROCRAND_PI_DOUBLE  (3.1415926535897932)
-#define ROCRAND_2PI (6.2831855f)
-#define ROCRAND_SQRT2 (1.4142135f)
-#define ROCRAND_SQRT2_DOUBLE (1.4142135623730951)
-
-#include <math.h>
-
-#ifndef FQUALIFIERS
-#define FQUALIFIERS __forceinline__ __device__
-#endif // FQUALIFIERS
-
-#if __HIP_DEVICE_COMPILE__ && (defined(__HIP_PLATFORM_HCC__) || (defined(__HIP_PLATFORM_NVCC__) && (__CUDA_ARCH__ >= 530)))
-#define ROCRAND_HALF_MATH_SUPPORTED
-#endif
-
-namespace rocrand_device {
-namespace detail {
-
-  #if ( defined(__gfx801__) || \
-        defined(__gfx802__) || \
-        defined(__gfx803__) || \
-        defined(__gfx810__) || \
-        defined(__gfx900__) || \
-        defined(__gfx902__) || \
-        defined(__gfx904__) || \
-        defined(__gfx906__) || \
-        defined(__gfx908__) || \
-        defined(__gfx909__) )
-  #ifndef ROCRAND_ENABLE_INLINE_ASM
-    #define ROCRAND_ENABLE_INLINE_ASM
-  #endif
-#else
-  #ifdef ROCRAND_ENABLE_INLINE_ASM
-    #undef ROCRAND_ENABLE_INLINE_ASM
-    #pragma warning "Disabled inline asm, because the build target does not support it."
-  #endif
-#endif
-
-FQUALIFIERS
-unsigned long long mad_u64_u32(const unsigned int x, const unsigned int y, const unsigned long long z)
-{
-  #if defined(__HIP_PLATFORM_HCC__) && defined(__HIP_DEVICE_COMPILE__) \
-    && defined(ROCRAND_ENABLE_INLINE_ASM)
-
-    unsigned long long r;
-    unsigned long long c; // carry bits, SGPR, unused
-    // x has "r" constraint. This allows to use both VGPR and SGPR
-    // (to save VGPR) as input.
-    // y and z have "v" constraints, because only one SGPR or literal
-    // can be read by the instruction.
-    asm volatile("v_mad_u64_u32 %0, %1, %2, %3, %4"
-      : "=v"(r), "=s"(c) : "r"(x), "v"(y), "v"(z)
-    );
-    return r;
-  #elif defined(__HIP_PLATFORM_NVCC__) && defined(__HIP_DEVICE_COMPILE__) \
-        && defined(ROCRAND_ENABLE_INLINE_ASM)
-
-    unsigned long long r;
-    asm("mad.wide.u32 %0, %1, %2, %3;"
-        : "=l"(r) : "r"(x), "r"(y), "l"(z)
-    );
-    return r;
-
-  #else // host code
-
-    return static_cast<unsigned long long>(x) * static_cast<unsigned long long>(y) + z;
-
-  #endif
-}
-
-// This helps access fields of engine's internal state which
-// saves floats and doubles generated using the Box–Muller transform
-template<typename Engine>
-struct engine_boxmuller_helper
-{
-    static FQUALIFIERS
-    bool has_float(const Engine * engine)
-    {
-        return engine->m_state.boxmuller_float_state != 0;
-    }
-
-    static FQUALIFIERS
-    float get_float(Engine * engine)
-    {
-        engine->m_state.boxmuller_float_state = 0;
-        return engine->m_state.boxmuller_float;
-    }
-
-    static FQUALIFIERS
-    void save_float(Engine * engine, float f)
-    {
-        engine->m_state.boxmuller_float_state = 1;
-        engine->m_state.boxmuller_float = f;
-    }
-
-    static FQUALIFIERS
-    bool has_double(const Engine * engine)
-    {
-        return engine->m_state.boxmuller_double_state != 0;
-    }
-
-    static FQUALIFIERS
-    float get_double(Engine * engine)
-    {
-        engine->m_state.boxmuller_double_state = 0;
-        return engine->m_state.boxmuller_double;
-    }
-
-    static FQUALIFIERS
-    void save_double(Engine * engine, double d)
-    {
-        engine->m_state.boxmuller_double_state = 1;
-        engine->m_state.boxmuller_double = d;
-    }
-};
-
-} // end namespace detail
-} // end namespace rocrand_device
-
-#endif // ROCRAND_COMMON_H_
diff --git a/include/externals/rocrand/rocrand_mrg32k3a.h b/include/externals/rocrand/rocrand_mrg32k3a.h
deleted file mode 100644
index ffa9ef08e4..0000000000
--- a/include/externals/rocrand/rocrand_mrg32k3a.h
+++ /dev/null
@@ -1,506 +0,0 @@
-// Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-
-#ifndef ROCRAND_MRG32K3A_H_
-#define ROCRAND_MRG32K3A_H_
-
-#ifndef FQUALIFIERS
-#define FQUALIFIERS __forceinline__ __device__
-#endif // FQUALIFIERS_
-
-#include "rocrand_common.h"
-#include "rocrand_mrg32k3a_precomputed.h"
-
-// Thomas Bradley, Parallelisation Techniques for Random Number Generators
-// https://www.nag.co.uk/IndustryArticles/gpu_gems_article.pdf
-
-#define ROCRAND_MRG32K3A_POW32 4294967296
-#define ROCRAND_MRG32K3A_M1 4294967087
-#define ROCRAND_MRG32K3A_M1C 209
-#define ROCRAND_MRG32K3A_M2 4294944443
-#define ROCRAND_MRG32K3A_M2C 22853
-#define ROCRAND_MRG32K3A_A12 1403580
-#define ROCRAND_MRG32K3A_A13 (4294967087 -  810728)
-#define ROCRAND_MRG32K3A_A13N 810728
-#define ROCRAND_MRG32K3A_A21 527612
-#define ROCRAND_MRG32K3A_A23 (4294944443 - 1370589)
-#define ROCRAND_MRG32K3A_A23N 1370589
-#define ROCRAND_MRG32K3A_NORM_DOUBLE (2.3283065498378288e-10) // 1/ROCRAND_MRG32K3A_M1
-#define ROCRAND_MRG32K3A_UINT_NORM (1.000000048661607) // (ROCRAND_MRG32K3A_POW32 - 1)/(ROCRAND_MRG32K3A_M1 - 1)
-
-/** \rocrand_internal \addtogroup rocranddevice
- *
- *  @{
- */
- /**
- * \def ROCRAND_MRG32K3A_DEFAULT_SEED
- * \brief Default seed for MRG32K3A PRNG.
- */
- #define ROCRAND_MRG32K3A_DEFAULT_SEED 12345ULL
- /** @} */ // end of group rocranddevice
-
-namespace rocrand_device {
-
-class mrg32k3a_engine
-{
-public:
-    struct mrg32k3a_state
-    {
-        unsigned int g1[3];
-        unsigned int g2[3];
-
-        #ifndef ROCRAND_DETAIL_MRG32K3A_BM_NOT_IN_STATE
-        // The Box–Muller transform requires two inputs to convert uniformly
-        // distributed real values [0; 1] to normally distributed real values
-        // (with mean = 0, and stddev = 1). Often user wants only one
-        // normally distributed number, to save performance and random
-        // numbers the 2nd value is saved for future requests.
-        unsigned int boxmuller_float_state; // is there a float in boxmuller_float
-        unsigned int boxmuller_double_state; // is there a double in boxmuller_double
-        float boxmuller_float; // normally distributed float
-        double boxmuller_double; // normally distributed double
-        #endif
-
-        FQUALIFIERS
-        ~mrg32k3a_state() { }
-    };
-
-    FQUALIFIERS
-    mrg32k3a_engine()
-    {
-        this->seed(ROCRAND_MRG32K3A_DEFAULT_SEED, 0, 0);
-    }
-
-    /// Initializes the internal state of the PRNG using
-    /// seed value \p seed, goes to \p subsequence -th subsequence,
-    /// and skips \p offset random numbers.
-    ///
-    /// New seed value should not be zero. If \p seed_value is equal
-    /// zero, value \p ROCRAND_MRG32K3A_DEFAULT_SEED is used instead.
-    ///
-    /// A subsequence is 2^67 numbers long.
-    FQUALIFIERS
-    mrg32k3a_engine(const unsigned long long seed,
-                    const unsigned long long subsequence,
-                    const unsigned long long offset)
-    {
-        this->seed(seed, subsequence, offset);
-    }
-
-    FQUALIFIERS
-    ~mrg32k3a_engine() { }
-
-    /// Reinitializes the internal state of the PRNG using new
-    /// seed value \p seed_value, skips \p subsequence subsequences
-    /// and \p offset random numbers.
-    ///
-    /// New seed value should not be zero. If \p seed_value is equal
-    /// zero, value \p ROCRAND_MRG32K3A_DEFAULT_SEED is used instead.
-    ///
-    /// A subsequence is 2^67 numbers long.
-    FQUALIFIERS
-    void seed(unsigned long long seed_value,
-              const unsigned long long subsequence,
-              const unsigned long long offset)
-    {
-        if(seed_value == 0)
-        {
-            seed_value = ROCRAND_MRG32K3A_DEFAULT_SEED;
-        }
-        unsigned int x = (unsigned int) seed_value ^ 0x55555555U;
-        unsigned int y = (unsigned int) ((seed_value >> 32) ^ 0xAAAAAAAAU);
-        m_state.g1[0] = mod_mul_m1(x, seed_value);
-        m_state.g1[1] = mod_mul_m1(y, seed_value);
-        m_state.g1[2] = mod_mul_m1(x, seed_value);
-        m_state.g2[0] = mod_mul_m2(y, seed_value);
-        m_state.g2[1] = mod_mul_m2(x, seed_value);
-        m_state.g2[2] = mod_mul_m2(y, seed_value);
-        this->restart(subsequence, offset);
-    }
-
-    /// Advances the internal state to skip \p offset numbers.
-    FQUALIFIERS
-    void discard(unsigned long long offset)
-    {
-        this->discard_impl(offset);
-    }
-
-    /// Advances the internal state to skip \p subsequence subsequences.
-    /// A subsequence is 2^67 numbers long.
-    FQUALIFIERS
-    void discard_subsequence(unsigned long long subsequence)
-    {
-        this->discard_subsequence_impl(subsequence);
-    }
-
-    /// Advances the internal state to skip \p sequence sequences.
-    /// A sequence is 2^127 numbers long.
-    FQUALIFIERS
-    void discard_sequence(unsigned long long sequence)
-    {
-        this->discard_sequence_impl(sequence);
-    }
-
-    FQUALIFIERS
-    void restart(const unsigned long long subsequence,
-                 const unsigned long long offset)
-    {
-        #ifndef ROCRAND_DETAIL_MRG32K3A_BM_NOT_IN_STATE
-        m_state.boxmuller_float_state = 0;
-        m_state.boxmuller_double_state = 0;
-        #endif
-        this->discard_subsequence_impl(subsequence);
-        this->discard_impl(offset);
-    }
-
-    FQUALIFIERS
-    unsigned int operator()()
-    {
-        return this->next();
-    }
-
-    // Returned value is in range [1, ROCRAND_MRG32K3A_M1],
-    // where ROCRAND_MRG32K3A_M1 < UINT_MAX
-    FQUALIFIERS
-    unsigned int next()
-    {
-        const unsigned int p1 = mod_m1(
-            detail::mad_u64_u32(
-                ROCRAND_MRG32K3A_A12,
-                m_state.g1[1],
-                detail::mad_u64_u32(
-                    ROCRAND_MRG32K3A_A13N,
-                    (ROCRAND_MRG32K3A_M1 - m_state.g1[0]),
-                    0
-                )
-            )
-        );
-
-        m_state.g1[0] = m_state.g1[1]; m_state.g1[1] = m_state.g1[2];
-        m_state.g1[2] = p1;
-
-        const unsigned int p2 = mod_m2(
-            detail::mad_u64_u32(
-                ROCRAND_MRG32K3A_A21,
-                m_state.g2[2],
-                detail::mad_u64_u32(
-                    ROCRAND_MRG32K3A_A23N,
-                    (ROCRAND_MRG32K3A_M2 - m_state.g2[0]),
-                    0
-                )
-            )
-        );
-
-        m_state.g2[0] = m_state.g2[1]; m_state.g2[1] = m_state.g2[2];
-        m_state.g2[2] = p2;
-
-        return (p1 - p2) + (p1 <= p2 ? ROCRAND_MRG32K3A_M1 : 0);
-    }
-
-protected:
-    // Advances the internal state to skip \p offset numbers.
-    // DOES NOT CALCULATE NEW ULONGLONG
-    FQUALIFIERS
-    void discard_impl(unsigned long long offset)
-    {
-        discard_state(offset);
-    }
-
-    // DOES NOT CALCULATE NEW ULONGLONG
-    FQUALIFIERS
-    void discard_subsequence_impl(unsigned long long subsequence)
-    {
-        int i = 0;
-
-        while(subsequence > 0) {
-            if (subsequence & 1) {
-                #if defined(__HIP_DEVICE_COMPILE__)
-                mod_mat_vec_m1(d_A1P67 + i, m_state.g1);
-                mod_mat_vec_m2(d_A2P67 + i, m_state.g2);
-                #else
-                mod_mat_vec_m1(h_A1P67 + i, m_state.g1);
-                mod_mat_vec_m2(h_A2P67 + i, m_state.g2);
-                #endif
-            }
-            subsequence >>= 1;
-            i += 9;
-        }
-    }
-
-    // DOES NOT CALCULATE NEW ULONGLONG
-    FQUALIFIERS
-    void discard_sequence_impl(unsigned long long sequence)
-    {
-        int i = 0;
-
-        while(sequence > 0) {
-            if (sequence & 1) {
-                #if defined(__HIP_DEVICE_COMPILE__)
-                mod_mat_vec_m1(d_A1P127 + i, m_state.g1);
-                mod_mat_vec_m2(d_A2P127 + i, m_state.g2);
-                #else
-                mod_mat_vec_m1(h_A1P127 + i, m_state.g1);
-                mod_mat_vec_m2(h_A2P127 + i, m_state.g2);
-                #endif
-            }
-            sequence >>= 1;
-            i += 9;
-        }
-    }
-
-    // Advances the internal state by offset times.
-    // DOES NOT CALCULATE NEW ULONGLONG
-    FQUALIFIERS
-    void discard_state(unsigned long long offset)
-    {
-        int i = 0;
-
-        while(offset > 0) {
-            if (offset & 1) {
-                #if defined(__HIP_DEVICE_COMPILE__)
-                mod_mat_vec_m1(d_A1 + i, m_state.g1);
-                mod_mat_vec_m2(d_A2 + i, m_state.g2);
-                #else
-                mod_mat_vec_m1(h_A1 + i, m_state.g1);
-                mod_mat_vec_m2(h_A2 + i, m_state.g2);
-                #endif
-            }
-            offset >>= 1;
-            i += 9;
-        }
-    }
-
-    // Advances the internal state to the next state
-    // DOES NOT CALCULATE NEW ULONGLONG
-    FQUALIFIERS
-    void discard_state()
-    {
-        discard_state(1);
-    }
-
-private:
-    FQUALIFIERS
-    void mod_mat_vec_m1(const unsigned long long * A,
-                        unsigned int * s)
-    {
-        unsigned long long x[3];
-
-        x[0] = mod_m1(mod_m1(A[0] * s[0])
-                    + mod_m1(A[1] * s[1])
-                    + mod_m1(A[2] * s[2]));
-
-        x[1] = mod_m1(mod_m1(A[3] * s[0])
-                    + mod_m1(A[4] * s[1])
-                    + mod_m1(A[5] * s[2]));
-
-        x[2] = mod_m1(mod_m1(A[6] * s[0])
-                    + mod_m1(A[7] * s[1])
-                    + mod_m1(A[8] * s[2]));
-
-        s[0] = x[0];
-        s[1] = x[1];
-        s[2] = x[2];
-    }
-
-    FQUALIFIERS
-    void mod_mat_vec_m2(const unsigned long long * A,
-                        unsigned int * s)
-    {
-        unsigned long long x[3];
-
-        x[0] = mod_m2(mod_m2(A[0] * s[0])
-                    + mod_m2(A[1] * s[1])
-                    + mod_m2(A[2] * s[2]));
-
-        x[1] = mod_m2(mod_m2(A[3] * s[0])
-                    + mod_m2(A[4] * s[1])
-                    + mod_m2(A[5] * s[2]));
-
-        x[2] = mod_m2(mod_m2(A[6] * s[0])
-                    + mod_m2(A[7] * s[1])
-                    + mod_m2(A[8] * s[2]));
-
-        s[0] = x[0];
-        s[1] = x[1];
-        s[2] = x[2];
-    }
-
-    FQUALIFIERS
-    unsigned long long mod_mul_m1(unsigned int i,
-                                  unsigned long long j)
-    {
-        long long hi, lo, temp1, temp2;
-
-        hi = i / 131072;
-        lo = i - (hi * 131072);
-        temp1 = mod_m1(hi * j) * 131072;
-        temp2 = mod_m1(lo * j);
-        lo = mod_m1(temp1 + temp2);
-
-        if (lo < 0)
-            lo += ROCRAND_MRG32K3A_M1;
-        return lo;
-    }
-
-    FQUALIFIERS
-    unsigned long long mod_m1(unsigned long long p)
-    {
-        p = detail::mad_u64_u32(ROCRAND_MRG32K3A_M1C, (p >> 32), p & (ROCRAND_MRG32K3A_POW32 - 1));
-        if (p >= ROCRAND_MRG32K3A_M1)
-            p -= ROCRAND_MRG32K3A_M1;
-
-        return p;
-    }
-
-    FQUALIFIERS
-    unsigned long long mod_mul_m2(unsigned int i,
-                                  unsigned long long j)
-    {
-        long long hi, lo, temp1, temp2;
-
-        hi = i / 131072;
-        lo = i - (hi * 131072);
-        temp1 = mod_m2(hi * j) * 131072;
-        temp2 = mod_m2(lo * j);
-        lo = mod_m2(temp1 + temp2);
-
-        if (lo < 0)
-            lo += ROCRAND_MRG32K3A_M2;
-        return lo;
-    }
-
-    FQUALIFIERS
-    unsigned long long mod_m2(unsigned long long p)
-    {
-        p = detail::mad_u64_u32(ROCRAND_MRG32K3A_M2C, (p >> 32), p & (ROCRAND_MRG32K3A_POW32 - 1));
-        p = detail::mad_u64_u32(ROCRAND_MRG32K3A_M2C, (p >> 32), p & (ROCRAND_MRG32K3A_POW32 - 1));
-        if (p >= ROCRAND_MRG32K3A_M2)
-            p -= ROCRAND_MRG32K3A_M2;
-
-        return p;
-    }
-
-protected:
-    // State
-    mrg32k3a_state m_state;
-
-    #ifndef ROCRAND_DETAIL_MRG32K3A_BM_NOT_IN_STATE
-    friend struct detail::engine_boxmuller_helper<mrg32k3a_engine>;
-    #endif
-
-}; // mrg32k3a_engine class
-
-} // end namespace rocrand_device
-
-/** \rocrand_internal \addtogroup rocranddevice
- *
- *  @{
- */
-
-/// \cond ROCRAND_KERNEL_DOCS_TYPEDEFS
-typedef rocrand_device::mrg32k3a_engine rocrand_state_mrg32k3a;
-/// \endcond
-
-/**
- * \brief Initializes MRG32K3A state.
- *
- * Initializes the MRG32K3A generator \p state with the given
- * \p seed, \p subsequence, and \p offset.
- *
- * \param seed - Value to use as a seed
- * \param subsequence - Subsequence to start at
- * \param offset - Absolute offset into subsequence
- * \param state - Pointer to state to initialize
- */
-FQUALIFIERS
-void rocrand_init(const unsigned long long seed,
-                  const unsigned long long subsequence,
-                  const unsigned long long offset,
-                  rocrand_state_mrg32k3a * state)
-{
-    *state = rocrand_state_mrg32k3a(seed, subsequence, offset);
-}
-
-/**
- * \brief Returns uniformly distributed random <tt>unsigned int</tt> value
- * from [0; 2^32 - 1] range.
- *
- * Generates and returns uniformly distributed random <tt>unsigned int</tt>
- * value from [0; 2^32 - 1] range using MRG32K3A generator in \p state.
- * State is incremented by one position.
- *
- * \param state - Pointer to a state to use
- *
- * \return Pseudorandom value (32-bit) as an <tt>unsigned int</tt>
- */
-FQUALIFIERS
-unsigned int rocrand(rocrand_state_mrg32k3a * state)
-{
-    // next() in [1, ROCRAND_MRG32K3A_M1]
-    return static_cast<unsigned int>((state->next() - 1) * ROCRAND_MRG32K3A_UINT_NORM);
-}
-
-/**
- * \brief Updates MRG32K3A state to skip ahead by \p offset elements.
- *
- * Updates the MRG32K3A state in \p state to skip ahead by \p offset elements.
- *
- * \param offset - Number of elements to skip
- * \param state - Pointer to state to update
- */
-FQUALIFIERS
-void skipahead(unsigned long long offset, rocrand_state_mrg32k3a * state)
-{
-    return state->discard(offset);
-}
-
-/**
- * \brief Updates MRG32K3A state to skip ahead by \p subsequence subsequences.
- *
- * Updates the MRG32K3A state in \p state to skip ahead by \p subsequence subsequences.
- * Each subsequence is 2^67 numbers long.
- *
- * \param subsequence - Number of subsequences to skip
- * \param state - Pointer to state to update
- */
-FQUALIFIERS
-void skipahead_subsequence(unsigned long long subsequence, rocrand_state_mrg32k3a * state)
-{
-    return state->discard_subsequence(subsequence);
-}
-
-/**
- * \brief Updates MRG32K3A state to skip ahead by \p sequence sequences.
- *
- * Updates the MRG32K3A state in \p state to skip ahead by \p sequence sequences.
- * Each sequence is 2^127 numbers long.
- *
- * \param sequence - Number of sequences to skip
- * \param state - Pointer to state to update
- */
-FQUALIFIERS
-void skipahead_sequence(unsigned long long sequence, rocrand_state_mrg32k3a * state)
-{
-    return state->discard_sequence(sequence);
-}
-
-#endif // ROCRAND_MRG32K3A_H_
-
-/** @} */ // end of group rocranddevice
diff --git a/include/externals/rocrand/rocrand_mrg32k3a_precomputed.h b/include/externals/rocrand/rocrand_mrg32k3a_precomputed.h
deleted file mode 100644
index 25ad28813f..0000000000
--- a/include/externals/rocrand/rocrand_mrg32k3a_precomputed.h
+++ /dev/null
@@ -1,859 +0,0 @@
-// Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-
-#ifndef ROCRAND_MRG32K3A_PRECOMPUTED_H_
-#define ROCRAND_MRG32K3A_PRECOMPUTED_H_
-
-// Auto-generated file. Do not edit!
-// Generated by tools/mrg32k3a_precomputed_generator
-
-#define MRG323A_DIM 64
-#define MRG323A_N 576
-
-static constexpr __device__ unsigned long long d_A1[MRG323A_N] =
-    {
-        0, 1, 0, 0, 0, 1, 4294156359, 1403580, 0,
-        0, 0, 1, 4294156359, 1403580, 0, 0, 4294156359, 1403580,
-        0, 4294156359, 1403580, 244671815, 2941890554, 4294156359, 149925673, 489343630, 2941890554,
-        1527363550, 2758233149, 1831234280, 4072640363, 939574583, 2758233149, 2064391165, 3228066636, 939574583,
-        736416029, 2961816100, 342112271, 387300998, 1062452522, 2961816100, 2955879160, 340793741, 1062452522,
-        1243502014, 2218748291, 1709215645, 2019641772, 3847560959, 2218748291, 3866010231, 2305448679, 3847560959,
-        3241775219, 3453352062, 3721871040, 4062454730, 3015754, 3453352062, 919711945, 613405362, 3015754,
-        1955221006, 1414472808, 1746037714, 3653507277, 1644962013, 1414472808, 3501544776, 2336229602, 1644962013,
-        1170096663, 49135452, 3441537107, 1857945175, 1649398389, 49135452, 333002869, 3109147376, 1649398389,
-        2299034194, 2297111910, 862649200, 1399961132, 996706937, 2297111910, 3439056503, 1481993076, 996706937,
-        4146310528, 458782589, 1007330283, 4241015765, 3979619964, 458782589, 553886495, 2186897562, 3979619964,
-        3630027893, 2130448350, 292773857, 1392525159, 1299285967, 2130448350, 2589171163, 1217405758, 1299285967,
-        892409263, 1999175811, 2979225418, 1996163538, 2148702503, 1999175811, 3922720782, 103819730, 2148702503,
-        1586003016, 2114210471, 3240775579, 2777288607, 1400478398, 2114210471, 3018215420, 535326008, 1400478398,
-        2188531273, 1783231160, 3576659343, 1908318389, 379210133, 1783231160, 554369329, 250053591, 379210133,
-        4022841636, 3951951872, 2143424240, 1046219306, 1591992468, 3951951872, 1510277444, 381333958, 1591992468,
-        2256493727, 3715182130, 642697923, 3615342722, 3975008370, 3715182130, 2405650329, 754337639, 3975008370,
-        1286664224, 627406673, 963516608, 1541344588, 460768826, 627406673, 1089892553, 2717717970, 460768826,
-        2956342842, 3471097641, 2353092905, 2996150472, 420480221, 3471097641, 2221681883, 372736411, 420480221,
-        420492906, 153526651, 3499730988, 2662640502, 3278195133, 153526651, 4086436419, 2510762118, 3278195133,
-        3310184147, 2228376089, 823220763, 3992771814, 1693168425, 2228376089, 2295790366, 1401872772, 1693168425,
-        2529428830, 1497104068, 4253248635, 3746310018, 630867741, 1497104068, 627043435, 721725795, 630867741,
-        2571072593, 3039669025, 1591031831, 526054481, 661344445, 3039669025, 4246010312, 735391270, 661344445,
-        1847312821, 4042890210, 4241772463, 606605705, 2644799309, 4042890210, 2658402822, 1342278931, 2644799309,
-        2409846784, 1096138313, 1416249993, 1501878241, 138013862, 1096138313, 1617749306, 1975136163, 138013862,
-        599453422, 73950522, 2965395603, 55354701, 3855242202, 73950522, 3981734504, 3354399019, 3855242202,
-        4271076381, 813410089, 3461955319, 1044920137, 3029005516, 813410089, 3501837362, 3321539504, 3029005516,
-        3058183515, 941408572, 1783998098, 1546486080, 4116985007, 941408572, 2247500745, 1460625377, 4116985007,
-        4216782514, 3352801941, 2315095646, 639029973, 94451952, 3352801941, 1242898773, 3964593332, 94451952,
-        2264905138, 1926285644, 1108147171, 2390706911, 385258225, 1926285644, 3569882325, 3728744670, 385258225,
-        270679073, 1065683096, 2992662885, 4196917281, 2886425156, 1065683096, 749134119, 1849148167, 2886425156,
-        35689930, 1378151623, 951629713, 673810920, 948843427, 1378151623, 3808868984, 927013635, 948843427,
-        1891490872, 1130489594, 3734864133, 1457450350, 3362920032, 1130489594, 638998846, 1401175590, 3362920032,
-        2254459023, 2384691454, 1730098031, 2844861718, 1807491073, 2384691454, 351423668, 1570264155, 1807491073,
-        3047429268, 4245359555, 2449575498, 1797081212, 1237196477, 4245359555, 143400628, 3663731096, 1237196477,
-        3313321106, 4263819658, 1047529624, 3719941673, 3155049403, 4263819658, 1981313839, 4281524426, 3155049403,
-        2005252417, 3263186729, 1535805957, 2951515865, 1729281525, 3263186729, 1141249417, 2268963059, 1729281525,
-        2367065164, 83908466, 4294308508, 1352516724, 1416676049, 83908466, 1040867745, 1304732377, 1416676049,
-        3214147257, 1434230503, 2944821434, 2753040912, 4041536918, 1434230503, 1317260239, 338830578, 4041536918,
-        300628476, 2054743463, 1499597869, 1762244284, 1422043015, 2054743463, 3581125669, 1207561803, 1422043015,
-        4171745404, 4064983592, 1934508265, 3049723261, 1744636487, 4064983592, 947753516, 3952135907, 1744636487,
-        1625369148, 3577024659, 2778677259, 1729967818, 1049600974, 3577024659, 2089137344, 1569794605, 1049600974,
-        1373068765, 3958611830, 569117280, 410042396, 3551255470, 3958611830, 869476379, 1680625376, 3551255470,
-        2108618602, 2543645250, 913717833, 2111984988, 1012482542, 2543645250, 2545745615, 3141042890, 1012482542,
-        1157293598, 584852249, 2272893205, 1631801979, 3013855247, 584852249, 3977310441, 82049263, 3013855247,
-        3580234334, 3137526662, 2403875621, 3580869206, 3670086228, 3137526662, 656744553, 1764904195, 3670086228,
-        2792496861, 3634185196, 3887031679, 3601823850, 3464838365, 3634185196, 3136165138, 2842987937, 3464838365,
-        1362557480, 3230022138, 4278720212, 3427386258, 3848976950, 3230022138, 2109817045, 2441486578, 3848976950,
-        1198519135, 2007945401, 3868481, 3335076429, 2082683147, 2007945401, 2341088247, 888193479, 2082683147,
-        3473925387, 3193380570, 565138859, 307060547, 782210925, 3193380570, 167617770, 2180014252, 782210925,
-        3811588895, 3303532086, 2766583698, 908630605, 2665400165, 3303532086, 2499994113, 3316180851, 2665400165,
-        4288926968, 3033075037, 1505732852, 1531633406, 645804125, 3033075037, 2942690261, 2205365640, 645804125,
-        3976196483, 3651411522, 1652430357, 1690405883, 1294990760, 3651411522, 209339647, 3088484327, 1294990760,
-        3171589548, 2291131070, 2093793287, 2997812074, 4093879780, 2291131070, 3255666800, 858124816, 4093879780,
-        4113016361, 2999667479, 3995043314, 1333973326, 4007774239, 2999667479, 3322921863, 4278103786, 4007774239,
-        925786347, 2109676036, 1879981040, 1701566570, 1489702270, 2109676036, 2719807628, 158549605, 1489702270,
-        2255405265, 3460246357, 218033453, 2135115875, 359516994, 3460246357, 3568862459, 3114762683, 359516994,
-        773148471, 4117539411, 3073622315, 3807175775, 186466108, 4117539411, 2842197411, 651334129, 186466108,
-        615242951, 1475251263, 3586439101, 1693917167, 3058812486, 1475251263, 568701600, 1164226398, 3058812486,
-        1632636204, 15370275, 2061555515, 4187505695, 1741164221, 15370275, 2882176274, 3978412194, 1741164221,
-        3446066703, 344820524, 74213775, 1008543583, 2579620192, 344820524, 3753911358, 1538453821, 2579620192,
-        3600859892, 1269921024, 4069458760, 2050939727, 2222725697, 1269921024, 3208347646, 690898125, 2222725697,
-        599407451, 2806239788, 1742216102, 975123999, 764869161, 2806239788, 2729710367, 1845257036, 764869161,
-        967330218, 3464884028, 3444447102, 580449578, 1343714307, 3464884028, 1775329096, 4027221761, 1343714307,
-
-    };
-
-static constexpr unsigned long long h_A1[MRG323A_N] =
-    {
-        0, 1, 0, 0, 0, 1, 4294156359, 1403580, 0,
-        0, 0, 1, 4294156359, 1403580, 0, 0, 4294156359, 1403580,
-        0, 4294156359, 1403580, 244671815, 2941890554, 4294156359, 149925673, 489343630, 2941890554,
-        1527363550, 2758233149, 1831234280, 4072640363, 939574583, 2758233149, 2064391165, 3228066636, 939574583,
-        736416029, 2961816100, 342112271, 387300998, 1062452522, 2961816100, 2955879160, 340793741, 1062452522,
-        1243502014, 2218748291, 1709215645, 2019641772, 3847560959, 2218748291, 3866010231, 2305448679, 3847560959,
-        3241775219, 3453352062, 3721871040, 4062454730, 3015754, 3453352062, 919711945, 613405362, 3015754,
-        1955221006, 1414472808, 1746037714, 3653507277, 1644962013, 1414472808, 3501544776, 2336229602, 1644962013,
-        1170096663, 49135452, 3441537107, 1857945175, 1649398389, 49135452, 333002869, 3109147376, 1649398389,
-        2299034194, 2297111910, 862649200, 1399961132, 996706937, 2297111910, 3439056503, 1481993076, 996706937,
-        4146310528, 458782589, 1007330283, 4241015765, 3979619964, 458782589, 553886495, 2186897562, 3979619964,
-        3630027893, 2130448350, 292773857, 1392525159, 1299285967, 2130448350, 2589171163, 1217405758, 1299285967,
-        892409263, 1999175811, 2979225418, 1996163538, 2148702503, 1999175811, 3922720782, 103819730, 2148702503,
-        1586003016, 2114210471, 3240775579, 2777288607, 1400478398, 2114210471, 3018215420, 535326008, 1400478398,
-        2188531273, 1783231160, 3576659343, 1908318389, 379210133, 1783231160, 554369329, 250053591, 379210133,
-        4022841636, 3951951872, 2143424240, 1046219306, 1591992468, 3951951872, 1510277444, 381333958, 1591992468,
-        2256493727, 3715182130, 642697923, 3615342722, 3975008370, 3715182130, 2405650329, 754337639, 3975008370,
-        1286664224, 627406673, 963516608, 1541344588, 460768826, 627406673, 1089892553, 2717717970, 460768826,
-        2956342842, 3471097641, 2353092905, 2996150472, 420480221, 3471097641, 2221681883, 372736411, 420480221,
-        420492906, 153526651, 3499730988, 2662640502, 3278195133, 153526651, 4086436419, 2510762118, 3278195133,
-        3310184147, 2228376089, 823220763, 3992771814, 1693168425, 2228376089, 2295790366, 1401872772, 1693168425,
-        2529428830, 1497104068, 4253248635, 3746310018, 630867741, 1497104068, 627043435, 721725795, 630867741,
-        2571072593, 3039669025, 1591031831, 526054481, 661344445, 3039669025, 4246010312, 735391270, 661344445,
-        1847312821, 4042890210, 4241772463, 606605705, 2644799309, 4042890210, 2658402822, 1342278931, 2644799309,
-        2409846784, 1096138313, 1416249993, 1501878241, 138013862, 1096138313, 1617749306, 1975136163, 138013862,
-        599453422, 73950522, 2965395603, 55354701, 3855242202, 73950522, 3981734504, 3354399019, 3855242202,
-        4271076381, 813410089, 3461955319, 1044920137, 3029005516, 813410089, 3501837362, 3321539504, 3029005516,
-        3058183515, 941408572, 1783998098, 1546486080, 4116985007, 941408572, 2247500745, 1460625377, 4116985007,
-        4216782514, 3352801941, 2315095646, 639029973, 94451952, 3352801941, 1242898773, 3964593332, 94451952,
-        2264905138, 1926285644, 1108147171, 2390706911, 385258225, 1926285644, 3569882325, 3728744670, 385258225,
-        270679073, 1065683096, 2992662885, 4196917281, 2886425156, 1065683096, 749134119, 1849148167, 2886425156,
-        35689930, 1378151623, 951629713, 673810920, 948843427, 1378151623, 3808868984, 927013635, 948843427,
-        1891490872, 1130489594, 3734864133, 1457450350, 3362920032, 1130489594, 638998846, 1401175590, 3362920032,
-        2254459023, 2384691454, 1730098031, 2844861718, 1807491073, 2384691454, 351423668, 1570264155, 1807491073,
-        3047429268, 4245359555, 2449575498, 1797081212, 1237196477, 4245359555, 143400628, 3663731096, 1237196477,
-        3313321106, 4263819658, 1047529624, 3719941673, 3155049403, 4263819658, 1981313839, 4281524426, 3155049403,
-        2005252417, 3263186729, 1535805957, 2951515865, 1729281525, 3263186729, 1141249417, 2268963059, 1729281525,
-        2367065164, 83908466, 4294308508, 1352516724, 1416676049, 83908466, 1040867745, 1304732377, 1416676049,
-        3214147257, 1434230503, 2944821434, 2753040912, 4041536918, 1434230503, 1317260239, 338830578, 4041536918,
-        300628476, 2054743463, 1499597869, 1762244284, 1422043015, 2054743463, 3581125669, 1207561803, 1422043015,
-        4171745404, 4064983592, 1934508265, 3049723261, 1744636487, 4064983592, 947753516, 3952135907, 1744636487,
-        1625369148, 3577024659, 2778677259, 1729967818, 1049600974, 3577024659, 2089137344, 1569794605, 1049600974,
-        1373068765, 3958611830, 569117280, 410042396, 3551255470, 3958611830, 869476379, 1680625376, 3551255470,
-        2108618602, 2543645250, 913717833, 2111984988, 1012482542, 2543645250, 2545745615, 3141042890, 1012482542,
-        1157293598, 584852249, 2272893205, 1631801979, 3013855247, 584852249, 3977310441, 82049263, 3013855247,
-        3580234334, 3137526662, 2403875621, 3580869206, 3670086228, 3137526662, 656744553, 1764904195, 3670086228,
-        2792496861, 3634185196, 3887031679, 3601823850, 3464838365, 3634185196, 3136165138, 2842987937, 3464838365,
-        1362557480, 3230022138, 4278720212, 3427386258, 3848976950, 3230022138, 2109817045, 2441486578, 3848976950,
-        1198519135, 2007945401, 3868481, 3335076429, 2082683147, 2007945401, 2341088247, 888193479, 2082683147,
-        3473925387, 3193380570, 565138859, 307060547, 782210925, 3193380570, 167617770, 2180014252, 782210925,
-        3811588895, 3303532086, 2766583698, 908630605, 2665400165, 3303532086, 2499994113, 3316180851, 2665400165,
-        4288926968, 3033075037, 1505732852, 1531633406, 645804125, 3033075037, 2942690261, 2205365640, 645804125,
-        3976196483, 3651411522, 1652430357, 1690405883, 1294990760, 3651411522, 209339647, 3088484327, 1294990760,
-        3171589548, 2291131070, 2093793287, 2997812074, 4093879780, 2291131070, 3255666800, 858124816, 4093879780,
-        4113016361, 2999667479, 3995043314, 1333973326, 4007774239, 2999667479, 3322921863, 4278103786, 4007774239,
-        925786347, 2109676036, 1879981040, 1701566570, 1489702270, 2109676036, 2719807628, 158549605, 1489702270,
-        2255405265, 3460246357, 218033453, 2135115875, 359516994, 3460246357, 3568862459, 3114762683, 359516994,
-        773148471, 4117539411, 3073622315, 3807175775, 186466108, 4117539411, 2842197411, 651334129, 186466108,
-        615242951, 1475251263, 3586439101, 1693917167, 3058812486, 1475251263, 568701600, 1164226398, 3058812486,
-        1632636204, 15370275, 2061555515, 4187505695, 1741164221, 15370275, 2882176274, 3978412194, 1741164221,
-        3446066703, 344820524, 74213775, 1008543583, 2579620192, 344820524, 3753911358, 1538453821, 2579620192,
-        3600859892, 1269921024, 4069458760, 2050939727, 2222725697, 1269921024, 3208347646, 690898125, 2222725697,
-        599407451, 2806239788, 1742216102, 975123999, 764869161, 2806239788, 2729710367, 1845257036, 764869161,
-        967330218, 3464884028, 3444447102, 580449578, 1343714307, 3464884028, 1775329096, 4027221761, 1343714307,
-
-    };
-
-static constexpr __device__ unsigned long long d_A2[MRG323A_N] =
-    {
-        0, 1, 0, 0, 0, 1, 4293573854, 0, 527612,
-        0, 0, 1, 4293573854, 0, 527612, 2706407399, 4293573854, 3497978192,
-        2706407399, 4293573854, 3497978192, 1431525864, 2706407399, 3281754271, 97673890, 1431525864, 1673476130,
-        3405842137, 2680076935, 893509979, 4035147174, 3405842137, 3280220074, 2623373296, 4035147174, 361718588,
-        818368950, 3790774567, 3542344109, 1817134745, 818368950, 3321940838, 3493477402, 1817134745, 2854655037,
-        498682467, 2928649385, 811441367, 1777037472, 498682467, 479207863, 3058260025, 1777037472, 1528225099,
-        3893311647, 3140922085, 64039185, 82107183, 3893311647, 2655465224, 1674879036, 82107183, 1089381262,
-        28639152, 3496041927, 2231910770, 3174683233, 28639152, 2828785870, 3681140872, 3174683233, 3910194649,
-        1463826069, 300842059, 3313769518, 1799677538, 1463826069, 3174861078, 1882279394, 1799677538, 3509975160,
-        2092194020, 184076987, 2202401252, 3103629604, 2092194020, 3409560232, 4257445059, 3103629604, 2390202783,
-        812917091, 2574011276, 4168802395, 209817750, 812917091, 2974870628, 3238802184, 209817750, 3692836406,
-        477309738, 3314523413, 3442242150, 2755731404, 477309738, 2782713347, 1606221490, 2755731404, 1033463096,
-        2155469603, 3326516116, 3843369786, 288604458, 2155469603, 571673571, 1501677614, 288604458, 2928213494,
-        2082469029, 749754403, 3963963316, 2764859700, 2082469029, 3576428059, 2840894706, 2764859700, 1782279859,
-        3760163766, 1041986082, 1799196192, 1022129134, 3760163766, 1332558840, 276873446, 1022129134, 3979423632,
-        1021313167, 1312544548, 1716381787, 3037868518, 1021313167, 199085085, 2582787611, 3037868518, 3539882179,
-        2569413030, 1631336015, 2594942403, 1030618503, 2569413030, 3467650326, 1998739584, 1030618503, 3174552073,
-        2334639309, 3114094203, 601680947, 2110199318, 2334639309, 678342865, 1649523168, 2110199318, 2154948056,
-        563657176, 191330473, 1641595774, 780563537, 563657176, 3029522338, 2037330914, 780563537, 2084602709,
-        3414769923, 1968799026, 2238126504, 832866376, 3414769923, 3754780168, 2165145850, 832866376, 1594768331,
-        1646861218, 2317984620, 2301581548, 2672536210, 1646861218, 359763062, 2391283983, 2672536210, 1885870777,
-        841254072, 3765813448, 1635365181, 2013240130, 841254072, 605925849, 3743932305, 2013240130, 400681955,
-        1930213004, 2072952279, 3077694794, 3579956569, 1930213004, 2478539210, 1960229502, 3579956569, 1455652656,
-        1097613522, 1784540933, 1194440107, 321747515, 1097613522, 1225209584, 74521379, 321747515, 4288531000,
-        143812745, 3254530816, 3514348856, 769295000, 143812745, 2468210728, 1927161272, 769295000, 522705580,
-        2692035063, 2596905012, 1643240704, 1103432342, 2692035063, 1446182108, 4161111774, 1103432342, 3076435551,
-        2375319030, 1391532370, 3742334018, 1202100604, 2375319030, 4098434768, 2327872488, 1202100604, 1471526950,
-        4269164791, 2795313144, 2507855960, 4245372460, 4269164791, 4094914553, 3873219634, 4245372460, 1473695507,
-        513890845, 1208902926, 2870530442, 1984873167, 513890845, 1257532340, 1212627640, 1984873167, 2354363842,
-        1848364568, 1552116673, 3496528455, 4160778291, 1848364568, 141769900, 3611019106, 4160778291, 596424080,
-        364070020, 3520039729, 837362349, 2544671570, 364070020, 2188646679, 163978331, 2544671570, 672947816,
-        1192700714, 3968150021, 298357363, 635565666, 1192700714, 2589432341, 2548654227, 635565666, 3531570992,
-        2709640529, 676525399, 875361870, 1315499519, 2709640529, 3842690720, 3300994644, 1315499519, 2446760804,
-        2742149264, 1410604392, 3032350755, 3774935330, 2742149264, 597633965, 4085935803, 3774935330, 3952463556,
-        3878579563, 845297523, 1721916511, 2077922420, 3878579563, 3651360351, 2177255734, 2077922420, 3791239282,
-        1570315355, 4252790045, 3522351060, 2324624266, 1570315355, 3594939336, 1725087354, 2324624266, 1338343327,
-        2305761589, 381933244, 3663579047, 1355307047, 2305761589, 313617972, 992174375, 1355307047, 3881593435,
-        1667857811, 1564715297, 2263851601, 3791771273, 1667857811, 4196134923, 3347975047, 3791771273, 615040705,
-        4093947334, 3454015638, 2815567716, 4261953004, 4093947334, 3973733876, 2979573134, 4261953004, 3757047667,
-        250120061, 570149551, 1513430926, 3178644752, 250120061, 1701869032, 4172515680, 3178644752, 4213855850,
-        4158106802, 3062358456, 1815738463, 1379176112, 4158106802, 3926509890, 2842564878, 1379176112, 2852219546,
-        931848746, 256263523, 2633569246, 3284646837, 931848746, 2567084715, 415258465, 3284646837, 2017565947,
-        1648005210, 1032291296, 3987397422, 1831496020, 1648005210, 2829448427, 1821082272, 1831496020, 2917140265,
-        4161327077, 489964129, 3870847744, 1669447863, 4161327077, 4292947198, 1522417114, 1669447863, 2652286672,
-        1270934555, 3136631324, 505612043, 2981474723, 1270934555, 2528619024, 625182639, 2981474723, 1008985039,
-        280996820, 143706137, 3013099060, 1797675893, 280996820, 3743985508, 1123794455, 1797675893, 2460119169,
-        919218027, 4154920441, 1125672685, 3933041881, 919218027, 474242849, 564891116, 3933041881, 2263904321,
-        2920112852, 1965329198, 1177141043, 2135250851, 2920112852, 969184056, 296035385, 2135250851, 4267827987,
-        1481142942, 4120754772, 1088557292, 265491023, 1481142942, 2860005744, 301796252, 265491023, 1935975979,
-        2111859033, 2813610100, 1001476468, 73849832, 2111859033, 3980799998, 3330206241, 73849832, 1933943506,
-        1781286360, 3661231931, 3509383709, 2753158871, 1781286360, 3119883109, 3576525143, 2753158871, 551079002,
-        1185024844, 587779104, 1004942725, 3763632860, 1185024844, 947424568, 3811666068, 3763632860, 2352253462,
-        1310227170, 218138208, 3172947233, 766129426, 1310227170, 1808643264, 2226659371, 766129426, 3853798112,
-        2230902378, 4243560874, 2491962392, 3836629116, 2230902378, 3637515403, 2846140932, 3836629116, 3083355464,
-        999448569, 1464488480, 3344426626, 946166795, 999448569, 340856814, 3686999436, 946166795, 3231079441,
-        1226155368, 3477563770, 550006884, 2378667355, 1226155368, 1493409040, 260364836, 2378667355, 4133888397,
-        1277901832, 310796286, 2818511068, 3088910653, 1277901832, 3303406025, 2507911914, 3088910653, 3712928074,
-        481918378, 339570348, 1728801469, 1623163429, 481918378, 2209094694, 3146982514, 1623163429, 508445538,
-        3138921230, 2381863183, 1992357430, 1024510915, 3138921230, 2122851650, 1453455184, 1024510915, 941946604,
-        2465372719, 1391015357, 3328905025, 1821933605, 2465372719, 1343489680, 3648970313, 1821933605, 1816599716,
-        118634664, 3358712512, 2492792220, 348833376, 118634664, 2495544591, 3235582254, 348833376, 4043157504,
-        2303067090, 3371139074, 1967771133, 598630070, 2303067090, 1819012637, 2049250561, 598630070, 4093044926,
-        3035321857, 3971176093, 226779704, 3361614254, 3035321857, 2807125404, 326640887, 3361614254, 3147308542,
-        1774298149, 4179629947, 3145006948, 1688753503, 1774298149, 94869516, 2327946901, 1688753503, 2786835219,
-
-    };
-
-static constexpr unsigned long long h_A2[MRG323A_N] =
-    {
-        0, 1, 0, 0, 0, 1, 4293573854, 0, 527612,
-        0, 0, 1, 4293573854, 0, 527612, 2706407399, 4293573854, 3497978192,
-        2706407399, 4293573854, 3497978192, 1431525864, 2706407399, 3281754271, 97673890, 1431525864, 1673476130,
-        3405842137, 2680076935, 893509979, 4035147174, 3405842137, 3280220074, 2623373296, 4035147174, 361718588,
-        818368950, 3790774567, 3542344109, 1817134745, 818368950, 3321940838, 3493477402, 1817134745, 2854655037,
-        498682467, 2928649385, 811441367, 1777037472, 498682467, 479207863, 3058260025, 1777037472, 1528225099,
-        3893311647, 3140922085, 64039185, 82107183, 3893311647, 2655465224, 1674879036, 82107183, 1089381262,
-        28639152, 3496041927, 2231910770, 3174683233, 28639152, 2828785870, 3681140872, 3174683233, 3910194649,
-        1463826069, 300842059, 3313769518, 1799677538, 1463826069, 3174861078, 1882279394, 1799677538, 3509975160,
-        2092194020, 184076987, 2202401252, 3103629604, 2092194020, 3409560232, 4257445059, 3103629604, 2390202783,
-        812917091, 2574011276, 4168802395, 209817750, 812917091, 2974870628, 3238802184, 209817750, 3692836406,
-        477309738, 3314523413, 3442242150, 2755731404, 477309738, 2782713347, 1606221490, 2755731404, 1033463096,
-        2155469603, 3326516116, 3843369786, 288604458, 2155469603, 571673571, 1501677614, 288604458, 2928213494,
-        2082469029, 749754403, 3963963316, 2764859700, 2082469029, 3576428059, 2840894706, 2764859700, 1782279859,
-        3760163766, 1041986082, 1799196192, 1022129134, 3760163766, 1332558840, 276873446, 1022129134, 3979423632,
-        1021313167, 1312544548, 1716381787, 3037868518, 1021313167, 199085085, 2582787611, 3037868518, 3539882179,
-        2569413030, 1631336015, 2594942403, 1030618503, 2569413030, 3467650326, 1998739584, 1030618503, 3174552073,
-        2334639309, 3114094203, 601680947, 2110199318, 2334639309, 678342865, 1649523168, 2110199318, 2154948056,
-        563657176, 191330473, 1641595774, 780563537, 563657176, 3029522338, 2037330914, 780563537, 2084602709,
-        3414769923, 1968799026, 2238126504, 832866376, 3414769923, 3754780168, 2165145850, 832866376, 1594768331,
-        1646861218, 2317984620, 2301581548, 2672536210, 1646861218, 359763062, 2391283983, 2672536210, 1885870777,
-        841254072, 3765813448, 1635365181, 2013240130, 841254072, 605925849, 3743932305, 2013240130, 400681955,
-        1930213004, 2072952279, 3077694794, 3579956569, 1930213004, 2478539210, 1960229502, 3579956569, 1455652656,
-        1097613522, 1784540933, 1194440107, 321747515, 1097613522, 1225209584, 74521379, 321747515, 4288531000,
-        143812745, 3254530816, 3514348856, 769295000, 143812745, 2468210728, 1927161272, 769295000, 522705580,
-        2692035063, 2596905012, 1643240704, 1103432342, 2692035063, 1446182108, 4161111774, 1103432342, 3076435551,
-        2375319030, 1391532370, 3742334018, 1202100604, 2375319030, 4098434768, 2327872488, 1202100604, 1471526950,
-        4269164791, 2795313144, 2507855960, 4245372460, 4269164791, 4094914553, 3873219634, 4245372460, 1473695507,
-        513890845, 1208902926, 2870530442, 1984873167, 513890845, 1257532340, 1212627640, 1984873167, 2354363842,
-        1848364568, 1552116673, 3496528455, 4160778291, 1848364568, 141769900, 3611019106, 4160778291, 596424080,
-        364070020, 3520039729, 837362349, 2544671570, 364070020, 2188646679, 163978331, 2544671570, 672947816,
-        1192700714, 3968150021, 298357363, 635565666, 1192700714, 2589432341, 2548654227, 635565666, 3531570992,
-        2709640529, 676525399, 875361870, 1315499519, 2709640529, 3842690720, 3300994644, 1315499519, 2446760804,
-        2742149264, 1410604392, 3032350755, 3774935330, 2742149264, 597633965, 4085935803, 3774935330, 3952463556,
-        3878579563, 845297523, 1721916511, 2077922420, 3878579563, 3651360351, 2177255734, 2077922420, 3791239282,
-        1570315355, 4252790045, 3522351060, 2324624266, 1570315355, 3594939336, 1725087354, 2324624266, 1338343327,
-        2305761589, 381933244, 3663579047, 1355307047, 2305761589, 313617972, 992174375, 1355307047, 3881593435,
-        1667857811, 1564715297, 2263851601, 3791771273, 1667857811, 4196134923, 3347975047, 3791771273, 615040705,
-        4093947334, 3454015638, 2815567716, 4261953004, 4093947334, 3973733876, 2979573134, 4261953004, 3757047667,
-        250120061, 570149551, 1513430926, 3178644752, 250120061, 1701869032, 4172515680, 3178644752, 4213855850,
-        4158106802, 3062358456, 1815738463, 1379176112, 4158106802, 3926509890, 2842564878, 1379176112, 2852219546,
-        931848746, 256263523, 2633569246, 3284646837, 931848746, 2567084715, 415258465, 3284646837, 2017565947,
-        1648005210, 1032291296, 3987397422, 1831496020, 1648005210, 2829448427, 1821082272, 1831496020, 2917140265,
-        4161327077, 489964129, 3870847744, 1669447863, 4161327077, 4292947198, 1522417114, 1669447863, 2652286672,
-        1270934555, 3136631324, 505612043, 2981474723, 1270934555, 2528619024, 625182639, 2981474723, 1008985039,
-        280996820, 143706137, 3013099060, 1797675893, 280996820, 3743985508, 1123794455, 1797675893, 2460119169,
-        919218027, 4154920441, 1125672685, 3933041881, 919218027, 474242849, 564891116, 3933041881, 2263904321,
-        2920112852, 1965329198, 1177141043, 2135250851, 2920112852, 969184056, 296035385, 2135250851, 4267827987,
-        1481142942, 4120754772, 1088557292, 265491023, 1481142942, 2860005744, 301796252, 265491023, 1935975979,
-        2111859033, 2813610100, 1001476468, 73849832, 2111859033, 3980799998, 3330206241, 73849832, 1933943506,
-        1781286360, 3661231931, 3509383709, 2753158871, 1781286360, 3119883109, 3576525143, 2753158871, 551079002,
-        1185024844, 587779104, 1004942725, 3763632860, 1185024844, 947424568, 3811666068, 3763632860, 2352253462,
-        1310227170, 218138208, 3172947233, 766129426, 1310227170, 1808643264, 2226659371, 766129426, 3853798112,
-        2230902378, 4243560874, 2491962392, 3836629116, 2230902378, 3637515403, 2846140932, 3836629116, 3083355464,
-        999448569, 1464488480, 3344426626, 946166795, 999448569, 340856814, 3686999436, 946166795, 3231079441,
-        1226155368, 3477563770, 550006884, 2378667355, 1226155368, 1493409040, 260364836, 2378667355, 4133888397,
-        1277901832, 310796286, 2818511068, 3088910653, 1277901832, 3303406025, 2507911914, 3088910653, 3712928074,
-        481918378, 339570348, 1728801469, 1623163429, 481918378, 2209094694, 3146982514, 1623163429, 508445538,
-        3138921230, 2381863183, 1992357430, 1024510915, 3138921230, 2122851650, 1453455184, 1024510915, 941946604,
-        2465372719, 1391015357, 3328905025, 1821933605, 2465372719, 1343489680, 3648970313, 1821933605, 1816599716,
-        118634664, 3358712512, 2492792220, 348833376, 118634664, 2495544591, 3235582254, 348833376, 4043157504,
-        2303067090, 3371139074, 1967771133, 598630070, 2303067090, 1819012637, 2049250561, 598630070, 4093044926,
-        3035321857, 3971176093, 226779704, 3361614254, 3035321857, 2807125404, 326640887, 3361614254, 3147308542,
-        1774298149, 4179629947, 3145006948, 1688753503, 1774298149, 94869516, 2327946901, 1688753503, 2786835219,
-
-    };
-
-static constexpr __device__ unsigned long long d_A1P67[MRG323A_N] =
-    {
-        82758667, 1871391091, 4127413238, 3672831523, 69195019, 1871391091, 3672091415, 3528743235, 69195019,
-        3361372532, 2329303404, 99651939, 2008671965, 2931758910, 2329303404, 1113529483, 2374097189, 2931758910,
-        1831590873, 1588259595, 1314332382, 2385989343, 2508077280, 1588259595, 1787615788, 661437137, 2508077280,
-        2326052247, 4183591379, 4049009082, 2604529491, 1453913233, 4183591379, 2311925423, 1805360390, 1453913233,
-        3956367490, 604461629, 1257432102, 794711716, 1155867175, 604461629, 1777070788, 429445904, 1155867175,
-        1686241617, 1257046062, 1427609439, 490376081, 387798431, 1257046062, 235551485, 1312672615, 387798431,
-        2362447880, 3445363024, 3160262066, 2426867845, 4194339866, 3445363024, 1046144413, 4177893681, 4194339866,
-        4251175413, 3559576374, 3107663662, 697539134, 1909472435, 3559576374, 280754246, 375835695, 1909472435,
-        1099512970, 712404985, 1571467521, 546519870, 1135109300, 712404985, 3325312332, 2352874613, 1135109300,
-        1945425936, 1653045514, 381988982, 3733376326, 414410025, 1653045514, 1181583679, 1185848176, 414410025,
-        2526336124, 3019211015, 4215964965, 2683163472, 4188191530, 3019211015, 2964651598, 293801056, 4188191530,
-        1444052678, 2253324417, 39719589, 1880267534, 2391992038, 2253324417, 987740265, 3691889508, 2391992038,
-        166599066, 2335494420, 1232261118, 2227597731, 2570600780, 2335494420, 2700034538, 3460843234, 2570600780,
-        2511338360, 1188954576, 1251401239, 2511664974, 292276982, 1188954576, 697844082, 3093661552, 292276982,
-        3624650744, 51993077, 3540268009, 3252828938, 3710319575, 51993077, 2858628849, 3910069381, 3710319575,
-        655966702, 754002362, 1646581402, 1958331075, 475572423, 754002362, 3248619000, 3228514800, 475572423,
-        2760311307, 4166372813, 741596417, 2282679206, 3090782630, 4166372813, 3242468721, 1628442374, 3090782630,
-        4265279407, 3532111852, 1754687396, 500404765, 2603727025, 3532111852, 1428367254, 3149485478, 2603727025,
-        2873769531, 2081104178, 596284397, 4153800443, 1261269623, 2081104178, 3967600061, 1830023157, 1261269623,
-        278611533, 2229285304, 3443204327, 3110641420, 77498444, 2229285304, 3904070810, 1070507239, 77498444,
-        544639534, 568528663, 2177189807, 2475829068, 121482268, 568528663, 876978915, 3116647617, 121482268,
-        1547862823, 2404658587, 4191448009, 2158188804, 2976916793, 2404658587, 168571747, 1691884706, 2976916793,
-        3208213311, 4212638780, 3235157352, 671148556, 2951207765, 4212638780, 2075145516, 2395485231, 2951207765,
-        4080517315, 2133433101, 4043998180, 2044221845, 867670560, 2133433101, 834432416, 3613001199, 867670560,
-        4102885735, 1319434267, 2678775073, 740092580, 607380970, 1319434267, 2198271844, 2610193258, 607380970,
-        1165218048, 1317690360, 1189150958, 399240205, 2507168618, 1317690360, 2988334517, 2687593413, 2507168618,
-        1028861702, 4082006648, 338232527, 1888486946, 1842080991, 4082006648, 3903826366, 3109935091, 1842080991,
-        614134826, 2261996505, 2888080641, 710199359, 2773979788, 2261996505, 1144301620, 2554371815, 2773979788,
-        4056173823, 1285620078, 357420018, 2423072612, 2309408315, 1285620078, 1533175115, 2760088020, 2309408315,
-        4264130267, 815015434, 3142242173, 180649975, 2500813569, 815015434, 3378723563, 829683767, 2500813569,
-        4174387531, 1030729435, 2812778314, 1752988797, 4044178729, 1030729435, 467969301, 554748104, 4044178729,
-        1348429235, 2928743274, 3776082629, 3607529209, 3069812185, 2928743274, 2542432347, 3208181168, 3069812185,
-        4064845753, 668285756, 3816217625, 3713143233, 1380634204, 668285756, 3533700508, 1192551435, 1380634204,
-        1515684518, 1706771705, 728123349, 3174850469, 2057456462, 1706771705, 3410402985, 2897339640, 2057456462,
-        3082272717, 531091457, 1390161328, 3895139973, 2171402857, 531091457, 4030688141, 3049703400, 2171402857,
-        1241147206, 3193892819, 1244284192, 65180262, 4065669017, 3193892819, 1484817937, 3661081858, 4065669017,
-        1438760812, 3491341751, 3414470157, 2805337292, 272266053, 3491341751, 824109230, 3202556526, 272266053,
-        135412706, 3627115412, 2345042216, 1565169824, 2166856449, 3627115412, 1026946745, 3467845248, 2166856449,
-        1889419951, 3256876154, 1240505488, 1254783743, 989966800, 3256876154, 1995297400, 3692472918, 989966800,
-        3206226875, 285700890, 496017472, 2515316194, 2129675196, 285700890, 1863853990, 2673457552, 2129675196,
-        4163770641, 255160418, 772100749, 1987092456, 3237660221, 255160418, 1394381051, 4216039401, 3237660221,
-        2133915627, 2713747584, 627765421, 2300605925, 35690583, 2713747584, 2918902946, 2638220304, 35690583,
-        2587549655, 998684270, 4292130625, 1791772791, 2820705344, 998684270, 124590158, 3831143549, 2820705344,
-        978482299, 3200877282, 497605289, 3717741518, 3737164414, 3200877282, 4046686626, 861393946, 3737164414,
-        2665561897, 300934584, 3179822945, 893043137, 2031413512, 300934584, 3806926970, 2413249929, 2031413512,
-        1417581911, 3071835354, 2575196237, 4101127251, 1375339216, 3071835354, 847617977, 3632503316, 1375339216,
-        2747488994, 3296604805, 898095468, 1742777145, 219265369, 3296604805, 823714885, 667779292, 219265369,
-        2640209692, 3040506537, 3626115220, 161827078, 852668118, 3040506537, 3856381322, 3360242076, 852668118,
-        3734246393, 4151553160, 4177051283, 266522866, 1731798531, 4151553160, 632196679, 3864297722, 1731798531,
-        1694175127, 1087914338, 2384195794, 2764925057, 505782858, 1087914338, 3235634082, 807915248, 505782858,
-        2402749950, 2353776151, 75909174, 890570951, 1752665661, 2353776151, 3120241607, 3862435696, 1752665661,
-        2427906178, 3580155704, 949770784, 226153695, 1230515664, 3580155704, 1988835001, 986791581, 1230515664,
-        1774047142, 3199155377, 3106427820, 1901920839, 4290900039, 3199155377, 4178980191, 280623348, 4290900039,
-        3567524348, 1934119675, 3188270128, 2997767678, 826363896, 1934119675, 262952343, 614326610, 826363896,
-        1625613062, 4288164505, 2481284279, 4273461426, 1177260757, 4288164505, 305959988, 4017252267, 1177260757,
-        337929267, 333342539, 418300166, 2944208672, 379097734, 333342539, 2084056909, 3625475947, 379097734,
-        1189899255, 1307754719, 1214919992, 3736721708, 3514751918, 1307754719, 732435953, 2021244538, 3514751918,
-        4089172695, 1533534334, 525643282, 1497577018, 1335684482, 1533534334, 2079007086, 3977541427, 1335684482,
-        3075256652, 2762754934, 3846844247, 3057872364, 3274545167, 2762754934, 4028573983, 938934351, 3274545167,
-        2597859300, 2880151048, 2523330453, 1121709186, 175667448, 2880151048, 4182510911, 1723133625, 175667448,
-        484148868, 1404283933, 2982534313, 3736767353, 3179865161, 1404283933, 391120388, 3758716888, 3179865161,
-        2138867468, 1128973399, 2133702321, 1613561693, 3622350766, 1128973399, 1500151924, 3759983985, 3622350766,
-        3027706760, 3786576552, 2698781808, 2810527099, 90498489, 3786576552, 4220122612, 1855245979, 90498489,
-        3739389517, 1110440720, 917457922, 2163873618, 3707591763, 1110440720, 2667061910, 2533383962, 3707591763,
-
-    };
-
-static constexpr unsigned long long h_A1P67[MRG323A_N] =
-    {
-        82758667, 1871391091, 4127413238, 3672831523, 69195019, 1871391091, 3672091415, 3528743235, 69195019,
-        3361372532, 2329303404, 99651939, 2008671965, 2931758910, 2329303404, 1113529483, 2374097189, 2931758910,
-        1831590873, 1588259595, 1314332382, 2385989343, 2508077280, 1588259595, 1787615788, 661437137, 2508077280,
-        2326052247, 4183591379, 4049009082, 2604529491, 1453913233, 4183591379, 2311925423, 1805360390, 1453913233,
-        3956367490, 604461629, 1257432102, 794711716, 1155867175, 604461629, 1777070788, 429445904, 1155867175,
-        1686241617, 1257046062, 1427609439, 490376081, 387798431, 1257046062, 235551485, 1312672615, 387798431,
-        2362447880, 3445363024, 3160262066, 2426867845, 4194339866, 3445363024, 1046144413, 4177893681, 4194339866,
-        4251175413, 3559576374, 3107663662, 697539134, 1909472435, 3559576374, 280754246, 375835695, 1909472435,
-        1099512970, 712404985, 1571467521, 546519870, 1135109300, 712404985, 3325312332, 2352874613, 1135109300,
-        1945425936, 1653045514, 381988982, 3733376326, 414410025, 1653045514, 1181583679, 1185848176, 414410025,
-        2526336124, 3019211015, 4215964965, 2683163472, 4188191530, 3019211015, 2964651598, 293801056, 4188191530,
-        1444052678, 2253324417, 39719589, 1880267534, 2391992038, 2253324417, 987740265, 3691889508, 2391992038,
-        166599066, 2335494420, 1232261118, 2227597731, 2570600780, 2335494420, 2700034538, 3460843234, 2570600780,
-        2511338360, 1188954576, 1251401239, 2511664974, 292276982, 1188954576, 697844082, 3093661552, 292276982,
-        3624650744, 51993077, 3540268009, 3252828938, 3710319575, 51993077, 2858628849, 3910069381, 3710319575,
-        655966702, 754002362, 1646581402, 1958331075, 475572423, 754002362, 3248619000, 3228514800, 475572423,
-        2760311307, 4166372813, 741596417, 2282679206, 3090782630, 4166372813, 3242468721, 1628442374, 3090782630,
-        4265279407, 3532111852, 1754687396, 500404765, 2603727025, 3532111852, 1428367254, 3149485478, 2603727025,
-        2873769531, 2081104178, 596284397, 4153800443, 1261269623, 2081104178, 3967600061, 1830023157, 1261269623,
-        278611533, 2229285304, 3443204327, 3110641420, 77498444, 2229285304, 3904070810, 1070507239, 77498444,
-        544639534, 568528663, 2177189807, 2475829068, 121482268, 568528663, 876978915, 3116647617, 121482268,
-        1547862823, 2404658587, 4191448009, 2158188804, 2976916793, 2404658587, 168571747, 1691884706, 2976916793,
-        3208213311, 4212638780, 3235157352, 671148556, 2951207765, 4212638780, 2075145516, 2395485231, 2951207765,
-        4080517315, 2133433101, 4043998180, 2044221845, 867670560, 2133433101, 834432416, 3613001199, 867670560,
-        4102885735, 1319434267, 2678775073, 740092580, 607380970, 1319434267, 2198271844, 2610193258, 607380970,
-        1165218048, 1317690360, 1189150958, 399240205, 2507168618, 1317690360, 2988334517, 2687593413, 2507168618,
-        1028861702, 4082006648, 338232527, 1888486946, 1842080991, 4082006648, 3903826366, 3109935091, 1842080991,
-        614134826, 2261996505, 2888080641, 710199359, 2773979788, 2261996505, 1144301620, 2554371815, 2773979788,
-        4056173823, 1285620078, 357420018, 2423072612, 2309408315, 1285620078, 1533175115, 2760088020, 2309408315,
-        4264130267, 815015434, 3142242173, 180649975, 2500813569, 815015434, 3378723563, 829683767, 2500813569,
-        4174387531, 1030729435, 2812778314, 1752988797, 4044178729, 1030729435, 467969301, 554748104, 4044178729,
-        1348429235, 2928743274, 3776082629, 3607529209, 3069812185, 2928743274, 2542432347, 3208181168, 3069812185,
-        4064845753, 668285756, 3816217625, 3713143233, 1380634204, 668285756, 3533700508, 1192551435, 1380634204,
-        1515684518, 1706771705, 728123349, 3174850469, 2057456462, 1706771705, 3410402985, 2897339640, 2057456462,
-        3082272717, 531091457, 1390161328, 3895139973, 2171402857, 531091457, 4030688141, 3049703400, 2171402857,
-        1241147206, 3193892819, 1244284192, 65180262, 4065669017, 3193892819, 1484817937, 3661081858, 4065669017,
-        1438760812, 3491341751, 3414470157, 2805337292, 272266053, 3491341751, 824109230, 3202556526, 272266053,
-        135412706, 3627115412, 2345042216, 1565169824, 2166856449, 3627115412, 1026946745, 3467845248, 2166856449,
-        1889419951, 3256876154, 1240505488, 1254783743, 989966800, 3256876154, 1995297400, 3692472918, 989966800,
-        3206226875, 285700890, 496017472, 2515316194, 2129675196, 285700890, 1863853990, 2673457552, 2129675196,
-        4163770641, 255160418, 772100749, 1987092456, 3237660221, 255160418, 1394381051, 4216039401, 3237660221,
-        2133915627, 2713747584, 627765421, 2300605925, 35690583, 2713747584, 2918902946, 2638220304, 35690583,
-        2587549655, 998684270, 4292130625, 1791772791, 2820705344, 998684270, 124590158, 3831143549, 2820705344,
-        978482299, 3200877282, 497605289, 3717741518, 3737164414, 3200877282, 4046686626, 861393946, 3737164414,
-        2665561897, 300934584, 3179822945, 893043137, 2031413512, 300934584, 3806926970, 2413249929, 2031413512,
-        1417581911, 3071835354, 2575196237, 4101127251, 1375339216, 3071835354, 847617977, 3632503316, 1375339216,
-        2747488994, 3296604805, 898095468, 1742777145, 219265369, 3296604805, 823714885, 667779292, 219265369,
-        2640209692, 3040506537, 3626115220, 161827078, 852668118, 3040506537, 3856381322, 3360242076, 852668118,
-        3734246393, 4151553160, 4177051283, 266522866, 1731798531, 4151553160, 632196679, 3864297722, 1731798531,
-        1694175127, 1087914338, 2384195794, 2764925057, 505782858, 1087914338, 3235634082, 807915248, 505782858,
-        2402749950, 2353776151, 75909174, 890570951, 1752665661, 2353776151, 3120241607, 3862435696, 1752665661,
-        2427906178, 3580155704, 949770784, 226153695, 1230515664, 3580155704, 1988835001, 986791581, 1230515664,
-        1774047142, 3199155377, 3106427820, 1901920839, 4290900039, 3199155377, 4178980191, 280623348, 4290900039,
-        3567524348, 1934119675, 3188270128, 2997767678, 826363896, 1934119675, 262952343, 614326610, 826363896,
-        1625613062, 4288164505, 2481284279, 4273461426, 1177260757, 4288164505, 305959988, 4017252267, 1177260757,
-        337929267, 333342539, 418300166, 2944208672, 379097734, 333342539, 2084056909, 3625475947, 379097734,
-        1189899255, 1307754719, 1214919992, 3736721708, 3514751918, 1307754719, 732435953, 2021244538, 3514751918,
-        4089172695, 1533534334, 525643282, 1497577018, 1335684482, 1533534334, 2079007086, 3977541427, 1335684482,
-        3075256652, 2762754934, 3846844247, 3057872364, 3274545167, 2762754934, 4028573983, 938934351, 3274545167,
-        2597859300, 2880151048, 2523330453, 1121709186, 175667448, 2880151048, 4182510911, 1723133625, 175667448,
-        484148868, 1404283933, 2982534313, 3736767353, 3179865161, 1404283933, 391120388, 3758716888, 3179865161,
-        2138867468, 1128973399, 2133702321, 1613561693, 3622350766, 1128973399, 1500151924, 3759983985, 3622350766,
-        3027706760, 3786576552, 2698781808, 2810527099, 90498489, 3786576552, 4220122612, 1855245979, 90498489,
-        3739389517, 1110440720, 917457922, 2163873618, 3707591763, 1110440720, 2667061910, 2533383962, 3707591763,
-
-    };
-
-static constexpr __device__ unsigned long long d_A2P67[MRG323A_N] =
-    {
-        1511326704, 3759209742, 1610795712, 4292754251, 1511326704, 3889917532, 3859662829, 4292754251, 3708466080,
-        972103006, 964807713, 878035866, 4248550197, 972103006, 1926628839, 1448629089, 4248550197, 3196114006,
-        3497384788, 3174249442, 3182508868, 3864816447, 3497384788, 3038399593, 2546884738, 3864816447, 2980208068,
-        1776335558, 1189944887, 4095757548, 3813600746, 1776335558, 789475914, 4119698302, 3813600746, 2145357457,
-        4022832294, 4130146837, 1942923647, 1675130777, 4022832294, 916677004, 4089786548, 1675130777, 116540512,
-        165639584, 1205513289, 2037453462, 1444587280, 165639584, 161923120, 2617085459, 1444587280, 2006913311,
-        3458099202, 3062421748, 4052486999, 1064270720, 3458099202, 230768332, 4056228301, 1064270720, 2219267779,
-        296275263, 3452455838, 2081462173, 1789143993, 296275263, 3463234943, 2097389984, 1789143993, 3447191459,
-        2828288883, 3866690251, 410553827, 1587005542, 2828288883, 1469478670, 2766486018, 1587005542, 2627363449,
-        3288027530, 412403981, 2458742268, 4267121909, 3288027530, 138566505, 420803572, 4267121909, 4094554844,
-        3844599430, 2430152838, 3283485436, 2486244684, 3844599430, 4252427633, 3560842909, 2486244684, 3960267499,
-        67933059, 1294996291, 2657888382, 513233413, 67933059, 1379805031, 44564058, 513233413, 86971645,
-        2732588524, 1866530072, 818237694, 2540507736, 2732588524, 3257104212, 1164400003, 2540507736, 1124501551,
-        4199239222, 3155848463, 2121388468, 1135554501, 4199239222, 2056492193, 3251740389, 1135554501, 2343537248,
-        550710036, 500329021, 1075236085, 356444753, 550710036, 1634965500, 58733535, 356444753, 1261552815,
-        708689546, 419139045, 2012018174, 706488081, 708689546, 1113760995, 585555005, 706488081, 76092226,
-        1293182265, 3168473803, 366230236, 3319068849, 1293182265, 1085259665, 1675229290, 3319068849, 3912300371,
-        3186089068, 4188864734, 1211781402, 756122322, 3186089068, 578262892, 2518961174, 756122322, 1658665581,
-        1347291439, 2050427676, 736113023, 4102191254, 1347291439, 878627148, 1293500383, 4102191254, 745646810,
-        4196897331, 3436564969, 1900167098, 3108887846, 4196897331, 2697923227, 1405263476, 3108887846, 314631094,
-        958383622, 3694638688, 1150087061, 3770009830, 958383622, 793326651, 533700213, 3770009830, 1513734026,
-        4119603367, 3479396923, 3534176399, 3765397477, 4119603367, 1458031003, 3380901602, 3765397477, 2684083587,
-        980937351, 2094378936, 448446028, 1421333909, 980937351, 3405683645, 323724368, 1421333909, 338680738,
-        2942968846, 4293637338, 3549906544, 527851489, 2942968846, 3852871282, 4209198933, 527851489, 1091268872,
-        1975983015, 2092556693, 611187071, 3982652344, 1975983015, 3001736262, 2055073597, 3982652344, 1875181995,
-        2970221269, 880904779, 2447465272, 2888742196, 2970221269, 3521651749, 3019977656, 2888742196, 2712717326,
-        419134859, 2976059897, 747864206, 4101695717, 419134859, 4264593116, 2657991148, 4101695717, 2542621682,
-        4043135299, 1612983166, 1149778656, 1267010518, 4043135299, 3496325546, 3094232897, 1267010518, 2949176293,
-        3949395794, 1774568686, 2123036003, 2182983404, 3949395794, 2355671350, 2820933455, 2182983404, 513963325,
-        3046911698, 2576744453, 2492729814, 4277866093, 3046911698, 3146977604, 2249371766, 4277866093, 3622293976,
-        1391529818, 423458502, 2587125255, 3536237833, 1391529818, 985347517, 157623850, 3536237833, 1015566287,
-        48329260, 2599277669, 821961664, 902187690, 48329260, 1716556555, 4019658974, 902187690, 950730510,
-        1318489562, 1530977112, 3713577419, 4270158447, 1318489562, 1654940598, 2679964938, 4270158447, 1337075195,
-        770600793, 3249576224, 3578552768, 2710443459, 770600793, 2990852339, 3098163705, 2710443459, 522138188,
-        2803285489, 1922250286, 3164022812, 477609731, 2803285489, 2140252218, 2252852611, 477609731, 3058519788,
-        208329741, 3633562083, 3548346666, 3892091460, 208329741, 516833304, 3440632377, 3892091460, 1638833719,
-        1816075033, 3570111203, 959489356, 3482051486, 1816075033, 861657108, 3119495098, 3482051486, 2576849579,
-        4240216888, 2891584407, 2102314945, 4064489450, 4240216888, 1427441010, 2441164913, 4064489450, 3558527186,
-        2918371295, 65155283, 3469357011, 3579773554, 2918371295, 3494391959, 3266584309, 3579773554, 3837485479,
-        2959420453, 1365016881, 4082486022, 236489012, 2959420453, 3802558529, 2687043642, 236489012, 2547086826,
-        4185325422, 2762854843, 3200044912, 3664909559, 4185325422, 3543921700, 4240262918, 3664909559, 2853212443,
-        2618500928, 4237264351, 1470046497, 1893990098, 2618500928, 2982567031, 3017062825, 1893990098, 3195556801,
-        1868464655, 3407681142, 1652841784, 1678569574, 1868464655, 4162480901, 1477016185, 1678569574, 4145063890,
-        792188465, 4251338402, 2219407026, 3840340879, 792188465, 3493367465, 2979958414, 3840340879, 2338974139,
-        478845700, 2378167062, 882114621, 1674533845, 478845700, 3572905305, 3571222880, 1674533845, 1242316901,
-        2636090868, 1972761498, 71690719, 1228103463, 2636090868, 1280685025, 3741735502, 1228103463, 994061750,
-        1156725261, 1100755307, 221922891, 2892200461, 1156725261, 1505716533, 2287613563, 2892200461, 3689457190,
-        1387244644, 3135090808, 1243609165, 1724967466, 1387244644, 3296353235, 1064364031, 1724967466, 2107521044,
-        2822471992, 2034317853, 2071407475, 170903528, 2822471992, 1322162887, 2524982332, 170903528, 2656231333,
-        3653936868, 3893194049, 2484299328, 1313746234, 3653936868, 1705346273, 1397638018, 1313746234, 4015529545,
-        4129760842, 1671665759, 1677834656, 3200005334, 4129760842, 3486207172, 2850728736, 3200005334, 3076201597,
-        1464411153, 277697599, 1610723613, 32183930, 1464411153, 1022607788, 2824425944, 32183930, 2093834863,
-        3492361727, 1027004383, 3167429889, 3674905362, 3492361727, 3572939265, 4270409313, 3674905362, 698814233,
-        880482061, 205175925, 4070445105, 2208329119, 880482061, 1933248566, 3741227945, 2208329119, 3962062826,
-        4184605179, 1189429800, 567967482, 107217966, 4184605179, 784865788, 549462420, 107217966, 3134382704,
-        2732536445, 1231107067, 3374588386, 409954030, 2732536445, 1044831206, 3398162498, 409954030, 3505648581,
-        2169560691, 1076348534, 637306236, 3704346564, 2169560691, 293694496, 632453145, 3704346564, 1609425246,
-        372115891, 3928812480, 2830541169, 3056527841, 372115891, 1924239834, 3044937468, 3056527841, 547142630,
-        1660852083, 3635660815, 1389092450, 1025573319, 1660852083, 3276803366, 4036331438, 1025573319, 4092197741,
-        1360732901, 2887812973, 4101068693, 52572783, 1360732901, 112458461, 2636566855, 52572783, 1136777988,
-        3455696508, 536919193, 3978804036, 3094157668, 3455696508, 3821833900, 2278849016, 3094157668, 2531965909,
-        2125991744, 890897326, 3790557569, 1433592392, 2125991744, 3671109604, 808215503, 1433592392, 2446306581,
-        3524411799, 932865240, 1838275365, 1789634890, 3524411799, 4130736474, 2252266098, 1789634890, 3048775967,
-        1773339925, 948403862, 1999624391, 983864203, 1773339925, 3734776305, 314407045, 983864203, 2648614071,
-
-    };
-
-static constexpr unsigned long long h_A2P67[MRG323A_N] =
-    {
-        1511326704, 3759209742, 1610795712, 4292754251, 1511326704, 3889917532, 3859662829, 4292754251, 3708466080,
-        972103006, 964807713, 878035866, 4248550197, 972103006, 1926628839, 1448629089, 4248550197, 3196114006,
-        3497384788, 3174249442, 3182508868, 3864816447, 3497384788, 3038399593, 2546884738, 3864816447, 2980208068,
-        1776335558, 1189944887, 4095757548, 3813600746, 1776335558, 789475914, 4119698302, 3813600746, 2145357457,
-        4022832294, 4130146837, 1942923647, 1675130777, 4022832294, 916677004, 4089786548, 1675130777, 116540512,
-        165639584, 1205513289, 2037453462, 1444587280, 165639584, 161923120, 2617085459, 1444587280, 2006913311,
-        3458099202, 3062421748, 4052486999, 1064270720, 3458099202, 230768332, 4056228301, 1064270720, 2219267779,
-        296275263, 3452455838, 2081462173, 1789143993, 296275263, 3463234943, 2097389984, 1789143993, 3447191459,
-        2828288883, 3866690251, 410553827, 1587005542, 2828288883, 1469478670, 2766486018, 1587005542, 2627363449,
-        3288027530, 412403981, 2458742268, 4267121909, 3288027530, 138566505, 420803572, 4267121909, 4094554844,
-        3844599430, 2430152838, 3283485436, 2486244684, 3844599430, 4252427633, 3560842909, 2486244684, 3960267499,
-        67933059, 1294996291, 2657888382, 513233413, 67933059, 1379805031, 44564058, 513233413, 86971645,
-        2732588524, 1866530072, 818237694, 2540507736, 2732588524, 3257104212, 1164400003, 2540507736, 1124501551,
-        4199239222, 3155848463, 2121388468, 1135554501, 4199239222, 2056492193, 3251740389, 1135554501, 2343537248,
-        550710036, 500329021, 1075236085, 356444753, 550710036, 1634965500, 58733535, 356444753, 1261552815,
-        708689546, 419139045, 2012018174, 706488081, 708689546, 1113760995, 585555005, 706488081, 76092226,
-        1293182265, 3168473803, 366230236, 3319068849, 1293182265, 1085259665, 1675229290, 3319068849, 3912300371,
-        3186089068, 4188864734, 1211781402, 756122322, 3186089068, 578262892, 2518961174, 756122322, 1658665581,
-        1347291439, 2050427676, 736113023, 4102191254, 1347291439, 878627148, 1293500383, 4102191254, 745646810,
-        4196897331, 3436564969, 1900167098, 3108887846, 4196897331, 2697923227, 1405263476, 3108887846, 314631094,
-        958383622, 3694638688, 1150087061, 3770009830, 958383622, 793326651, 533700213, 3770009830, 1513734026,
-        4119603367, 3479396923, 3534176399, 3765397477, 4119603367, 1458031003, 3380901602, 3765397477, 2684083587,
-        980937351, 2094378936, 448446028, 1421333909, 980937351, 3405683645, 323724368, 1421333909, 338680738,
-        2942968846, 4293637338, 3549906544, 527851489, 2942968846, 3852871282, 4209198933, 527851489, 1091268872,
-        1975983015, 2092556693, 611187071, 3982652344, 1975983015, 3001736262, 2055073597, 3982652344, 1875181995,
-        2970221269, 880904779, 2447465272, 2888742196, 2970221269, 3521651749, 3019977656, 2888742196, 2712717326,
-        419134859, 2976059897, 747864206, 4101695717, 419134859, 4264593116, 2657991148, 4101695717, 2542621682,
-        4043135299, 1612983166, 1149778656, 1267010518, 4043135299, 3496325546, 3094232897, 1267010518, 2949176293,
-        3949395794, 1774568686, 2123036003, 2182983404, 3949395794, 2355671350, 2820933455, 2182983404, 513963325,
-        3046911698, 2576744453, 2492729814, 4277866093, 3046911698, 3146977604, 2249371766, 4277866093, 3622293976,
-        1391529818, 423458502, 2587125255, 3536237833, 1391529818, 985347517, 157623850, 3536237833, 1015566287,
-        48329260, 2599277669, 821961664, 902187690, 48329260, 1716556555, 4019658974, 902187690, 950730510,
-        1318489562, 1530977112, 3713577419, 4270158447, 1318489562, 1654940598, 2679964938, 4270158447, 1337075195,
-        770600793, 3249576224, 3578552768, 2710443459, 770600793, 2990852339, 3098163705, 2710443459, 522138188,
-        2803285489, 1922250286, 3164022812, 477609731, 2803285489, 2140252218, 2252852611, 477609731, 3058519788,
-        208329741, 3633562083, 3548346666, 3892091460, 208329741, 516833304, 3440632377, 3892091460, 1638833719,
-        1816075033, 3570111203, 959489356, 3482051486, 1816075033, 861657108, 3119495098, 3482051486, 2576849579,
-        4240216888, 2891584407, 2102314945, 4064489450, 4240216888, 1427441010, 2441164913, 4064489450, 3558527186,
-        2918371295, 65155283, 3469357011, 3579773554, 2918371295, 3494391959, 3266584309, 3579773554, 3837485479,
-        2959420453, 1365016881, 4082486022, 236489012, 2959420453, 3802558529, 2687043642, 236489012, 2547086826,
-        4185325422, 2762854843, 3200044912, 3664909559, 4185325422, 3543921700, 4240262918, 3664909559, 2853212443,
-        2618500928, 4237264351, 1470046497, 1893990098, 2618500928, 2982567031, 3017062825, 1893990098, 3195556801,
-        1868464655, 3407681142, 1652841784, 1678569574, 1868464655, 4162480901, 1477016185, 1678569574, 4145063890,
-        792188465, 4251338402, 2219407026, 3840340879, 792188465, 3493367465, 2979958414, 3840340879, 2338974139,
-        478845700, 2378167062, 882114621, 1674533845, 478845700, 3572905305, 3571222880, 1674533845, 1242316901,
-        2636090868, 1972761498, 71690719, 1228103463, 2636090868, 1280685025, 3741735502, 1228103463, 994061750,
-        1156725261, 1100755307, 221922891, 2892200461, 1156725261, 1505716533, 2287613563, 2892200461, 3689457190,
-        1387244644, 3135090808, 1243609165, 1724967466, 1387244644, 3296353235, 1064364031, 1724967466, 2107521044,
-        2822471992, 2034317853, 2071407475, 170903528, 2822471992, 1322162887, 2524982332, 170903528, 2656231333,
-        3653936868, 3893194049, 2484299328, 1313746234, 3653936868, 1705346273, 1397638018, 1313746234, 4015529545,
-        4129760842, 1671665759, 1677834656, 3200005334, 4129760842, 3486207172, 2850728736, 3200005334, 3076201597,
-        1464411153, 277697599, 1610723613, 32183930, 1464411153, 1022607788, 2824425944, 32183930, 2093834863,
-        3492361727, 1027004383, 3167429889, 3674905362, 3492361727, 3572939265, 4270409313, 3674905362, 698814233,
-        880482061, 205175925, 4070445105, 2208329119, 880482061, 1933248566, 3741227945, 2208329119, 3962062826,
-        4184605179, 1189429800, 567967482, 107217966, 4184605179, 784865788, 549462420, 107217966, 3134382704,
-        2732536445, 1231107067, 3374588386, 409954030, 2732536445, 1044831206, 3398162498, 409954030, 3505648581,
-        2169560691, 1076348534, 637306236, 3704346564, 2169560691, 293694496, 632453145, 3704346564, 1609425246,
-        372115891, 3928812480, 2830541169, 3056527841, 372115891, 1924239834, 3044937468, 3056527841, 547142630,
-        1660852083, 3635660815, 1389092450, 1025573319, 1660852083, 3276803366, 4036331438, 1025573319, 4092197741,
-        1360732901, 2887812973, 4101068693, 52572783, 1360732901, 112458461, 2636566855, 52572783, 1136777988,
-        3455696508, 536919193, 3978804036, 3094157668, 3455696508, 3821833900, 2278849016, 3094157668, 2531965909,
-        2125991744, 890897326, 3790557569, 1433592392, 2125991744, 3671109604, 808215503, 1433592392, 2446306581,
-        3524411799, 932865240, 1838275365, 1789634890, 3524411799, 4130736474, 2252266098, 1789634890, 3048775967,
-        1773339925, 948403862, 1999624391, 983864203, 1773339925, 3734776305, 314407045, 983864203, 2648614071,
-
-    };
-
-static constexpr __device__ unsigned long long d_A1P127[MRG323A_N] =
-    {
-        2427906178, 3580155704, 949770784, 226153695, 1230515664, 3580155704, 1988835001, 986791581, 1230515664,
-        1774047142, 3199155377, 3106427820, 1901920839, 4290900039, 3199155377, 4178980191, 280623348, 4290900039,
-        3567524348, 1934119675, 3188270128, 2997767678, 826363896, 1934119675, 262952343, 614326610, 826363896,
-        1625613062, 4288164505, 2481284279, 4273461426, 1177260757, 4288164505, 305959988, 4017252267, 1177260757,
-        337929267, 333342539, 418300166, 2944208672, 379097734, 333342539, 2084056909, 3625475947, 379097734,
-        1189899255, 1307754719, 1214919992, 3736721708, 3514751918, 1307754719, 732435953, 2021244538, 3514751918,
-        4089172695, 1533534334, 525643282, 1497577018, 1335684482, 1533534334, 2079007086, 3977541427, 1335684482,
-        3075256652, 2762754934, 3846844247, 3057872364, 3274545167, 2762754934, 4028573983, 938934351, 3274545167,
-        2597859300, 2880151048, 2523330453, 1121709186, 175667448, 2880151048, 4182510911, 1723133625, 175667448,
-        484148868, 1404283933, 2982534313, 3736767353, 3179865161, 1404283933, 391120388, 3758716888, 3179865161,
-        2138867468, 1128973399, 2133702321, 1613561693, 3622350766, 1128973399, 1500151924, 3759983985, 3622350766,
-        3027706760, 3786576552, 2698781808, 2810527099, 90498489, 3786576552, 4220122612, 1855245979, 90498489,
-        3739389517, 1110440720, 917457922, 2163873618, 3707591763, 1110440720, 2667061910, 2533383962, 3707591763,
-        1545226000, 1812182123, 3693349190, 3422065122, 3291428549, 1812182123, 1193168720, 2072837757, 3291428549,
-        3230096243, 2131723358, 3262178024, 2882890127, 4088518247, 2131723358, 3991553306, 1282224087, 4088518247,
-        301207261, 1722796810, 3697719854, 3350228505, 3410986694, 1722796810, 3684514720, 2846958957, 3410986694,
-        1532963114, 4236235786, 3871128158, 3540401964, 1285250577, 4236235786, 1105070646, 2764245175, 1285250577,
-        210906218, 3068599594, 3034582784, 340633153, 4004365908, 3068599594, 4238928187, 2299166464, 4004365908,
-        2274701639, 3955606166, 3081246407, 3199954992, 3948054919, 3955606166, 2399101442, 3438340286, 3948054919,
-        504137100, 1182303684, 201533985, 4188299661, 3042453580, 1182303684, 2578519273, 2674782930, 3042453580,
-        1382964588, 2578452047, 3140440866, 261861891, 1076783073, 2578452047, 1634588989, 164438428, 1076783073,
-        2529186343, 526867394, 3102803247, 2687252475, 2908898908, 526867394, 1213100579, 86050422, 2908898908,
-        2690118316, 538108523, 790337895, 4193870709, 1053552056, 538108523, 1635227281, 4002399925, 1053552056,
-        2123712957, 4205383007, 1812304090, 1095349745, 166243972, 4205383007, 428569070, 2128782357, 166243972,
-        1330151766, 3569679412, 4107175982, 3808641551, 3621125056, 3569679412, 4262164578, 1927692878, 3621125056,
-        3606295184, 2442739556, 3894922338, 1629626641, 2729678535, 2442739556, 3379124758, 4279360935, 2729678535,
-        1052092278, 4249024666, 919210106, 3253349463, 3629539480, 4249024666, 852514024, 4025926501, 3629539480,
-        12394571, 1252747620, 2133571953, 4227339509, 3197545170, 1252747620, 1884529704, 1976203831, 3197545170,
-        2986331025, 2671019282, 2847338542, 3173738401, 3542657885, 2671019282, 745203060, 1546667401, 3542657885,
-        2613012997, 2311336951, 2911336433, 1493974713, 92565032, 2311336951, 2786645250, 257065974, 92565032,
-        3424925004, 2776053372, 2204068573, 3770626858, 2509257810, 2776053372, 2979919489, 1146336783, 2509257810,
-        1474384834, 827894421, 515339473, 1373055755, 1949809417, 827894421, 3088339524, 1194193824, 1949809417,
-        1825805135, 1289872272, 3700877161, 3433422861, 4062509844, 1289872272, 3019008744, 2060641859, 4062509844,
-        3842597153, 4253338264, 3424495942, 698444416, 60268595, 4253338264, 4096010585, 47309624, 60268595,
-        2662288323, 2043518992, 1593435980, 1330201507, 3618850300, 2043518992, 2538793204, 271787962, 3618850300,
-        741020448, 997594656, 2398808739, 1160477043, 1522130854, 997594656, 3036916315, 2847712653, 1522130854,
-        2654964886, 1889728930, 53329096, 2042322941, 1621136330, 1889728930, 1553642730, 784545882, 1621136330,
-        1715219514, 2831829177, 929124824, 997274536, 404228189, 2831829177, 1386575385, 4107238699, 404228189,
-        3928131551, 2912523524, 1840499723, 4216003022, 2970489088, 2912523524, 1158689953, 1425511081, 2970489088,
-        2807004452, 2510299562, 271603006, 2505735035, 2370490899, 2510299562, 10873814, 2450376936, 2370490899,
-        2000734342, 1113679064, 2502160539, 1475266926, 2787925323, 1113679064, 1475797635, 3044470744, 2787925323,
-        1457157056, 1252556678, 3073232607, 1926798761, 3639907189, 1252556678, 2067740348, 2256217204, 3639907189,
-        3740999688, 1035400458, 3162437311, 4126312242, 686702830, 1035400458, 1699805291, 667792040, 686702830,
-        2422495016, 3203768688, 1858240466, 848719394, 4092709154, 3203768688, 659945473, 1863075174, 4092709154,
-        246817944, 871751352, 2834051003, 3976202597, 3721214025, 871751352, 783929942, 745295675, 3721214025,
-        3811740424, 3603608092, 2365398362, 3826150877, 2906557036, 3603608092, 2300510686, 966815948, 2906557036,
-        2816329160, 18201123, 3367710570, 437309679, 2220769388, 18201123, 1346863388, 705296543, 2220769388,
-        3310028953, 1662315499, 132645114, 2572908401, 3105849797, 1662315499, 1937586849, 1735620028, 3105849797,
-        461386353, 1359675853, 3599822966, 106675209, 2044154050, 1359675853, 1787730088, 1149892630, 2044154050,
-        3303902397, 345146034, 1417149696, 2231869247, 1116882637, 345146034, 1846832385, 79626976, 1116882637,
-        2765049417, 3117782790, 1805260159, 3796182890, 1101141726, 3117782790, 224270120, 1004001443, 1101141726,
-        89118668, 2494198515, 1356989069, 2490435731, 997151755, 2494198515, 1175528637, 3444341166, 997151755,
-        2340639019, 510225634, 286119182, 2045217287, 1194574818, 510225634, 2662281592, 1728500627, 1194574818,
-        210787847, 1189120688, 2848040407, 1087786165, 2343328484, 1189120688, 3465141330, 2893041005, 2343328484,
-        3438170226, 3236285682, 962036916, 2873263091, 215280489, 3236285682, 730413847, 1474823842, 215280489,
-        1566461658, 133010024, 2886695328, 2835827516, 653809404, 133010024, 3082882924, 3710942807, 653809404,
-        4201558916, 1263786956, 326001602, 762846463, 621546357, 1263786956, 2697142404, 1156650856, 621546357,
-        2655768102, 2339029465, 2430211448, 2669906627, 403962847, 2339029465, 1483118807, 639660658, 403962847,
-        3508595200, 4228486662, 754946994, 1913148390, 3500531602, 4228486662, 24637, 3773159052, 3500531602,
-        4024866227, 1143874914, 3205058469, 2970344133, 2873927273, 1143874914, 2167114735, 4095476435, 2873927273,
-        1479401095, 2958366486, 3027708794, 2704486034, 3574053987, 2958366486, 3630964515, 1276667706, 3574053987,
-        2035927380, 1363628533, 818363998, 3023327955, 3968427114, 1363628533, 1284825950, 2871663372, 3968427114,
-        3827747418, 3897287251, 4106993377, 1527779946, 3221052941, 3897287251, 4178727866, 4281160673, 3221052941,
-        1174358892, 2835476193, 959978619, 850076464, 3774782533, 2835476193, 3880910680, 3237990203, 3774782533,
-
-    };
-
-static constexpr unsigned long long h_A1P127[MRG323A_N] =
-    {
-        2427906178, 3580155704, 949770784, 226153695, 1230515664, 3580155704, 1988835001, 986791581, 1230515664,
-        1774047142, 3199155377, 3106427820, 1901920839, 4290900039, 3199155377, 4178980191, 280623348, 4290900039,
-        3567524348, 1934119675, 3188270128, 2997767678, 826363896, 1934119675, 262952343, 614326610, 826363896,
-        1625613062, 4288164505, 2481284279, 4273461426, 1177260757, 4288164505, 305959988, 4017252267, 1177260757,
-        337929267, 333342539, 418300166, 2944208672, 379097734, 333342539, 2084056909, 3625475947, 379097734,
-        1189899255, 1307754719, 1214919992, 3736721708, 3514751918, 1307754719, 732435953, 2021244538, 3514751918,
-        4089172695, 1533534334, 525643282, 1497577018, 1335684482, 1533534334, 2079007086, 3977541427, 1335684482,
-        3075256652, 2762754934, 3846844247, 3057872364, 3274545167, 2762754934, 4028573983, 938934351, 3274545167,
-        2597859300, 2880151048, 2523330453, 1121709186, 175667448, 2880151048, 4182510911, 1723133625, 175667448,
-        484148868, 1404283933, 2982534313, 3736767353, 3179865161, 1404283933, 391120388, 3758716888, 3179865161,
-        2138867468, 1128973399, 2133702321, 1613561693, 3622350766, 1128973399, 1500151924, 3759983985, 3622350766,
-        3027706760, 3786576552, 2698781808, 2810527099, 90498489, 3786576552, 4220122612, 1855245979, 90498489,
-        3739389517, 1110440720, 917457922, 2163873618, 3707591763, 1110440720, 2667061910, 2533383962, 3707591763,
-        1545226000, 1812182123, 3693349190, 3422065122, 3291428549, 1812182123, 1193168720, 2072837757, 3291428549,
-        3230096243, 2131723358, 3262178024, 2882890127, 4088518247, 2131723358, 3991553306, 1282224087, 4088518247,
-        301207261, 1722796810, 3697719854, 3350228505, 3410986694, 1722796810, 3684514720, 2846958957, 3410986694,
-        1532963114, 4236235786, 3871128158, 3540401964, 1285250577, 4236235786, 1105070646, 2764245175, 1285250577,
-        210906218, 3068599594, 3034582784, 340633153, 4004365908, 3068599594, 4238928187, 2299166464, 4004365908,
-        2274701639, 3955606166, 3081246407, 3199954992, 3948054919, 3955606166, 2399101442, 3438340286, 3948054919,
-        504137100, 1182303684, 201533985, 4188299661, 3042453580, 1182303684, 2578519273, 2674782930, 3042453580,
-        1382964588, 2578452047, 3140440866, 261861891, 1076783073, 2578452047, 1634588989, 164438428, 1076783073,
-        2529186343, 526867394, 3102803247, 2687252475, 2908898908, 526867394, 1213100579, 86050422, 2908898908,
-        2690118316, 538108523, 790337895, 4193870709, 1053552056, 538108523, 1635227281, 4002399925, 1053552056,
-        2123712957, 4205383007, 1812304090, 1095349745, 166243972, 4205383007, 428569070, 2128782357, 166243972,
-        1330151766, 3569679412, 4107175982, 3808641551, 3621125056, 3569679412, 4262164578, 1927692878, 3621125056,
-        3606295184, 2442739556, 3894922338, 1629626641, 2729678535, 2442739556, 3379124758, 4279360935, 2729678535,
-        1052092278, 4249024666, 919210106, 3253349463, 3629539480, 4249024666, 852514024, 4025926501, 3629539480,
-        12394571, 1252747620, 2133571953, 4227339509, 3197545170, 1252747620, 1884529704, 1976203831, 3197545170,
-        2986331025, 2671019282, 2847338542, 3173738401, 3542657885, 2671019282, 745203060, 1546667401, 3542657885,
-        2613012997, 2311336951, 2911336433, 1493974713, 92565032, 2311336951, 2786645250, 257065974, 92565032,
-        3424925004, 2776053372, 2204068573, 3770626858, 2509257810, 2776053372, 2979919489, 1146336783, 2509257810,
-        1474384834, 827894421, 515339473, 1373055755, 1949809417, 827894421, 3088339524, 1194193824, 1949809417,
-        1825805135, 1289872272, 3700877161, 3433422861, 4062509844, 1289872272, 3019008744, 2060641859, 4062509844,
-        3842597153, 4253338264, 3424495942, 698444416, 60268595, 4253338264, 4096010585, 47309624, 60268595,
-        2662288323, 2043518992, 1593435980, 1330201507, 3618850300, 2043518992, 2538793204, 271787962, 3618850300,
-        741020448, 997594656, 2398808739, 1160477043, 1522130854, 997594656, 3036916315, 2847712653, 1522130854,
-        2654964886, 1889728930, 53329096, 2042322941, 1621136330, 1889728930, 1553642730, 784545882, 1621136330,
-        1715219514, 2831829177, 929124824, 997274536, 404228189, 2831829177, 1386575385, 4107238699, 404228189,
-        3928131551, 2912523524, 1840499723, 4216003022, 2970489088, 2912523524, 1158689953, 1425511081, 2970489088,
-        2807004452, 2510299562, 271603006, 2505735035, 2370490899, 2510299562, 10873814, 2450376936, 2370490899,
-        2000734342, 1113679064, 2502160539, 1475266926, 2787925323, 1113679064, 1475797635, 3044470744, 2787925323,
-        1457157056, 1252556678, 3073232607, 1926798761, 3639907189, 1252556678, 2067740348, 2256217204, 3639907189,
-        3740999688, 1035400458, 3162437311, 4126312242, 686702830, 1035400458, 1699805291, 667792040, 686702830,
-        2422495016, 3203768688, 1858240466, 848719394, 4092709154, 3203768688, 659945473, 1863075174, 4092709154,
-        246817944, 871751352, 2834051003, 3976202597, 3721214025, 871751352, 783929942, 745295675, 3721214025,
-        3811740424, 3603608092, 2365398362, 3826150877, 2906557036, 3603608092, 2300510686, 966815948, 2906557036,
-        2816329160, 18201123, 3367710570, 437309679, 2220769388, 18201123, 1346863388, 705296543, 2220769388,
-        3310028953, 1662315499, 132645114, 2572908401, 3105849797, 1662315499, 1937586849, 1735620028, 3105849797,
-        461386353, 1359675853, 3599822966, 106675209, 2044154050, 1359675853, 1787730088, 1149892630, 2044154050,
-        3303902397, 345146034, 1417149696, 2231869247, 1116882637, 345146034, 1846832385, 79626976, 1116882637,
-        2765049417, 3117782790, 1805260159, 3796182890, 1101141726, 3117782790, 224270120, 1004001443, 1101141726,
-        89118668, 2494198515, 1356989069, 2490435731, 997151755, 2494198515, 1175528637, 3444341166, 997151755,
-        2340639019, 510225634, 286119182, 2045217287, 1194574818, 510225634, 2662281592, 1728500627, 1194574818,
-        210787847, 1189120688, 2848040407, 1087786165, 2343328484, 1189120688, 3465141330, 2893041005, 2343328484,
-        3438170226, 3236285682, 962036916, 2873263091, 215280489, 3236285682, 730413847, 1474823842, 215280489,
-        1566461658, 133010024, 2886695328, 2835827516, 653809404, 133010024, 3082882924, 3710942807, 653809404,
-        4201558916, 1263786956, 326001602, 762846463, 621546357, 1263786956, 2697142404, 1156650856, 621546357,
-        2655768102, 2339029465, 2430211448, 2669906627, 403962847, 2339029465, 1483118807, 639660658, 403962847,
-        3508595200, 4228486662, 754946994, 1913148390, 3500531602, 4228486662, 24637, 3773159052, 3500531602,
-        4024866227, 1143874914, 3205058469, 2970344133, 2873927273, 1143874914, 2167114735, 4095476435, 2873927273,
-        1479401095, 2958366486, 3027708794, 2704486034, 3574053987, 2958366486, 3630964515, 1276667706, 3574053987,
-        2035927380, 1363628533, 818363998, 3023327955, 3968427114, 1363628533, 1284825950, 2871663372, 3968427114,
-        3827747418, 3897287251, 4106993377, 1527779946, 3221052941, 3897287251, 4178727866, 4281160673, 3221052941,
-        1174358892, 2835476193, 959978619, 850076464, 3774782533, 2835476193, 3880910680, 3237990203, 3774782533,
-
-    };
-
-static constexpr __device__ unsigned long long d_A2P127[MRG323A_N] =
-    {
-        1464411153, 277697599, 1610723613, 32183930, 1464411153, 1022607788, 2824425944, 32183930, 2093834863,
-        3492361727, 1027004383, 3167429889, 3674905362, 3492361727, 3572939265, 4270409313, 3674905362, 698814233,
-        880482061, 205175925, 4070445105, 2208329119, 880482061, 1933248566, 3741227945, 2208329119, 3962062826,
-        4184605179, 1189429800, 567967482, 107217966, 4184605179, 784865788, 549462420, 107217966, 3134382704,
-        2732536445, 1231107067, 3374588386, 409954030, 2732536445, 1044831206, 3398162498, 409954030, 3505648581,
-        2169560691, 1076348534, 637306236, 3704346564, 2169560691, 293694496, 632453145, 3704346564, 1609425246,
-        372115891, 3928812480, 2830541169, 3056527841, 372115891, 1924239834, 3044937468, 3056527841, 547142630,
-        1660852083, 3635660815, 1389092450, 1025573319, 1660852083, 3276803366, 4036331438, 1025573319, 4092197741,
-        1360732901, 2887812973, 4101068693, 52572783, 1360732901, 112458461, 2636566855, 52572783, 1136777988,
-        3455696508, 536919193, 3978804036, 3094157668, 3455696508, 3821833900, 2278849016, 3094157668, 2531965909,
-        2125991744, 890897326, 3790557569, 1433592392, 2125991744, 3671109604, 808215503, 1433592392, 2446306581,
-        3524411799, 932865240, 1838275365, 1789634890, 3524411799, 4130736474, 2252266098, 1789634890, 3048775967,
-        1773339925, 948403862, 1999624391, 983864203, 1773339925, 3734776305, 314407045, 983864203, 2648614071,
-        321802921, 1099164995, 2112167358, 3760936985, 321802921, 1003573324, 3758858458, 3760936985, 4014658840,
-        2196438580, 805386227, 4266375092, 4124675351, 2196438580, 2527961345, 94452540, 4124675351, 2825656399,
-        66735368, 2228005807, 4186703168, 2624855312, 66735368, 2708679078, 4098470056, 2624855312, 1773862183,
-        3072642883, 2746897053, 2690305546, 1105106652, 3072642883, 4047666135, 2862886282, 1105106652, 3597347398,
-        232906611, 3873338256, 4051554873, 3027413363, 232906611, 3159432673, 3872967050, 3027413363, 987156327,
-        1160686753, 3676603152, 1635979789, 1447386846, 1160686753, 2670438424, 816212890, 1447386846, 4288868534,
-        3825238244, 1445162354, 2362389441, 3440193648, 3825238244, 3520937545, 2652790808, 3440193648, 405299994,
-        1984094858, 532165989, 2027397575, 1455977136, 1984094858, 2433255524, 1039994763, 1455977136, 2069333087,
-        3680843319, 2332949611, 3516795313, 2033851810, 3680843319, 3843367307, 3686294589, 2033851810, 3912995069,
-        967423689, 1724183394, 635932799, 641380480, 967423689, 2145297779, 1723000412, 641380480, 455633660,
-        2130938335, 1534972306, 2511584766, 273828453, 2130938335, 3112810093, 4084843716, 273828453, 1399334152,
-        168278549, 541167592, 190177712, 403188859, 168278549, 2092073970, 58789558, 403188859, 2777887189,
-        634843389, 4082275720, 2092828966, 351187677, 634843389, 1312056270, 3347241070, 351187677, 2417192332,
-        443276110, 1113643788, 271102234, 3083745876, 443276110, 3370743767, 4200577503, 3083745876, 3298601960,
-        3533393557, 764977733, 3400275098, 144639933, 3533393557, 2646475951, 77963866, 144639933, 3794766611,
-        4064854722, 1198665008, 2872196602, 3274748603, 4064854722, 4164637970, 4238693771, 3274748603, 1981721347,
-        2279220396, 2355957139, 1417574285, 885864931, 2279220396, 1344421653, 1895527787, 885864931, 3726919367,
-        2898100178, 2427331008, 348923199, 3175444953, 2898100178, 4290541487, 246118669, 3175444953, 3410622769,
-        284442065, 4064194676, 2295560707, 4182706556, 284442065, 3696899246, 1201342255, 4182706556, 1145356382,
-        656615546, 442908965, 3724738272, 1624967553, 656615546, 798014134, 1157949454, 1624967553, 496247378,
-        265689579, 675056541, 3009083380, 3820679930, 265689579, 2961990151, 562287964, 3820679930, 1853486796,
-        1675739167, 2319843005, 760605578, 4161492847, 1675739167, 226142150, 1017447188, 4161492847, 3431158427,
-        1759873736, 2334568602, 2154570180, 1812793060, 1759873736, 2111094408, 1168460586, 1812793060, 2495653141,
-        317621194, 868104288, 664971082, 2340275074, 317621194, 2168960688, 725706104, 2340275074, 3532023115,
-        3926931954, 2907684453, 615601328, 1132340715, 3926931954, 676995757, 1154819290, 1132340715, 1662727700,
-        3921782078, 3376494857, 2969567377, 475345024, 3921782078, 4206379953, 1795936544, 475345024, 934679595,
-        3119292228, 741613041, 2083352304, 1047885963, 3119292228, 1581078542, 1065969969, 1047885963, 661718928,
-        3643472111, 2870554228, 3995474529, 3804264051, 3643472111, 1366457944, 1246805564, 3804264051, 993186530,
-        796711791, 3878204845, 3160293932, 255632881, 796711791, 3778927111, 3472564181, 255632881, 388382377,
-        1776984101, 1742284034, 3449763933, 1349354417, 1776984101, 1264780832, 715722511, 1349354417, 1213319489,
-        4261866865, 1914382786, 201872335, 614207188, 4261866865, 1853554849, 2046042882, 614207188, 3193186353,
-        2210205512, 2847073169, 3324925707, 1251969297, 2210205512, 3491451503, 470400916, 1251969297, 2184392547,
-        1523590942, 2391111113, 68341529, 295466806, 1523590942, 4143310876, 3527253079, 295466806, 4059123142,
-        1406902110, 3735012720, 1774518130, 1814959027, 1406902110, 1560544267, 346472965, 1814959027, 964257199,
-        855309653, 4208503105, 1518467541, 2025248418, 855309653, 4148125749, 1349947330, 2025248418, 1168504873,
-        2375338156, 3629519168, 409696181, 252401654, 2375338156, 3992097193, 2793725401, 252401654, 1350184085,
-        873141039, 3885583138, 361604799, 3554143374, 873141039, 894746180, 1919765327, 3554143374, 876210854,
-        246368794, 1703793169, 2317362874, 2300930144, 246368794, 2560214589, 2016163623, 2300930144, 1504276775,
-        1574610921, 2147546631, 4103450226, 107416526, 1574610921, 1773803959, 1402542742, 107416526, 550063800,
-        363388665, 592194244, 1746615522, 2637234667, 363388665, 4031408742, 2895130475, 2637234667, 296510335,
-        3997368560, 3047771871, 3178383826, 1160174754, 3997368560, 4027094919, 1234984211, 1160174754, 4226264344,
-        3303179301, 4243968063, 3235964171, 1776841674, 3303179301, 2867287469, 1500495759, 1776841674, 1708226553,
-        1482944153, 3192311574, 354466071, 3932773012, 1482944153, 389193591, 3350181058, 3932773012, 3398059015,
-        640968550, 3226860971, 922372912, 1254989667, 640968550, 2383815228, 2027371896, 1254989667, 2925300409,
-        2313146046, 3910187183, 1377591475, 1689291784, 2313146046, 4255405993, 1650609719, 1689291784, 1897624297,
-        3656310954, 882924050, 2702189958, 3185020283, 3656310954, 1923190496, 2449669145, 3185020283, 4235849984,
-        377232416, 1498446142, 4229103619, 3926377906, 377232416, 600268838, 511317726, 3926377906, 216160452,
-        1969399344, 3273966859, 4220943579, 3952111894, 1969399344, 575096961, 3815277103, 3952111894, 792177412,
-        2957238169, 1410010554, 1523740068, 3949237584, 2957238169, 74149658, 2564746147, 3949237584, 2557663578,
-        3377318569, 1927835240, 2556102508, 3022040116, 3377318569, 2549406364, 2387074241, 3022040116, 1477293711,
-        257306870, 1748489735, 547809226, 3708493374, 257306870, 4183546362, 4435502, 3708493374, 1607696753,
-
-    };
-
-static constexpr unsigned long long h_A2P127[MRG323A_N] =
-    {
-        1464411153, 277697599, 1610723613, 32183930, 1464411153, 1022607788, 2824425944, 32183930, 2093834863,
-        3492361727, 1027004383, 3167429889, 3674905362, 3492361727, 3572939265, 4270409313, 3674905362, 698814233,
-        880482061, 205175925, 4070445105, 2208329119, 880482061, 1933248566, 3741227945, 2208329119, 3962062826,
-        4184605179, 1189429800, 567967482, 107217966, 4184605179, 784865788, 549462420, 107217966, 3134382704,
-        2732536445, 1231107067, 3374588386, 409954030, 2732536445, 1044831206, 3398162498, 409954030, 3505648581,
-        2169560691, 1076348534, 637306236, 3704346564, 2169560691, 293694496, 632453145, 3704346564, 1609425246,
-        372115891, 3928812480, 2830541169, 3056527841, 372115891, 1924239834, 3044937468, 3056527841, 547142630,
-        1660852083, 3635660815, 1389092450, 1025573319, 1660852083, 3276803366, 4036331438, 1025573319, 4092197741,
-        1360732901, 2887812973, 4101068693, 52572783, 1360732901, 112458461, 2636566855, 52572783, 1136777988,
-        3455696508, 536919193, 3978804036, 3094157668, 3455696508, 3821833900, 2278849016, 3094157668, 2531965909,
-        2125991744, 890897326, 3790557569, 1433592392, 2125991744, 3671109604, 808215503, 1433592392, 2446306581,
-        3524411799, 932865240, 1838275365, 1789634890, 3524411799, 4130736474, 2252266098, 1789634890, 3048775967,
-        1773339925, 948403862, 1999624391, 983864203, 1773339925, 3734776305, 314407045, 983864203, 2648614071,
-        321802921, 1099164995, 2112167358, 3760936985, 321802921, 1003573324, 3758858458, 3760936985, 4014658840,
-        2196438580, 805386227, 4266375092, 4124675351, 2196438580, 2527961345, 94452540, 4124675351, 2825656399,
-        66735368, 2228005807, 4186703168, 2624855312, 66735368, 2708679078, 4098470056, 2624855312, 1773862183,
-        3072642883, 2746897053, 2690305546, 1105106652, 3072642883, 4047666135, 2862886282, 1105106652, 3597347398,
-        232906611, 3873338256, 4051554873, 3027413363, 232906611, 3159432673, 3872967050, 3027413363, 987156327,
-        1160686753, 3676603152, 1635979789, 1447386846, 1160686753, 2670438424, 816212890, 1447386846, 4288868534,
-        3825238244, 1445162354, 2362389441, 3440193648, 3825238244, 3520937545, 2652790808, 3440193648, 405299994,
-        1984094858, 532165989, 2027397575, 1455977136, 1984094858, 2433255524, 1039994763, 1455977136, 2069333087,
-        3680843319, 2332949611, 3516795313, 2033851810, 3680843319, 3843367307, 3686294589, 2033851810, 3912995069,
-        967423689, 1724183394, 635932799, 641380480, 967423689, 2145297779, 1723000412, 641380480, 455633660,
-        2130938335, 1534972306, 2511584766, 273828453, 2130938335, 3112810093, 4084843716, 273828453, 1399334152,
-        168278549, 541167592, 190177712, 403188859, 168278549, 2092073970, 58789558, 403188859, 2777887189,
-        634843389, 4082275720, 2092828966, 351187677, 634843389, 1312056270, 3347241070, 351187677, 2417192332,
-        443276110, 1113643788, 271102234, 3083745876, 443276110, 3370743767, 4200577503, 3083745876, 3298601960,
-        3533393557, 764977733, 3400275098, 144639933, 3533393557, 2646475951, 77963866, 144639933, 3794766611,
-        4064854722, 1198665008, 2872196602, 3274748603, 4064854722, 4164637970, 4238693771, 3274748603, 1981721347,
-        2279220396, 2355957139, 1417574285, 885864931, 2279220396, 1344421653, 1895527787, 885864931, 3726919367,
-        2898100178, 2427331008, 348923199, 3175444953, 2898100178, 4290541487, 246118669, 3175444953, 3410622769,
-        284442065, 4064194676, 2295560707, 4182706556, 284442065, 3696899246, 1201342255, 4182706556, 1145356382,
-        656615546, 442908965, 3724738272, 1624967553, 656615546, 798014134, 1157949454, 1624967553, 496247378,
-        265689579, 675056541, 3009083380, 3820679930, 265689579, 2961990151, 562287964, 3820679930, 1853486796,
-        1675739167, 2319843005, 760605578, 4161492847, 1675739167, 226142150, 1017447188, 4161492847, 3431158427,
-        1759873736, 2334568602, 2154570180, 1812793060, 1759873736, 2111094408, 1168460586, 1812793060, 2495653141,
-        317621194, 868104288, 664971082, 2340275074, 317621194, 2168960688, 725706104, 2340275074, 3532023115,
-        3926931954, 2907684453, 615601328, 1132340715, 3926931954, 676995757, 1154819290, 1132340715, 1662727700,
-        3921782078, 3376494857, 2969567377, 475345024, 3921782078, 4206379953, 1795936544, 475345024, 934679595,
-        3119292228, 741613041, 2083352304, 1047885963, 3119292228, 1581078542, 1065969969, 1047885963, 661718928,
-        3643472111, 2870554228, 3995474529, 3804264051, 3643472111, 1366457944, 1246805564, 3804264051, 993186530,
-        796711791, 3878204845, 3160293932, 255632881, 796711791, 3778927111, 3472564181, 255632881, 388382377,
-        1776984101, 1742284034, 3449763933, 1349354417, 1776984101, 1264780832, 715722511, 1349354417, 1213319489,
-        4261866865, 1914382786, 201872335, 614207188, 4261866865, 1853554849, 2046042882, 614207188, 3193186353,
-        2210205512, 2847073169, 3324925707, 1251969297, 2210205512, 3491451503, 470400916, 1251969297, 2184392547,
-        1523590942, 2391111113, 68341529, 295466806, 1523590942, 4143310876, 3527253079, 295466806, 4059123142,
-        1406902110, 3735012720, 1774518130, 1814959027, 1406902110, 1560544267, 346472965, 1814959027, 964257199,
-        855309653, 4208503105, 1518467541, 2025248418, 855309653, 4148125749, 1349947330, 2025248418, 1168504873,
-        2375338156, 3629519168, 409696181, 252401654, 2375338156, 3992097193, 2793725401, 252401654, 1350184085,
-        873141039, 3885583138, 361604799, 3554143374, 873141039, 894746180, 1919765327, 3554143374, 876210854,
-        246368794, 1703793169, 2317362874, 2300930144, 246368794, 2560214589, 2016163623, 2300930144, 1504276775,
-        1574610921, 2147546631, 4103450226, 107416526, 1574610921, 1773803959, 1402542742, 107416526, 550063800,
-        363388665, 592194244, 1746615522, 2637234667, 363388665, 4031408742, 2895130475, 2637234667, 296510335,
-        3997368560, 3047771871, 3178383826, 1160174754, 3997368560, 4027094919, 1234984211, 1160174754, 4226264344,
-        3303179301, 4243968063, 3235964171, 1776841674, 3303179301, 2867287469, 1500495759, 1776841674, 1708226553,
-        1482944153, 3192311574, 354466071, 3932773012, 1482944153, 389193591, 3350181058, 3932773012, 3398059015,
-        640968550, 3226860971, 922372912, 1254989667, 640968550, 2383815228, 2027371896, 1254989667, 2925300409,
-        2313146046, 3910187183, 1377591475, 1689291784, 2313146046, 4255405993, 1650609719, 1689291784, 1897624297,
-        3656310954, 882924050, 2702189958, 3185020283, 3656310954, 1923190496, 2449669145, 3185020283, 4235849984,
-        377232416, 1498446142, 4229103619, 3926377906, 377232416, 600268838, 511317726, 3926377906, 216160452,
-        1969399344, 3273966859, 4220943579, 3952111894, 1969399344, 575096961, 3815277103, 3952111894, 792177412,
-        2957238169, 1410010554, 1523740068, 3949237584, 2957238169, 74149658, 2564746147, 3949237584, 2557663578,
-        3377318569, 1927835240, 2556102508, 3022040116, 3377318569, 2549406364, 2387074241, 3022040116, 1477293711,
-        257306870, 1748489735, 547809226, 3708493374, 257306870, 4183546362, 4435502, 3708493374, 1607696753,
-
-    };
-
-
-#endif // ROCRAND_MRG32K3A_PRECOMPUTED_H_
diff --git a/include/targets/omptarget/mrg32k3a.h b/include/targets/omptarget/mrg32k3a.h
new file mode 100644
index 0000000000..d42ac43891
--- /dev/null
+++ b/include/targets/omptarget/mrg32k3a.h
@@ -0,0 +1,206 @@
+/*
+   An implementation of MRG32k3a based on constexpr.
+   Original algorithm from
+      Pierre L'Ecuyer
+      Good Parameter Sets for Combined Multiple Recursive Random Number Generators
+      Operations Research, 47, 1 (1999), 159-164.
+ */
+
+#pragma once
+
+#include<cmath>
+#include<cstdint>
+#include<iostream>
+
+namespace quda
+{
+  namespace target
+  {
+    namespace rng
+    {
+      template <typename E,int N>
+      struct Arr {
+        constexpr E& operator[](int i) {return d[i];}
+        constexpr const E& operator[](int i) const {return d[i];}
+        E d[N];
+      };
+
+      template <typename E>
+      struct Arr3 {
+        constexpr Arr3() {};
+        constexpr Arr3(E x,E y,E z):d{x,y,z} {};
+        constexpr E& operator[](int i) {return d[i];}
+        constexpr const E& operator[](int i) const {return d[i];}
+        constexpr bool operator==(const Arr3& x) const {return d[0]==x.d[0] && d[1]==x.d[1] && d[2]==x.d[2];}
+        E d[3];
+      };
+
+      template <typename E>
+      inline std::ostream& operator<<(std::ostream& o, const Arr3<E>& a)
+      {
+        return o<<'['<<a[0]<<' '<<a[1]<<' '<<a[2]<<']';
+      }
+
+      using State = Arr3<uint32_t>;
+      using Trans = Arr3<State>;
+
+      struct MRG32k3a {
+        State s1,s2;
+      };
+
+      inline std::ostream& operator<<(std::ostream& o, const MRG32k3a& prn)
+      {
+        return o<<"MRG32k3a("<<prn.s1<<' '<<prn.s2<<')';
+      }
+
+      inline constexpr Trans squaremod(Trans a, uint64_t m)
+      {
+        Trans x;
+        for(int i=0;i<3;++i){
+          Arr3<uint64_t> t{0u,0u,0u};
+          for(int k=0;k<3;++k){
+            uint64_t aik = (uint64_t)a[i][k];
+            t[0] += aik*(uint64_t)a[k][0] % m;
+            t[1] += aik*(uint64_t)a[k][1] % m;
+            t[2] += aik*(uint64_t)a[k][2] % m;
+          }
+          x[i][0] = (uint32_t)(t[0] % m);
+          x[i][1] = (uint32_t)(t[1] % m);
+          x[i][2] = (uint32_t)(t[2] % m);
+        }
+        return x;
+      }
+
+      template <int p>
+      inline constexpr Arr<Trans,p> squaremodarray(Trans a, uint64_t m)
+      {
+        Arr<Trans,p> x;
+        x[0] = a;
+        for(int i=1;i<p;++i)
+          x[i] = squaremod(x[i-1], m);
+        return x;
+      }
+
+      inline void matvecmod(Trans a, State& v, uint64_t m)
+      {
+        uint64_t v0 = (uint64_t)v[0];
+        uint64_t v1 = (uint64_t)v[1];
+        uint64_t v2 = (uint64_t)v[2];
+        v[0] = (uint32_t)((((uint64_t)a[0][0]*v0) % m + ((uint64_t)a[0][1]*v1) % m + ((uint64_t)a[0][2]*v2) % m) % m);
+        v[1] = (uint32_t)((((uint64_t)a[1][0]*v0) % m + ((uint64_t)a[1][1]*v1) % m + ((uint64_t)a[1][2]*v2) % m) % m);
+        v[2] = (uint32_t)((((uint64_t)a[2][0]*v0) % m + ((uint64_t)a[2][1]*v1) % m + ((uint64_t)a[2][2]*v2) % m) % m);
+      }
+
+      constexpr double norm = 2.328306549295728e-10;
+      constexpr double m1 = 4294967087.0;
+      constexpr double m2 = 4294944443.0;
+      constexpr double a12 = 1403580.0;
+      constexpr double a13n = 810728.0;
+      constexpr double a21 = 527612.0;
+      constexpr double a23n = 1370589.0;
+      constexpr uint32_t defaultSEED = 12345U;
+      constexpr int subsequenceBase = 76;
+
+      constexpr Trans a1 {
+        State {0u,                  1u,            0u},
+        State {0u,                  0u,            1u},
+        State {(uint32_t)(m1-a13n), (uint32_t)a12, 0u}};
+      constexpr Trans a2 {
+        State {0u,                  1u, 0u},
+        State {0u,                  0u, 1u},
+        State {(uint32_t)(m2-a23n), 0u, (uint32_t)a21}};
+
+      constexpr int maxpower2 = 190;
+      constexpr Arr<Trans,maxpower2> a1sq = squaremodarray<maxpower2>(a1, (uint64_t)m1);
+      constexpr Arr<Trans,maxpower2> a2sq = squaremodarray<maxpower2>(a2, (uint64_t)m2);
+
+      static_assert(
+        a1sq[76] ==
+          Trans {
+            State {82758667u,   1871391091u, 4127413238u},
+            State {3672831523u, 69195019u,   1871391091u},
+            State {3672091415u, 3528743235u, 69195019u}},
+        "a1sq[76] wrong!");
+      static_assert(
+        a2sq[76] ==
+          Trans {
+            State {1511326704u, 3759209742u, 1610795712u},
+            State {4292754251u, 1511326704u, 3889917532u},
+            State {3859662829u, 4292754251u, 3708466080u}},
+        "a2sq[76] wrong!");
+
+      inline void skip(MRG32k3a& prn, uint64_t offset, int base = 0)
+      {
+        int i = 0;
+        uint64_t s = offset;
+        while(s>0){
+          if(s&1u){
+            matvecmod(a1sq[base+i], prn.s1, (uint64_t)m1);
+            matvecmod(a2sq[base+i], prn.s2, (uint64_t)m2);
+          }
+          s >>= 1u;
+          ++i;
+        }
+      }
+
+      inline void seed(MRG32k3a& prn, uint64_t seed, uint64_t subsequence)
+      {
+        if(seed){
+          const uint64_t d1 = (uint64_t)defaultSEED * (uint64_t)((uint32_t)seed ^ 0x55555555u);
+          const uint64_t d2 = (uint64_t)defaultSEED * (uint64_t)((uint32_t)(seed >> 32u) ^ 0xAAAAAAAAu);
+          prn.s1[0] = (uint32_t)(d1 % (uint64_t)m1);
+          prn.s1[1] = (uint32_t)(d2 % (uint64_t)m1);
+          prn.s1[2] = (uint32_t)(d1 % (uint64_t)m1);
+          prn.s2[0] = (uint32_t)(d2 % (uint64_t)m2);
+          prn.s2[1] = (uint32_t)(d1 % (uint64_t)m2);
+          prn.s2[2] = (uint32_t)(d2 % (uint64_t)m2);
+        }else{
+          prn.s1[0] = defaultSEED;
+          prn.s1[1] = defaultSEED;
+          prn.s1[2] = defaultSEED;
+          prn.s2[0] = defaultSEED;
+          prn.s2[1] = defaultSEED;
+          prn.s2[2] = defaultSEED;
+        }
+        skip(prn, subsequence, subsequenceBase);
+      }
+
+      inline double uniform(MRG32k3a& prn)
+      {
+        double p1,p2;
+        p1 = a12 * (double)(prn.s1[1]) - a13n * (double)(prn.s1[0]);
+        p1 = std::fmod(p1, m1);
+        if(p1<0.0)
+          p1 += m1;
+        prn.s1[0] = prn.s1[1];
+        prn.s1[1] = prn.s1[2];
+        prn.s1[2] = (uint32_t)p1;
+
+        p2 = a21 * (double)(prn.s2[2]) - a23n * (double)(prn.s2[0]);
+        p2 = std::fmod(p2, m2);
+        if(p2<0.0)
+          p2 += m2;
+        prn.s2[0] = prn.s2[1];
+        prn.s2[1] = prn.s2[2];
+        prn.s2[2] = (uint32_t)p2;
+
+        if(p1<=p2)
+          return (p1 - p2 + m1) * norm;
+        else
+          return (p1 - p2) * norm;
+      }
+
+      template <typename R>
+      inline void gaussian(MRG32k3a& prn, R& x, R& y)
+      {
+        constexpr R TINY = 9.999999999999999e-308;
+        R v,p,r;
+        v = (R)uniform(prn);
+        p = (R)uniform(prn) * (R)2.0 * (R)3.141592653589793238462643383279502884;
+        r = std::sqrt((R)(-2.0) * std::log(v + TINY));
+        x = r * std::sin(p);
+        y = r * std::cos(p);
+      }
+    }
+  }
+}
diff --git a/include/targets/omptarget/random_helper.h b/include/targets/omptarget/random_helper.h
index 7ea84123f7..86b9c15f8d 100644
--- a/include/targets/omptarget/random_helper.h
+++ b/include/targets/omptarget/random_helper.h
@@ -1,15 +1,16 @@
 #pragma once
 
 #include <random_quda.h>
+#include <mrg32k3a.h>
 
-// We don't use the internal Box-Muller state.
-#define ROCRAND_DETAIL_MRG32K3A_BM_NOT_IN_STATE
-#include <externals/rocrand/rocrand_mrg32k3a.h>
-
-namespace quda {
+namespace quda
+{
 
   struct RNGState {
-    rocrand_state_mrg32k3a state;
+    target::rng::MRG32k3a state;
+    bool has_extf, has_extd;
+    float extf;
+    double extd;
   };
 
   /**
@@ -22,14 +23,18 @@ namespace quda {
   inline void random_init(unsigned long long seed, unsigned long long sequence,
 			  unsigned long long offset, RNGState &state)
   {
-    //curand_init(seed, sequence, offset, &state.state);
-    state.state.seed(seed, sequence, offset);
+    target::rng::seed(state.state, seed, sequence);
+    target::rng::skip(state.state, offset);
+    state.has_extf = 0;
+    state.has_extd = 0;
+    state.extf = 0.0f;
+    state.extd = 0.0;
   }
 
-  template<class Real>
-    struct uniform { };
-  template<>
-    struct uniform<float> {
+  template <class Real> struct uniform {
+  };
+
+  template <> struct uniform<float> {
 
     /**
      * \brief Return a uniform deviate between 0 and 1
@@ -37,8 +42,7 @@ namespace quda {
      */
     static inline float rand(RNGState &state)
     {
-      //curand_uniform(&state.state);
-      return ROCRAND_2POW32_INV + (rocrand(&state.state) * ROCRAND_2POW32_INV);
+      return (float)target::rng::uniform(state.state);
     }
 
     /**
@@ -49,22 +53,19 @@ namespace quda {
      */
     static inline float rand(RNGState &state, float a, float b)
     {
-      //return a + (b - a) * rand(state);
-      return a + (b - a) * (ROCRAND_2POW32_INV + (rocrand(&state.state) * ROCRAND_2POW32_INV));
+      return a + (b - a) * (float)target::rng::uniform(state.state);
     }
 
   };
 
-  template<>
-    struct uniform<double> {
+  template <> struct uniform<double> {
     /**
      * \brief Return a uniform deviate between 0 and 1
      * @param [in,out] the RNG State
      */
     static inline double rand(RNGState &state)
     {
-      //curand_uniform_double(&state.state);
-      return ROCRAND_2POW32_INV_DOUBLE + (rocrand(&state.state) * ROCRAND_2POW32_INV_DOUBLE);
+      return target::rng::uniform(state.state);
     }
 
     /**
@@ -75,44 +76,51 @@ namespace quda {
      */
     static inline double rand(RNGState &state, double a, double b)
     {
-      //return a + (b - a) * rand(state);
-      return a + (b - a) * (ROCRAND_2POW32_INV_DOUBLE + (rocrand(&state.state) * ROCRAND_2POW32_INV_DOUBLE));
+      return a + (b - a) * target::rng::uniform(state.state);
     }
   };
 
-  template<class Real>
-    struct normal { };
+  template <class Real> struct normal {
+  };
 
-  template<>
-    struct normal<float> {
+  template <> struct normal<float> {
     /**
      * \brief return a gaussian normal deviate with mean of 0
      * @param [in,out] state
      */
     static inline float rand(RNGState &state)
     {
-      //curand_normal(&state.state);
-      float u = ROCRAND_2POW32_INV + (rocrand(&state.state) * ROCRAND_2POW32_INV);
-      float v = (ROCRAND_2POW32_INV + (rocrand(&state.state) * ROCRAND_2POW32_INV)) * ROCRAND_2PI;
-      float s = sqrtf(-2.0f * logf(u));
-      return sinf(v) * s;
+      if(state.has_extf){
+        state.has_extf = 0;
+        return state.extf;
+      }else{
+        float x,y;
+        target::rng::gaussian(state.state, x, y);
+        state.has_extf = 1;
+        state.extf = y;
+        return x;
+      }
     }
   };
 
-  template<>
-    struct normal<double> {
+  template <> struct normal<double> {
     /**
      * \brief return a gaussian (normal) deviate with a mean of 0
      * @param [in,out] state
      */
     static inline double rand(RNGState &state)
     {
-      //curand_normal_double(&state.state);
-      double u = ROCRAND_2POW32_INV_DOUBLE + (rocrand(&state.state) * ROCRAND_2POW32_INV_DOUBLE);
-      double v = (ROCRAND_2POW32_INV_DOUBLE + (rocrand(&state.state) * ROCRAND_2POW32_INV_DOUBLE)) * ROCRAND_PI_DOUBLE * 2.;
-      double s = sqrt(-2. * log(u));
-      return sin(v) * s;
+      if(state.has_extd){
+        state.has_extd = 0;
+        return state.extd;
+      }else{
+        double x,y;
+        target::rng::gaussian(state.state, x, y);
+        state.has_extd = 1;
+        state.extd = y;
+        return x;
+      }
     }
   };
 
-}
+} // namespace quda

From db9d29b3d6dddc79ea8f301b7cf265ef051c6deb Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 20 Apr 2022 21:56:38 +0000
Subject: [PATCH 111/198] omp: revert generic/fast_intdiv.h to upstream version

---
 include/targets/generic/fast_intdiv.h | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/include/targets/generic/fast_intdiv.h b/include/targets/generic/fast_intdiv.h
index 71fe4761b8..2ac6878b0d 100644
--- a/include/targets/generic/fast_intdiv.h
+++ b/include/targets/generic/fast_intdiv.h
@@ -112,7 +112,7 @@ class int_fastdiv
   }
 
   __host__ __device__ __forceinline__
-    friend int operator/(const int n, const int_fastdiv& divisor);
+    friend int operator/(const int divident, const int_fastdiv& divisor);
 };
 
 template <bool is_device> struct mul_hi {
@@ -135,7 +135,7 @@ int operator/(const int n, const int_fastdiv& divisor)
   return q;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator%(const int n, const int_fastdiv& divisor)
 {
   int quotient = n / divisor;
@@ -143,61 +143,61 @@ int operator%(const int n, const int_fastdiv& divisor)
   return remainder;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator/(const unsigned int n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator%(const unsigned int n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator/(const short n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator%(const short n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator/(const unsigned short n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator%(const unsigned short n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator/(const char n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator%(const char n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator/(const unsigned char n, const int_fastdiv& divisor)
 {
   return ((int)n) / divisor;
 }
 
-__host__ __device__ __forceinline__ static
+__host__ __device__ __forceinline__
 int operator%(const unsigned char n, const int_fastdiv& divisor)
 {
   return ((int)n) % divisor;

From 6329d5735394736dd27289791cbbd5636bd78098 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 18 Apr 2022 17:01:38 +0000
Subject: [PATCH 112/198] omp: use "omp for" for block reduce

---
 .../targets/omptarget/block_reduce_helper.h   | 48 +++++--------------
 1 file changed, 11 insertions(+), 37 deletions(-)

diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index 7c108d632c..86880c3384 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -214,58 +214,32 @@ namespace quda
     __device__ inline T operator()(const T &value_, bool async, int batch, bool all, const reducer_t &r,
                                    const param_t &)
     {
-#if 1
-      using block_reduce_t = cub::BlockReduce<T, param_t::block_size_x, cub::BLOCK_REDUCE_WARP_REDUCTIONS,
-                                              param_t::block_size_y, param_t::block_size_z, __COMPUTE_CAPABILITY__>;
-      using tempStorage = typename block_reduce_t::TempStorage;
-      static_assert(param_t::batch_size*sizeof(tempStorage) <= sizeof(target::omptarget::shared_cache.addr[0])*(8192+4096+2048), "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, do we have that many?
-      tempStorage *storage = (tempStorage*)&target::omptarget::shared_cache.addr[128];  // FIXME arbitrary
-      // typename block_reduce_t::TempStorage storage[param_t::batch_size];
-      block_reduce_t block_reduce(storage[batch]);
-      if (!async){
-        #pragma omp barrier
-        // __syncthreads(); // only synchronize if we are not pipelining
-      }
-      T value = reducer_t::do_sum ? block_reduce.Sum(value_) : block_reduce.Reduce(value_, r);
-      if (all) {
-        QUDA_RT_CONSTS;
-        T &value_shared = reinterpret_cast<T &>(storage[batch]);
-        if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) value_shared = value;
-        #pragma omp barrier
-        value = value_shared;
-      }
-#else
-      // this block of code only works for a single batch the size of the thread block
-      // test with QUDA/BlasTest.verify/reDotProduct_block_double_double --nsrc 2 --msrc 2
-      constexpr int nthr = param_t::block_size_x*param_t::block_size_y*param_t::block_size_z;
+      constexpr int nthr = param_t::block_size_x*param_t::block_size_y*(param_t::batch_size>param_t::block_size_z?param_t::batch_size:param_t::block_size_z);
       int tid = omp_get_thread_num();
       static_assert(nthr*sizeof(T) <= sizeof(target::omptarget::shared_cache.addr[0])*8192, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, do we have that many?
       T *storage = (T*)&target::omptarget::shared_cache.addr[128];  // FIXME arbitrary
+      // #pragma omp barrier
       storage[tid] = value_;
+      // #pragma omp barrier
       int offset = 1;
       constexpr int bs = param_t::batch_size;
       static_assert(nthr%bs==0, "Block size not divisible by batch_size");
       constexpr int subblock = nthr/bs;
-      do{
-        #pragma omp for collapse(2)
-        for(int b=0;b<nthr;b+=subblock)
-          for(int i=0;i<subblock-offset;i+=2*offset)
-            storage[b+i] = r(storage[b+i], storage[b+i+offset]);
-        offset *= 2;
-      }while(offset<subblock);
-/*
-      int b = subblock*batch;
+      auto t0 = r.init();
       do{
         #pragma omp for
-        for(int i=0;i<subblock-offset;i+=2*offset)
-          storage[b+i] = r(storage[b+i], storage[b+i+offset]);
+        for(int i=0;i<nthr;++i){
+          auto j = i+offset;
+          auto t = i/subblock==j/subblock ? storage[j] : t0;
+          storage[i] = r(storage[i], t);
+        }
+        // #pragma omp barrier
         offset *= 2;
       }while(offset<subblock);
-*/
+      // #pragma omp barrier
       T value = storage[tid];
       if(all)
         value = storage[subblock*(tid/subblock)];
-#endif
       return value;
     }
   };

From b2b4c5aab6a8ea502e4326cac9b5f96a6d765dac Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 21 Apr 2022 19:49:18 +0000
Subject: [PATCH 113/198] omp/mrg32k3a: use numeric_limits::min for TINY

---
 include/targets/omptarget/mrg32k3a.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/include/targets/omptarget/mrg32k3a.h b/include/targets/omptarget/mrg32k3a.h
index d42ac43891..0dacbfcf4f 100644
--- a/include/targets/omptarget/mrg32k3a.h
+++ b/include/targets/omptarget/mrg32k3a.h
@@ -11,6 +11,7 @@
 #include<cmath>
 #include<cstdint>
 #include<iostream>
+#include<limits>
 
 namespace quda
 {
@@ -193,7 +194,7 @@ namespace quda
       template <typename R>
       inline void gaussian(MRG32k3a& prn, R& x, R& y)
       {
-        constexpr R TINY = 9.999999999999999e-308;
+        constexpr R TINY = std::numeric_limits<R>::min();
         R v,p,r;
         v = (R)uniform(prn);
         p = (R)uniform(prn) * (R)2.0 * (R)3.141592653589793238462643383279502884;

From c04ab7263f5ea2415699cbd4bdba7f57f843ff69 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 6 May 2022 21:18:36 +0000
Subject: [PATCH 114/198] omp: update function types in comm_target.cpp

---
 lib/targets/omptarget/comm_target.cpp | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/targets/omptarget/comm_target.cpp b/lib/targets/omptarget/comm_target.cpp
index fd43158016..90e28e17ab 100644
--- a/lib/targets/omptarget/comm_target.cpp
+++ b/lib/targets/omptarget/comm_target.cpp
@@ -17,24 +17,27 @@ namespace quda
     return 0;
   }
 
-  void comm_create_neighbor_memory(void *remote[QUDA_MAX_DIM][2], void *local)
+  void comm_create_neighbor_memory(array_2d<void *, QUDA_MAX_DIM, 2> &remote, void *local)
   {
     ompwip("unimplemented");
   }
 
-  void comm_destroy_neighbor_memory(void *remote[QUDA_MAX_DIM][2])
+  void comm_destroy_neighbor_memory(array_2d<void *, QUDA_MAX_DIM, 2> &remote)
   {
     ompwip("unimplemented");
   }
 
-  void comm_create_neighbor_event(qudaEvent_t remote[2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
+  void comm_create_neighbor_event(array_2d<qudaEvent_t, QUDA_MAX_DIM, 2> &remote,
+                                  array_2d<qudaEvent_t, QUDA_MAX_DIM, 2> &local)
   {
     ompwip("unimplemented");
   }
 
-  void comm_destroy_neighbor_event(qudaEvent_t [2][QUDA_MAX_DIM], qudaEvent_t local[2][QUDA_MAX_DIM])
+  void comm_destroy_neighbor_event(array_2d<qudaEvent_t, QUDA_MAX_DIM, 2> &,
+                                   array_2d<qudaEvent_t, QUDA_MAX_DIM, 2> &local)
   {
     ompwip("unimplemented");
   }
 
 } // namespace quda
+

From e2f9f7306da7e221a7fa3b9f3cd5fe32836fe234 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 18 May 2022 22:10:48 +0000
Subject: [PATCH 115/198] omp: reworked shared cache system, replaced single
 address passing

---
 include/kernels/coarse_op_kernel.cuh          |  2 +-
 include/shared_memory_cache_helper.cuh        |  5 +--
 .../targets/omptarget/block_reduce_helper.h   | 16 +++++-----
 .../omptarget/block_reduction_kernel.h        |  6 ++--
 include/targets/omptarget/kernel.h            | 21 ++++++++-----
 include/targets/omptarget/quda_api.h          | 31 +++++++++++++++++--
 include/targets/omptarget/reduce_helper.h     | 23 ++++++++------
 include/targets/omptarget/reduction_kernel.h  | 12 ++++---
 include/targets/omptarget/target_device.h     |  2 +-
 lib/targets/omptarget/target_omptarget.cmake  |  4 +--
 10 files changed, 84 insertions(+), 38 deletions(-)

diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 19c8f235da..6971975040 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -1373,7 +1373,7 @@ QUDA_UNROLL
       __shared__ complex<storeType> Y[Arg::max_color_height_per_block][Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
 #else
       typedef complex<storeType> TyX0[Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
-      TyX0 *X = reinterpret_cast<TyX0 *>(target::omptarget::shared_cache.addr);
+      TyX0 *X = reinterpret_cast<TyX0 *>(target::omptarget::get_shared_cache());
       TyX0 *Y = X+Arg::max_color_height_per_block;
 #endif
 
diff --git a/include/shared_memory_cache_helper.cuh b/include/shared_memory_cache_helper.cuh
index 902d0f3cfd..6bbc0448f2 100644
--- a/include/shared_memory_cache_helper.cuh
+++ b/include/shared_memory_cache_helper.cuh
@@ -76,7 +76,7 @@ namespace quda
         extern __shared__ int cache_[];
         return reinterpret_cast<atom_t*>(cache_);
 #else
-        return reinterpret_cast<atom_t*>(target::omptarget::shared_cache.addr);
+        return reinterpret_cast<atom_t*>(target::omptarget::get_shared_cache());
 #endif
       }
     };
@@ -92,7 +92,8 @@ namespace quda
         static __shared__ atom_t cache_[n_element * block_size_x * block_size_y * block_size_z];
         return reinterpret_cast<atom_t*>(cache_);
 #else
-        return reinterpret_cast<atom_t*>(target::omptarget::shared_cache.addr);
+        static_assert(n_element * block_size_x * block_size_y * block_size_z * sizeof(atom_t) <= device::max_shared_memory_size(), "not enough shared memory");
+        return reinterpret_cast<atom_t*>(target::omptarget::get_shared_cache());
 #endif
       }
     };
diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index 86880c3384..fde393026f 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -171,16 +171,16 @@ namespace quda
     {
       using warp_reduce_t = cub::WarpReduce<T, param_t::width, __COMPUTE_CAPABILITY__>;
       using tempStorage = typename warp_reduce_t::TempStorage;
-      static_assert(sizeof(tempStorage) <= sizeof(target::omptarget::shared_cache.addr[0])*1024, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, 1024 is used below
-      tempStorage *dummy_storage = (tempStorage*)&target::omptarget::shared_cache.addr[1024];
+      static_assert(sizeof(tempStorage) <= sizeof(target::omptarget::get_shared_cache()[0])*1024, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, 1024 is used below
+      tempStorage *dummy_storage = (tempStorage*)&target::omptarget::get_shared_cache()[1024];
       warp_reduce_t warp_reduce(*dummy_storage);
       T value = reducer_t::do_sum ? warp_reduce.Sum(value_) : warp_reduce.Reduce(value_, r);
 
       if (all) {
         using warp_scan_t = cub::WarpScan<T, param_t::width, __COMPUTE_CAPABILITY__>;
         using tempStorage = typename warp_scan_t::TempStorage;
-        static_assert(sizeof(tempStorage) <= sizeof(target::omptarget::shared_cache.addr[0])*1024, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, 1024 is arbitrary
-        tempStorage *dummy_storage = (tempStorage*)&target::omptarget::shared_cache.addr[2048];
+        static_assert(sizeof(tempStorage) <= sizeof(target::omptarget::get_shared_cache()[0])*1024, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, 1024 is arbitrary
+        tempStorage *dummy_storage = (tempStorage*)&target::omptarget::get_shared_cache()[2048];
         // typename warp_scan_t::TempStorage dummy_storage;
         warp_scan_t warp_scan(*dummy_storage);
         value = warp_scan.Broadcast(value, 0);
@@ -216,11 +216,11 @@ namespace quda
     {
       constexpr int nthr = param_t::block_size_x*param_t::block_size_y*(param_t::batch_size>param_t::block_size_z?param_t::batch_size:param_t::block_size_z);
       int tid = omp_get_thread_num();
-      static_assert(nthr*sizeof(T) <= sizeof(target::omptarget::shared_cache.addr[0])*8192, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, do we have that many?
-      T *storage = (T*)&target::omptarget::shared_cache.addr[128];  // FIXME arbitrary
-      // #pragma omp barrier
+      static_assert(nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
+      T *storage = (T*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
+      #pragma omp barrier
       storage[tid] = value_;
-      // #pragma omp barrier
+      #pragma omp barrier
       int offset = 1;
       constexpr int bs = param_t::batch_size;
       static_assert(nthr%bs==0, "Block size not divisible by batch_size");
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 0ddb3a9195..44a400370d 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -123,10 +123,11 @@ namespace quda
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -165,10 +166,11 @@ namespace quda
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index de4ca6e037..274363395d 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -59,10 +59,11 @@ namespace quda
     // const int ty = arg.threads.y;
     // const int tz = arg.threads.z;
     // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -97,10 +98,11 @@ namespace quda
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -168,10 +170,11 @@ namespace quda
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -205,10 +208,11 @@ namespace quda
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
         int cache[device::max_shared_memory_size()/sizeof(int)];
-        target::omptarget::shared_cache.addr = cache;
+        target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -278,10 +282,11 @@ namespace quda
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -315,10 +320,11 @@ namespace quda
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -357,10 +363,11 @@ namespace quda
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index 90a58deadc..c016f30467 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -93,9 +93,36 @@ namespace quda {
   namespace target {
     namespace omptarget {
       struct SharedCache{
-        int *addr;
+        int **addr;
+        int num_teams;
       };
-      extern SharedCache shared_cache;
+      inline void prepare_shared_cache(int num_teams)
+      {
+        extern SharedCache shared_cache;
+        static int init = 0;
+        if(!init){
+          init = 1;
+          shared_cache.addr = (int**)omp_target_alloc(num_teams*sizeof(void*), omp_get_default_device());
+          shared_cache.num_teams = num_teams;
+        }
+        if(shared_cache.num_teams<num_teams){
+          omp_target_free(shared_cache.addr, omp_get_default_device());
+          shared_cache.addr = (int**)omp_target_alloc(num_teams*sizeof(void*), omp_get_default_device());
+          shared_cache.num_teams = num_teams;
+        }
+        #pragma omp target update to(shared_cache)
+      }
+      inline void save_shared_cache(int ptr[])
+      {
+        extern SharedCache shared_cache;
+        shared_cache.addr[omp_get_team_num()] = ptr;
+      }
+      inline int *get_shared_cache(void)
+      {
+        extern SharedCache shared_cache;
+        #pragma omp barrier
+        return shared_cache.addr[omp_get_team_num()];
+      }
 
       struct LaunchParam{
         dim3 block;
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 3cb285a8ba..2db117b84b 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -125,12 +125,10 @@ namespace quda
       = std::min(Arg::max_n_batch_block, device::max_block_size() / (block_size_x * block_size_y));
     using BlockReduce = BlockReduce<T, block_size_x, block_size_y, n_batch_block, true>;
     // bool isLastBlockDone[n_batch_block];
-    static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::shared_cache.addr[0])*64, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 128 is used in block_reduce_helper.h:/tempStorage/
-    bool *isLastBlockDone = (bool*)target::omptarget::shared_cache.addr;
-    bool *hasLastBlockDone = (bool*)&target::omptarget::shared_cache.addr[64];
+    static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::get_shared_cache()[0])*64, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 128 is used in block_reduce_helper.h:/tempStorage/
+    bool *isLastBlockDone = (bool*)target::omptarget::get_shared_cache();
+    bool *hasLastBlockDone = (bool*)&target::omptarget::get_shared_cache()[64];
     // printf("team %d thread %d isLastBlockDone %p\n", omp_get_team_num(), omp_get_thread_num(), isLastBlockDone);
-    if (target::thread_idx().x == 0 && target::thread_idx().y == 0 && target::thread_idx().z == 0)
-      *hasLastBlockDone = 0;
 
     T aggregate = BlockReduce(target::thread_idx().z).Reduce(in, r);
     // printf("team %d thread %d  r %g  aggregate %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&in), *(double*)(&aggregate));
@@ -148,11 +146,17 @@ namespace quda
 
       // determine if last block
       isLastBlockDone[target::thread_idx().z] = (value == (target::grid_dim().x - 1));
-      if(value == (target::grid_dim().x - 1))
-        *hasLastBlockDone = 1;
     }
-
-    __syncthreads();
+    #pragma omp barrier
+    #pragma omp single
+    {
+      *hasLastBlockDone = 0;
+      for(int i=0;i<target::block_dim().z;++i)
+        if(isLastBlockDone[i]){
+          *hasLastBlockDone = 1;
+          break;
+        }
+    }
 
     // finish the reduction if last block
     if (*hasLastBlockDone) {
@@ -181,6 +185,7 @@ namespace quda
           arg.count[idx] = 0; // set to zero for next time
         }
       }
+      #pragma omp barrier
     }  // hasLastBlockDone
   }
 
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index d95310833c..cfb19b4594 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -80,10 +80,11 @@ namespace quda
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       char buffer[sizeof(Arg)];
@@ -111,10 +112,11 @@ namespace quda
   {
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       Reduction2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
@@ -183,10 +185,11 @@ namespace quda
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       char buffer[sizeof(Arg)];
@@ -214,10 +217,11 @@ namespace quda
   {
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
       int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::shared_cache.addr = cache;
+      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       MultiReduction_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 0d64dcf629..c902e402f3 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -9,7 +9,7 @@
 #endif
 
 #ifndef QUDA_MAX_SHARED_MEMORY_SIZE
-#define QUDA_MAX_SHARED_MEMORY_SIZE 32*1024
+#define QUDA_MAX_SHARED_MEMORY_SIZE 40*1024
 #endif
 
 namespace quda {
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index 9b794c808e..18dd7250f5 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -25,8 +25,8 @@ set_property(CACHE QUDA_MAX_BLOCK_SIZE PROPERTY STRINGS 256 512 768 1024)
 target_compile_definitions(quda PUBLIC QUDA_MAX_BLOCK_SIZE=${QUDA_MAX_BLOCK_SIZE})
 message(STATUS "Using maximum team size: ${QUDA_MAX_BLOCK_SIZE}")
 
-set(QUDA_MAX_SHARED_MEMORY_SIZE 32768 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
-set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 16384 32768 49152)
+set(QUDA_MAX_SHARED_MEMORY_SIZE 40960 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
+set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152)
 target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
 message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")
 

From 2d6335f0f1a576be93b4657a55329182383acbc2 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 18 May 2022 23:01:53 +0000
Subject: [PATCH 116/198] omp: allocate global memory for shared local memory

---
 .../omptarget/block_reduction_kernel.h        |  6 -----
 include/targets/omptarget/kernel.h            | 21 ---------------
 include/targets/omptarget/quda_api.h          | 27 +++----------------
 include/targets/omptarget/reduction_kernel.h  | 12 ---------
 lib/targets/omptarget/quda_api.cpp            | 17 ++++++++++++
 5 files changed, 20 insertions(+), 63 deletions(-)

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 44a400370d..1ef255ef81 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -123,11 +123,8 @@ namespace quda
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -166,11 +163,8 @@ namespace quda
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 274363395d..c9a2e50daa 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -59,11 +59,8 @@ namespace quda
     // const int ty = arg.threads.y;
     // const int tz = arg.threads.z;
     // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -98,11 +95,8 @@ namespace quda
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -170,11 +164,8 @@ namespace quda
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -208,11 +199,8 @@ namespace quda
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-        int cache[device::max_shared_memory_size()/sizeof(int)];
-        target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -282,11 +270,8 @@ namespace quda
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -320,11 +305,8 @@ namespace quda
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     // printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
@@ -363,11 +345,8 @@ namespace quda
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
       #pragma omp parallel num_threads(ld)
       {
         // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index c016f30467..2ad380aa7d 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -93,35 +93,14 @@ namespace quda {
   namespace target {
     namespace omptarget {
       struct SharedCache{
-        int **addr;
+        int *addr;
         int num_teams;
+        int cache_length;
       };
-      inline void prepare_shared_cache(int num_teams)
-      {
-        extern SharedCache shared_cache;
-        static int init = 0;
-        if(!init){
-          init = 1;
-          shared_cache.addr = (int**)omp_target_alloc(num_teams*sizeof(void*), omp_get_default_device());
-          shared_cache.num_teams = num_teams;
-        }
-        if(shared_cache.num_teams<num_teams){
-          omp_target_free(shared_cache.addr, omp_get_default_device());
-          shared_cache.addr = (int**)omp_target_alloc(num_teams*sizeof(void*), omp_get_default_device());
-          shared_cache.num_teams = num_teams;
-        }
-        #pragma omp target update to(shared_cache)
-      }
-      inline void save_shared_cache(int ptr[])
-      {
-        extern SharedCache shared_cache;
-        shared_cache.addr[omp_get_team_num()] = ptr;
-      }
       inline int *get_shared_cache(void)
       {
         extern SharedCache shared_cache;
-        #pragma omp barrier
-        return shared_cache.addr[omp_get_team_num()];
+        return &shared_cache.addr[omp_get_team_num()*shared_cache.cache_length];
       }
 
       struct LaunchParam{
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index cfb19b4594..c34d698a2b 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -80,11 +80,8 @@ namespace quda
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       char buffer[sizeof(Arg)];
@@ -112,11 +109,8 @@ namespace quda
   {
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       Reduction2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
@@ -185,11 +179,8 @@ namespace quda
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
     Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       char buffer[sizeof(Arg)];
@@ -217,11 +208,8 @@ namespace quda
   {
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    target::omptarget::prepare_shared_cache(gd);
     #pragma omp target teams num_teams(gd) thread_limit(ld)
     {
-      int cache[device::max_shared_memory_size()/sizeof(int)];
-      target::omptarget::save_shared_cache(cache);
     #pragma omp parallel num_threads(ld)
     {
       MultiReduction_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index ed5d23f1c4..ff3fde7007 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -4,6 +4,7 @@
 #include <quda_internal.h>
 #include <timer.h>
 #include <device.h>
+#include <target_device.h>
 
 namespace quda
 {
@@ -39,6 +40,22 @@ namespace quda
         launch_param.grid = tp.grid;
         launch_param.block = tp.block;
         #pragma omp target update to(launch_param)
+
+        static int init = 0;
+        int num_teams = tp.grid.x*tp.grid.y*tp.grid.z;
+        if(!init){
+          init = 1;
+          shared_cache.addr = (int*)omp_target_alloc(num_teams*device::max_shared_memory_size(), omp_get_default_device());
+          shared_cache.num_teams = num_teams;
+          shared_cache.cache_length = device::max_shared_memory_size()/sizeof(shared_cache.addr[0]);
+        }
+        if(shared_cache.num_teams<num_teams){
+          omp_target_free(shared_cache.addr, omp_get_default_device());
+          shared_cache.addr = (int*)omp_target_alloc(num_teams*device::max_shared_memory_size(), omp_get_default_device());
+          shared_cache.num_teams = num_teams;
+          shared_cache.cache_length = device::max_shared_memory_size()/sizeof(shared_cache.addr[0]);
+        }
+        #pragma omp target update to(shared_cache)
       }
 
       static inline int

From 1bdd388137ddc8fba21429f533077fe3e9d41d8d Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 20 May 2022 23:07:59 +0000
Subject: [PATCH 117/198] omp: fix min/max for arrays

---
 include/reducer.h                         |  4 ++--
 include/targets/omptarget/math_helper.cuh | 17 +++++++++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/include/reducer.h b/include/reducer.h
index d52e7a7d47..af930e9b7b 100644
--- a/include/reducer.h
+++ b/include/reducer.h
@@ -91,7 +91,7 @@ namespace quda
     using reducer_t = maximum<T>;
     template <typename U> static inline void comm_reduce(std::vector<U> &a) { comm_allreduce_max(a); }
     __device__ __host__ static inline T init() { return low<T>::value(); }
-    __device__ __host__ static inline T apply(T a, T b) { return std::max(a, b); }
+    __device__ __host__ static inline T apply(T a, T b) { return max(a, b); }
     __device__ __host__ inline T operator()(T a, T b) const { return apply(a, b); }
   };
 
@@ -104,7 +104,7 @@ namespace quda
     using reducer_t = minimum<T>;
     template <typename U> static inline void comm_reduce(std::vector<U> &a) { comm_allreduce_min(a); }
     __device__ __host__ static inline T init() { return high<T>::value(); }
-    __device__ __host__ static inline T apply(T a, T b) { return std::min(a, b); }
+    __device__ __host__ static inline T apply(T a, T b) { return min(a, b); }
     __device__ __host__ inline T operator()(T a, T b) const { return apply(a, b); }
   };
 
diff --git a/include/targets/omptarget/math_helper.cuh b/include/targets/omptarget/math_helper.cuh
index 4891d8f7ac..faad4c9588 100644
--- a/include/targets/omptarget/math_helper.cuh
+++ b/include/targets/omptarget/math_helper.cuh
@@ -5,6 +5,23 @@
 
 namespace quda {
 
+  /**
+   * @brief Maximum of two numbers
+   * @param a first number
+   * @param b second number
+   */
+  template<typename T>
+  inline __host__ __device__ T max(const T &a, const T &b) { return a > b ? a : b; }
+
+  /**
+   * @brief Minimum of two numbers
+   * @param a first number
+   * @param b second number
+   */
+  template<typename T>
+  inline __host__ __device__ T min(const T &a, const T &b) { return a < b ? a : b; }
+
+
   /**
    * @brief Combined sin and cos calculation in QUDA NAMESPACE
    * @param a the angle

From da4a8806adc488712f03b333d0642d4be271369f Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 20 May 2022 23:14:40 +0000
Subject: [PATCH 118/198] omp: cmake allows larger parameters for testing

---
 lib/targets/omptarget/target_omptarget.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index 18dd7250f5..7c87d19f01 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -16,7 +16,7 @@ set_property(CACHE QUDA_GPU_ARCH PROPERTY STRINGS xehp)
 message(STATUS "Building for GPU Architectures: ${QUDA_GPU_ARCH}")
 
 set(QUDA_WARP_SIZE 16 CACHE STRING "OpenMP target logical warp size")
-set_property(CACHE QUDA_WARP_SIZE PROPERTY STRINGS 8 16 32)
+set_property(CACHE QUDA_WARP_SIZE PROPERTY STRINGS 8 16 32 64)
 target_compile_definitions(quda PUBLIC QUDA_WARP_SIZE=${QUDA_WARP_SIZE})
 message(STATUS "Using logical warp size: ${QUDA_WARP_SIZE}")
 
@@ -26,7 +26,7 @@ target_compile_definitions(quda PUBLIC QUDA_MAX_BLOCK_SIZE=${QUDA_MAX_BLOCK_SIZE
 message(STATUS "Using maximum team size: ${QUDA_MAX_BLOCK_SIZE}")
 
 set(QUDA_MAX_SHARED_MEMORY_SIZE 40960 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
-set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152)
+set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152 65536 98304 131072 196608 262144)
 target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
 message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")
 

From f0fc798e3be8ad7c5676623d16ac63e866db50be Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 27 May 2022 19:26:52 +0000
Subject: [PATCH 119/198] omp: disable simulation of get_num_threads in target

---
 lib/targets/omptarget/target_omptarget.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index 7c87d19f01..b4f2d2e5e7 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -32,7 +32,7 @@ message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}
 
 # ######################################################################################################################
 # define omptarget flags
-set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51)
+set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 -mllvm -vpo-paropt-simulate-get-num-threads-in-target=false)
 message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
 
 # QUDA_HASH for tunecache

From 6e0b0641ad1ece2ab00f8211d66df9340271eac3 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 27 May 2022 19:27:52 +0000
Subject: [PATCH 120/198] omp: use firstprivate on teams to get thread private
 copy (non-stardard behavior?)

---
 .../omptarget/block_reduction_kernel.h        | 14 ++---
 include/targets/omptarget/kernel.h            | 52 +++++++++----------
 include/targets/omptarget/reduction_kernel.h  | 28 +++++-----
 3 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 1ef255ef81..863dac46fd 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -120,10 +120,10 @@ namespace quda
     // const int ty = arg.threads.y;
     // const int tz = arg.threads.z;
     // printf("BlockKernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
-    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
     {
     #pragma omp parallel num_threads(ld)
     {
@@ -132,12 +132,12 @@ namespace quda
       //          "omp reports: teams %d threads %d\n",
       //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
       //          omp_get_num_teams(), omp_get_num_threads());
-      char buffer[sizeof(Arg)];
-      memcpy(buffer, (void *)dparg, sizeof(Arg));
-      BlockKernel2D_impl<Functor, Arg>(*(Arg *)buffer);
+      // char buffer[sizeof(Arg)];
+      // memcpy(buffer, (void *)dparg, sizeof(Arg));
+      BlockKernel2D_impl<Functor, Arg>(arg);
     }
     }
-    omp_target_free(dparg, omp_get_default_device());
+    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index c9a2e50daa..1fb2140a02 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -52,14 +52,14 @@ namespace quda
   {
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
-    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
     // const int tx = arg.threads.x;
     // const int ty = arg.threads.y;
     // const int tz = arg.threads.z;
     // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
     {
       #pragma omp parallel num_threads(ld)
       {
@@ -68,13 +68,13 @@ namespace quda
         //          "omp reports: teams %d threads %d\n",
         //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
         //          omp_get_num_teams(), omp_get_num_threads());
-        char buffer[sizeof(Arg)];
-        memcpy(buffer, (void *)dparg, sizeof(Arg));
-        Kernel1D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
+        // char buffer[sizeof(Arg)];
+        // memcpy(buffer, (void *)dparg, sizeof(Arg));
+        Kernel1D_impl<Functor, Arg, grid_stride>(arg);
       }
     }
     // printf("Kernel1D: exited\n");
-    omp_target_free(dparg, omp_get_default_device());
+    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -161,10 +161,10 @@ namespace quda
     const int ty = arg.threads.y;
     const int tz = arg.threads.z;
     // printf("Kernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
-    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
     {
       #pragma omp parallel num_threads(ld)
       {
@@ -173,12 +173,12 @@ namespace quda
         //          "omp reports: teams %d threads %d\n",
         //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
         //          omp_get_num_teams(), omp_get_num_threads());
-        char buffer[sizeof(Arg)];
-        memcpy(buffer, (void *)dparg, sizeof(Arg));
-        Kernel2D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
+        // char buffer[sizeof(Arg)];
+        // memcpy(buffer, (void *)dparg, sizeof(Arg));
+        Kernel2D_impl<Functor, Arg, grid_stride>(arg);
       }
     }
-    omp_target_free(dparg, omp_get_default_device());
+    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -267,10 +267,10 @@ namespace quda
     const int ty = arg.threads.y;
     const int tz = arg.threads.z;
     // printf("Kernel3D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
-    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
     {
       #pragma omp parallel num_threads(ld)
       {
@@ -279,12 +279,12 @@ namespace quda
         //          "omp reports: teams %d threads %d\n",
         //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
         //          omp_get_num_teams(), omp_get_num_threads());
-        char buffer[sizeof(Arg)];
-        memcpy(buffer, (void *)dparg, sizeof(Arg));
-        Kernel3D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
+        // char buffer[sizeof(Arg)];
+        // memcpy(buffer, (void *)dparg, sizeof(Arg));
+        Kernel3D_impl<Functor, Arg, grid_stride>(arg);
       }
     }
-    omp_target_free(dparg, omp_get_default_device());
+    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -342,10 +342,10 @@ namespace quda
     const int ty = arg.threads.y;
     const int tz = arg.threads.z;
     // printf("raw_kernel: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
     // printf("dparg %p\n", dparg);
-    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
     {
       #pragma omp parallel num_threads(ld)
       {
@@ -354,11 +354,11 @@ namespace quda
         //          "omp reports: teams %d threads %d\n",
         //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
         //          omp_get_num_teams(), omp_get_num_threads());
-        Functor<Arg> f(*dparg);
+        Functor<Arg> f(&arg);
         f();
       }
     }
-    omp_target_free(dparg, omp_get_default_device());
+    // omp_target_free(dparg, omp_get_default_device());
   }
 
 }
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index c34d698a2b..5e06887fd8 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -78,18 +78,18 @@ namespace quda
   {
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
     {
     #pragma omp parallel num_threads(ld)
     {
-      char buffer[sizeof(Arg)];
-      memcpy(buffer, (void *)dparg, sizeof(Arg));
-      Reduction2D_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
+      // char buffer[sizeof(Arg)];
+      // memcpy(buffer, (void *)dparg, sizeof(Arg));
+      Reduction2D_impl<Functor, Arg, grid_stride>(arg);
     }
     }
-    omp_target_free(dparg, omp_get_default_device());
+    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -177,18 +177,18 @@ namespace quda
   {
     const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
     const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(dparg)
+    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
+    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
     {
     #pragma omp parallel num_threads(ld)
     {
-      char buffer[sizeof(Arg)];
-      memcpy(buffer, (void *)dparg, sizeof(Arg));
-      MultiReduction_impl<Functor, Arg, grid_stride>(*(Arg *)buffer);
+      // char buffer[sizeof(Arg)];
+      // memcpy(buffer, (void *)dparg, sizeof(Arg));
+      MultiReduction_impl<Functor, Arg, grid_stride>(arg);
     }
     }
-    omp_target_free(dparg, omp_get_default_device());
+    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**

From 493e2e2d421f7c40eb5c5e2ed392d913e9ec5170 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 31 May 2022 20:22:08 +0000
Subject: [PATCH 121/198] omp: update reductions to upstream API

Note: some tests fail likely due to unidentified race condition.
---
 .../targets/omptarget/block_reduce_helper.h   | 216 +++---------------
 .../omptarget/block_reduction_kernel.h        |   4 +-
 include/targets/omptarget/reduce_helper.h     |  42 ++--
 include/targets/omptarget/reduction_kernel.h  |  21 +-
 include/targets/omptarget/target_device.h     |  61 +++--
 lib/targets/omptarget/comm_target.cpp         |   2 +-
 lib/targets/omptarget/target_omptarget.cmake  |   4 +-
 7 files changed, 93 insertions(+), 257 deletions(-)

diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index fde393026f..8a8ccd1a5d 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -10,144 +10,43 @@
    warp- and block-level reductions, using the CUB library
  */
 
-// ensures we use shfl_sync and not shfl when compiling with clang
-#if defined(__clang__) && defined(__CUDA__)
-#define CUB_USE_COOPERATIVE_GROUPS
-#endif
-
 using namespace quda;
 
-namespace cub {
-
-  // Naive implementation of cub collectives in OpenMP target.
-  // FIXME: use team shared memory
-
-  enum BlockReduceAlgorithm {BLOCK_REDUCE_WARP_REDUCTIONS};
-
-  constexpr int get_warp_steps(int s)
-  {
-    // log2(s)
-    return s==1?0:s==2?1:s==4?2:s==8?3:s==16?4:s==32?5:s==64?6:-1;
-  }
-
-// https://github.com/NVlabs/cub/blob/1.8.0/cub/warp/specializations/warp_reduce_smem.cuh
-template <typename T, int warp_threads, int ptx_arch = __COMPUTE_CAPABILITY__>
-struct WarpReduce
-{
-  static constexpr int warp_steps = get_warp_steps(warp_threads);
-  static_assert(warp_steps>=0, "unsupported warp_threads");
-  static constexpr int half_warp_threads = 1 << (warp_steps-1);
-  static constexpr int warp_smem_elments = warp_threads + half_warp_threads;
-  struct TempStorage
-  {
-    T reduce[warp_smem_elments];
-  };
-  TempStorage &temp_storage;
-  int tid,lane_id;
-  inline WarpReduce(TempStorage &temp_storage) :
-    temp_storage(temp_storage),
-    tid(omp_get_thread_num()),
-    lane_id(tid%warp_threads)
-  {}
-  template <typename ReductionOp>
-  inline T Reduce(T input, int valid_items, ReductionOp reduction_op)
-  {
-    QUDA_UNROLL
-    for(int i=0;i<warp_steps;++i){
-      temp_storage.reduce[lane_id] = input;
-      const int offset = 1 << i;
-      // this is bad, but we don't have warp barrier
-      #pragma omp barrier
-      if((lane_id + offset) < valid_items)
-        input = reduction_op(input, temp_storage.reduce[lane_id+offset]);
-    }
-    return input;
-  }
-  inline T Sum(T input, int valid_items = warp_threads)  // FIXME warp_num_valid < warp_threads
-  {
-    QUDA_UNROLL
-    for(int i=0;i<warp_steps;++i){
-      temp_storage.reduce[lane_id] = input;
-      const int offset = 1 << i;
-      // this is bad, but we don't have warp barrier
-      #pragma omp barrier
-      if((lane_id + offset) < valid_items)
-        input += temp_storage.reduce[lane_id+offset];
-    }
-    return input;
-  }
-};
-
-template <typename T, int warp_threads, int ptx_arch = __COMPUTE_CAPABILITY__>
-struct WarpScan
+namespace quda
 {
-  struct TempStorage
-  {
-    T v;
-  };
-  TempStorage &temp_storage;
-  int lane_id;
-  inline WarpScan(TempStorage &temp_storage) :
-    temp_storage(temp_storage),
-    lane_id(omp_get_thread_num()%warp_threads)
-  {}
-  inline T Broadcast(T input, unsigned int src_lane)
-  {
-    if (lane_id == src_lane)
-      temp_storage.v = input;
-    #pragma omp barrier
-    return temp_storage.v;
-  }
-};
 
-// https://github.com/NVlabs/cub/blob/1.8.0/cub/block/specializations/block_reduce_warp_reductions.cuh
-template <typename T, int block_dim_x, BlockReduceAlgorithm alg = BLOCK_REDUCE_WARP_REDUCTIONS, int block_dim_y = 1, int block_dim_z = 1, int ptx_arch = __COMPUTE_CAPABILITY__>
-struct BlockReduce
-{
-  static constexpr int block_threads = block_dim_x*block_dim_y*block_dim_z;
-  static constexpr int warp_threads = device::warp_size();
-  static constexpr int warps = (block_threads + warp_threads - 1) / warp_threads;
-  static constexpr int logical_warp_size = std::min(block_threads, warp_threads);
-  static constexpr bool even_warp_multiple = block_threads%logical_warp_size == 0;
-  typedef WarpReduce<T, logical_warp_size, ptx_arch> WarpReduce_;
-  struct TempStorage
+  namespace target
   {
-    typename WarpReduce_::TempStorage warp_reduce[warps];
-    T warp_aggregates[warps];
-  };
-  TempStorage &temp_storage;
-  int tid,lane_id,warp_id;
-  inline BlockReduce(TempStorage &temp_storage) :
-    temp_storage(temp_storage),
-    tid(omp_get_thread_num()),
-    lane_id(tid%logical_warp_size),
-    warp_id(tid/logical_warp_size)
-  {}
-  template <typename ReductionOp>
-  inline T Reduce(T input, ReductionOp reduction_op)
-  {
-    int warp_offset = warp_id * logical_warp_size;
-    int warp_num_valid = (even_warp_multiple || (warp_offset + logical_warp_size <= block_threads)) ? logical_warp_size : block_threads - warp_offset;
-    T warp_aggregate = WarpReduce_(temp_storage.warp_reduce[warp_id]).Reduce(input, warp_num_valid, reduction_op);
-
-    if (lane_id == 0) temp_storage.warp_aggregates[warp_id] = warp_aggregate;
-    #pragma omp barrier
-    if (tid == 0) {
-QUDA_UNROLL
-      for(int i=1;i<warps;++i) warp_aggregate = reduction_op(warp_aggregate, temp_storage.warp_aggregates[i]);
+    /**
+       @brief OpenMP reduction over a group of consecutive threads smaller than omp_num_threads()
+     */
+    template <typename T, typename reducer_t>
+    inline T any_reduce(const reducer_t &r, const T &value, const int batch, const int block_size, const bool all, const bool async)
+    {
+      constexpr max_nthr = device::max_block_size();
+      static_assert(max_nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
+      T *storage = (T*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
+      const int nthr = omp_get_num_threads();
+      const int tid = omp_get_thread_num();
+      if(!async){ // only synchronize if we are not pipelining
+        #pragma omp barrier
+      }
+      storage[tid] = value;
+      #pragma omp barrier
+      int offset = 1;
+      const auto t0 = r.init();
+      do{
+        #pragma omp for
+        for(int i=0;i<nthr;++i){  // TODO: benchmark against i+=offset+1
+          const auto j = i+offset;
+          const auto t = i/block_size==j/block_size ? storage[j] : t0;
+          storage[i] = r(storage[i], t);
+        }
+        offset *= 2;
+      }while(offset<block_size);
+      return all ? storage[block_size*batch] : storage[tid];
     }
-    return warp_aggregate;
   }
-  inline T Sum(T input)
-  {
-    return Reduce(input, plus<T>());
-  }
-};
-
-}
-
-namespace quda
-{
 
   // pre-declaration of warp_reduce that we wish to specialize
   template <bool> struct warp_reduce;
@@ -169,24 +68,9 @@ namespace quda
     template <typename T, typename reducer_t, typename param_t>
     __device__ inline T operator()(const T &value_, bool all, const reducer_t &r, const param_t &)
     {
-      using warp_reduce_t = cub::WarpReduce<T, param_t::width, __COMPUTE_CAPABILITY__>;
-      using tempStorage = typename warp_reduce_t::TempStorage;
-      static_assert(sizeof(tempStorage) <= sizeof(target::omptarget::get_shared_cache()[0])*1024, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, 1024 is used below
-      tempStorage *dummy_storage = (tempStorage*)&target::omptarget::get_shared_cache()[1024];
-      warp_reduce_t warp_reduce(*dummy_storage);
-      T value = reducer_t::do_sum ? warp_reduce.Sum(value_) : warp_reduce.Reduce(value_, r);
-
-      if (all) {
-        using warp_scan_t = cub::WarpScan<T, param_t::width, __COMPUTE_CAPABILITY__>;
-        using tempStorage = typename warp_scan_t::TempStorage;
-        static_assert(sizeof(tempStorage) <= sizeof(target::omptarget::get_shared_cache()[0])*1024, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, 1024 is arbitrary
-        tempStorage *dummy_storage = (tempStorage*)&target::omptarget::get_shared_cache()[2048];
-        // typename warp_scan_t::TempStorage dummy_storage;
-        warp_scan_t warp_scan(*dummy_storage);
-        value = warp_scan.Broadcast(value, 0);
-      }
-
-      return value;
+      constexpr int block_size = device::warp_size();
+      const int batch = omp_get_thread_num() / block_size;
+      return target::any_reduce(r, value_, batch, block_size, all, false);
     }
   };
 
@@ -194,7 +78,7 @@ namespace quda
   template <bool> struct block_reduce;
 
   /**
-     @brief CUDA specialization of block_reduce, utilizing cub::BlockReduce
+     @brief CUDA specialization of block_reduce, utilizing target::omptarget::get_shared_cache() and omp for
   */
   template <> struct block_reduce<true> {
 
@@ -211,36 +95,10 @@ namespace quda
        @return The block-wide reduced value
      */
     template <typename T, typename reducer_t, typename param_t>
-    __device__ inline T operator()(const T &value_, bool async, int batch, bool all, const reducer_t &r,
-                                   const param_t &)
+    __device__ inline T operator()(const T &value_, bool async, int batch, bool all, const reducer_t &r, const param_t &)
     {
-      constexpr int nthr = param_t::block_size_x*param_t::block_size_y*(param_t::batch_size>param_t::block_size_z?param_t::batch_size:param_t::block_size_z);
-      int tid = omp_get_thread_num();
-      static_assert(nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
-      T *storage = (T*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
-      #pragma omp barrier
-      storage[tid] = value_;
-      #pragma omp barrier
-      int offset = 1;
-      constexpr int bs = param_t::batch_size;
-      static_assert(nthr%bs==0, "Block size not divisible by batch_size");
-      constexpr int subblock = nthr/bs;
-      auto t0 = r.init();
-      do{
-        #pragma omp for
-        for(int i=0;i<nthr;++i){
-          auto j = i+offset;
-          auto t = i/subblock==j/subblock ? storage[j] : t0;
-          storage[i] = r(storage[i], t);
-        }
-        // #pragma omp barrier
-        offset *= 2;
-      }while(offset<subblock);
-      // #pragma omp barrier
-      T value = storage[tid];
-      if(all)
-        value = storage[subblock*(tid/subblock)];
-      return value;
+      const auto block_size = target::block_size<param_t::block_dim>();
+      return target::any_reduce(r, value_, batch, block_size, all, async);
     }
   };
 
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 863dac46fd..7478c2ff3a 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -109,7 +109,7 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __launch_bounds__(Arg::launch_bounds || Arg::block_size > 512 ?
+  __launch_bounds__(Arg::launch_bounds ?
                       Arg::block_size :
                       0) __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
@@ -155,7 +155,7 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __launch_bounds__(Arg::launch_bounds || Arg::block_size > 512 ?
+  __launch_bounds__(Arg::launch_bounds ?
                       Arg::block_size :
                       0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D()
   {
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 2db117b84b..37d46a379e 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -5,19 +5,13 @@
 #include <block_reduce_helper.h>
 #include <kernel_helper.h>
 
-#ifdef QUAD_SUM
-using device_reduce_t = doubledouble;
-#else
-using device_reduce_t = double;
-#endif
-
 using count_t = unsigned int;
 
 namespace quda
 {
 
   // declaration of reduce function
-  template <int block_size_x, int block_size_y = 1, typename Reducer, typename Arg, typename T>
+  template <typename Reducer, typename Arg, typename T>
   __device__ inline void reduce(Arg &arg, const Reducer &r, const T &in, const int idx = 0);
 
   /**
@@ -31,7 +25,7 @@ namespace quda
   template <typename T, bool use_kernel_arg = true> struct ReduceArg : kernel_param<use_kernel_arg> {
     using reduce_t = T;
 
-    template <int, int, typename Reducer, typename Arg, typename I>
+    template <typename Reducer, typename Arg, typename I>
     friend __device__ void reduce(Arg &, const Reducer &, const I &, const int);
     qudaError_t launch_error; /** only do complete if no launch error to avoid hang */
     static constexpr unsigned int max_n_batch_block
@@ -111,19 +105,23 @@ namespace quda
      variant which require explicit host-device synchronization to
      signal the completion of the reduction to the host.
 
-     @param arg The kernel argument, this must derive from ReduceArg
-     @param r Instance of the reducer to be used in this reduction
-     @param in The input per-thread data to be reduced
-     @param idx In the case of multiple reductions, idx identifies
-     which reduction this thread block corresponds to.  Typically idx
-     will be constant along constant block_idx().y and block_idx().z.
+     The reduce function supports:
+     - a global reduction across the x thread dimension
+     - a local block reduction across the y thread dimension
+     - the z thread dimension is a batching dimension in the case of independent reductions
+
+     @param[in,out] arg The kernel argument, this must derive from ReduceArg
+     @param[in] r Instance of the reducer to be used in this reduction
+     @param[in] in The input per-thread data to be reduced
+     @param[in] idx In the case of multiple reductions, idx identifies
+     which reduction this thread block corresponds to and should be
+     constant along the x and y thread dimensions.
   */
-  template <int block_size_x, int block_size_y, typename Reducer, typename Arg, typename T>
+  template <typename Reducer, typename Arg, typename T>
   __device__ inline void reduce(Arg &arg, const Reducer &r, const T &in, const int idx)
   {
-    constexpr auto n_batch_block
-      = std::min(Arg::max_n_batch_block, device::max_block_size() / (block_size_x * block_size_y));
-    using BlockReduce = BlockReduce<T, block_size_x, block_size_y, n_batch_block, true>;
+    constexpr auto n_batch_block = std::min(Arg::max_n_batch_block, device::max_block_size());
+    using BlockReduce = BlockReduce<T, Reducer::reduce_block_dim, n_batch_block>;
     // bool isLastBlockDone[n_batch_block];
     static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::get_shared_cache()[0])*64, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 128 is used in block_reduce_helper.h:/tempStorage/
     bool *isLastBlockDone = (bool*)target::omptarget::get_shared_cache();
@@ -133,7 +131,7 @@ namespace quda
     T aggregate = BlockReduce(target::thread_idx().z).Reduce(in, r);
     // printf("team %d thread %d  r %g  aggregate %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&in), *(double*)(&aggregate));
 
-    if (target::thread_idx().x == 0 && target::thread_idx().y == 0) {
+    if (target::thread_idx_linear<2>() == 0) {
       arg.partial[idx * target::grid_dim().x + target::block_idx().x] = aggregate;
       // __threadfence(); // flush result
 
@@ -163,11 +161,11 @@ namespace quda
       bool thisSubBlock = isLastBlockDone[target::thread_idx().z];
       T sum = r.init();
       if (thisSubBlock) {
-        auto i = target::thread_idx().y * block_size_x + target::thread_idx().x;
+        auto i = target::thread_idx_linear<2>();
         while (i < target::grid_dim().x) {
           sum = r(sum, const_cast<T &>(static_cast<volatile T *>(arg.partial)[idx * target::grid_dim().x + i]));
           // printf("team %d thread %d  sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
-          i += block_size_x * block_size_y;
+          i += target::block_size<2>();
         }
       }
 
@@ -175,7 +173,7 @@ namespace quda
 
       // write out the final reduced value
       if (thisSubBlock) {
-        if (target::thread_idx().y * block_size_x + target::thread_idx().x == 0) {
+        if (target::thread_idx_linear<2>() == 0) {
           // printf("team %d thread %d  final sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
           if (arg.get_output_async_buffer()) {
             arg.get_output_async_buffer()[idx] = sum;
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 5e06887fd8..0ecca5a9e0 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -6,23 +6,6 @@
 namespace quda
 {
 
-  /**
-     @brief This class is derived from the arg class that the functor
-     creates and curries in the block size.  This allows the block
-     size to be set statically at launch time in the actual argument
-     class that is passed to the kernel.
-
-     @tparam block_size_x x-dimension block-size
-     @tparam block_size_y y-dimension block-size
-     @tparam Arg Kernel argument struct
-  */
-  template <int block_size_x_, int block_size_y_, typename Arg_> struct ReduceKernelArg : Arg_ {
-    using Arg = Arg_;
-    static constexpr int block_size_x = block_size_x_;
-    static constexpr int block_size_y = block_size_y_;
-    ReduceKernelArg(const Arg &arg) : Arg(arg) { }
-  };
-
   /**
      @brief Reduction2D_impl is the implementation of the generic 2-d
      reduction kernel.  Functors that utilize this kernel have two
@@ -58,7 +41,7 @@ namespace quda
     }
 
     // perform final inter-block reduction and write out result
-    reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value);
+    reduce(arg, t, value);
   }
 
   /**
@@ -157,7 +140,7 @@ namespace quda
     }
 
     // perform final inter-block reduction and write out result
-    reduce<Arg::block_size_x, Arg::block_size_y>(arg, t, value, j);
+    reduce(arg, t, value, j);
   }
 
   /**
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index c902e402f3..6bbda27c4f 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -92,7 +92,34 @@ namespace quda {
       return dim3(n%target::omptarget::launch_param.block.x, (n/target::omptarget::launch_param.block.x)%target::omptarget::launch_param.block.y, n/(target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y));
     }
 
-  }
+    /**
+       @brief Helper function that returns a linear thread index within a thread block.
+    */
+    template <int dim> __device__ __host__ inline auto thread_idx_linear()
+    {
+      const auto n = (unsigned int)omp_get_thread_num();
+      switch (dim) {
+      case 1: return n%target::omptarget::launch_param.block.x;
+      case 2: return n%(target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y);
+      case 3:
+      default: return n;
+      }
+    }
+
+    /**
+       @brief Helper function that returns the total number thread in a thread block
+    */
+    template <int dim> __device__ __host__ inline auto block_size()
+    {
+      switch (dim) {
+      case 1: return target::omptarget::launch_param.block.x;
+      case 2: return target::omptarget::launch_param.block.y * target::omptarget::launch_param.block.x;
+      case 3:
+      default: return target::omptarget::launch_param.block.z * target::omptarget::launch_param.block.y * target::omptarget::launch_param.block.x;
+      }
+    }
+
+  } // namespace target
 
   namespace device {
 
@@ -114,39 +141,9 @@ namespace quda {
     template <int block_size_y = 1, int block_size_z = 1>
       constexpr unsigned int max_block_size()
       {
-        return QUDA_MAX_BLOCK_SIZE / (block_size_y * block_size_z);
+        return std::max(warp_size(), QUDA_MAX_BLOCK_SIZE / (block_size_y * block_size_z));
       }
 
-    /**
-       @brief Helper function that returns the maximum number of threads
-       in a block in the x dimension for reduction kernels.
-    */
-    template <int block_size_y = 1, int block_size_z = 1>
-    constexpr unsigned int max_reduce_block_size()
-      {
-#ifdef QUDA_FAST_COMPILE_REDUCE
-        // This is the specialized variant used when we have fast-compilation mode enabled
-        return warp_size();
-#else
-        return max_block_size<block_size_y, block_size_z>();
-#endif
-      }
-
-    /**
-       @brief Helper function that returns the maximum number of threads
-       in a block in the x dimension for reduction kernels.
-    */
-    template <int block_size_y = 1, int block_size_z = 1>
-    constexpr unsigned int max_multi_reduce_block_size()
-    {
-#ifdef QUDA_FAST_COMPILE_REDUCE
-      // This is the specialized variant used when we have fast-compilation mode enabled
-      return warp_size();
-#else
-      return max_block_size<block_size_y, block_size_z>();
-#endif
-    }
-
     /**
        @brief Helper function that returns the maximum size of a
        __constant__ buffer on the target architecture.  For CUDA,
diff --git a/lib/targets/omptarget/comm_target.cpp b/lib/targets/omptarget/comm_target.cpp
index 90e28e17ab..7b3d46c445 100644
--- a/lib/targets/omptarget/comm_target.cpp
+++ b/lib/targets/omptarget/comm_target.cpp
@@ -22,7 +22,7 @@ namespace quda
     ompwip("unimplemented");
   }
 
-  void comm_destroy_neighbor_memory(array_2d<void *, QUDA_MAX_DIM, 2> &remote)
+  void comm_destroy_neighbor_memory(array_2d<void *, QUDA_MAX_DIM, 2> &)
   {
     ompwip("unimplemented");
   }
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index b4f2d2e5e7..d258e8df90 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -25,8 +25,8 @@ set_property(CACHE QUDA_MAX_BLOCK_SIZE PROPERTY STRINGS 256 512 768 1024)
 target_compile_definitions(quda PUBLIC QUDA_MAX_BLOCK_SIZE=${QUDA_MAX_BLOCK_SIZE})
 message(STATUS "Using maximum team size: ${QUDA_MAX_BLOCK_SIZE}")
 
-set(QUDA_MAX_SHARED_MEMORY_SIZE 40960 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
-set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152 65536 98304 131072 196608 262144)
+set(QUDA_MAX_SHARED_MEMORY_SIZE 65536 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
+set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152 65536 98304 131072 196608 262144 393216 524288)
 target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
 message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")
 

From f95738fb14d0825339eacb3470a1c30370256c55 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 31 May 2022 23:37:41 +0000
Subject: [PATCH 122/198] omp: use large shared memory size (in global mem
 space) by default

---
 lib/targets/omptarget/target_omptarget.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index d258e8df90..b0c3c1f868 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -25,7 +25,7 @@ set_property(CACHE QUDA_MAX_BLOCK_SIZE PROPERTY STRINGS 256 512 768 1024)
 target_compile_definitions(quda PUBLIC QUDA_MAX_BLOCK_SIZE=${QUDA_MAX_BLOCK_SIZE})
 message(STATUS "Using maximum team size: ${QUDA_MAX_BLOCK_SIZE}")
 
-set(QUDA_MAX_SHARED_MEMORY_SIZE 65536 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
+set(QUDA_MAX_SHARED_MEMORY_SIZE 393216 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
 set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152 65536 98304 131072 196608 262144 393216 524288)
 target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
 message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")

From 7de35929e466f175ecb2dbf09c4efaa0467d9709 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 31 May 2022 23:38:39 +0000
Subject: [PATCH 123/198] omp: thread/block_idx use uint

---
 include/targets/omptarget/target_device.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 6bbda27c4f..7b53809a54 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -77,7 +77,7 @@ namespace quda {
     */
     __device__ __host__ inline dim3 block_idx()
     {
-      const int n = omp_get_team_num();
+      const auto n = (unsigned int)omp_get_team_num();
       return dim3(n%target::omptarget::launch_param.grid.x, (n/target::omptarget::launch_param.grid.x)%target::omptarget::launch_param.grid.y, n/(target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y));
     }
 
@@ -88,7 +88,7 @@ namespace quda {
     */
     __device__ __host__ inline dim3 thread_idx()
     {
-      const int n = omp_get_thread_num();
+      const auto n = (unsigned int)omp_get_thread_num();
       return dim3(n%target::omptarget::launch_param.block.x, (n/target::omptarget::launch_param.block.x)%target::omptarget::launch_param.block.y, n/(target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y));
     }
 

From 078fe8ab2b4bb0a5eb7ea7153c4e752bbc205eb4 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 1 Jun 2022 15:08:27 +0000
Subject: [PATCH 124/198] omp: check shared cache allocation

---
 lib/targets/omptarget/quda_api.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index ff3fde7007..f319a7491c 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -45,13 +45,17 @@ namespace quda
         int num_teams = tp.grid.x*tp.grid.y*tp.grid.z;
         if(!init){
           init = 1;
-          shared_cache.addr = (int*)omp_target_alloc(num_teams*device::max_shared_memory_size(), omp_get_default_device());
+          shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), omp_get_default_device());
+          if(!shared_cache.addr)
+            errorQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
           shared_cache.num_teams = num_teams;
           shared_cache.cache_length = device::max_shared_memory_size()/sizeof(shared_cache.addr[0]);
         }
         if(shared_cache.num_teams<num_teams){
           omp_target_free(shared_cache.addr, omp_get_default_device());
-          shared_cache.addr = (int*)omp_target_alloc(num_teams*device::max_shared_memory_size(), omp_get_default_device());
+          shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), omp_get_default_device());
+          if(!shared_cache.addr)
+            errorQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
           shared_cache.num_teams = num_teams;
           shared_cache.cache_length = device::max_shared_memory_size()/sizeof(shared_cache.addr[0]);
         }

From 53f70d7f5f8ece9487d8d3512e5306ae25ed5aad Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 1 Jun 2022 17:06:16 +0000
Subject: [PATCH 125/198] omp: fix race condition in omp for; replace omp for
 with manual reductions

---
 .../targets/omptarget/block_reduce_helper.h   | 50 ++++++++++++++-----
 1 file changed, 38 insertions(+), 12 deletions(-)

diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index 8a8ccd1a5d..5bbd4aaa10 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -21,30 +21,56 @@ namespace quda
        @brief OpenMP reduction over a group of consecutive threads smaller than omp_num_threads()
      */
     template <typename T, typename reducer_t>
-    inline T any_reduce(const reducer_t &r, const T &value, const int batch, const int block_size, const bool all, const bool async)
+    inline T any_reduce(const reducer_t &r, const T &value_, const int batch, const int block_size, const bool all, const bool async)
     {
       constexpr max_nthr = device::max_block_size();
       static_assert(max_nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
       T *storage = (T*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
-      const int nthr = omp_get_num_threads();
       const int tid = omp_get_thread_num();
+      const auto v0 = r.init();
+#if 1
+      const auto batch_boundary = block_size*(batch+1);
+      auto value = value_;
+      for(int offset=1;offset<block_size;offset*=2){
+        if(offset>1 || !async){ // only synchronize if we are not pipelining
+          #pragma omp barrier
+        }
+        storage[tid] = value;
+        const auto j = tid+offset;
+        #pragma omp barrier
+        const auto& v = j<batch_boundary ? storage[j] : v0;
+        value = r(value, v);
+      }
+      if(all){
+        if(tid==block_size*batch)
+          storage[tid] = value;
+        #pragma omp barrier
+        value = storage[block_size*batch];
+        if(!async){
+          #pragma omp barrier
+        }
+      }
+#else
+      const int nthr = omp_get_num_threads();
       if(!async){ // only synchronize if we are not pipelining
         #pragma omp barrier
       }
-      storage[tid] = value;
+      storage[tid] = value_;
       #pragma omp barrier
-      int offset = 1;
-      const auto t0 = r.init();
-      do{
+      for(int offset=1;offset<block_size;offset*=2){
         #pragma omp for
-        for(int i=0;i<nthr;++i){  // TODO: benchmark against i+=offset+1
+        for(int i=0;i<nthr;i+=2*offset){
           const auto j = i+offset;
-          const auto t = i/block_size==j/block_size ? storage[j] : t0;
-          storage[i] = r(storage[i], t);
+          const auto batch_boundary = block_size*(1+i/block_size);
+          const auto& u = storage[i];
+          const auto& v = j<batch_boundary ? storage[j] : v0;
+          const auto& z = r(u, v);
+          storage[i] = z;
         }
-        offset *= 2;
-      }while(offset<block_size);
-      return all ? storage[block_size*batch] : storage[tid];
+      }
+      const auto& value = storage[block_size*batch];
+#endif
+      return value;
     }
   }
 

From b776f9fbb937e680c0d4b17d7347b4234edf7253 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 1 Jun 2022 18:15:05 +0000
Subject: [PATCH 126/198] omp: test implementations of block reduce

---
 .../targets/omptarget/block_reduce_helper.h   | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index 5bbd4aaa10..b112e77cc8 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -27,9 +27,10 @@ namespace quda
       static_assert(max_nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
       T *storage = (T*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
       const int tid = omp_get_thread_num();
-      const auto v0 = r.init();
+      const auto& v0 = r.init();
 #if 1
-      const auto batch_boundary = block_size*(batch+1);
+      const auto batch_begin = block_size*batch;
+      const auto batch_end = batch_begin+block_size;
       auto value = value_;
       for(int offset=1;offset<block_size;offset*=2){
         if(offset>1 || !async){ // only synchronize if we are not pipelining
@@ -38,8 +39,17 @@ namespace quda
         storage[tid] = value;
         const auto j = tid+offset;
         #pragma omp barrier
-        const auto& v = j<batch_boundary ? storage[j] : v0;
+        const auto& v = j<batch_end ? storage[j] : v0;
         value = r(value, v);
+/*
+        const auto oid = (tid-batch_begin)%(offset*2);
+        if(oid==offset)
+          storage[tid] = value;
+        const auto j = tid+offset;
+        #pragma omp barrier
+        if(oid==0 && j<batch_end)
+          value = r(value, storage[j]);
+*/
       }
       if(all){
         if(tid==block_size*batch)
@@ -61,9 +71,9 @@ namespace quda
         #pragma omp for
         for(int i=0;i<nthr;i+=2*offset){
           const auto j = i+offset;
-          const auto batch_boundary = block_size*(1+i/block_size);
+          const auto batch_end = block_size*(1+i/block_size);
           const auto& u = storage[i];
-          const auto& v = j<batch_boundary ? storage[j] : v0;
+          const auto& v = j<batch_end ? storage[j] : v0;
           const auto& z = r(u, v);
           storage[i] = z;
         }

From 2f8380a7120a02689dc3c44e1b16bc8f143f0be8 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 3 Jun 2022 02:30:48 +0000
Subject: [PATCH 127/198] omp: using share memory for warp_combine

---
 include/targets/omptarget/warp_collective.h | 25 +++++++++++++++------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/include/targets/omptarget/warp_collective.h b/include/targets/omptarget/warp_collective.h
index 8f4c203e6e..4796f2d158 100644
--- a/include/targets/omptarget/warp_collective.h
+++ b/include/targets/omptarget/warp_collective.h
@@ -12,22 +12,33 @@ namespace quda
   template <> struct warp_combine_impl<true> {
     template <typename T> __device__ inline T operator()(T &x, int warp_split)
     {
-#if 0
-      // OMP FIXME
+      using R = typename T::value_type;
+      constexpr max_nthr = device::max_block_size();
+      static_assert(max_nthr*sizeof(R) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
+      R *storage = (R*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
+      const int tid = omp_get_thread_num();
+
       constexpr int warp_size = device::warp_size();
+      const int wid = tid / warp_size;
+      const int warpend = (wid+1)*warp_size;
+      const int split_size = warp_size / warp_split;
       if (warp_split > 1) {
 QUDA_UNROLL
         for (int i = 0; i < x.size(); i++) {
           // reduce down to the first group of column-split threads
 QUDA_UNROLL
-          for (int offset = warp_size / 2; offset >= warp_size / warp_split; offset /= 2) {
-            // TODO - add support for non-converged warps
-            x[i].real(x[i].real() + __shfl_down_sync(device::warp_converged_mask(), x[i].real(), offset));
-            x[i].imag(x[i].imag() + __shfl_down_sync(device::warp_converged_mask(), x[i].imag(), offset));
+          for (int offset = warp_size / 2; offset >= split_size; offset /= 2) {
+            #pragma omp barrier
+            storage[tid] = x[i];
+            const auto tid_offset = tid + offset;
+            const auto load_id = tid_offset >= warpend ? tid_offset-warp_size : tid_offset;
+            #pragma omp barrier
+            const auto& y = storage[load_id];
+            x[i].real(x[i].real() + y.real());
+            x[i].imag(x[i].imag() + y.imag());
           }
         }
       }
-#endif
       return x;
     }
   };

From 834cbcf4cc5a070e5d476bf7329069b619186e3a Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 3 Jun 2022 02:31:42 +0000
Subject: [PATCH 128/198] omp: change the constraint on qudaLaunchKernel a bit

---
 include/targets/omptarget/tunable_kernel.h | 25 +++++++++++++---------
 lib/targets/omptarget/quda_api.cpp         | 18 ++++++++++------
 2 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index a2be594283..b96f10c3d1 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -14,30 +14,35 @@ namespace quda {
   namespace target {
     namespace omptarget {
       // defined in ../../../lib/targets/omptarget/quda_api.cpp:/qudaSetupLaunchParameter
-      void qudaSetupLaunchParameter(const TuneParam &);
+      int qudaSetupLaunchParameter(const TuneParam &);
       void set_runtime_error(int error, const char *api_func, const char *func, const char *file,
                              const char *line, bool allow_error = false);
     }
   }
 
   template <typename Arg>
-  static bool acceptThreads(const TuneParam &tp, const Arg &arg)
+  inline bool acceptThreads(const TuneParam &tp, const Arg &arg)
   {
-    bool r = arg.threads.x<tp.block.x ||    // trivial cases where arg.threads.x == 1 or few
-       arg.threads.y<tp.block.y ||
-       arg.threads.z<tp.block.z ||
-      (arg.threads.x%(tp.grid.x*tp.block.x)==0 &&
-       arg.threads.y%(tp.grid.y*tp.block.y)==0 &&
-       arg.threads.z%(tp.grid.z*tp.block.z)==0);
+    bool r = tp.block.x*tp.block.y*tp.block.z<=device::max_block_size() &&
+       (arg.threads.x<tp.block.x ||    // trivial cases where arg.threads.x == 1 or few
+        arg.threads.y<tp.block.y ||
+        arg.threads.z<tp.block.z ||
+        (arg.threads.x%tp.block.x==0 &&
+         arg.threads.y%tp.block.y==0 &&
+         arg.threads.z%tp.block.z==0));
     if(!r && getVerbosity() >= QUDA_VERBOSE)
       ompwip("WARNING: rejecting threads setup arg %d %d %d tp (%d %d %d)x(%d %d %d)",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
     return r;
   }
+  inline bool acceptTeams(const TuneParam &tp)
+  {
+    const auto nt = tp.grid.x*tp.grid.y*tp.grid.z;
+    return nt <= device::max_grid_size(0);
+  }
   template <typename... Arg>
   qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, qudaStream_t stream, const Arg &... arg)
   {
-    if(acceptThreads(tp, arg...)){
-      target::omptarget::qudaSetupLaunchParameter(tp);
+    if(acceptThreads(tp, arg...) && acceptTeams(tp) && 0==target::omptarget::qudaSetupLaunchParameter(tp)){
       reinterpret_cast<void(*)(Arg...)>(const_cast<void*>(func))(arg...);
       return QUDA_SUCCESS;
     }else
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index f319a7491c..dcd69b707d 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -35,7 +35,7 @@ namespace quda
       LaunchParam launch_param;
       #pragma omp declare target to(launch_param)
 
-      void qudaSetupLaunchParameter(const TuneParam &tp)
+      int qudaSetupLaunchParameter(const TuneParam &tp)
       {
         launch_param.grid = tp.grid;
         launch_param.block = tp.block;
@@ -44,22 +44,28 @@ namespace quda
         static int init = 0;
         int num_teams = tp.grid.x*tp.grid.y*tp.grid.z;
         if(!init){
-          init = 1;
           shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), omp_get_default_device());
-          if(!shared_cache.addr)
-            errorQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
+          if(!shared_cache.addr){
+            // warnQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
+            return -1;
+          }
+          init = 1;
           shared_cache.num_teams = num_teams;
           shared_cache.cache_length = device::max_shared_memory_size()/sizeof(shared_cache.addr[0]);
         }
         if(shared_cache.num_teams<num_teams){
           omp_target_free(shared_cache.addr, omp_get_default_device());
           shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), omp_get_default_device());
-          if(!shared_cache.addr)
-            errorQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
+          if(!shared_cache.addr){
+            // warnQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
+            init = 0;
+            return -1;
+          }
           shared_cache.num_teams = num_teams;
           shared_cache.cache_length = device::max_shared_memory_size()/sizeof(shared_cache.addr[0]);
         }
         #pragma omp target update to(shared_cache)
+        return 0;
       }
 
       static inline int

From 97872e206008f29de6d27c7ea4c6119af89f4ec1 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 9 Jun 2022 23:33:30 +0000
Subject: [PATCH 129/198] omp: update math_helper

---
 include/targets/omptarget/math_helper.cuh | 51 +++++++++++------------
 1 file changed, 24 insertions(+), 27 deletions(-)

diff --git a/include/targets/omptarget/math_helper.cuh b/include/targets/omptarget/math_helper.cuh
index faad4c9588..a5de3897cc 100644
--- a/include/targets/omptarget/math_helper.cuh
+++ b/include/targets/omptarget/math_helper.cuh
@@ -11,7 +11,7 @@ namespace quda {
    * @param b second number
    */
   template<typename T>
-  inline __host__ __device__ T max(const T &a, const T &b) { return a > b ? a : b; }
+  inline T max(const T &a, const T &b) { return a > b ? a : b; }
 
   /**
    * @brief Minimum of two numbers
@@ -19,7 +19,7 @@ namespace quda {
    * @param b second number
    */
   template<typename T>
-  inline __host__ __device__ T min(const T &a, const T &b) { return a < b ? a : b; }
+  inline T min(const T &a, const T &b) { return a < b ? a : b; }
 
 
   /**
@@ -29,38 +29,35 @@ namespace quda {
    * @param c pointer to the storage for the result of the cos
    */
   template<typename T>
-  inline __host__ __device__ void sincos(const T& a, T* s, T* c) { *s = std::sin(a); *c = std::cos(a); }
+  inline void sincos(const T& a, T* s, T* c) { *s = std::sin(a); *c = std::cos(a); }
 
   /**
-   * @brief Reciprocal square root function (rsqrt)
-   * @param a the argument  (In|out)
-   *
-   * some math libraries provide a fast inverse sqrt() function.
-   * this implementation trusts the compiler.
+   * @brief Combined sinpi and cospi calculation in QUDA NAMESPACE
+   * @param a the angle
+   * @param s pointer to the storage for the result of the sin
+   * @param c pointer to the storage for the result of the cos
+   */
+  template <typename T> inline void sincospi(const T& a, T *s, T *c) { quda::sincos(a * static_cast<T>(M_PI), s, c); }
+
+  /**
+   * @brief Sine pi calculation in QUDA NAMESPACE.
+   * @param a the angle
+   * @return result of the sin(a * pi)
    */
-  template<typename T> inline __host__ __device__ T rsqrt(T a) { return static_cast<T>(1.0) / std::sqrt(a); }
+  template <typename T> inline T sinpi(T a) { return sin(a * static_cast<T>(M_PI)); }
 
-    /**
-     Generic wrapper for Trig functions -- used in gauge field order 
-    */
-  template <bool isFixed, typename T>
-  struct Trig {
-    __device__ __host__ static T Atan2( const T &a, const T &b) { return std::atan2(a,b); }
-    __device__ __host__ static T Sin( const T &a ) { return std::sin(a); }
-    __device__ __host__ static T Cos( const T &a ) { return std::cos(a); }
-    __device__ __host__ static void SinCos(const T &a, T *s, T *c) { sincos(a, s, c); }
-  };
+  /**
+   * @brief Cosine pi calculation in QUDA NAMESPACE.
+   * @param a the angle
+   * @return result of the cos(a * pi)
+   */
+  template <typename T> inline T cospi(T a) { return cos(a * static_cast<T>(M_PI)); }
 
   /**
-     Specialization of Trig functions using fixed b/c gauge reconstructs are -1 -> 1 instead of -Pi -> Pi
+   * @brief Reciprocal square root function (rsqrt)
+   * @param a the argument  (In|out)
    */
-  template <>
-    struct Trig<true,float> {
-    __device__ __host__ static float Atan2( const float &a, const float &b) { return std::atan2(a,b)/M_PI; }
-    __device__ __host__ static float Sin(const float &a) { return std::sin(a * static_cast<float>(M_PI)); }
-    __device__ __host__ static float Cos(const float &a) { return std::cos(a * static_cast<float>(M_PI)); }
-    __device__ __host__ static void SinCos(const float &a, float *s, float *c) { sincos(a * static_cast<float>(M_PI), s, c); }
-  };
+  template<typename T> inline T rsqrt(T a) { return static_cast<T>(1.0) / std::sqrt(a); }
 
   /*
     @brief Fast power function that works for negative "a" argument

From 67b504712f94dc622b105cd8abfd5053af1b69ca Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 9 Jun 2022 23:34:35 +0000
Subject: [PATCH 130/198] omp: block_reduce array element iteratively with
 limited shared mem

---
 .../targets/omptarget/block_reduce_helper.h   | 34 +++++++++++++++++--
 lib/targets/omptarget/target_omptarget.cmake  |  6 ++--
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index b112e77cc8..27c3cb62b9 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -17,14 +17,19 @@ namespace quda
 
   namespace target
   {
+    template <typename T>
+    constexpr bool enough_shared_mem(void)
+    {
+      constexpr auto max_nthr = device::max_block_size();
+      return max_nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128;  // FIXME arbitrary, the number is arbitrary, offset 128 below & in reduce_helper.h:/reduce
+    }
     /**
        @brief OpenMP reduction over a group of consecutive threads smaller than omp_num_threads()
      */
     template <typename T, typename reducer_t>
-    inline T any_reduce(const reducer_t &r, const T &value_, const int batch, const int block_size, const bool all, const bool async)
+    inline T any_reduce_impl(const reducer_t &r, const T &value_, const int batch, const int block_size, const bool all, const bool async)
     {
-      constexpr max_nthr = device::max_block_size();
-      static_assert(max_nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
+      static_assert(enough_shared_mem<T>(), "Shared cache not large enough for tempStorage");
       T *storage = (T*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
       const int tid = omp_get_thread_num();
       const auto& v0 = r.init();
@@ -82,6 +87,29 @@ namespace quda
 #endif
       return value;
     }
+    template <typename T, typename R>
+    inline T any_reduce(const R &r, const T &value_, const int batch, const int block_size, const bool all, const bool async)
+    {
+      if constexpr (enough_shared_mem<T>())
+        return any_reduce_impl(r, value_, batch, block_size, all, async);
+      else{
+        using V = typename T::value_type;
+        if constexpr (
+            std::is_same_v<typename R::reducer_t,plus<typename R::reduce_t>> &&
+            std::is_same_v<T,typename R::reduce_t> &&
+            std::is_same_v<T,array<V,T::N>>){
+          const constexpr plus<V> re {};
+          auto value = value_;
+          value[0] = any_reduce_impl(re, value_[0], batch, block_size, all, async);
+QUDA_UNROLL
+          for (int i = 1; i < value.size(); i++) {
+            value[i] = any_reduce_impl(re, value_[i], batch, block_size, all, false);
+          }
+          return value;
+        }else
+          static_assert(sizeof(T)==0, "unimplemented reduction");  // let me fail at compile time
+      }
+    }
   }
 
   // pre-declaration of warp_reduce that we wish to specialize
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index b0c3c1f868..eb1b6078cd 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -15,7 +15,7 @@ set_property(CACHE QUDA_GPU_ARCH PROPERTY STRINGS xehp)
 
 message(STATUS "Building for GPU Architectures: ${QUDA_GPU_ARCH}")
 
-set(QUDA_WARP_SIZE 16 CACHE STRING "OpenMP target logical warp size")
+set(QUDA_WARP_SIZE 32 CACHE STRING "OpenMP target logical warp size")
 set_property(CACHE QUDA_WARP_SIZE PROPERTY STRINGS 8 16 32 64)
 target_compile_definitions(quda PUBLIC QUDA_WARP_SIZE=${QUDA_WARP_SIZE})
 message(STATUS "Using logical warp size: ${QUDA_WARP_SIZE}")
@@ -25,8 +25,8 @@ set_property(CACHE QUDA_MAX_BLOCK_SIZE PROPERTY STRINGS 256 512 768 1024)
 target_compile_definitions(quda PUBLIC QUDA_MAX_BLOCK_SIZE=${QUDA_MAX_BLOCK_SIZE})
 message(STATUS "Using maximum team size: ${QUDA_MAX_BLOCK_SIZE}")
 
-set(QUDA_MAX_SHARED_MEMORY_SIZE 393216 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
-set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152 65536 98304 131072 196608 262144 393216 524288)
+set(QUDA_MAX_SHARED_MEMORY_SIZE 40960 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
+set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152 65536)
 target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
 message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")
 

From 6cf1c6836c7c8bc705fda369007eefee98ae702d Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 24 Jun 2022 20:53:32 +0000
Subject: [PATCH 131/198] omp: block_reduce split large arrays into smaller
 ones

---
 .../targets/omptarget/block_reduce_helper.h   | 37 +++++++++----------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index 27c3cb62b9..2269057016 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -6,8 +6,8 @@
 /**
    @file block_reduce_helper.h
 
-   @section This files contains the CUDA device specialziations for
-   warp- and block-level reductions, using the CUB library
+   @section This files contains the OpenMP target specializations for
+   warp- and block-level reductions
  */
 
 using namespace quda;
@@ -46,15 +46,6 @@ namespace quda
         #pragma omp barrier
         const auto& v = j<batch_end ? storage[j] : v0;
         value = r(value, v);
-/*
-        const auto oid = (tid-batch_begin)%(offset*2);
-        if(oid==offset)
-          storage[tid] = value;
-        const auto j = tid+offset;
-        #pragma omp barrier
-        if(oid==0 && j<batch_end)
-          value = r(value, storage[j]);
-*/
       }
       if(all){
         if(tid==block_size*batch)
@@ -94,17 +85,23 @@ namespace quda
         return any_reduce_impl(r, value_, batch, block_size, all, async);
       else{
         using V = typename T::value_type;
+        constexpr auto N = T::N;
         if constexpr (
             std::is_same_v<typename R::reducer_t,plus<typename R::reduce_t>> &&
             std::is_same_v<T,typename R::reduce_t> &&
-            std::is_same_v<T,array<V,T::N>>){
-          const constexpr plus<V> re {};
+            std::is_same_v<T,array<V,N>>){
+          // make sure the implementation is still compatible: ../../array.h
+          constexpr auto N0 = N/2;
+          constexpr auto N1 = N-N0;
+          using T0 = array<V,N0>;
+          using T1 = array<V,N1>;
+          const constexpr plus<T0> r0 {};
+          const constexpr plus<T1> r1 {};
           auto value = value_;
-          value[0] = any_reduce_impl(re, value_[0], batch, block_size, all, async);
-QUDA_UNROLL
-          for (int i = 1; i < value.size(); i++) {
-            value[i] = any_reduce_impl(re, value_[i], batch, block_size, all, false);
-          }
+          // recurse to myself
+          reinterpret_cast<T0&>(value[0]) = any_reduce(r0, reinterpret_cast<const T0&>(value_[0]), batch, block_size, all, async);
+          // recurse with async==false
+          reinterpret_cast<T1&>(value[N0]) = any_reduce(r1, reinterpret_cast<const T1&>(value_[N0]), batch, block_size, all, false);
           return value;
         }else
           static_assert(sizeof(T)==0, "unimplemented reduction");  // let me fail at compile time
@@ -116,7 +113,7 @@ QUDA_UNROLL
   template <bool> struct warp_reduce;
 
   /**
-     @brief CUDA specialization of warp_reduce, utilizing cub::WarpReduce
+     @brief OpenMP target specialization of warp_reduce
   */
   template <> struct warp_reduce<true> {
 
@@ -142,7 +139,7 @@ QUDA_UNROLL
   template <bool> struct block_reduce;
 
   /**
-     @brief CUDA specialization of block_reduce, utilizing target::omptarget::get_shared_cache() and omp for
+     @brief OpenMP target  specialization of block_reduce
   */
   template <> struct block_reduce<true> {
 

From 7f47a402b7ae7eeb48479d613891277a81f35abb Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 24 Jun 2022 21:09:35 +0000
Subject: [PATCH 132/198] omp: use omp_target_alloc_host for pinned/mapped
 malloc

---
 lib/targets/omptarget/malloc.cpp | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index a25e0d5610..fcfbcf39ee 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -316,8 +316,7 @@ namespace quda
   void *pinned_malloc_(const char *func, const char *file, int line, size_t size)
   {
     MemAlloc a(func, file, line);
-    void *ptr = aligned_malloc(a, size);
-    ompwip("ERROR: pinned_malloc_ uses aligned_malloc");
+    void *ptr = omp_target_alloc_host(size, omp_get_default_device());
     if(!ptr)
       errorQuda("Failed to register pinned memory of size %zu (%s:%d in %s())\n", size, file, line, func);
 /*
@@ -362,7 +361,8 @@ namespace quda
     a.size = a.base_size = size;
     void *ptr;
     if(0<omp_get_num_devices()){
-      ptr = omp_target_alloc_shared(size, omp_get_default_device());   // FIXME non-portable
+      // ptr = omp_target_alloc_shared(size, omp_get_default_device());   // FIXME non-portable
+      ptr = omp_target_alloc_host(size, omp_get_default_device());   // FIXME non-portable
     }else{
       warningQuda("%s:%d %s() mapped malloc without a device", file, line, func);
       ptr = aligned_malloc(a, size);
@@ -809,13 +809,11 @@ namespace quda
       } else {
         ptr = quda::pinned_malloc_(func, file, line, nbytes);
       }
-      ompwip("ERROR: untested pool::pinned_malloc_: %p",ptr);
       return ptr;
     }
 
     void pinned_free_(const char *func, const char *file, int line, void *ptr)
     {
-      ompwip("ERROR: untested pool::pinnd_free_: %p",ptr);
       if (pinned_memory_pool) {
         if (!pinnedSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
         pinnedCache.insert(std::make_pair(pinnedSize[ptr], ptr));
@@ -849,13 +847,11 @@ namespace quda
       } else {
         ptr = quda::device_malloc_(func, file, line, nbytes);
       }
-      // ompwip("pool::device_malloc_: %p",ptr);
       return ptr;
     }
 
     void device_free_(const char *func, const char *file, int line, void *ptr)
     {
-      // ompwip("pool::device_free_: %p",ptr);
       if (device_memory_pool) {
         if (!deviceSize.count(ptr)) { errorQuda("Attempt to free invalid pointer"); }
         deviceCache.insert(std::make_pair(deviceSize[ptr], ptr));

From ac5771b9f820516e684f84a9f8478658d35b2a53 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 24 Jun 2022 21:38:54 +0000
Subject: [PATCH 133/198] omp: clean up kernel launch

---
 .../omptarget/block_reduction_kernel.h        |  39 +---
 include/targets/omptarget/kernel.h            | 169 +++++-------------
 include/targets/omptarget/reduction_kernel.h  |  46 ++---
 include/targets/omptarget/tunable_kernel.h    |  54 ++----
 4 files changed, 87 insertions(+), 221 deletions(-)

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 7478c2ff3a..433905ad9f 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -114,30 +114,15 @@ namespace quda
                       0) __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    // const int tx = arg.threads.x;
-    // const int ty = arg.threads.y;
-    // const int tz = arg.threads.z;
-    // printf("BlockKernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // printf("dparg %p\n", dparg);
-    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    {
     #pragma omp parallel num_threads(ld)
     {
-      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-      //          "omp reports: teams %d threads %d\n",
-      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-      //          omp_get_num_teams(), omp_get_num_threads());
-      // char buffer[sizeof(Arg)];
-      // memcpy(buffer, (void *)dparg, sizeof(Arg));
       BlockKernel2D_impl<Functor, Arg>(arg);
     }
-    }
-    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -157,24 +142,18 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __launch_bounds__(Arg::launch_bounds ?
                       Arg::block_size :
-                      0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D()
+                      0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D(void)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld)
-    {
     #pragma omp parallel num_threads(ld)
     {
-      // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-      //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-      //          "omp reports: teams %d threads %d\n",
-      //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-      //          omp_get_num_teams(), omp_get_num_threads());
       BlockKernel2D_impl<Functor, Arg>(device::get_arg<Arg>());
     }
-    }
   }
 
 }
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 1fb2140a02..a87a0598a8 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -50,31 +50,15 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // printf("dparg %p\n", dparg);
-    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
-    // const int tx = arg.threads.x;
-    // const int ty = arg.threads.y;
-    // const int tz = arg.threads.z;
-    // printf("Kernel1D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
+    #pragma omp parallel num_threads(ld)
     {
-      #pragma omp parallel num_threads(ld)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        // char buffer[sizeof(Arg)];
-        // memcpy(buffer, (void *)dparg, sizeof(Arg));
-        Kernel1D_impl<Functor, Arg, grid_stride>(arg);
-      }
+      Kernel1D_impl<Functor, Arg, grid_stride>(arg);
     }
-    // printf("Kernel1D: exited\n");
-    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -90,22 +74,16 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D()
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D(void)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    // printf("Kernel1D: launch parameter: gd %d ld %d\n", gd, ld);
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp parallel num_threads(ld)
     {
-      #pragma omp parallel num_threads(ld)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
-      }
+      Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
     }
   }
 
@@ -155,30 +133,15 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    const int tx = arg.threads.x;
-    const int ty = arg.threads.y;
-    const int tz = arg.threads.z;
-    // printf("Kernel2D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // printf("dparg %p\n", dparg);
-    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
+    #pragma omp parallel num_threads(ld)
     {
-      #pragma omp parallel num_threads(ld)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        // char buffer[sizeof(Arg)];
-        // memcpy(buffer, (void *)dparg, sizeof(Arg));
-        Kernel2D_impl<Functor, Arg, grid_stride>(arg);
-      }
+      Kernel2D_impl<Functor, Arg, grid_stride>(arg);
     }
-    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -194,22 +157,16 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D()
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D(void)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    // printf("Kernel2D: launch parameter: gd %d ld %d\n", gd, ld);
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp parallel num_threads(ld)
     {
-      #pragma omp parallel num_threads(ld)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
-      }
+      Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
     }
   }
 
@@ -261,30 +218,15 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    const int tx = arg.threads.x;
-    const int ty = arg.threads.y;
-    const int tz = arg.threads.z;
-    // printf("Kernel3D: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // printf("dparg %p\n", dparg);
-    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
+    #pragma omp parallel num_threads(ld)
     {
-      #pragma omp parallel num_threads(ld)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        // char buffer[sizeof(Arg)];
-        // memcpy(buffer, (void *)dparg, sizeof(Arg));
-        Kernel3D_impl<Functor, Arg, grid_stride>(arg);
-      }
+      Kernel3D_impl<Functor, Arg, grid_stride>(arg);
     }
-    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -300,22 +242,16 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D()
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D(void)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    // printf("Kernel3D: launch parameter: gd %d ld %d\n", gd, ld);
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp parallel num_threads(ld)
     {
-      #pragma omp parallel num_threads(ld)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
-      }
+      Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
     }
   }
 
@@ -336,29 +272,16 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool dummy = false>
   __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    const int tx = arg.threads.x;
-    const int ty = arg.threads.y;
-    const int tz = arg.threads.z;
-    // printf("raw_kernel: launch parameter: gd %d ld %d tx %d ty %d tz %d\n", gd, ld, tx, ty, tz);
-    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // printf("dparg %p\n", dparg);
-    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
+    #pragma omp parallel num_threads(ld)
     {
-      #pragma omp parallel num_threads(ld)
-      {
-        // if(omp_get_team_num()==0 && omp_get_thread_num()==0)
-        //   printf("In target: launch_param.grid %d %d %d block %d %d %d\n"
-        //          "omp reports: teams %d threads %d\n",
-        //          launch_param.grid.x, launch_param.grid.y, launch_param.grid.z, launch_param.block.x, launch_param.block.y, launch_param.block.z,
-        //          omp_get_num_teams(), omp_get_num_threads());
-        Functor<Arg> f(&arg);
-        f();
-      }
+      Functor<Arg> f(&arg);
+      f();
     }
-    // omp_target_free(dparg, omp_get_default_device());
   }
 
 }
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 0ecca5a9e0..c737e753a8 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -59,20 +59,15 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    {
     #pragma omp parallel num_threads(ld)
     {
-      // char buffer[sizeof(Arg)];
-      // memcpy(buffer, (void *)dparg, sizeof(Arg));
       Reduction2D_impl<Functor, Arg, grid_stride>(arg);
     }
-    }
-    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -88,17 +83,17 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D()
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D(void)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld)
-    {
     #pragma omp parallel num_threads(ld)
     {
       Reduction2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
     }
-    }
   }
 
   /**
@@ -158,20 +153,15 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
-    // Arg *dparg = (Arg*)omp_target_alloc(sizeof(Arg), omp_get_default_device());
-    // omp_target_memcpy(dparg, (void *)(&arg), sizeof(Arg), 0, 0, omp_get_default_device(), omp_get_initial_device());
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    {
     #pragma omp parallel num_threads(ld)
     {
-      // char buffer[sizeof(Arg)];
-      // memcpy(buffer, (void *)dparg, sizeof(Arg));
       MultiReduction_impl<Functor, Arg, grid_stride>(arg);
     }
-    }
-    // omp_target_free(dparg, omp_get_default_device());
   }
 
   /**
@@ -187,17 +177,17 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction()
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction(void)
   {
-    const int gd = target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y*target::omptarget::launch_param.grid.z;
-    const int ld = target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y*target::omptarget::launch_param.block.z;
+    const auto& grid = target::omptarget::launch_param.grid;
+    const auto& block = target::omptarget::launch_param.block;
+    const int gd = grid.x*grid.y*grid.z;
+    const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld)
-    {
     #pragma omp parallel num_threads(ld)
     {
       MultiReduction_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
     }
-    }
   }
 
 } // namespace quda
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index b96f10c3d1..798304aaf1 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -5,10 +5,6 @@
 #include <kernel_helper.h>
 #include <kernel.h>
 
-#ifdef JITIFY
-#include <jitify_helper2.cuh>
-#endif
-
 namespace quda {
 
   namespace target {
@@ -34,21 +30,6 @@ namespace quda {
       ompwip("WARNING: rejecting threads setup arg %d %d %d tp (%d %d %d)x(%d %d %d)",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
     return r;
   }
-  inline bool acceptTeams(const TuneParam &tp)
-  {
-    const auto nt = tp.grid.x*tp.grid.y*tp.grid.z;
-    return nt <= device::max_grid_size(0);
-  }
-  template <typename... Arg>
-  qudaError_t qudaLaunchKernel(const void *func, const TuneParam &tp, qudaStream_t stream, const Arg &... arg)
-  {
-    if(acceptThreads(tp, arg...) && acceptTeams(tp) && 0==target::omptarget::qudaSetupLaunchParameter(tp)){
-      reinterpret_cast<void(*)(Arg...)>(const_cast<void*>(func))(arg...);
-      return QUDA_SUCCESS;
-    }else
-      target::omptarget::set_runtime_error(QUDA_ERROR, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
-      return QUDA_ERROR;
-  }
 
   class TunableKernel : public Tunable
   {
@@ -60,28 +41,21 @@ namespace quda {
     virtual unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
 
     template <template <typename> class Functor, bool grid_stride, typename Arg>
-    std::enable_if_t<device::use_kernel_arg<Arg>(), qudaError_t>
-      launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
-    {
-#ifdef JITIFY
-      launch_error = launch_jitify<Functor, grid_stride, Arg>(kernel.name, tp, stream, arg);
-#else
-      launch_error = qudaLaunchKernel(kernel.func, tp, stream, arg);
-#endif
-      return launch_error;
-    }
-
-    template <template <typename> class Functor, bool grid_stride, typename Arg>
-    std::enable_if_t<!device::use_kernel_arg<Arg>(), qudaError_t>
-      launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
+    qudaError_t launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
-#ifdef JITIFY
-      launch_error = launch_jitify<Functor, grid_stride, Arg>(kernel.name, tp, stream, arg);
-#else
-      static_assert(sizeof(Arg) <= device::max_constant_size(), "Parameter struct is greater than max constant size");
-      qudaMemcpyAsync(device::get_constant_buffer<Arg>(), &arg, sizeof(Arg), qudaMemcpyHostToDevice, stream);
-      launch_error = qudaLaunchKernel(kernel.func, tp, stream, arg);
-#endif
+      if (acceptThreads(tp, arg) && 0==target::omptarget::qudaSetupLaunchParameter(tp)) {
+        if constexpr (device::use_kernel_arg<Arg>()) {
+          reinterpret_cast<void(*)(Arg)>(const_cast<void*>(kernel.func))(arg);
+        } else {
+          static_assert(sizeof(Arg) <= device::max_constant_size(), "Parameter struct is greater than max constant size");
+          qudaMemcpyAsync(device::get_constant_buffer<Arg>(), &arg, sizeof(Arg), qudaMemcpyHostToDevice, stream);
+          reinterpret_cast<void(*)(void)>(const_cast<void*>(kernel.func))();
+        }
+        launch_error = QUDA_SUCCESS;
+      } else {
+        target::omptarget::set_runtime_error(QUDA_ERROR, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
+        launch_error = QUDA_ERROR;
+      }
       return launch_error;
     }
 

From f6392616c171f9e5d418156786c9ac2f393188d1 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 24 Jun 2022 23:07:31 +0000
Subject: [PATCH 134/198] omp: reduce max_kernel_arg_size to 2048

---
 include/dslash_helper.cuh                 | 2 +-
 include/targets/omptarget/target_device.h | 2 +-
 lib/hisq_paths_force_quda.cu              | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index d845c053b9..2423fe536d 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -10,7 +10,7 @@
 #include <dslash_shmem.h>
 #include <shmem_pack_helper.cuh>
 
-#if defined(_NVHPC_CUDA)
+#if defined(_NVHPC_CUDA) || defined(QUDA_TARGET_OMPTARGET)
 #include <constant_kernel_arg.h>
 constexpr bool use_kernel_arg = false;
 #else
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 7b53809a54..88567248e6 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -156,7 +156,7 @@ namespace quda {
        the kernel arguments passed to a kernel on the target
        architecture.
     */
-    constexpr size_t max_kernel_arg_size() { return 4096; }
+    constexpr size_t max_kernel_arg_size() { return 2048; }
 
     /**
        @brief Helper function that returns the bank width of the
diff --git a/lib/hisq_paths_force_quda.cu b/lib/hisq_paths_force_quda.cu
index 5febd467b2..3574a9c4d5 100644
--- a/lib/hisq_paths_force_quda.cu
+++ b/lib/hisq_paths_force_quda.cu
@@ -4,6 +4,7 @@
 #include <ks_improved_force.h>
 #include <tune_quda.h>
 #include <instantiate.h>
+#include <constant_kernel_arg.h>
 #include <tunable_nd.h>
 #include <kernels/hisq_paths_force.cuh>
 

From 8cba780000f880491b0d0c9bb5e470f327baf7e7 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 24 Jun 2022 23:37:46 +0000
Subject: [PATCH 135/198] omp: use a single buffer for arg and extern in header

---
 .../targets/omptarget/constant_kernel_arg.h    | 18 ++++--------------
 include/targets/omptarget/quda_api.h           |  1 -
 lib/targets/omptarget/quda_api.cpp             |  6 ++++++
 3 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/include/targets/omptarget/constant_kernel_arg.h b/include/targets/omptarget/constant_kernel_arg.h
index 59bb3ee328..0b2c6e1061 100644
--- a/include/targets/omptarget/constant_kernel_arg.h
+++ b/include/targets/omptarget/constant_kernel_arg.h
@@ -21,21 +21,12 @@ namespace quda
   {
 
     /**
-       @brief The __constant__ buffer used for kernel parameters
+       @brief The buffer used for kernel parameters
     */
-#if defined(__CUDACC_RDC__) && !defined(QUDA_CONSTANT_DEFINE)
-    // rdc is enabled when NVSHMEM is enabled, so we need to make the
-    // buffer as extern and define it in one place only
-    extern __constant__ char buffer[max_constant_size()];
-#else
-    #pragma omp declare target
-    __constant__ char buffer[max_constant_size()];
-    #pragma omp end declare target
-#endif
+    extern char buffer[max_constant_size()];
 
     /**
-       @brief Helper function that returns kernel argument from
-       __constant__ memory.
+       @brief Helper function that returns kernel argument from buffer.
      */
     template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), Arg &> get_arg()
     {
@@ -43,8 +34,7 @@ namespace quda
     }
 
     /**
-       @brief Helper function that returns a pointer to the
-       __constant__ memory buffer.
+       @brief Helper function that returns a pointer to the buffer.
      */
     template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), void *> get_constant_buffer()
     {
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index 2ad380aa7d..485105dbda 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -8,7 +8,6 @@
 #define __host__
 #define __device__
 #define __global__
-#define __constant__ static
 #define __launch_bounds__(...)
 #define __syncthreads() _Pragma("omp barrier")
 #define __threadfence() _Pragma("omp barrier")
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index dcd69b707d..f768c792e0 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -26,6 +26,12 @@ namespace quda
     return rtn;
   }
 
+  namespace device {
+    #pragma omp declare target
+    char buffer[max_constant_size()];
+    #pragma omp end declare target
+  }
+
   namespace target {
     namespace omptarget {
       #pragma omp declare target

From b14af6dbdfb29d0d28025f9eaee875d36f719e5c Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 29 Jun 2022 21:12:49 +0000
Subject: [PATCH 136/198] omp: clean up malloc

---
 lib/targets/omptarget/malloc.cpp | 62 +++++++++++++++++++-------------
 1 file changed, 38 insertions(+), 24 deletions(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index fcfbcf39ee..d0507a814d 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -263,17 +263,18 @@ namespace quda
   void *device_pinned_malloc_(const char *func, const char *file, int line, size_t size)
   {
     if (!comm_peer2peer_present()) return device_malloc_(func, file, line, size);
-    ompwip("ERROR: device_pinned_malloc_ with !comm_peer2peer_present() unimplemented");
-    return nullptr;
-/*
 
     MemAlloc a(func, file, line);
-    void *ptr;
-
     a.size = a.base_size = size;
+    void *ptr;
 
-    CUresult err = cuMemAlloc((CUdeviceptr *)&ptr, size);
-    if (err != CUDA_SUCCESS) {
+    if(0<omp_get_num_devices())
+      ptr = omp_target_alloc_device(size, omp_get_default_device());
+    else{
+      warningQuda("%s:%d %s() allocate on host instead of device", file, line, func);
+      ptr = aligned_alloc(64, size);
+    }
+    if (!ptr) {
       errorQuda("Failed to allocate device memory of size %zu (%s:%d in %s())\n", size, file, line, func);
     }
     track_malloc(DEVICE_PINNED, a, ptr);
@@ -281,7 +282,6 @@ namespace quda
     cudaMemset(ptr, 0xff, size);
 #endif
     return ptr;
-*/
   }
 
   /**
@@ -417,14 +417,19 @@ namespace quda
    * defined in malloc_quda.h
    */
   void *managed_malloc_(const char *func, const char *file, int line, size_t size)
-  {ompwip("ERROR: managed_malloc_ unimplemented return nullptr");return nullptr;/*
+  {
     MemAlloc a(func, file, line);
     void *ptr;
 
     a.size = a.base_size = size;
 
-    cudaError_t err = cudaMallocManaged(&ptr, size);
-    if (err != cudaSuccess) {
+    if(0<omp_get_num_devices()){
+      ptr = omp_target_alloc_shared(size, omp_get_default_device());
+    }else{
+      warningQuda("%s:%d %s() managed malloc without a device", file, line, func);
+      ptr = aligned_malloc(a, size);
+    }
+    if (!ptr) {
       errorQuda("Failed to allocate managed memory of size %zu (%s:%d in %s())\n", size, file, line, func);
     }
     track_malloc(MANAGED, a, ptr);
@@ -432,7 +437,7 @@ namespace quda
     cudaMemset(ptr, 0xff, size);
 #endif
     return ptr;
-  */}
+  }
 
   /**
    * Free device memory allocated with device_malloc().  This function
@@ -441,7 +446,7 @@ namespace quda
    */
   void device_free_(const char *func, const char *file, int line, void *ptr)
   {
-    if (use_managed_memory()) {ompwip("ERROR: unimplemented");
+    if (use_managed_memory()) {
       managed_free_(func, file, line, ptr);
       return;
     }
@@ -473,7 +478,6 @@ namespace quda
    */
   void device_pinned_free_(const char *func, const char *file, int line, void *ptr)
   {
-    ompwip("ERROR: device_pinned_free_ untested code path: %p",ptr); print_trace();
     if (!comm_peer2peer_present()) {
       device_free_(func, file, line, ptr);
       return;
@@ -500,15 +504,17 @@ namespace quda
    * malloc_quda.h
    */
   void managed_free_(const char *func, const char *file, int line, void *ptr)
-  {ompwip("ERROR: managed_free_ unimplemented doing nothing: %p",ptr);/*
+  {
     if (!ptr) { errorQuda("Attempt to free NULL managed pointer (%s:%d in %s())\n", file, line, func); }
     if (!alloc[MANAGED].count(ptr)) {
       errorQuda("Attempt to free invalid managed pointer (%s:%d in %s())\n", file, line, func);
     }
-    cudaError_t err = cudaFree(ptr);
-    if (err != cudaSuccess) { errorQuda("Failed to free device memory (%s:%d in %s())\n", file, line, func); }
+    if(0<omp_get_num_devices())
+      omp_target_free(ptr, omp_get_default_device());
+    else
+      free(ptr);
     track_free(MANAGED, ptr);
-  */}
+  }
 
   /**
    * Free host memory allocated with safe_malloc(), pinned_malloc(),
@@ -522,20 +528,25 @@ namespace quda
       track_free(HOST, ptr);
       free(ptr);
     } else if (alloc[PINNED].count(ptr)) {
-      ompwip("WARNING: free PINNED ptr: %p",ptr);
 /*
       cudaError_t err = cudaHostUnregister(ptr);
       if (err != cudaSuccess) { errorQuda("Failed to unregister pinned memory (%s:%d in %s())\n", file, line, func); }
 */
       track_free(PINNED, ptr);
-      free(ptr);
+      if(0<omp_get_num_devices())
+        omp_target_free(ptr, omp_get_default_device());
+      else
+        free(ptr);
     } else if (alloc[MAPPED].count(ptr)) {
 #ifdef HOST_ALLOC
 #ifdef OMPTARGET_MAPPED_USE_ASSOCIATE_PTR
       ompwip("ERROR: HOST_ALLOC alloc[MAPPED].count(ptr) untested code path: %p",ptr); print_trace();
       free(ptr);
 #else
-      omp_target_free(ptr, omp_get_default_device());
+      if(0<omp_get_num_devices())
+        omp_target_free(ptr, omp_get_default_device());
+      else
+        free(ptr);
 #endif
 /*
       cudaError_t err = cudaFreeHost(ptr);
@@ -552,7 +563,10 @@ namespace quda
       }
       free(ptr);
 #else
-      omp_target_free(ptr, omp_get_default_device());
+      if(0<omp_get_num_devices())
+        omp_target_free(ptr, omp_get_default_device());
+      else
+        free(ptr);
 #endif
 /*
       cudaError_t err = cudaHostUnregister(ptr);
@@ -706,7 +720,7 @@ namespace quda
 
   void register_pinned_(const char *func, const char *file, int line, void *ptr, size_t bytes)
   {
-    ompwip("ERROR: register_pinned_ unimplemented: %p",ptr);
+    ompwip("WARNING: register_pinned_ unimplemented: %p",ptr);
 /*
     auto error = cudaHostRegister(ptr, bytes, cudaHostRegisterDefault);
     if (error != cudaSuccess) {
@@ -718,7 +732,7 @@ namespace quda
 
   void unregister_pinned_(const char *func, const char *file, int line, void *ptr)
   {
-    ompwip("ERROR: unregister_pinned_ unimplemented: %p",ptr);
+    ompwip("WARNING: unregister_pinned_ unimplemented: %p",ptr);
 /*
     auto error = cudaHostUnregister(ptr);
     if (error != cudaSuccess) {

From a3abf65697ac94a402a558841be6a4228feb5b16 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 29 Jun 2022 21:14:13 +0000
Subject: [PATCH 137/198] omp: remove device::get_arg, directly passing arg
 pointer to kernels as an argument

---
 .../targets/omptarget/block_reduction_kernel.h |  6 +++---
 .../targets/omptarget/constant_kernel_arg.h    | 12 ++----------
 include/targets/omptarget/kernel.h             | 18 +++++++++---------
 include/targets/omptarget/reduction_kernel.h   | 12 ++++++------
 include/targets/omptarget/target_device.h      |  9 ---------
 include/targets/omptarget/tunable_kernel.h     |  5 +++--
 lib/targets/omptarget/quda_api.cpp             |  9 ++++++---
 7 files changed, 29 insertions(+), 42 deletions(-)

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 433905ad9f..b0b4aa2ee8 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -142,17 +142,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __launch_bounds__(Arg::launch_bounds ?
                       Arg::block_size :
-                      0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D(void)
+                      0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg *argp)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
     const auto& grid = target::omptarget::launch_param.grid;
     const auto& block = target::omptarget::launch_param.block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
     #pragma omp parallel num_threads(ld)
     {
-      BlockKernel2D_impl<Functor, Arg>(device::get_arg<Arg>());
+      BlockKernel2D_impl<Functor, Arg>(*argp);
     }
   }
 
diff --git a/include/targets/omptarget/constant_kernel_arg.h b/include/targets/omptarget/constant_kernel_arg.h
index 0b2c6e1061..bd8ed7b976 100644
--- a/include/targets/omptarget/constant_kernel_arg.h
+++ b/include/targets/omptarget/constant_kernel_arg.h
@@ -23,22 +23,14 @@ namespace quda
     /**
        @brief The buffer used for kernel parameters
     */
-    extern char buffer[max_constant_size()];
-
-    /**
-       @brief Helper function that returns kernel argument from buffer.
-     */
-    template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), Arg &> get_arg()
-    {
-      return reinterpret_cast<Arg &>(buffer);
-    }
+    extern void *constant_arg_buffer;
 
     /**
        @brief Helper function that returns a pointer to the buffer.
      */
     template <typename Arg> constexpr std::enable_if_t<!use_kernel_arg<Arg>(), void *> get_constant_buffer()
     {
-      return omp_get_mapped_ptr(buffer, omp_get_default_device());
+      return constant_arg_buffer;
     }
 
   } // namespace device
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index a87a0598a8..5cbc10989c 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -74,16 +74,16 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D(void)
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D(Arg *argp)
   {
     const auto& grid = target::omptarget::launch_param.grid;
     const auto& block = target::omptarget::launch_param.block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
     #pragma omp parallel num_threads(ld)
     {
-      Kernel1D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+      Kernel1D_impl<Functor, Arg, grid_stride>(*argp);
     }
   }
 
@@ -157,16 +157,16 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D(void)
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D(Arg *argp)
   {
     const auto& grid = target::omptarget::launch_param.grid;
     const auto& block = target::omptarget::launch_param.block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
     #pragma omp parallel num_threads(ld)
     {
-      Kernel2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+      Kernel2D_impl<Functor, Arg, grid_stride>(*argp);
     }
   }
 
@@ -242,16 +242,16 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D(void)
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D(Arg *argp)
   {
     const auto& grid = target::omptarget::launch_param.grid;
     const auto& block = target::omptarget::launch_param.block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
     #pragma omp parallel num_threads(ld)
     {
-      Kernel3D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+      Kernel3D_impl<Functor, Arg, grid_stride>(*argp);
     }
   }
 
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index c737e753a8..0721dc82ce 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -83,16 +83,16 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D(void)
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D(Arg *argp)
   {
     const auto& grid = target::omptarget::launch_param.grid;
     const auto& block = target::omptarget::launch_param.block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
     #pragma omp parallel num_threads(ld)
     {
-      Reduction2D_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+      Reduction2D_impl<Functor, Arg, grid_stride>(*argp);
     }
   }
 
@@ -177,16 +177,16 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction(void)
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction(Arg *argp)
   {
     const auto& grid = target::omptarget::launch_param.grid;
     const auto& block = target::omptarget::launch_param.block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
     #pragma omp parallel num_threads(ld)
     {
-      MultiReduction_impl<Functor, Arg, grid_stride>(device::get_arg<Arg>());
+      MultiReduction_impl<Functor, Arg, grid_stride>(*argp);
     }
   }
 
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 88567248e6..fc76a8f893 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -181,15 +181,6 @@ namespace quda {
       return (sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg);
     }
 
-    /**
-       @brief Helper function that returns kernel argument from
-       __constant__ memory.  Note this is the dummy implementation,
-       and is present only to keep the compiler happy in the
-       translation units where constant memory is not used.
-     */
-    template <typename Arg>
-      constexpr std::enable_if_t<use_kernel_arg<Arg>(), const Arg&> get_arg() { return reinterpret_cast<Arg&>(nullptr); }
-
     /**
        @brief Helper function that returns a pointer to the
        __constant__ memory buffer.  Note this is the dummy
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index 798304aaf1..4e2b7b682d 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -48,8 +48,9 @@ namespace quda {
           reinterpret_cast<void(*)(Arg)>(const_cast<void*>(kernel.func))(arg);
         } else {
           static_assert(sizeof(Arg) <= device::max_constant_size(), "Parameter struct is greater than max constant size");
-          qudaMemcpyAsync(device::get_constant_buffer<Arg>(), &arg, sizeof(Arg), qudaMemcpyHostToDevice, stream);
-          reinterpret_cast<void(*)(void)>(const_cast<void*>(kernel.func))();
+          Arg *argp = reinterpret_cast<Arg*>(device::get_constant_buffer<Arg>());
+          memcpy(argp, &arg, sizeof(Arg));
+          reinterpret_cast<void(*)(Arg*)>(const_cast<void*>(kernel.func))(argp);
         }
         launch_error = QUDA_SUCCESS;
       } else {
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index f768c792e0..667e6a1f84 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -27,9 +27,7 @@ namespace quda
   }
 
   namespace device {
-    #pragma omp declare target
-    char buffer[max_constant_size()];
-    #pragma omp end declare target
+    void *constant_arg_buffer;
   }
 
   namespace target {
@@ -50,6 +48,11 @@ namespace quda
         static int init = 0;
         int num_teams = tp.grid.x*tp.grid.y*tp.grid.z;
         if(!init){
+          device::constant_arg_buffer = omp_target_alloc_host(device::max_constant_size(), omp_get_default_device());
+          if(!device::constant_arg_buffer){
+            warningQuda("failed to allocate %lu bytes host memory for kernel arguments.", device::max_constant_size());
+            return -1;
+          }
           shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), omp_get_default_device());
           if(!shared_cache.addr){
             // warnQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);

From 3d434c5f20900521faa66d8687f5f6f848e252a6 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 29 Jun 2022 22:06:38 +0000
Subject: [PATCH 138/198] omp: use constant_kernel_arg for coarse_op

---
 lib/coarse_op.cuh          | 1 +
 lib/staggered_coarse_op.cu | 1 +
 2 files changed, 2 insertions(+)

diff --git a/lib/coarse_op.cuh b/lib/coarse_op.cuh
index fbefd474a0..b69cd118f5 100644
--- a/lib/coarse_op.cuh
+++ b/lib/coarse_op.cuh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <constant_kernel_arg.h>
 #include <kernels/coarse_op_kernel.cuh>
 #include <uint_to_char.h>
 #include <coarse_op_mma_launch.h>
diff --git a/lib/staggered_coarse_op.cu b/lib/staggered_coarse_op.cu
index 6e7e21d635..6d42d48ff2 100644
--- a/lib/staggered_coarse_op.cu
+++ b/lib/staggered_coarse_op.cu
@@ -2,6 +2,7 @@
 
 #include <transfer.h>
 #include <gauge_field.h>
+#include <constant_kernel_arg.h>
 #include <tunable_nd.h>
 #include <instantiate.h>
 

From 1622d0f8ddbaccbadf1cb73591057db527dd5ef2 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 7 Jul 2022 20:45:35 +0000
Subject: [PATCH 139/198] omp: fix max_nthr type in warp_collective

---
 include/targets/omptarget/warp_collective.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/targets/omptarget/warp_collective.h b/include/targets/omptarget/warp_collective.h
index 4796f2d158..993acc9d5f 100644
--- a/include/targets/omptarget/warp_collective.h
+++ b/include/targets/omptarget/warp_collective.h
@@ -13,7 +13,7 @@ namespace quda
     template <typename T> __device__ inline T operator()(T &x, int warp_split)
     {
       using R = typename T::value_type;
-      constexpr max_nthr = device::max_block_size();
+      constexpr auto max_nthr = device::max_block_size();
       static_assert(max_nthr*sizeof(R) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
       R *storage = (R*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
       const int tid = omp_get_thread_num();

From 03aeb636a99b8c9c57622aff22742c3df0407fb5 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 19 Jul 2022 22:08:39 +0000
Subject: [PATCH 140/198] omp: use MKL cgetrf/i and ?gemm, with the 32 bit
 integer interface

---
 lib/targets/omptarget/CMakeLists.txt          |   2 +-
 ..._lapack_cublas.cpp => blas_lapack_mkl.cpp} | 215 +++++++++---------
 lib/targets/omptarget/target_omptarget.cmake  |   6 +
 3 files changed, 109 insertions(+), 114 deletions(-)
 rename lib/targets/omptarget/{blas_lapack_cublas.cpp => blas_lapack_mkl.cpp} (69%)

diff --git a/lib/targets/omptarget/CMakeLists.txt b/lib/targets/omptarget/CMakeLists.txt
index a81643e172..9d273d5b6a 100644
--- a/lib/targets/omptarget/CMakeLists.txt
+++ b/lib/targets/omptarget/CMakeLists.txt
@@ -1,2 +1,2 @@
 # add target specific files / options
-target_sources(quda_cpp PRIVATE  blas_lapack_cublas.cpp comm_target.cpp device.cpp malloc.cpp quda_api.cpp)
+target_sources(quda_cpp PRIVATE  blas_lapack_mkl.cpp comm_target.cpp device.cpp malloc.cpp quda_api.cpp)
diff --git a/lib/targets/omptarget/blas_lapack_cublas.cpp b/lib/targets/omptarget/blas_lapack_mkl.cpp
similarity index 69%
rename from lib/targets/omptarget/blas_lapack_cublas.cpp
rename to lib/targets/omptarget/blas_lapack_mkl.cpp
index 35f328da86..aa6cd5cec5 100644
--- a/lib/targets/omptarget/blas_lapack_cublas.cpp
+++ b/lib/targets/omptarget/blas_lapack_mkl.cpp
@@ -2,7 +2,8 @@
 #include <blas_lapack.h>
 #include <timer.h>
 #ifdef NATIVE_LAPACK_LIB
-#include <cublas_v2.h>
+#include <mkl.h>
+#include <mkl_omp_offload.h>
 #include <malloc_quda.h>
 #endif
 
@@ -21,36 +22,8 @@ namespace quda
     namespace native
     {
 
-#ifdef NATIVE_LAPACK_LIB
-      static cublasHandle_t handle;
-#endif
-      static bool cublas_init = false;
-
-      void init()
-      {
-        if (!cublas_init) {
-#ifdef NATIVE_LAPACK_LIB
-          cublasStatus_t error = cublasCreate(&handle);
-          if (error != CUBLAS_STATUS_SUCCESS)
-            errorQuda("cublasCreate failed with error %d", error);
-          else
-            printfQuda("cublasCreated successfully\n");
-          cublas_init = true;
-#endif
-        }
-      }
-
-      void destroy()
-      {
-        if (cublas_init) {
-#ifdef NATIVE_LAPACK_LIB
-          cublasStatus_t error = cublasDestroy(handle);
-          if (error != CUBLAS_STATUS_SUCCESS)
-            errorQuda("\nError indestroying cublas context, error code = %d\n", error);
-          cublas_init = false;
-#endif
-        }
-      }
+      void init() {}
+      void destroy() {}
 
 #ifdef _DEBUG
       template <typename EigenMatrix, typename Float>
@@ -78,7 +51,6 @@ namespace quda
       long long BatchInvertMatrix(void *Ainv, void *A, const int n, const uint64_t batch, QudaPrecision prec,
                                   QudaFieldLocation location)
       {
-        init();
         if (getVerbosity() >= QUDA_VERBOSE)
           printfQuda("BatchInvertMatrix (native - cuBLAS): Nc = %d, batch = %lu\n", n, batch);
 
@@ -99,29 +71,38 @@ namespace quda
         if (location == QUDA_CUDA_FIELD_LOCATION) qudaMemcpy((void *)A_h, A_d, size, qudaMemcpyDeviceToHost);
 #endif
 
+        static_assert(sizeof(int)==sizeof(MKL_INT), "MKL_INT must be int.");
+
         int *dipiv = static_cast<int *>(pool_device_malloc(batch * n * sizeof(int)));
         int *dinfo_array = static_cast<int *>(pool_device_malloc(batch * sizeof(int)));
         int *info_array = static_cast<int *>(pool_pinned_malloc(batch * sizeof(int)));
         memset(info_array, '0', batch * sizeof(int)); // silence memcheck warnings
 
         if (prec == QUDA_SINGLE_PRECISION) {
-          typedef cuFloatComplex C;
+          typedef MKL_Complex8 C;
           C **A_array = static_cast<C **>(pool_device_malloc(2 * batch * sizeof(C *)));
           C **Ainv_array = A_array + batch;
           C **A_array_h = static_cast<C **>(pool_pinned_malloc(2 * batch * sizeof(C *)));
           C **Ainv_array_h = A_array_h + batch;
+          MKL_INT **ipiv_array = static_cast<MKL_INT **>(pool_device_malloc(batch * sizeof(MKL_INT *)));
+          MKL_INT **ipiv_array_h = static_cast<MKL_INT **>(pool_pinned_malloc(batch * sizeof(MKL_INT *)));
           for (uint64_t i = 0; i < batch; i++) {
             A_array_h[i] = static_cast<C *>(A_d) + i * n * n;
             Ainv_array_h[i] = static_cast<C *>(Ainv_d) + i * n * n;
+            ipiv_array_h[i] = dipiv + i * n;
           }
           qudaMemcpy(A_array, A_array_h, 2 * batch * sizeof(C *), qudaMemcpyHostToDevice);
-
-          cublasStatus_t error = cublasCgetrfBatched(handle, n, A_array, n, dipiv, dinfo_array, batch);
+          qudaMemcpy(ipiv_array, ipiv_array_h, batch * sizeof(MKL_INT *), qudaMemcpyHostToDevice);
+
+          MKL_INT n_array = n;
+          MKL_INT group_size = batch;
+          MKL_INT group_count = (MKL_INT)1;
+          #pragma omp target variant dispatch use_device_ptr(A_array, ipiv_array, dinfo_array)
+          {
+            cgetrf_batch(&n_array, &n_array, A_array, &n_array, ipiv_array, &group_count, &group_size, dinfo_array);
+          }
           flops += batch * FLOPS_CGETRF(n, n);
 
-          if (error != CUBLAS_STATUS_SUCCESS)
-            errorQuda("\nError in LU decomposition (cublasCgetrfBatched), error code = %d\n", error);
-
           qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), qudaMemcpyDeviceToHost);
           for (uint64_t i = 0; i < batch; i++) {
             if (info_array[i] < 0) {
@@ -132,12 +113,12 @@ namespace quda
             }
           }
 
-          error = cublasCgetriBatched(handle, n, (const C **)A_array, n, dipiv, Ainv_array, n, dinfo_array, batch);
+          #pragma omp target variant dispatch use_device_ptr(A_array, Ainv_array, ipiv_array, dinfo_array)
+          {
+            cgetri_oop_batch(&n_array, (const C**)A_array, &n_array, (const MKL_INT**)ipiv_array, Ainv_array, &n_array, &group_count, &group_size, dinfo_array);
+          }
           flops += batch * FLOPS_CGETRI(n);
 
-          if (error != CUBLAS_STATUS_SUCCESS)
-            errorQuda("\nError in matrix inversion (cublasCgetriBatched), error code = %d\n", error);
-
           qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), qudaMemcpyDeviceToHost);
 
           for (uint64_t i = 0; i < batch; i++) {
@@ -151,6 +132,8 @@ namespace quda
 
           pool_device_free(A_array);
           pool_pinned_free(A_array_h);
+          pool_device_free(ipiv_array);
+          pool_pinned_free(ipiv_array_h);
 
 #ifdef _DEBUG
           // Debug code: Copy computed Ainv to host
@@ -328,19 +311,19 @@ namespace quda
           qudaMemcpy(C_d, C_data, sizeCarr, qudaMemcpyHostToDevice);
         }
 
-        cublasOperation_t trans_a = CUBLAS_OP_N;
+        CBLAS_TRANSPOSE trans_a = CblasNoTrans;
         switch (blas_param.trans_a) {
-        case QUDA_BLAS_OP_N: trans_a = CUBLAS_OP_N; break;
-        case QUDA_BLAS_OP_T: trans_a = CUBLAS_OP_T; break;
-        case QUDA_BLAS_OP_C: trans_a = CUBLAS_OP_C; break;
+        case QUDA_BLAS_OP_N: trans_a = CblasNoTrans; break;
+        case QUDA_BLAS_OP_T: trans_a = CblasTrans; break;
+        case QUDA_BLAS_OP_C: trans_a = CblasConjTrans; break;
         default: errorQuda("Unknown QUDA_BLAS_OP type %d\n", blas_param.trans_a);
         }
 
-        cublasOperation_t trans_b = CUBLAS_OP_N;
+        CBLAS_TRANSPOSE trans_b = CblasNoTrans;
         switch (blas_param.trans_b) {
-        case QUDA_BLAS_OP_N: trans_b = CUBLAS_OP_N; break;
-        case QUDA_BLAS_OP_T: trans_b = CUBLAS_OP_T; break;
-        case QUDA_BLAS_OP_C: trans_b = CUBLAS_OP_C; break;
+        case QUDA_BLAS_OP_N: trans_b = CblasNoTrans; break;
+        case QUDA_BLAS_OP_T: trans_b = CblasTrans; break;
+        case QUDA_BLAS_OP_C: trans_b = CblasConjTrans; break;
         default: errorQuda("Unknown QUDA_BLAS_OP type %d\n", blas_param.trans_b);
         }
         //-------------------------------------------------------------------------
@@ -349,56 +332,60 @@ namespace quda
         //-------------------------------------------------------------------------
         if (blas_param.data_type == QUDA_BLAS_DATATYPE_Z) {
 
-          typedef cuDoubleComplex Z;
+          typedef MKL_Complex16 Z;
+          static_assert(sizeof(Z)==sizeof(double2), "MKL_Complex16 and double2 must be the same.");
 
-          const Z alpha = make_double2((double)(static_cast<std::complex<double>>(blas_param.alpha).real()),
-                                       (double)(static_cast<std::complex<double>>(blas_param.alpha).imag()));
+          const double2 alpha = make_double2((double)(static_cast<std::complex<double>>(blas_param.alpha).real()),
+                                             (double)(static_cast<std::complex<double>>(blas_param.alpha).imag()));
 
-          const Z beta = make_double2((double)(static_cast<std::complex<double>>(blas_param.beta).real()),
-                                      (double)(static_cast<std::complex<double>>(blas_param.beta).imag()));
+          const double2 beta = make_double2((double)(static_cast<std::complex<double>>(blas_param.beta).real()),
+                                            (double)(static_cast<std::complex<double>>(blas_param.beta).imag()));
 
-          cublasStatus_t error;
           if (batch > 1) {
-            error = cublasZgemmStridedBatched(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
-                                              &alpha, (Z *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
-                                              (Z *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
-                                              (Z *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
-
-            if (error != CUBLAS_STATUS_SUCCESS)
-              errorQuda("\nError in cuBLASZGEMMStridedBatched, error code = %d\n", error);
+            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
+            {
+              cblas_zgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                        &alpha, (Z *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                        (Z *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
+                                        (Z *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
+            }
           } else {
-            error = cublasZgemm(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
-                                (Z *)A_d + blas_param.a_offset, blas_param.lda, (Z *)B_d + blas_param.b_offset,
-                                blas_param.ldb, &beta, (Z *)C_d + blas_param.c_offset, blas_param.ldc);
-
-            if (error != CUBLAS_STATUS_SUCCESS) errorQuda("\nError in cuBLASZGEMM, error code = %d\n", error);
+            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
+            {
+              cblas_zgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
+                          (Z *)A_d + blas_param.a_offset, blas_param.lda, (Z *)B_d + blas_param.b_offset,
+                          blas_param.ldb, &beta, (Z *)C_d + blas_param.c_offset, blas_param.ldc);
+            }
           }
+          flops += batch * FLOPS_CGEMM(blas_param.m, blas_param.n, blas_param.k);
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_C) {
 
-          typedef cuFloatComplex C;
+          typedef MKL_Complex8 C;
+          static_assert(sizeof(C)==sizeof(float2), "MKL_Complex8 and float2 must be the same.");
 
-          const C alpha = make_float2((float)(static_cast<std::complex<double>>(blas_param.alpha).real()),
-                                      (float)(static_cast<std::complex<double>>(blas_param.alpha).imag()));
+          const float2 alpha = make_float2((float)(static_cast<std::complex<double>>(blas_param.alpha).real()),
+                                           (float)(static_cast<std::complex<double>>(blas_param.alpha).imag()));
 
-          const C beta = make_float2((float)(static_cast<std::complex<double>>(blas_param.beta).real()),
-                                     (float)(static_cast<std::complex<double>>(blas_param.beta).imag()));
+          const float2 beta = make_float2((float)(static_cast<std::complex<double>>(blas_param.beta).real()),
+                                          (float)(static_cast<std::complex<double>>(blas_param.beta).imag()));
 
-          cublasStatus_t error;
           if (batch > 1) {
-            error = cublasCgemmStridedBatched(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
-                                              &alpha, (C *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
-                                              (C *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
-                                              (C *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
-
-            if (error != CUBLAS_STATUS_SUCCESS)
-              errorQuda("\nError in cuBLASCGEMMStridedBatched, error code = %d\n", error);
+            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
+            {
+              cblas_cgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                         &alpha, (C *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                         (C *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
+                                         (C *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
+            }
           } else {
-            error = cublasCgemm(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
-                                (C *)A_d + blas_param.a_offset, blas_param.lda, (C *)B_d + blas_param.b_offset,
-                                blas_param.ldb, &beta, (C *)C_d + blas_param.c_offset, blas_param.ldc);
-
-            if (error != CUBLAS_STATUS_SUCCESS) errorQuda("\nError in cuBLASCGEMMBatched, error code = %d\n", error);
+            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
+            {
+              cblas_cgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
+                          (C *)A_d + blas_param.a_offset, blas_param.lda, (C *)B_d + blas_param.b_offset,
+                          blas_param.ldb, &beta, (C *)C_d + blas_param.c_offset, blas_param.ldc);
+            }
           }
+          flops += batch * FLOPS_CGEMM(blas_param.m, blas_param.n, blas_param.k);
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_D) {
 
           typedef double D;
@@ -406,22 +393,23 @@ namespace quda
           const D alpha = (D)(static_cast<std::complex<double>>(blas_param.alpha).real());
           const D beta = (D)(static_cast<std::complex<double>>(blas_param.beta).real());
 
-          cublasStatus_t error;
           if (batch > 1) {
-            error = cublasDgemmStridedBatched(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
-                                              &alpha, (D *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
-                                              (D *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
-                                              (D *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
-
-            if (error != CUBLAS_STATUS_SUCCESS)
-              errorQuda("\nError in cuBLASDGEMMStridedBatched, error code = %d\n", error);
+            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
+            {
+              cblas_dgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                        alpha, (D *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                        (D *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, beta,
+                                        (D *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
+            }
           } else {
-            error = cublasDgemm(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
-                                (D *)A_d + blas_param.a_offset, blas_param.lda, (D *)B_d + blas_param.b_offset,
-                                blas_param.ldb, &beta, (D *)C_d + blas_param.c_offset, blas_param.ldc);
-
-            if (error != CUBLAS_STATUS_SUCCESS) errorQuda("\nError in cuBLASDGEMMBatched, error code = %d\n", error);
+            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
+            {
+              cblas_dgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, alpha,
+                          (D *)A_d + blas_param.a_offset, blas_param.lda, (D *)B_d + blas_param.b_offset,
+                          blas_param.ldb, beta, (D *)C_d + blas_param.c_offset, blas_param.ldc);
+            }
           }
+          flops += batch * FLOPS_SGEMM(blas_param.m, blas_param.n, blas_param.k);
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_S) {
 
           typedef float S;
@@ -429,24 +417,25 @@ namespace quda
           const S alpha = (S)(static_cast<std::complex<float>>(blas_param.alpha).real());
           const S beta = (S)(static_cast<std::complex<float>>(blas_param.beta).real());
 
-          cublasStatus_t error;
           if (batch > 1) {
-            error = cublasSgemmStridedBatched(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
-                                              &alpha, (S *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
-                                              (S *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
-                                              (S *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
-
-            if (error != CUBLAS_STATUS_SUCCESS)
-              errorQuda("\nError in cuBLASSGEMMStridedBatched, error code = %d\n", error);
+            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
+            {
+              cblas_sgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                        alpha, (S *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                        (S *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, beta,
+                                        (S *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
+            }
           } else {
-            error = cublasSgemm(handle, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
-                                (S *)A_d + blas_param.a_offset, blas_param.lda, (S *)B_d + blas_param.b_offset,
-                                blas_param.ldb, &beta, (S *)C_d + blas_param.c_offset, blas_param.ldc);
-
-            if (error != CUBLAS_STATUS_SUCCESS) errorQuda("\nError in cuBLASSGEMMBatched, error code = %d\n", error);
+            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
+            {
+              cblas_sgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, alpha,
+                          (S *)A_d + blas_param.a_offset, blas_param.lda, (S *)B_d + blas_param.b_offset,
+                          blas_param.ldb, beta, (S *)C_d + blas_param.c_offset, blas_param.ldc);
+            }
           }
+          flops += batch * FLOPS_SGEMM(blas_param.m, blas_param.n, blas_param.k);
         } else {
-          errorQuda("cublasGEMM type %d not implemented\n", blas_param.data_type);
+          errorQuda("MKL GEMM type %d not implemented\n", blas_param.data_type);
         }
         //-------------------------------------------------------------------------
 
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index eb1b6078cd..e9ae16f79d 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -70,6 +70,12 @@ set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES COMPILE_FLAGS "-x c++")
 # target_link_libraries(quda PUBLIC hip::hiprand roc::rocrand hip::hipcub roc::rocprim_hip)
 # target_link_libraries(quda PUBLIC roc::hipblas roc::rocblas)
 
+set(OMPTARGET_MKL_LIBRARY "-lmkl_sycl -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread")
+
+if(${QUDA_BUILD_NATIVE_LAPACK} STREQUAL "ON")
+  target_link_libraries(quda PUBLIC ${OMPTARGET_MKL_LIBRARY})
+endif()
+
 # if(QUDA_GAUGE_ALG)
 #   target_include_directories(quda PUBLIC ${ROCM_PATH}/hipfft/include)
 #   target_link_libraries(quda PUBLIC hip::hipfft)

From 8b1a57f84c2bbfba745a3c19854bb0c222e884f0 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 21 Jul 2022 18:05:48 +0000
Subject: [PATCH 141/198] omp: reinstate #pragma unroll

---
 include/blas_helper.cuh                       |  24 +--
 include/clover_field_order.h                  |  62 +++---
 include/color_spinor.h                        | 172 ++++++++--------
 include/color_spinor_field_order.h            |  52 ++---
 include/float_vector.h                        |   6 +-
 include/gauge_field_order.h                   |  56 ++---
 include/gauge_fix_ovr_hit_devf.cuh            |  10 +-
 include/index_helper.cuh                      |  12 +-
 include/kernels/blas_core.cuh                 |  28 +--
 include/kernels/block_orthogonalize.cuh       |  26 +--
 include/kernels/clover_compute.cuh            |   6 +-
 include/kernels/clover_invert.cuh             |   2 +-
 include/kernels/clover_outer_product.cuh      |   2 +-
 .../kernels/clover_sigma_outer_product.cuh    |   2 +-
 include/kernels/coarse_op_kernel.cuh          | 194 +++++++++---------
 include/kernels/coarse_op_preconditioned.cuh  |   6 +-
 include/kernels/color_spinor_pack.cuh         |  10 +-
 include/kernels/contraction.cuh               |   4 +-
 include/kernels/copy_clover.cuh               |   4 +-
 include/kernels/copy_field_offset.cuh         |   6 +-
 include/kernels/dslash_clover_helper.cuh      |   4 +-
 include/kernels/dslash_coarse.cuh             |  40 ++--
 include/kernels/dslash_mdw_fused.cuh          |  16 +-
 include/kernels/dslash_staggered.cuh          |   2 +-
 .../dslash_twisted_clover_preconditioned.cuh  |   2 +-
 .../dslash_twisted_mass_preconditioned.cuh    |   2 +-
 include/kernels/dslash_wilson.cuh             |   2 +-
 include/kernels/dslash_wilson_clover.cuh      |   2 +-
 .../dslash_wilson_clover_hasenbusch_twist.cuh |   2 +-
 ...clover_hasenbusch_twist_preconditioned.cuh |   4 +-
 .../dslash_wilson_clover_preconditioned.cuh   |   2 +-
 include/kernels/gauge_det_trace.cuh           |   6 +-
 include/kernels/gauge_fix_fft.cuh             |  12 +-
 include/kernels/gauge_fix_ovr.cuh             |   8 +-
 include/kernels/gauge_heatbath.cuh            |   6 +-
 include/kernels/gauge_plaq.cuh                |   6 +-
 include/kernels/gauge_qcharge.cuh             |   4 +-
 include/kernels/gauge_utils.cuh               |   2 +-
 include/kernels/hisq_paths_force.cuh          |   6 +-
 include/kernels/laplace.cuh                   |   2 +-
 include/kernels/momentum.cuh                  |   2 +-
 include/kernels/multi_blas_core.cuh           |  12 +-
 include/kernels/multi_reduce_core.cuh         |   8 +-
 include/kernels/pgauge_init.cuh               |  12 +-
 include/kernels/prolongator.cuh               |  16 +-
 include/kernels/reduce_core.cuh               |  36 ++--
 include/kernels/restrictor.cuh                |  24 +--
 .../kernels/staggered_coarse_op_kernel.cuh    |   6 +-
 .../staggered_kd_apply_xinv_kernel.cuh        |  10 +-
 include/kernels/staggered_outer_product.cuh   |   2 +-
 include/linalg.cuh                            |  30 +--
 include/matrix_tile.cuh                       |  62 +++---
 include/quda_api.h                            |   4 -
 include/quda_matrix.h                         | 122 +++++------
 include/shared_memory_cache_helper.cuh        |   4 +-
 include/su3_project.cuh                       |   8 +-
 include/targets/generic/aos.h                 |   4 +-
 include/targets/omptarget/quda_api.h          |   4 -
 include/targets/omptarget/warp_collective.h   |   4 +-
 59 files changed, 588 insertions(+), 596 deletions(-)

diff --git a/include/blas_helper.cuh b/include/blas_helper.cuh
index 33af170333..ce311456bc 100644
--- a/include/blas_helper.cuh
+++ b/include/blas_helper.cuh
@@ -208,10 +208,10 @@ namespace quda
       {
         norm_t max_[n];
         // two-pass to increase ILP (assumes length divisible by two, e.g. complex-valued)
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < n; i++) max_[i] = fmaxf(fabsf((norm_t)v[i].real()), fabsf((norm_t)v[i].imag()));
         norm_t scale = 0.0;
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < n; i++) scale = fmaxf(max_[i], scale);
         norm = scale * fixedInvMaxValue<store_t>::value;
         return fdividef(fixedMaxValue<store_t>::value, scale);
@@ -235,12 +235,12 @@ QUDA_UNROLL
           array<real, len> v_;
 
           constexpr int M = len / N;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < M; i++) {
             // first load from memory
             Vector vecTmp = vector_load<Vector>(data.spinor, parity * data.cb_offset + x + data.stride * i);
             // now copy into output and scale
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < N; j++) copy_and_scale(v_[i * N + j], reinterpret_cast<store_t *>(&vecTmp)[j], nrm);
           }
 
@@ -259,10 +259,10 @@ QUDA_UNROLL
           memcpy(&nrm, &vecTmp.w, sizeof(norm_t));
 
           // now copy into output and scale
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < len; i++) copy_and_scale(v_[i], reinterpret_cast<store_t *>(&vecTmp)[i], nrm);
 
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < n; i++) { v[i] = complex<real>(v_[2 * i + 0], v_[2 * i + 1]); }
         }
       }
@@ -285,13 +285,13 @@ QUDA_UNROLL
 
           if constexpr (isFixed<store_t>::value) {
             real scale_inv = store_norm<isFixed<store_t>::value, real, n>(v, data.norm[x + parity * data.cb_norm_offset]);
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < n; i++) {
               v_[2 * i + 0] = scale_inv * v[i].real();
               v_[2 * i + 1] = scale_inv * v[i].imag();
             }
           } else {
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < n; i++) {
               v_[2 * i + 0] = v[i].real();
               v_[2 * i + 1] = v[i].imag();
@@ -299,11 +299,11 @@ QUDA_UNROLL
           }
 
           constexpr int M = len / N;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < M; i++) {
             Vector vecTmp;
             // first do scalar copy converting into storage type
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < N; j++) copy_scaled(reinterpret_cast<store_t *>(&vecTmp)[j], v_[i * N + j]);
             // second do vectorized copy into memory
             vector_store(data.spinor, parity * data.cb_offset + x + data.stride * i, vecTmp);
@@ -315,7 +315,7 @@ QUDA_UNROLL
           norm_t norm;
           norm_t scale_inv = store_norm<isFixed<store_t>::value, real, n>(v, norm);
           array<real, len> v_;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < n; i++) {
             v_[2 * i + 0] = scale_inv * v[i].real();
             v_[2 * i + 1] = scale_inv * v[i].imag();
@@ -323,7 +323,7 @@ QUDA_UNROLL
 
           Vector vecTmp;
           memcpy(&vecTmp.w, &norm, sizeof(norm_t)); // pack the norm
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < len; i++) copy_scaled(reinterpret_cast<store_t *>(&vecTmp)[i], v_[i]);
           // second do vectorized copy into memory
           vector_store(data.spinor, parity * cb_offset + x, vecTmp);
diff --git a/include/clover_field_order.h b/include/clover_field_order.h
index f73a6ed74f..7c17b25b51 100644
--- a/include/clover_field_order.h
+++ b/include/clover_field_order.h
@@ -143,13 +143,13 @@ namespace quda {
 
       template <typename T1, typename T2> __device__ __host__ inline void unpack(T1 &out, const T2 &in) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < compressed_block_size(); i++) out[unpack_idx(i)] = in[i];
 
           // first reconstruct second set of diagonal elements before we reconstruct the first set
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < 3; i++) out[i + 3] = diagonal - out[i];
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < 3; i++) out[i + 0] = diagonal + out[i];
 
         out[30] = -out[6];
@@ -162,10 +162,10 @@ QUDA_UNROLL
 
       template <typename T1, typename T2> __device__ __host__ inline void pack(T1 &out, const T2 &in) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < compressed_block_size(); i++) out[i] = in[unpack_idx(i)];
           // remove diagonal constant
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < 3; i++) out[i] -= diagonal;
         out[3] = 0.0; // intentionally zero this so that it can't contribute to the max element
       }
@@ -184,13 +184,13 @@ QUDA_UNROLL
 
       template <typename T1, typename T2> __device__ __host__ inline void unpack(T1 &out, const T2 &in) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < block; i++) out[i] = in[i];
       }
 
       template <typename T1, typename T2> __device__ __host__ inline void pack(T1 &out, const T2 &in) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < block; i++) out[i] = in[i];
       }
     };
@@ -673,23 +673,23 @@ QUDA_UNROLL
           // the chiral block size may not be exactly divisible by N, in which case we need to over-size the read 
           array<real, M * N> tmp;             // array storing the elements read in
 
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < M; i++) {
             // first load from memory
             Vector vecTmp = vector_load<Vector>(clover, parity * offset + x + volumeCB * (chirality * M_offset + i));
 
             // second do scalar copy converting into register type
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < N; j++) { copy_and_scale(tmp[i * N + j], reinterpret_cast<Float *>(&vecTmp)[j], nrm); }
           }
 
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < compressed_block; i++) v[i] = tmp[i + chirality * compressed_block % N];
         }
 
         __device__ __host__ inline void raw_load(real v[2 * compressed_block], int x, int parity) const
         {
-QUDA_UNROLL
+#pragma unroll
           for (int ch = 0; ch < 2; ch++) raw_load(v + ch * compressed_block, x, parity, ch);
         }
 
@@ -707,7 +707,7 @@ QUDA_UNROLL
           recon.unpack(v, tmp);
 
           if (add_rho) {
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 6; i++) v[i] += rho;
           }
         }
@@ -725,14 +725,14 @@ QUDA_UNROLL
           // in which case we write the divisble part first and then
           // deal with the remainder afterwards
           array<real, compressed_block> tmp;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < compressed_block; i++) tmp[i] = isFixed<Float>::value ? v[i] * nrm_inv : v[i];
 
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < M_offset; i++) {
             Vector vecTmp;
             // first do scalar copy converting into storage type
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < N; j++)
               copy_scaled(reinterpret_cast<Float *>(&vecTmp)[j], tmp[chirality * M_rem + i * N + j]);
             // second do vectorized copy into memory
@@ -742,7 +742,7 @@ QUDA_UNROLL
           if (M_rem) {
             typename VectorType<Float, std::max(M_rem, 1)>::type vecTmp;
             // first do scalar copy converting into storage type
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < M_rem; j++)
               copy_scaled(reinterpret_cast<Float *>(&vecTmp)[j], tmp[(1 - chirality) * M_offset * N + j]);
 
@@ -754,7 +754,7 @@ QUDA_UNROLL
 
         __device__ __host__ inline void raw_save(const real v[2 * compressed_block], int x, int parity) const
         {
-QUDA_UNROLL
+#pragma unroll
           for (int ch = 0; ch < 2; ch++) raw_save(v + ch * compressed_block, x, parity, ch);
         }
 
@@ -781,7 +781,7 @@ QUDA_UNROLL
          */
         __device__ __host__ inline void load(real v[], int x, int parity) const
         {
-QUDA_UNROLL
+#pragma unroll
           for (int chirality = 0; chirality < 2; chirality++) load(&v[chirality * block], x, parity, chirality);
         }
 
@@ -794,7 +794,7 @@ QUDA_UNROLL
          */
         __device__ __host__ inline void save(const real v[], int x, int parity) const
         {
-QUDA_UNROLL
+#pragma unroll
           for (int chirality = 0; chirality < 2; chirality++) save(&v[chirality * block], x, parity, chirality);
         }
 
@@ -855,14 +855,14 @@ QUDA_UNROLL
 	  // factor of 0.5 comes from basis change
           Float v_[length];
           block_load<Float, length>(v_, &clover[parity * offset + x * length]);
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < length; i++) v[i] = 0.5 * v_[i];
         }
 
         __device__ __host__ inline void save(const RegType v[length], int x, int parity) const
         {
           Float v_[length];
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < length; i++) v_[i] = 2.0 * v[i];
           block_store<Float, length>(&clover[parity * offset + x * length], v_);
         }
@@ -902,16 +902,16 @@ QUDA_UNROLL
 
         __device__ __host__ inline void load(RegType v[length], int x, int parity) const {
 	  // the factor of 0.5 comes from a basis change
-QUDA_UNROLL
+#pragma unroll
           for (int chirality = 0; chirality < 2; chirality++) {
             // set diagonal elements
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 6; i++) {
               v[chirality*36 + i] = 0.5*diag[((i*2 + chirality)*2 + parity)*volumeCB + x];
             }
 
             // the off diagonal elements
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 30; i++) {
               int z = i%2;
 	      int off = i/2;
@@ -924,16 +924,16 @@ QUDA_UNROLL
         __device__ __host__ inline void save(const RegType v[length], int x, int parity) const
         {
           // the factor of 2.0 comes from undoing the basis change
-QUDA_UNROLL
+#pragma unroll
           for (int chirality = 0; chirality < 2; chirality++) {
             // set diagonal elements
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 6; i++) {
               diag[((i*2 + chirality)*2 + parity)*volumeCB + x] = 2.0*v[chirality*36 + i];
             }
 
             // the off diagonal elements
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 30; i++) {
               int z = i%2;
 	      int off = i/2;
@@ -993,18 +993,18 @@ QUDA_UNROLL
 
           // flip the sign of the imaginary components
           int sign[36];
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < 6; i++) sign[i] = 1;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 6; i < 36; i += 2) {
             if ( (i >= 10 && i<= 15) || (i >= 18 && i <= 29) )  { sign[i] = -1; sign[i+1] = -1; }
 	    else { sign[i] = 1; sign[i+1] = -1; }
           }
 
           const int M = length / 2;
-QUDA_UNROLL
+#pragma unroll
           for (int chirality = 0; chirality < 2; chirality++)
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < M; i++)
               v[chirality * M + i] = sign[i] * clover[parity][x * length + chirality * M + bq[i]];
         }
diff --git a/include/color_spinor.h b/include/color_spinor.h
index c893db412f..3f7e32ff74 100644
--- a/include/color_spinor.h
+++ b/include/color_spinor.h
@@ -28,27 +28,27 @@ namespace quda {
 
     __device__ __host__ inline ColorSpinor<Float, Nc, Ns>()
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < size; i++) { data[i] = 0; }
       }
 
       __device__ __host__ inline ColorSpinor<Float, Nc, Ns> operator-() const
       {
         ColorSpinor<Float, Nc, Ns> a;
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < size; i++) { a.data[i] = -data[i]; }
         return a;
       }
 
       __device__ __host__ inline ColorSpinor<Float, Nc, Ns>& operator+=(const ColorSpinor<Float, Nc, Ns> &a) {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < size; i++) { data[i] += a.data[i]; }
         return *this;
       }
 
       template <typename T> __device__ __host__ inline ColorSpinor<Float, Nc, Ns> &operator*=(const T &a)
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < size; i++) { data[i] *= a; }
         return *this;
       }
@@ -56,7 +56,7 @@ QUDA_UNROLL
       __device__ __host__ inline ColorSpinor<Float, Nc, Ns> &operator-=(const ColorSpinor<Float, Nc, Ns> &a)
       {
         if (this != &a) {
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < Nc * Ns; i++) { data[i] -= a.data[i]; }
         }
         return *this;
@@ -128,19 +128,19 @@ QUDA_UNROLL
 
       __device__ __host__ inline ColorSpinor<Float, Nc, 4>()
       {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < size; i++) { data[i] = 0; }
       }
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 4>& operator+=(const ColorSpinor<Float, Nc, 4> &a) {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < size; i++) { data[i] += a.data[i]; }
       return *this;
     }
 
     template <typename T> __device__ __host__ inline ColorSpinor<Float, Nc, 4> &operator*=(const T &a)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < size; i++) { data[i] *= a; }
       return *this;
     }
@@ -156,7 +156,7 @@ QUDA_UNROLL
 
       switch (dim) {
       case 0: // x dimension
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(3, i));
           a(1, i) = i_(t(2, i));
@@ -165,7 +165,7 @@ QUDA_UNROLL
         }
         break;
       case 1: // y dimension
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = t(3, i);
           a(1, i) = -t(2, i);
@@ -174,7 +174,7 @@ QUDA_UNROLL
         }
         break;
       case 2: // z dimension
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(2, i));
           a(1, i) = -i_(t(3, i));
@@ -183,7 +183,7 @@ QUDA_UNROLL
         }
         break;
       case 3: // t dimension
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = t(0, i);
           a(1, i) = t(1, i);
@@ -192,7 +192,7 @@ QUDA_UNROLL
         }
         break;
       case 4: // gamma_5
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = t(2, i);
           a(1, i) = t(3, i);
@@ -216,7 +216,7 @@ QUDA_UNROLL
 
       switch (dim) {
       case 0: // x dimension
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = -t(3, i);
           a(1, i) = -t(2, i);
@@ -225,7 +225,7 @@ QUDA_UNROLL
         }
         break;
       case 1: // y dimension
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(3, i));
           a(1, i) = -i_(t(2, i));
@@ -234,7 +234,7 @@ QUDA_UNROLL
         }
         break;
       case 2: // z dimension
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = -t(2, i);
           a(1, i) = t(3, i);
@@ -243,7 +243,7 @@ QUDA_UNROLL
         }
         break;
       case 3: // t dimension
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(0, i));
           a(1, i) = i_(t(1, i));
@@ -252,7 +252,7 @@ QUDA_UNROLL
         }
         break;
       case 4: // gamma_5
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<Nc; i++) {
           a(0, i) = i_(t(2, i));
           a(1, i) = i_(t(3, i));
@@ -271,9 +271,9 @@ QUDA_UNROLL
     */
     __device__ __host__ inline ColorSpinor<Float,Nc,2> chiral_project(int chirality) const {
       ColorSpinor<Float,Nc,2> proj;
-QUDA_UNROLL
+#pragma unroll
       for (int s=0; s<Ns/2; s++) {
-QUDA_UNROLL
+#pragma unroll
 	for (int c=0; c<Nc; c++) {
 	  proj(s,c) = (*this)(chirality*Ns/2+s,c);
 	}
@@ -295,14 +295,14 @@ QUDA_UNROLL
       case 0: // x dimension
 	switch (sign) {
 	case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) + i_(t(3, i));
             proj(1, i) = t(1, i) + i_(t(2, i));
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) - i_(t(3, i));
             proj(1, i) = t(1, i) - i_(t(2, i));
@@ -313,14 +313,14 @@ QUDA_UNROLL
       case 1: // y dimension
 	switch (sign) {
 	case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) + t(3, i);
             proj(1, i) = t(1, i) - t(2, i);
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) - t(3, i);
             proj(1, i) = t(1, i) + t(2, i);
@@ -331,14 +331,14 @@ QUDA_UNROLL
       case 2: // z dimension
 	switch (sign) {
 	case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) + i_(t(2, i));
             proj(1, i) = t(1, i) - i_(t(3, i));
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = t(0, i) - i_(t(2, i));
             proj(1, i) = t(1, i) + i_(t(3, i));
@@ -349,14 +349,14 @@ QUDA_UNROLL
       case 3: // t dimension
 	switch (sign) {
 	case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = static_cast<Float>(2.0) * t(0, i);
             proj(1, i) = static_cast<Float>(2.0) * t(1, i);
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             proj(0, i) = static_cast<Float>(2.0) * t(2, i);
             proj(1, i) = static_cast<Float>(2.0) * t(3, i);
@@ -367,14 +367,14 @@ QUDA_UNROLL
       case 4:
         switch (sign) {
         case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < Nc; i++) {
             proj(0, i) = t(0, i) + t(2, i);
             proj(1, i) = t(1, i) + t(3, i);
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < Nc; i++) {
             proj(0, i) = t(0, i) - t(2, i);
             proj(1, i) = t(1, i) - t(3, i);
@@ -432,7 +432,7 @@ QUDA_UNROLL
       case 0:
 	switch(nu) {
 	case 1:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) =  j*b(0,i);
 	    a(1,i) = -j*b(1,i);
@@ -441,7 +441,7 @@ QUDA_UNROLL
 	  }
 	  break;
 	case 2:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -b(1,i);
 	    a(1,i) =  b(0,i);
@@ -450,7 +450,7 @@ QUDA_UNROLL
 	  }
 	  break;
 	case 3:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -j*b(3,i);
 	    a(1,i) = -j*b(2,i);
@@ -463,7 +463,7 @@ QUDA_UNROLL
       case 1:
 	switch(nu) {
 	case 0:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -j*b(0,i);
 	    a(1,i) =  j*b(1,i);
@@ -472,7 +472,7 @@ QUDA_UNROLL
 	  }
 	  break;
 	case 2:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = j*b(1,i);
 	    a(1,i) = j*b(0,i);
@@ -481,7 +481,7 @@ QUDA_UNROLL
 	  }
 	  break;
 	case 3:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -b(3,i);
 	    a(1,i) =  b(2,i);
@@ -494,7 +494,7 @@ QUDA_UNROLL
       case 2:
 	switch(nu) {
 	case 0:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) =  b(1,i);
 	    a(1,i) = -b(0,i);
@@ -503,7 +503,7 @@ QUDA_UNROLL
 	  }
 	  break;
 	case 1:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -j*b(1,i);
 	    a(1,i) = -j*b(0,i);
@@ -512,7 +512,7 @@ QUDA_UNROLL
 	  }
 	  break;
 	case 3:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = -j*b(2,i);
 	    a(1,i) =  j*b(3,i);
@@ -525,7 +525,7 @@ QUDA_UNROLL
       case 3:
 	switch(nu) {
 	case 0:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) = j*b(3,i);
 	    a(1,i) = j*b(2,i);
@@ -534,7 +534,7 @@ QUDA_UNROLL
 	  }
 	  break;
 	case 1:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) =  b(3,i);
 	    a(1,i) = -b(2,i);
@@ -543,7 +543,7 @@ QUDA_UNROLL
 	  }
 	  break;
 	case 2:
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    a(0,i) =  j*b(2,i);
 	    a(1,i) = -j*b(3,i);
@@ -605,7 +605,7 @@ QUDA_UNROLL
     */
     __device__ __host__ inline void toNonRel() {
       ColorSpinor<Float,Nc,Ns> a;
-QUDA_UNROLL
+#pragma unroll
       for (int c=0; c<Nc; c++) {
 	a(0,c) =  (*this)(1,c)+(*this)(3,c);
 	a(1,c) = -(*this)(2,c)-(*this)(0,c);
@@ -620,7 +620,7 @@ QUDA_UNROLL
     */
     __device__ __host__ inline void toRel() {
       ColorSpinor<Float,Nc,Ns> a;
-QUDA_UNROLL
+#pragma unroll
       for (int c=0; c<Nc; c++) {
 	a(0,c) = -(*this)(1,c)-(*this)(3,c);
 	a(1,c) =  (*this)(2,c)+(*this)(0,c);
@@ -651,19 +651,19 @@ QUDA_UNROLL
     complex<Float> data[size];
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 2>() {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < size; i++) { data[i] = 0; }
     }
 
     __device__ __host__ inline ColorSpinor<Float, Nc, 2>& operator+=(const ColorSpinor<Float, Nc, 2> &a) {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < size; i++) { data[i] += a.data[i]; }
       return *this;
     }
 
     template <typename T> __device__ __host__ inline ColorSpinor<Float, Nc, 2> &operator*=(const T &a)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < size; i++) { data[i] *= a; }
       return *this;
     }
@@ -674,9 +674,9 @@ QUDA_UNROLL
     */
     __device__ __host__ inline ColorSpinor<Float,Nc,4> chiral_reconstruct(int chirality) const {
       ColorSpinor<Float,Nc,4> recon;
-QUDA_UNROLL
+#pragma unroll
       for (int s=0; s<Ns; s++) {
-QUDA_UNROLL
+#pragma unroll
 	for (int c=0; c<Nc; c++) {
 	  recon(chirality*Ns+s,c) = (*this)(s,c);
 	}
@@ -699,7 +699,7 @@ QUDA_UNROLL
       case 0: // x dimension
 	switch (sign) {
 	case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -708,7 +708,7 @@ QUDA_UNROLL
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -721,7 +721,7 @@ QUDA_UNROLL
       case 1: // y dimension
 	switch (sign) {
 	case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -730,7 +730,7 @@ QUDA_UNROLL
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -743,7 +743,7 @@ QUDA_UNROLL
       case 2: // z dimension
 	switch (sign) {
 	case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -752,7 +752,7 @@ QUDA_UNROLL
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -765,7 +765,7 @@ QUDA_UNROLL
       case 3: // t dimension
 	switch (sign) {
 	case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -774,7 +774,7 @@ QUDA_UNROLL
           }
 	  break;
 	case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<Nc; i++) {
 	    recon(0,i) = 0;
 	    recon(1,i) = 0;
@@ -787,7 +787,7 @@ QUDA_UNROLL
       case 4:
         switch (sign) {
         case 1: // positive projector
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -796,7 +796,7 @@ QUDA_UNROLL
           }
           break;
         case -1: // negative projector
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < Nc; i++) {
             recon(0, i) = t(0, i);
             recon(1, i) = t(1, i);
@@ -872,7 +872,7 @@ QUDA_UNROLL
   __device__ __host__ inline void caxpy(const complex<Float> &a, const ColorSpinor<Float, Nc, Ns> &x,
                                         ColorSpinor<Float, Nc, Ns> &y)
   {
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < Nc * Ns; i++) {
       y(i).real(a.real() * x(i).real() + y(i).real());
       y(i).real(-a.imag() * x(i).imag() + y(i).real());
@@ -890,7 +890,7 @@ QUDA_UNROLL
   template <typename Float, int Nc, int Ns> __device__ __host__ inline Float norm2(const ColorSpinor<Float, Nc, Ns> &a)
   {
     Float nrm = 0.0;
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < Nc * Ns; i++) {
       nrm += a(i).real() * a(i).real();
       nrm += a(i).imag() * a(i).imag();
@@ -910,7 +910,7 @@ QUDA_UNROLL
                                                           const ColorSpinor<Float, Nc, Ns> &b, int sa, int sb)
   {
     complex<Float> dot = cmul(a(sa, 0), b(sb, 0));
-QUDA_UNROLL
+#pragma unroll
     for (int c = 1; c < Nc; c++) dot = cmac(a(sa, c), b(sb, c), dot);
     return dot;
   }
@@ -927,9 +927,9 @@ QUDA_UNROLL
                                                          const ColorSpinor<Float, Nc, Ns> &b)
   {
     complex<Float> dot;
-QUDA_UNROLL
+#pragma unroll
     for (int s = 0; s < Ns; s++) {
-QUDA_UNROLL
+#pragma unroll
       for (int c = 0; c < Nc; c++) {
         if (s == 0 && c == 0)
           dot = cmul(conj(a(s, c)), b(s, c));
@@ -953,7 +953,7 @@ QUDA_UNROLL
                                                          const ColorSpinor<Float, Nc, Nsb> &b, int sa, int sb)
   {
     complex<Float> dot = cmul(conj(a(sa, 0)), b(sb, 0));
-QUDA_UNROLL
+#pragma unroll
     for (int c = 1; c < Nc; c++) dot = cmac(conj(a(sa, c)), b(sb, c), dot);
     return dot;
   }
@@ -1008,9 +1008,9 @@ QUDA_UNROLL
     Matrix<complex<Float>, Nc> out;
 
     // outer product over color
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < Nc; i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int j = 0; j < Nc; j++) {
         // trace over spin (manual unroll for perf)
         out(j, i).real(a(0, j).real() * b(0, i).real());
@@ -1019,7 +1019,7 @@ QUDA_UNROLL
         out(j, i).imag(out(j, i).imag() - a(0, j).real() * b(0, i).imag());
         // out(j,i) = a(0,j) * conj(b(0,i));
 
-QUDA_UNROLL
+#pragma unroll
 	for (int s=1; s<Ns; s++) {
 	  out(j,i).real( out(j,i).real() + a(s,j).real() * b(s,i).real() );
 	  out(j,i).real( out(j,i).real() + a(s,j).imag() * b(s,i).imag() );
@@ -1046,9 +1046,9 @@ QUDA_UNROLL
     Matrix<complex<Float>, Nc> out;
 
     // outer product over color
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < Nc; i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int j = 0; j < Nc; j++) {
         // trace over spin (manual unroll for perf)
         out(j, i).real(a(0, j).real() * b(0, i).real());
@@ -1072,9 +1072,9 @@ QUDA_UNROLL
 
     ColorSpinor<Float,Nc,Ns> z;
 
-QUDA_UNROLL
+#pragma unroll
     for (int i=0; i<Nc; i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int s=0; s<Ns; s++) {
 	z.data[s*Nc + i] = x.data[s*Nc + i] + y.data[s*Nc + i];
       }
@@ -1094,9 +1094,9 @@ QUDA_UNROLL
 
     ColorSpinor<Float,Nc,Ns> z;
 
-QUDA_UNROLL
+#pragma unroll
     for (int i=0; i<Nc; i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int s=0; s<Ns; s++) {
 	z.data[s*Nc + i] = x.data[s*Nc + i] - y.data[s*Nc + i];
       }
@@ -1116,9 +1116,9 @@ QUDA_UNROLL
 
     ColorSpinor<Float,Nc,Ns> y;
 
-QUDA_UNROLL
+#pragma unroll
     for (int i=0; i<Nc; i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int s=0; s<Ns; s++) {
 	y.data[s*Nc + i] = a * x.data[s*Nc + i];
       }
@@ -1138,18 +1138,18 @@ QUDA_UNROLL
 
     ColorSpinor<Float,Nc,Ns> y;
 
-QUDA_UNROLL
+#pragma unroll
     for (int i=0; i<Nc; i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int s=0; s<Ns; s++) {
 	y.data[s*Nc + i].x  = A(i,0).real() * x.data[s*Nc + 0].real();
 	y.data[s*Nc + i].x -= A(i,0).imag() * x.data[s*Nc + 0].imag();
 	y.data[s*Nc + i].y  = A(i,0).real() * x.data[s*Nc + 0].imag();
 	y.data[s*Nc + i].y += A(i,0).imag() * x.data[s*Nc + 0].real();
       }
-QUDA_UNROLL
+#pragma unroll
       for (int j=1; j<Nc; j++) {
-QUDA_UNROLL
+#pragma unroll
 	for (int s=0; s<Ns; s++) {
 	  y.data[s*Nc + i].x += A(i,j).real() * x.data[s*Nc + j].real();
 	  y.data[s*Nc + i].x -= A(i,j).imag() * x.data[s*Nc + j].imag();
@@ -1175,18 +1175,18 @@ QUDA_UNROLL
   {
     ColorSpinor<Float, Nc, Ns> z;
 
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < Nc; i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int s = 0; s < Ns; s++) {
         z.data[s * Nc + i].x = y.data[s * Nc + i].real() + A(i, 0).real() * x.data[s * Nc + 0].real();
         z.data[s * Nc + i].x -= A(i, 0).imag() * x.data[s * Nc + 0].imag();
         z.data[s * Nc + i].y = y.data[s * Nc + i].imag() + A(i, 0).real() * x.data[s * Nc + 0].imag();
         z.data[s * Nc + i].y += A(i, 0).imag() * x.data[s * Nc + 0].real();
       }
-QUDA_UNROLL
+#pragma unroll
       for (int j = 1; j < Nc; j++) {
-QUDA_UNROLL
+#pragma unroll
         for (int s = 0; s < Ns; s++) {
           z.data[s * Nc + i].x += A(i, j).real() * x.data[s * Nc + j].real();
           z.data[s * Nc + i].x -= A(i, j).imag() * x.data[s * Nc + j].imag();
@@ -1211,7 +1211,7 @@ QUDA_UNROLL
     ColorSpinor<Float,Nc,Ns> y;
     constexpr int N = Ns * Nc;
 
-QUDA_UNROLL
+#pragma unroll
     for (int i=0; i<N; i++) {
       if (i==0) {
 	y.data[i].x  = A(i,0).real() * x.data[0].real();
@@ -1222,7 +1222,7 @@ QUDA_UNROLL
 	y.data[i].y  = A(i,0).real() * x.data[0].imag();
 	y.data[i].y += A(i,0).imag() * x.data[0].real();
       }
-QUDA_UNROLL
+#pragma unroll
       for (int j=1; j<N; j++) {
 	if (i==j) {
 	  y.data[i].x += A(i,j).real() * x.data[j].real();
diff --git a/include/color_spinor_field_order.h b/include/color_spinor_field_order.h
index bd6eac1369..233d926869 100644
--- a/include/color_spinor_field_order.h
+++ b/include/color_spinor_field_order.h
@@ -233,7 +233,7 @@ namespace quda
         using vec_t = typename VectorType<Float, 2>::type;
         constexpr int N = nSpin * nColor * nVec;
         constexpr int M = nSpinBlock * nColor * nVec;
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < M; i++) {
           vec_t tmp
             = vector_load<vec_t>(reinterpret_cast<const vec_t *>(in + parity * offset_cb), x_cb * N + chi * M + i);
@@ -291,7 +291,7 @@ QUDA_UNROLL
       {
         using vec_t = typename VectorType<Float, 2>::type;
         constexpr int M = nSpinBlock * nColor * nVec;
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < M; i++) {
           vec_t tmp = vector_load<vec_t>(reinterpret_cast<const vec_t *>(in + parity * offset_cb),
                                          (chi * M + i) * stride + x_cb);
@@ -339,7 +339,7 @@ QUDA_UNROLL
       {
         using vec_t = typename VectorType<Float, 4>::type;
         constexpr int M = (nSpinBlock * nColor * nVec * 2) / 4;
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < M; i++) {
           vec_t tmp = vector_load<vec_t>(reinterpret_cast<const vec_t *>(in + parity * offset_cb),
                                          (chi * M + i) * stride + x_cb);
@@ -393,13 +393,13 @@ QUDA_UNROLL
         constexpr int N = nSpin * nColor * nVec * 2; // real numbers in the loaded vector
         constexpr int M = N / 8;
         Float tmp[N];
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < M; i++) {
           vec_t ld_tmp = vector_load<vec_t>(reinterpret_cast<const vec_t *>(in + parity * offset_cb), i * stride + x_cb);
           memcpy(&tmp[i * 8], &ld_tmp, sizeof(vec_t));
         }
         constexpr int N_chi = N / (nSpin / nSpinBlock);
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < N_chi; i++)
           out[i] = complex<Float>(tmp[chi * N_chi + 2 * i + 0], tmp[chi * N_chi + 2 * i + 1]);
       }
@@ -744,11 +744,11 @@ QUDA_UNROLL
           complex<storeFloat> tmp[nSpinBlock * nColor * nVec];
           accessor.template load<nSpinBlock>(tmp, v, parity, x_cb, chi);
           Float norm_ = block_float ? norm[parity * norm_offset + x_cb] : scale_inv;
-QUDA_UNROLL
+#pragma unroll
           for (int s = 0; s < nSpinBlock; s++) {
-QUDA_UNROLL
+#pragma unroll
             for (int c = 0; c < nColor; c++) {
-QUDA_UNROLL
+#pragma unroll
               for (int v = 0; v < nVec; v++) {
                 int k = (s * nColor + c) * nVec + v;
                 out[k] = norm_ * complex<Float>(static_cast<Float>(tmp[k].real()), static_cast<Float>(tmp[k].imag()));
@@ -1036,16 +1036,16 @@ QUDA_UNROLL
         real v[length];
         norm_type nrm = isFixed<Float>::value ? vector_load<float>(norm, x + parity * norm_offset) : 0.0;
 
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < M; i++) {
           // first load from memory
           Vector vecTmp = vector_load<Vector>(field, parity * offset + x + volumeCB * i);
           // now copy into output and scale
-QUDA_UNROLL
+#pragma unroll
           for (int j = 0; j < N; j++) copy_and_scale(v[i * N + j], reinterpret_cast<Float *>(&vecTmp)[j], nrm);
         }
 
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < length / 2; i++) out[i] = complex(v[2 * i + 0], v[2 * i + 1]);
       }
 
@@ -1053,7 +1053,7 @@ QUDA_UNROLL
       {
         real v[length];
 
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < length / 2; i++) {
           v[2 * i + 0] = in[i].real();
           v[2 * i + 1] = in[i].imag();
@@ -1062,24 +1062,24 @@ QUDA_UNROLL
         if (isFixed<Float>::value) {
           norm_type max_[length / 2];
           // two-pass to increase ILP (assumes length divisible by two, e.g. complex-valued)
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < length / 2; i++)
             max_[i] = fmaxf(fabsf((norm_type)v[i]), fabsf((norm_type)v[i + length / 2]));
           norm_type scale = 0.0;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < length / 2; i++) scale = fmaxf(max_[i], scale);
           norm[x + parity * norm_offset] = scale * fixedInvMaxValue<Float>::value;
 
           real scale_inv = fdividef(fixedMaxValue<Float>::value, scale);
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < length; i++) v[i] = v[i] * scale_inv;
         }
 
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < M; i++) {
           Vector vecTmp;
           // first do scalar copy converting into storage type
-QUDA_UNROLL
+#pragma unroll
           for (int j = 0; j < N; j++) copy_scaled(reinterpret_cast<Float *>(&vecTmp)[j], v[i * N + j]);
           // second do vectorized copy into memory
           vector_store(field, parity * offset + x + volumeCB * i, vecTmp);
@@ -1106,16 +1106,16 @@ QUDA_UNROLL
         norm_type nrm
           = isFixed<Float>::value ? vector_load<float>(ghost_norm[2 * dim + dir], parity * faceVolumeCB[dim] + x) : 0.0;
 
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < M_ghost; i++) {
           GhostVector vecTmp = vector_load<GhostVector>(
             ghost[2 * dim + dir], parity * faceVolumeCB[dim] * M_ghost + i * faceVolumeCB[dim] + x);
-QUDA_UNROLL
+#pragma unroll
           for (int j = 0; j < N_ghost; j++)
             copy_and_scale(v[i * N_ghost + j], reinterpret_cast<Float *>(&vecTmp)[j], nrm);
         }
 
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < length_ghost / 2; i++) out[i] = complex(v[2 * i + 0], v[2 * i + 1]);
       }
 
@@ -1123,7 +1123,7 @@ QUDA_UNROLL
                                                 int parity = 0) const
       {
         real v[length_ghost];
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < length_ghost / 2; i++) {
           v[2 * i + 0] = in[i].real();
           v[2 * i + 1] = in[i].imag();
@@ -1132,24 +1132,24 @@ QUDA_UNROLL
         if (isFixed<Float>::value) {
           norm_type max_[length_ghost / 2];
           // two-pass to increase ILP (assumes length divisible by two, e.g. complex-valued)
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < length_ghost / 2; i++)
             max_[i] = fmaxf((norm_type)fabsf((norm_type)v[i]), (norm_type)fabsf((norm_type)v[i + length_ghost / 2]));
           norm_type scale = 0.0;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < length_ghost / 2; i++) scale = fmaxf(max_[i], scale);
           ghost_norm[2 * dim + dir][parity * faceVolumeCB[dim] + x] = scale * fixedInvMaxValue<Float>::value;
 
           real scale_inv = fdividef(fixedMaxValue<Float>::value, scale);
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < length_ghost; i++) v[i] = v[i] * scale_inv;
         }
 
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < M_ghost; i++) {
           GhostVector vecTmp;
           // first do scalar copy converting into storage type
-QUDA_UNROLL
+#pragma unroll
           for (int j = 0; j < N_ghost; j++) copy_scaled(reinterpret_cast<Float *>(&vecTmp)[j], v[i * N_ghost + j]);
           // second do vectorized copy into memory
           vector_store(ghost[2 * dim + dir], parity * faceVolumeCB[dim] * M_ghost + i * faceVolumeCB[dim] + x, vecTmp);
diff --git a/include/float_vector.h b/include/float_vector.h
index 64b3ea34b8..5f7d7daaaa 100644
--- a/include/float_vector.h
+++ b/include/float_vector.h
@@ -39,7 +39,7 @@ namespace quda {
   __device__ __host__ inline array<T, n> operator+(const array<T, n> &a, const array<T, n> &b)
   {
     array<T, n> c;
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < n; i++) c[i] = a[i] + b[i];
     return c;
   }
@@ -69,7 +69,7 @@ QUDA_UNROLL
   template <typename T, int n> __device__ __host__ inline array<T, n> zero()
   {
     array<T, n> v;
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < n; i++) v[i] = zero<T>();
     return v;
   }
@@ -115,7 +115,7 @@ QUDA_UNROLL
     static inline __host__ __device__ array<T, N> value()
     {
       array<T, N> v;
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < N; i++) v[i] = low<T>::value();
       return v;
     }
diff --git a/include/gauge_field_order.h b/include/gauge_field_order.h
index 299ce7a1ce..0feec78c50 100644
--- a/include/gauge_field_order.h
+++ b/include/gauge_field_order.h
@@ -956,13 +956,13 @@ namespace quda {
         __device__ __host__ inline void Pack(real out[N], const complex in[N / 2]) const
         {
           if constexpr (isFixed<Float>::value) {
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < N / 2; i++) {
               out[2 * i + 0] = scale_inv * in[i].real();
               out[2 * i + 1] = scale_inv * in[i].imag();
             }
           } else {
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < N / 2; i++) {
               out[2 * i + 0] = in[i].real();
               out[2 * i + 1] = in[i].imag();
@@ -975,10 +975,10 @@ QUDA_UNROLL
                                                const int *) const
         {
           if constexpr (isFixed<Float>::value) {
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < N / 2; i++) { out[i] = scale * complex(in[2 * i + 0], in[2 * i + 1]); }
           } else {
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < N / 2; i++) { out[i] = complex(in[2 * i + 0], in[2 * i + 1]); }
           }
         }
@@ -1072,7 +1072,7 @@ QUDA_UNROLL
 
         __device__ __host__ inline void Pack(real out[12], const complex in[9]) const
         {
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < 6; i++) {
             out[2 * i + 0] = in[i].real();
             out[2 * i + 1] = in[i].imag();
@@ -1083,7 +1083,7 @@ QUDA_UNROLL
         __device__ __host__ inline void Unpack(complex out[9], const real in[12], int idx, int dir, real, const I *X,
                                                const int *R) const
         {
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < 6; i++) out[i] = complex(in[2 * i + 0], in[2 * i + 1]);
 
           const real u0 = dir < 3 ?
@@ -1128,7 +1128,7 @@ QUDA_UNROLL
 
         __device__ __host__ inline void Pack(real out[10], const complex in[9]) const
         {
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < 2; i++) {
             out[2 * i + 0] = in[i + 1].real();
             out[2 * i + 1] = in[i + 1].imag();
@@ -1183,7 +1183,7 @@ QUDA_UNROLL
         __device__ __host__ inline void Unpack(complex out[9], const real in[12], int, int, real phase, const I *,
                                                const int *) const
         {
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < 6; i++) out[i] = complex(in[2 * i + 0], in[2 * i + 1]);
 
           out[6] = cmul(out[2], out[4]);
@@ -1226,7 +1226,7 @@ QUDA_UNROLL
           return expI3Phase.real() > 0 ? 1 : -1;
 #else // phase from determinant
           Matrix<complex, 3> a;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < 9; i++) a(i) = scale_inv * in[i];
           const complex det = getDeterminant(a);
           return phase = arg(det) / static_cast<real>(3.0 * M_PI);
@@ -1289,7 +1289,7 @@ QUDA_UNROLL
           real u0 = u.real();
           real u0_inv = u.imag();
 
-QUDA_UNROLL
+#pragma unroll
           for (int i = 1; i <= 3; i++)
             out[i] = complex(in[2 * i + 0], in[2 * i + 1]); // these elements are copied directly
 
@@ -1355,7 +1355,7 @@ QUDA_UNROLL
 
           // Rearrange {{b1,b2,b3},{a1,a2,a3},{-c1,-c2,-c3}} back
           // to {{a1,a2,a3},{b1,b2,b3},{c1,c2,c3}}
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < 3; i++) {
             const auto tmp = out[i];
             out[i] = out[i + 3];
@@ -1410,7 +1410,7 @@ QUDA_UNROLL
           return expI3Phase.real() > 0 ? 1 : -1;
 #else // phase from determinant
           Matrix<complex, 3> a;
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < 9; i++) a(i) = scale_inv * in[i];
           const complex det = getDeterminant(a);
           real phase = arg(det) / static_cast<real>(3.0 * M_PI);
@@ -1429,10 +1429,10 @@ QUDA_UNROLL
             sincospi(static_cast<real>(-phase), &cos_sin[1], &cos_sin[0]);
             complex z(cos_sin[0], cos_sin[1]);
             z *= scale_inv;
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 9; i++) su3[i] = cmul(z, in[i]);
           } else {
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 9; i++) { su3[i] = phase * in[i]; }
           }
           reconstruct_8.Pack(out, su3);
@@ -1450,10 +1450,10 @@ QUDA_UNROLL
             sincospi(static_cast<real>(phase), &cos_sin[1], &cos_sin[0]);
             complex z(cos_sin[0], cos_sin[1]);
             z *= scale;
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 9; i++) out[i] = cmul(z, out[i]);
           } else { // stagic phase
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < 18; i++) { out[i] *= phase; }
           }
         }
@@ -1543,12 +1543,12 @@ QUDA_UNROLL
         const int M = reconLen / N;
         real tmp[reconLen];
 
-QUDA_UNROLL
+#pragma unroll
         for (int i=0; i<M; i++){
           // first load from memory
           Vector vecTmp = vector_load<Vector>(gauge, parity * offset + (dir * M + i) * stride + x);
           // second do copy converting into register type
-QUDA_UNROLL
+#pragma unroll
           for (int j = 0; j < N; j++) copy(tmp[i * N + j], reinterpret_cast<Float *>(&vecTmp)[j]);
         }
 
@@ -1571,11 +1571,11 @@ QUDA_UNROLL
         real tmp[reconLen];
         reconstruct.Pack(tmp, v);
 
-QUDA_UNROLL
+#pragma unroll
         for (int i=0; i<M; i++){
 	  Vector vecTmp;
 	  // first do copy converting into storage type
-QUDA_UNROLL
+#pragma unroll
 	  for (int j=0; j<N; j++) copy(reinterpret_cast<Float*>(&vecTmp)[j], tmp[i*N+j]);
 	  // second do vectorized copy into memory
           vector_store(gauge, parity * offset + x + (dir * M + i) * stride, vecTmp);
@@ -1610,13 +1610,13 @@ QUDA_UNROLL
           const int M = reconLen / N;
           real tmp[reconLen];
 
-QUDA_UNROLL
+#pragma unroll
           for (int i=0; i<M; i++) {
 	    // first do vectorized copy from memory into registers
             Vector vecTmp = vector_load<Vector>(
                 ghost[dir] + parity * faceVolumeCB[dir] * (M * N + hasPhase), i * faceVolumeCB[dir] + x);
             // second do copy converting into register type
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < N; j++) copy(tmp[i * N + j], reinterpret_cast<Float *>(&vecTmp)[j]);
           }
           real phase = 0.;
@@ -1643,11 +1643,11 @@ QUDA_UNROLL
           real tmp[reconLen];
           reconstruct.Pack(tmp, v);
 
-QUDA_UNROLL
+#pragma unroll
           for (int i=0; i<M; i++) {
 	    Vector vecTmp;
 	    // first do copy converting into storage type
-QUDA_UNROLL
+#pragma unroll
 	    for (int j=0; j<N; j++) copy(reinterpret_cast<Float*>(&vecTmp)[j], tmp[i*N+j]);
 	    // second do vectorized copy into memory
 	    vector_store(ghost[dir]+parity*faceVolumeCB[dir]*(M*N + hasPhase), i*faceVolumeCB[dir]+x, vecTmp);
@@ -1699,13 +1699,13 @@ QUDA_UNROLL
         const int M = reconLen / N;
         real tmp[reconLen];
 
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<M; i++) {
 	  // first do vectorized copy from memory
 	  Vector vecTmp = vector_load<Vector>(ghost[dim] + ((dir*2+parity)*geometry+g)*R[dim]*faceVolumeCB[dim]*(M*N + hasPhase),
 					      +i*R[dim]*faceVolumeCB[dim]+buff_idx);
 	  // second do copy converting into register type
-QUDA_UNROLL
+#pragma unroll
 	  for (int j=0; j<N; j++) copy(tmp[i*N+j], reinterpret_cast<Float*>(&vecTmp)[j]);
 	}
         real phase = 0.;
@@ -1725,11 +1725,11 @@ QUDA_UNROLL
         real tmp[reconLen];
         reconstruct.Pack(tmp, v);
 
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<M; i++) {
 	    Vector vecTmp;
 	    // first do copy converting into storage type
-QUDA_UNROLL
+#pragma unroll
 	    for (int j=0; j<N; j++) copy(reinterpret_cast<Float*>(&vecTmp)[j], tmp[i*N+j]);
 	    // second do vectorized copy to memory
 	    vector_store(ghost[dim] + ((dir*2+parity)*geometry+g)*R[dim]*faceVolumeCB[dim]*(M*N + hasPhase),
diff --git a/include/gauge_fix_ovr_hit_devf.cuh b/include/gauge_fix_ovr_hit_devf.cuh
index 6668420fd7..0d7bf84c90 100644
--- a/include/gauge_fix_ovr_hit_devf.cuh
+++ b/include/gauge_fix_ovr_hit_devf.cuh
@@ -175,7 +175,7 @@ namespace quda {
       if (mu == 0) {
         Float a0, a1, a2, a3;
         a0 = 0.0; a1 = 0.0; a2 = 0.0; a3 = 0.0;
-      QUDA_UNROLL
+      #pragma unroll
         for (int i = 0; i < gauge_dir; i++) {
           a0 += elems[tid + i * blockSize] + elems[tid + (i + 4) * blockSize];
           a1 += elems[tid + i * blockSize + blockSize * 8] + elems[tid + (i + 4) * blockSize + blockSize * 8];
@@ -252,7 +252,7 @@ namespace quda {
         elems[tid + blockSize * 3] = -(link(p,p).y - link(q,q).y);
       }
 
-QUDA_UNROLL
+#pragma unroll
       for (int mu_ = 1; mu_ < gauge_dir; mu_++) {
         cache.sync();
         if (mu_ == mu) {
@@ -263,7 +263,7 @@ QUDA_UNROLL
         }
       }
 
-QUDA_UNROLL
+#pragma unroll
       for (int mu_ = 4; mu_ < 4 + gauge_dir; mu_++) {
         cache.sync();
         if (mu_ == mu) {
@@ -435,7 +435,7 @@ QUDA_UNROLL
       if (mu == 0) {
         Float a0, a1, a2, a3;
         a0 = 0.0; a1 = 0.0; a2 = 0.0; a3 = 0.0;
-      QUDA_UNROLL
+      #pragma unroll
         for ( int i = 0; i < gauge_dir; i++ ) {
           a0 += elems[tid + i * blockSize];
           a1 += elems[tid + i * blockSize + blockSize * 4];
@@ -508,7 +508,7 @@ QUDA_UNROLL
         elems[tid + blockSize * 3] = (link1(p,p).y - link1(q,q).y) - (link(p,p).y - link(q,q).y);
       }
 
-QUDA_UNROLL
+#pragma unroll
       for (int mu_ = 1; mu_ < gauge_dir; mu_++) {
         cache.sync();
         if (mu_ == mu) {
diff --git a/include/index_helper.cuh b/include/index_helper.cuh
index 23679e2c05..e1a14225c8 100644
--- a/include/index_helper.cuh
+++ b/include/index_helper.cuh
@@ -14,7 +14,7 @@ namespace quda {
   template <typename I, typename J, typename K>
   __device__ __host__ inline int linkIndexShift(const I &x, const J &dx, const K &X) {
     int y[4];
-QUDA_UNROLL
+#pragma unroll
     for ( int i = 0; i < 4; i++ ) y[i] = (x[i] + dx[i] + X[i]) % X[i];
     int idx = (((y[3] * X[2] + y[2]) * X[1] + y[1]) * X[0] + y[0]) >> 1;
     return idx;
@@ -31,7 +31,7 @@ QUDA_UNROLL
    */
   template <typename I, typename J, typename K, typename L>
   __device__ __host__ inline int linkIndexShift(I &y, const J &x, const K &dx, const L &X) {
-QUDA_UNROLL
+#pragma unroll
     for ( int i = 0; i < 4; i++ ) y[i] = (x[i] + dx[i] + X[i]) % X[i];
     int idx = (((y[3] * X[2] + y[2]) * X[1] + y[1]) * X[0] + y[0]) >> 1;
     return idx;
@@ -78,7 +78,7 @@ QUDA_UNROLL
   __device__ __host__ inline int linkIndexDn(const Coord &x, const I X[4], const int mu)
   {
     int y[4];
-QUDA_UNROLL
+#pragma unroll
     for ( int i = 0; i < 4; i++ ) y[i] = x[i];
     y[mu] = (y[mu] + n + X[mu]) % X[mu];
     int idx = (((y[3] * X[2] + y[2]) * X[1] + y[1]) * X[0] + y[0]) >> 1;
@@ -98,7 +98,7 @@ QUDA_UNROLL
   __device__ __host__ inline auto linkIndexHop(const Coord &x, const I X[4], const int mu, int nFace)
   {
     int y[4];
-QUDA_UNROLL
+#pragma unroll
     for ( int i = 0; i < 4; i++ ) y[i] = x[i];
     switch (mu) {
     case 0: y[0] = (y[0] + nFace + X[0]) % X[0]; break;
@@ -146,7 +146,7 @@ QUDA_UNROLL
   template <typename I>
   __device__ __host__ inline int linkNormalIndexP1(const int x[], const I X[4], const int mu) {
     int y[4];
-QUDA_UNROLL
+#pragma unroll
     for ( int i = 0; i < 4; i++ ) y[i] = x[i];
     y[mu] = (y[mu] + 1 + X[mu]) % X[mu];
     int idx = ((y[3] * X[2] + y[2]) * X[1] + y[1]) * X[0] + y[0];
@@ -289,7 +289,7 @@ QUDA_UNROLL
     x[2] = (zb - x[3] * X[2]);
     int x1odd = (x[1] + x[2] + x[3] + parity) & 1;
     x[0] = (2 * cb_index + x1odd  - za * X[0]);
-QUDA_UNROLL
+#pragma unroll
     for (int d=0; d<4; d++) x[d] += R[d];
     return;
   }
diff --git a/include/kernels/blas_core.cuh b/include/kernels/blas_core.cuh
index 68e4a22389..38c3d912ba 100644
--- a/include/kernels/blas_core.cuh
+++ b/include/kernels/blas_core.cuh
@@ -100,7 +100,7 @@ namespace quda
       axpbyz_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &v) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) v[i] = a * x[i] + b * y[i];
       }                                  // use v not z to ensure same precision as y
       constexpr int flops() const { return 3; }   //! flops per element
@@ -116,7 +116,7 @@ QUDA_UNROLL
       ax_(const real &a, const real &, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) x[i] *= a;
       }
       constexpr int flops() const { return 1; }   //! flops per element
@@ -132,7 +132,7 @@ QUDA_UNROLL
       caxpy_(const complex<real> &a, const complex<real> &, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) y[i] = cmac(a, x[i], y[i]);
       }
       constexpr int flops() const { return 4; }   //! flops per element
@@ -165,7 +165,7 @@ QUDA_UNROLL
       caxpby_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) _caxpby(a, x[i], b, y[i]);
       }
       constexpr int flops() const { return 7; }   //! flops per element
@@ -183,7 +183,7 @@ QUDA_UNROLL
       axpbypczw_(const real &a, const real &b, const real &c) : a(a), b(b), c(c) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &w, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           w[i] = a * x[i] + b * y[i] + c * z[i];
         }
@@ -203,7 +203,7 @@ QUDA_UNROLL
       axpyBzpcx_(const real &a, const real &b, const real &c) : a(a), b(b), c(c) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] += a * x[i];
           x[i] = b * z[i] + c * x[i];
@@ -223,7 +223,7 @@ QUDA_UNROLL
       axpyZpbx_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] += a * x[i];
           x[i] = z[i] + b * x[i];
@@ -263,7 +263,7 @@ QUDA_UNROLL
       caxpyBzpx_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           x[i] = cmac(b, z[i], x[i]);
@@ -283,7 +283,7 @@ QUDA_UNROLL
       caxpyBxpz_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           z[i] = cmac(b, x[i], z[i]);
@@ -303,7 +303,7 @@ QUDA_UNROLL
       caxpbypzYmbw_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &w, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           z[i] = cmac(a, x[i], z[i]);
           z[i] = cmac(b, y[i], z[i]);
@@ -324,7 +324,7 @@ QUDA_UNROLL
       cabxpyAx_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a.real()), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           x[i] *= a;
           y[i] = cmac(b, x[i], y[i]);
@@ -345,7 +345,7 @@ QUDA_UNROLL
       caxpyxmaz_(const complex<real> &a, const complex<real> &, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           x[i] = cmac(-a, z[i], x[i]);
@@ -385,7 +385,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           x[i] = cmac(-a, z[i], x[i]);
@@ -409,7 +409,7 @@ QUDA_UNROLL
       tripleCGUpdate_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &w, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] += a * w[i];
           z[i] -= a * x[i];
diff --git a/include/kernels/block_orthogonalize.cuh b/include/kernels/block_orthogonalize.cuh
index af6ee39dfc..1ee52dc5a5 100644
--- a/include/kernels/block_orthogonalize.cuh
+++ b/include/kernels/block_orthogonalize.cuh
@@ -113,17 +113,17 @@ namespace quda {
 
     __device__ __host__ inline void load(ColorSpinor<real, nColor, spinBlock> &v, int parity, int x_cb, int chirality, int i)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int s = 0; s < spinBlock; s++)
-QUDA_UNROLL
+#pragma unroll
         for (int c = 0; c < nColor; c++) v(s, c) = arg.V(parity, x_cb, chirality * spinBlock + s, c, i);
     }
 
     __device__ __host__ inline void save(int parity, int x_cb, int chirality, int i, const ColorSpinor<real, nColor, spinBlock> &v)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int s = 0; s < spinBlock; s++)
-QUDA_UNROLL
+#pragma unroll
         for (int c = 0; c < nColor; c++) arg.V(parity, x_cb, chirality * spinBlock + s, c, i) = v(s, c);
     }
 
@@ -163,14 +163,14 @@ QUDA_UNROLL
             if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
             if (n == 0) { // load from B on first Gram-Schmidt, otherwise V.
               if (chirality == 0) {
-QUDA_UNROLL
+#pragma unroll
                 for (int m = 0; m < mVec; m++) arg.B[j+m].template load<spinBlock>(v[m][tx].data, parity[tx], x_cb[tx], 0);
               } else {
-QUDA_UNROLL
+#pragma unroll
                 for (int m = 0; m < mVec; m++) arg.B[j+m].template load<spinBlock>(v[m][tx].data, parity[tx], x_cb[tx], 1);
               }
             } else {
-QUDA_UNROLL
+#pragma unroll
               for (int m = 0; m < mVec; m++) load(v[m][tx], parity[tx], x_cb[tx], chirality, j + m);
             }
           }
@@ -183,7 +183,7 @@ QUDA_UNROLL
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
               load(vi[tx], parity[tx], x_cb[tx], chirality, i);
 
-QUDA_UNROLL
+#pragma unroll
               for (int m = 0; m < mVec; m++) dot[m] += innerProduct(vi[tx], v[m][tx]);
             }
 
@@ -192,19 +192,19 @@ QUDA_UNROLL
             // subtract the blocks to orthogonalise
             for (int tx = 0; tx < n_sites_per_thread; tx++) {
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
-QUDA_UNROLL
+#pragma unroll
               for (int m = 0; m < mVec; m++) caxpy(-complex<real>(dot[m].real(), dot[m].imag()), vi[tx], v[m][tx]);
             }
           } // i
 
           // now orthogonalize over the block diagonal and normalize each entry
-QUDA_UNROLL
+#pragma unroll
           for (int m = 0; m < mVec; m++) {
 
             dot_t dot{0};
             for (int tx = 0; tx < n_sites_per_thread; tx++) {
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
-QUDA_UNROLL
+#pragma unroll
               for (int i = 0; i < m; i++) dot[i] += innerProduct(v[i][tx], v[m][tx]);
             }
             
@@ -213,7 +213,7 @@ QUDA_UNROLL
             sum_t nrm = 0.0;
             for (int tx = 0; tx < n_sites_per_thread; tx++) {
               if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
-QUDA_UNROLL
+#pragma unroll
               for (int i = 0; i < m; i++) caxpy(-complex<real>(dot[i].real(), dot[i].imag()), v[i][tx], v[m][tx]); // subtract the blocks to orthogonalise
               nrm += norm2(v[m][tx]);
             }
@@ -229,7 +229,7 @@ QUDA_UNROLL
 
           for (int tx = 0; tx < n_sites_per_thread; tx++) {
             if (x_offset_cb[tx] >= arg.aggregate_size_cb) break;
-QUDA_UNROLL
+#pragma unroll
             for (int m = 0; m < mVec; m++) save(parity[tx], x_cb[tx], chirality, j + m, v[m][tx]);
           }
         } // j
diff --git a/include/kernels/clover_compute.cuh b/include/kernels/clover_compute.cuh
index 7677b04236..67f5560845 100644
--- a/include/kernels/clover_compute.cuh
+++ b/include/kernels/clover_compute.cuh
@@ -72,7 +72,7 @@ namespace quda {
 
       // Load the field-strength tensor from global memory
       Link F[6];
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<6; ++i) F[i] = arg.f(i, x_cb, parity);
 
       Complex I(0.0,1.0);
@@ -84,12 +84,12 @@ QUDA_UNROLL
       block2[1] = arg.coeff*(F[1]-F[4] - I*(F[2]+F[3])); // 126 floating-point ops
 
       // This uses lots of unnecessary memory
-QUDA_UNROLL
+#pragma unroll
       for (int ch=0; ch<2; ++ch) {
         HMatrix<real,N> A;
         // c = 0(1) => positive(negative) chiral block
         // Compute real diagonal elements
-QUDA_UNROLL
+#pragma unroll
         for (int i=0; i<N/2; ++i) {
           A(i+0,i+0) = 1.0 - block1[ch](i,i).real();
           A(i+3,i+3) = 1.0 + block1[ch](i,i).real();
diff --git a/include/kernels/clover_invert.cuh b/include/kernels/clover_invert.cuh
index a559ef08e4..42292b7a83 100644
--- a/include/kernels/clover_invert.cuh
+++ b/include/kernels/clover_invert.cuh
@@ -54,7 +54,7 @@ namespace quda
       using Mat = HMatrix<real, N>;
       double trLogA = 0.0;
 
-QUDA_UNROLL
+#pragma unroll
       for (int ch = 0; ch < 2; ch++) {
         Mat A = arg.clover(x_cb, parity, ch);
         A *= static_cast<real>(2.0); // factor of two is inherent to QUDA clover storage
diff --git a/include/kernels/clover_outer_product.cuh b/include/kernels/clover_outer_product.cuh
index 4acf4ca6b8..a915521e54 100644
--- a/include/kernels/clover_outer_product.cuh
+++ b/include/kernels/clover_outer_product.cuh
@@ -63,7 +63,7 @@ namespace quda {
       Spinor A = arg.inA(x_cb, 0);
       Spinor C = arg.inC(x_cb, 0);
 
-QUDA_UNROLL
+#pragma unroll
       for (int dim=0; dim<4; ++dim) {
         int shift[4] = {0, 0, 0, 0};
         shift[dim] = 1;
diff --git a/include/kernels/clover_sigma_outer_product.cuh b/include/kernels/clover_sigma_outer_product.cuh
index 1ae9f015b4..91d2de8ef5 100644
--- a/include/kernels/clover_sigma_outer_product.cuh
+++ b/include/kernels/clover_sigma_outer_product.cuh
@@ -47,7 +47,7 @@ namespace quda
     using Link = Matrix<complex<typename Arg::real>, Arg::nColor>;
     Link result;
 
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < Arg::nvector; i++) {
       const Spinor A = arg.inA[i](x_cb, parity);
       const Spinor B = arg.inB[i](x_cb, parity);
diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 6971975040..6308b7ca57 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -270,11 +270,11 @@ namespace quda {
 
       int ghost_idx = ghostFaceIndex<1>(coord, arg.x_size, arg.dim, nFace);
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of gauge field
         auto U = make_tile_A<complex, false>(tile);
         U.load(arg.U, arg.dim, parity, x_cb, i0, k);
-QUDA_UNROLL
+#pragma unroll
         for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
           auto W = make_tile_B<complex, true>(tile);
           W.loadCS(Wacc, arg.dim, 1, (parity+1)&1, ghost_idx, s, k, j0);
@@ -286,11 +286,11 @@ QUDA_UNROLL
 
       int y_cb = linkIndexHop(coord, arg.x_size, arg.dim, nFace);
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of gauge field
         auto U = make_tile_A<complex, false>(tile);
         U.load(arg.U, arg.dim, parity, x_cb, i0, k);
-QUDA_UNROLL
+#pragma unroll
         for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
           auto W = make_tile_B<complex, false>(tile);
           W.loadCS(Wacc, 0, 0, (parity+1)&1, y_cb, s, k, j0);
@@ -300,7 +300,7 @@ QUDA_UNROLL
     }
 
     real uv_max = static_cast<real>(0.0);
-QUDA_UNROLL
+#pragma unroll
     for (int s = 0; s < uvSpin; s++) {
       if constexpr (Arg::compute_max) {
         uv_max = fmax(UV[s].abs_max(), uv_max);
@@ -531,13 +531,13 @@ QUDA_UNROLL
 
     if (arg.dir == QUDA_IN_PLACE) {
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of coarse clover field
-QUDA_UNROLL
+#pragma unroll
         for (int s_col = 0; s_col < Arg::fineSpin; s_col++) {
           auto W = make_tile_B<complex, false>(tile);
           W.loadCS(Wacc, 0, 0, parity, x_cb, s_col, k, j0);
-QUDA_UNROLL
+#pragma unroll
           for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
             auto C = make_tile_A<complex, false>(tile);
             C.load(arg.C, 0, parity, x_cb, s, s_col, i0, k);
@@ -550,13 +550,13 @@ QUDA_UNROLL
 
       int ghost_idx = ghostFaceIndex<1>(coord, arg.x_size, arg.dim, nFace);
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of gauge field
-QUDA_UNROLL
+#pragma unroll
         for (int s_col=0; s_col<Arg::fineSpin; s_col++) {
           auto W = make_tile_B<complex, true>(tile);
           W.loadCS(Wacc, arg.dim, 1, (parity+1)&1, ghost_idx, s_col, k, j0);
-QUDA_UNROLL
+#pragma unroll
           for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
             // on coarse lattice, if forwards then use forwards links
             auto U = make_tile_A<complex, false>(tile);
@@ -571,13 +571,13 @@ QUDA_UNROLL
 
       int y_cb = linkIndexHop(coord, arg.x_size, arg.dim, nFace);
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < TileType::k; k += TileType::K) { // Fine Color columns of gauge field
-QUDA_UNROLL
+#pragma unroll
         for (int s_col = 0; s_col < Arg::fineSpin; s_col++) {
           auto W = make_tile_B<complex, false>(tile);
           W.loadCS(Wacc, 0, 0, (parity+1)&1, y_cb, s_col, k, j0);
-QUDA_UNROLL
+#pragma unroll
           for (int s = 0; s < Arg::fineSpin; s++) {  //Fine Spin
             // on coarse lattice, if forwards then use forwards links
             auto U = make_tile_A<complex, false>(tile);
@@ -590,7 +590,7 @@ QUDA_UNROLL
     }
 
     real uv_max = static_cast<real>(0.0);
-QUDA_UNROLL
+#pragma unroll
     for (int s = 0; s < uvSpin; s++) {
       if constexpr (Arg::compute_max) {
         uv_max = fmax(UV[s].abs_max(), uv_max);
@@ -703,11 +703,11 @@ QUDA_UNROLL
       constexpr int N = Arg::fineSpin * Arg::fineColor / 2;
       HMatrix<real, N> A;
 
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < N; i++) {
         int s_i = i / Arg::fineColor;
         int c_i = i % Arg::fineColor;
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j <= i; j++) {
           int s_j = j / Arg::fineColor;
           int c_j = j % Arg::fineColor;
@@ -720,9 +720,9 @@ QUDA_UNROLL
       }
 
       ColorSpinor<real, Arg::fineColor, Arg::fineSpin / 2> V;
-QUDA_UNROLL
+#pragma unroll
       for (int s = 0; s < Arg::fineSpin / 2; s++) {
-QUDA_UNROLL
+#pragma unroll
         for (int c = 0; c < Arg::fineColor; c++) { V(s, c) = arg.V(parity, x_cb, 2 * ch + s, c, ic_c); }
       }
 
@@ -735,16 +735,16 @@ QUDA_UNROLL
 #endif
 
       if (!Arg::compute_max) {
-QUDA_UNROLL
+#pragma unroll
         for (int s = 0; s < Arg::fineSpin / 2; s++) {
-QUDA_UNROLL
+#pragma unroll
           for (int ic = 0; ic < Arg::fineColor; ic++) { arg.AV(parity, x_cb, 2 * ch + s, ic, ic_c) = AV(s, ic); }
         }
       } else {
         real max = static_cast<real>(0.0);
-QUDA_UNROLL
+#pragma unroll
         for (int s = 0; s < Arg::fineSpin / 2; s++) {
-QUDA_UNROLL
+#pragma unroll
           for (int ic = 0; ic < Arg::fineColor; ic++) {
             auto abs_max = fmax(abs(AV(s, ic).real()), abs(AV(s, ic).imag()));
             max = fmax(abs_max, max);
@@ -776,17 +776,17 @@ QUDA_UNROLL
       complex<real> fp(1./(1.+arg.mu*arg.mu),-arg.mu/(1.+arg.mu*arg.mu));
       complex<real> fm(1./(1.+arg.mu*arg.mu),+arg.mu/(1.+arg.mu*arg.mu));
 
-QUDA_UNROLL
+#pragma unroll
       for (int s = 0; s < Arg::fineSpin/2; s++) {
-QUDA_UNROLL
+#pragma unroll
         for (int c = 0; c < Arg::fineColor; c++) {
           arg.AV(parity,x_cb,s,c,v) = arg.V(parity,x_cb,s,c,v) * fp;
         }
       }
 
-QUDA_UNROLL
+#pragma unroll
       for (int s = Arg::fineSpin/2; s < Arg::fineSpin; s++) {
-QUDA_UNROLL
+#pragma unroll
         for (int c = 0; c < Arg::fineColor; c++) {
           arg.AV(parity,x_cb,s,c,v) = arg.V(parity,x_cb,s,c,v) * fm;
         }
@@ -822,11 +822,11 @@ QUDA_UNROLL
       constexpr int N = Arg::fineSpin * Arg::fineColor / 2;
       HMatrix<real, N> A;
 
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < N; i++) {
         int s_i = i / Arg::fineColor;
         int c_i = i % Arg::fineColor;
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j <= i; j++) {
           int s_j = j / Arg::fineColor;
           int c_j = j % Arg::fineColor;
@@ -839,9 +839,9 @@ QUDA_UNROLL
 
       ColorSpinor<real, Arg::fineColor, Arg::fineSpin / 2> V;
 
-QUDA_UNROLL
+#pragma unroll
       for (int s = 0; s < Arg::fineSpin / 2; s++) {
-QUDA_UNROLL
+#pragma unroll
         for (int c = 0; c < Arg::fineColor; c++) {
           V(s, c) = arg.V(parity, x_cb, 2 * ch + s, c, ic_c);
         }
@@ -857,11 +857,11 @@ QUDA_UNROLL
       if (!clover::dynamic_inverse()) {
         // load in the clover inverse matrix
         HMatrix<real, N> Ainv;
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < N; i++) {
           int s_i = i / Arg::fineColor;
           int c_i = i % Arg::fineColor;
-QUDA_UNROLL
+#pragma unroll
           for (int j = 0; j <= i; j++) {
             int s_j = j / Arg::fineColor;
             int c_j = j % Arg::fineColor;
@@ -871,16 +871,16 @@ QUDA_UNROLL
         auto AV = Ainv * UV;
 
         if (!Arg::compute_max) {
-QUDA_UNROLL
+#pragma unroll
           for (int s = 0; s < Arg::fineSpin / 2; s++)
-QUDA_UNROLL
+#pragma unroll
             for (int c = 0; c < Arg::fineColor; c++)
               arg.AV(parity, x_cb, 2 * ch + s, c, ic_c) = AV(s, c);
         } else {
           real max = static_cast<real>(0.0);
-QUDA_UNROLL
+#pragma unroll
           for (int s = 0; s < Arg::fineSpin / 2; s++) {
-QUDA_UNROLL
+#pragma unroll
             for (int c = 0; c < Arg::fineColor; c++) {
               auto abs_max = fmax(abs(AV(s, c).real()), abs(AV(s, c).imag()));
               max = fmax(abs_max, max);
@@ -897,16 +897,16 @@ QUDA_UNROLL
         const auto AV = cholesky.solve(UV);
 
         if (!Arg::compute_max) {
-QUDA_UNROLL
+#pragma unroll
           for (int s = 0; s < Arg::fineSpin / 2; s++)
-QUDA_UNROLL
+#pragma unroll
             for (int c = 0; c < Arg::fineColor; c++)
               arg.AV(parity, x_cb, 2 * ch + s, c, ic_c) = AV(s, c);
         } else {
           real max = static_cast<real>(0.0);
-QUDA_UNROLL
+#pragma unroll
           for (int s = 0; s < Arg::fineSpin / 2; s++) {
-QUDA_UNROLL
+#pragma unroll
             for (int c = 0; c < Arg::fineColor; c++) {
               auto abs_max = fmax(abs(AV(s, c).real()), abs(AV(s, c).imag()));
               max = fmax(abs_max, max);
@@ -1068,7 +1068,7 @@ QUDA_UNROLL
     using TileType = typename Arg::vuvTileType;
     auto &tile = arg.vuvTile;
 
-QUDA_UNROLL
+#pragma unroll
     for (int s = 0; s < Arg::fineSpin; s++) { // Loop over fine spin
 
       //Spin part of the color matrix.  Will always consist
@@ -1086,7 +1086,7 @@ QUDA_UNROLL
       const int s_col = gamma.getcol(s);
       const int s_c_col = 1 - s_c_row; // always off-diagonal relative to row coord
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < TileType::k; k += TileType::K) { // Sum over fine color
 
         if (arg.dir == QUDA_BACKWARDS) {
@@ -1101,9 +1101,9 @@ QUDA_UNROLL
 
           //Off-diagonal Spin (backward link / positive projector applied)
           auto gammaV = make_tile_A<complex, false>(tile);
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < TileType::K; i++)
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < TileType::M; j++)
               gammaV(j, i) = gamma.apply(s, conj(V(i, j)));
 
@@ -1122,9 +1122,9 @@ QUDA_UNROLL
           //Off-diagonal Spin (forward link / negative projector applied)
           auto gammaAV = make_tile_A<complex, false>(tile);
 
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < TileType::K; i++)
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < TileType::M; j++)
               gammaAV(j, i) = -gamma.apply(s, conj(AV(i, j)));
 
@@ -1320,13 +1320,13 @@ QUDA_UNROLL
     using TileType = typename Arg::vuvTileType;
     auto &tile = arg.vuvTile;
 
-QUDA_UNROLL
+#pragma unroll
     for (int k = 0; k < TileType::k; k += TileType::K) { // Sum over fine color
-QUDA_UNROLL
+#pragma unroll
       for (int s = 0; s < Arg::fineSpin; s++) {
         auto AV = make_tile_At<complex, false>(tile);
         AV.loadCS(arg.AV, 0, 0, parity, x_cb, s, k, i0);
-QUDA_UNROLL
+#pragma unroll
         for (int s_col = 0; s_col < Arg::fineSpin; s_col++) { // which chiral block
           auto UV = make_tile_B<complex, false>(tile);
           UV.loadCS(arg.UV, 0, 0, parity, x_cb, s_col*Arg::fineSpin+s, k, j0);
@@ -1386,9 +1386,9 @@ QUDA_UNROLL
       int i_block0 = (threadIdx.y / (arg.parity_flip ? 1 : 2)) * TileType::M;
       int j_block0 = threadIdx.z * TileType::N;
 
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < TileType::M; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < TileType::N; j++) {
           if (tx < Arg::coarseSpin*Arg::coarseSpin) {
             if (pack.dir != QUDA_IN_PLACE) Y[i_block0+i][j_block0+j][x_][s_row][s_col] = 0;
@@ -1399,24 +1399,24 @@ QUDA_UNROLL
 
       __syncthreads();
 
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < TileType::M; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < TileType::N; j++) {
 
           if (pack.dir == QUDA_IN_PLACE || isDiagonal) {
-QUDA_UNROLL
+#pragma unroll
             for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-QUDA_UNROLL
+#pragma unroll
               for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
                 atomic_helper<real, storeType>(&X[i_block0+i][j_block0+j][x_][s_row][s_col],
                                                arg.X_atomic, vuv[s_row*Arg::coarseSpin+s_col](i,j));
               }
             }
           } else {
-QUDA_UNROLL
+#pragma unroll
             for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-QUDA_UNROLL
+#pragma unroll
               for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
                 atomic_helper<real, storeType>(&Y[i_block0+i][j_block0+j][x_][s_row][s_col],
                                                arg.Y_atomic, vuv[s_row*Arg::coarseSpin+s_col](i,j));
@@ -1430,9 +1430,9 @@ QUDA_UNROLL
 
       if (tx < Arg::coarseSpin*Arg::coarseSpin && (parity == 0 || arg.parity_flip == 1) ) {
 
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < TileType::M; i++) {
-QUDA_UNROLL
+#pragma unroll
           for (int j = 0; j < TileType::N; j++) {
             if (pack.dir == QUDA_IN_PLACE) {
               // same as dir == QUDA_FORWARDS
@@ -1487,25 +1487,25 @@ QUDA_UNROLL
 
     if (arg.dir == QUDA_IN_PLACE) {
       // same as dir == QUDA_FORWARDS
-QUDA_UNROLL
+#pragma unroll
       for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-QUDA_UNROLL
+#pragma unroll
         for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < TileType::M; i++)
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < TileType::N; j++)
               arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,s_row,s_col,i0+i,j0+j,vuv[s_row*Arg::coarseSpin+s_col](i,j));
         }
       }
     } else if (!isDiagonal) {
-QUDA_UNROLL
+#pragma unroll
       for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-QUDA_UNROLL
+#pragma unroll
         for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < TileType::M; i++)
-QUDA_UNROLL
+#pragma unroll
             for (int j = 0; j < TileType::N; j++)
               arg.Y_atomic.atomicAdd(dim_index,coarse_parity,coarse_x_cb,s_row,s_col,i0+i,j0+j,vuv[s_row*Arg::coarseSpin+s_col](i,j));
         }
@@ -1513,25 +1513,25 @@ QUDA_UNROLL
     } else {
 
       if (arg.dir == QUDA_BACKWARDS) {
-QUDA_UNROLL
+#pragma unroll
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-QUDA_UNROLL
+#pragma unroll
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < TileType::M; i++)
-QUDA_UNROLL
+#pragma unroll
               for (int j = 0; j < TileType::N; j++)
                 arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,s_col,s_row,j0+j,i0+i,conj(vuv[s_row*Arg::coarseSpin+s_col](i,j)));
           }
         }
       } else {
-QUDA_UNROLL
+#pragma unroll
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-QUDA_UNROLL
+#pragma unroll
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
-QUDA_UNROLL
+#pragma unroll
             for (int i = 0; i < TileType::M; i++)
-QUDA_UNROLL
+#pragma unroll
               for (int j = 0; j < TileType::N; j++)
                 arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,s_row,s_col,i0+i,j0+j,vuv[s_row*Arg::coarseSpin+s_col](i,j));
           }
@@ -1539,15 +1539,15 @@ QUDA_UNROLL
       }
 
       if (!arg.bidirectional) {
-QUDA_UNROLL
+#pragma unroll
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { // Chiral row block
-QUDA_UNROLL
+#pragma unroll
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { // Chiral column block
             if (s_row != s_col) vuv[s_row * Arg::coarseSpin + s_col] *= static_cast<real>(-1.0);
             if (Arg::fineSpin != 1 || s_row != s_col) {
-QUDA_UNROLL
+#pragma unroll
               for (int i = 0; i < TileType::M; i++)
-QUDA_UNROLL
+#pragma unroll
                 for (int j = 0; j < TileType::N; j++)
                   arg.X_atomic.atomicAdd(0,coarse_parity,coarse_x_cb,s_row,s_col,i0+i,j0+j,vuv[s_row*Arg::coarseSpin+s_col](i,j));
             }
@@ -1587,7 +1587,7 @@ QUDA_UNROLL
     multiplyVUV(vuv, arg, parity, x_cb, i0, j0);
 
     if (isDiagonal && !isFromCoarseClover) {
-QUDA_UNROLL
+#pragma unroll
       for (int s2=0; s2<Arg::coarseSpin*Arg::coarseSpin; s2++) vuv[s2] *= -arg.kappa;
     }
 
@@ -1745,20 +1745,20 @@ QUDA_UNROLL
       for (int i = 0; i < Arg::coarseSpin * Arg::coarseSpin; i++) X[i] = 0.0;
 
       // If Nspin = 4, then the clover term has structure C_{\mu\nu} = \gamma_{\mu\nu}C^{\mu\nu}
-QUDA_UNROLL
+#pragma unroll
       for (int chi = 0; chi < 2; chi++) {
 
-QUDA_UNROLL
+#pragma unroll
         for (int s_row = 0; s_row < Arg::fineSpin / 2; s_row++) { // Loop over fine spin row within a chiral block
           const int s_c = arg.spin_map(chi * Arg::fineSpin / 2 + s_row, parity);
           // On the fine lattice, the clover field is chirally blocked, so loop over rows/columns
           // in the same chiral block.
-QUDA_UNROLL
+#pragma unroll
           for (int s_col = 0; s_col < Arg::fineSpin / 2; s_col++) { // Loop over fine spin column within a chiral block
-QUDA_UNROLL
+#pragma unroll
             for (int ic = 0; ic < Arg::fineColor; ic++) { // Sum over fine color row
               complex<real> CV = 0.0;
-QUDA_UNROLL
+#pragma unroll
               for (int jc = 0; jc < Arg::fineColor; jc++) {  // Sum over fine color column
                 CV = cmac(arg.C(parity, x_cb, chi, s_row, s_col, ic, jc), arg.V(parity, x_cb, chi * Arg::fineSpin / 2 + s_col, jc, c_col), CV);
               } // Fine color column
@@ -1770,9 +1770,9 @@ QUDA_UNROLL
 
       }
 
-QUDA_UNROLL
+#pragma unroll
       for (int si = 0; si < Arg::coarseSpin; si++) {
-QUDA_UNROLL
+#pragma unroll
         for (int sj = 0; sj < Arg::coarseSpin; sj++) {
           arg.X_atomic.atomicAdd(0, coarse_parity, coarse_x_cb, si, sj, c_row, c_col, X[si*Arg::coarseSpin+sj]);
         }
@@ -1801,11 +1801,11 @@ QUDA_UNROLL
       int parity = parity_c_col / Arg::coarseColor;
       int c_col = parity_c_col % Arg::coarseColor;
 
-QUDA_UNROLL
+#pragma unroll
       for (int d=0; d<4; d++) {
-QUDA_UNROLL
+#pragma unroll
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { //Spin row
-QUDA_UNROLL
+#pragma unroll
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { //Spin column
             if (s_row == s_col && Arg::coarseSpin != 1)
               arg.Y(d+4, parity, x_cb, s_row, s_col, c_row, c_col) = arg.Y(d, parity, x_cb, s_row, s_col, c_row, c_col);
@@ -1918,9 +1918,9 @@ QUDA_UNROLL
         int d_in = arg.dim_index % in.geometry;
         int d_out = arg.dim_index % arg.Y.geometry;
 
-QUDA_UNROLL
+#pragma unroll
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { //Spin row
-QUDA_UNROLL
+#pragma unroll
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { //Spin column
             complex<real> M = in(d_in,parity,x_cb,s_row,s_col,c_row,c_col);
             arg.Y(d_out,parity,x_cb,s_row,s_col,c_row,c_col) = M;
@@ -1931,9 +1931,9 @@ QUDA_UNROLL
         int d_in = arg.dim_index % in.geometry;
         int d_out = arg.dim_index % arg.X.geometry;
 
-QUDA_UNROLL
+#pragma unroll
         for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { //Spin row
-QUDA_UNROLL
+#pragma unroll
           for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { //Spin column
             complex<real> M = in(d_in,parity,x_cb,s_row,s_col,c_row,c_col);
             arg.X(d_out,parity,x_cb,s_row,s_col,c_row,c_col) = M;
@@ -1963,9 +1963,9 @@ QUDA_UNROLL
       int c_col = parity_c_col % Arg::coarseColor; // color col index
       int parity = parity_c_col / Arg::coarseColor;
 
-QUDA_UNROLL
+#pragma unroll
       for (int s_row = 0; s_row < Arg::coarseSpin; s_row++) { //Spin row
-QUDA_UNROLL
+#pragma unroll
         for (int s_col = 0; s_col < Arg::coarseSpin; s_col++) { //Spin column
           complex<real> M = arg.Y(arg.dim_index,parity,x_cb,s_row,s_col,c_row,c_col);
           arg.Y(arg.dim_index,parity,x_cb,s_row,s_col,c_row,c_col) = arg.rescale*M;
diff --git a/include/kernels/coarse_op_preconditioned.cuh b/include/kernels/coarse_op_preconditioned.cuh
index 0777c79779..7917266b50 100644
--- a/include/kernels/coarse_op_preconditioned.cuh
+++ b/include/kernels/coarse_op_preconditioned.cuh
@@ -60,7 +60,7 @@ namespace quda {
 
       auto yHat = make_tile_C<complex,true>(arg.tile);
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < Arg::yhatTileType::k; k += Arg::yhatTileType::K) {
         auto Y = make_tile_A<complex, true>(arg.tile);
         Y.load(arg.Y, d, 1-parity, ghost_idx, i0, k);
@@ -82,7 +82,7 @@ QUDA_UNROLL
 
       auto yHat = make_tile_C<complex,false>(arg.tile);
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < Arg::yhatTileType::k; k += Arg::yhatTileType::K) {
         auto Y = make_tile_A<complex, false>(arg.tile);
         Y.load(arg.Y, d, 1-parity, back_idx, i0, k);
@@ -102,7 +102,7 @@ QUDA_UNROLL
     { // now do the forwards links X^{-1} * Y^{-\mu}
       auto yHat = make_tile_C<complex, false>(arg.tile);
 
-QUDA_UNROLL
+#pragma unroll
       for (int k = 0; k < Arg::yhatTileType::k; k += Arg::yhatTileType::K) {
         auto X = make_tile_A<complex, false>(arg.tile);
         X.load(arg.Xinv, 0, parity, x_cb, i0, k);
diff --git a/include/kernels/color_spinor_pack.cuh b/include/kernels/color_spinor_pack.cuh
index e60da7ad6b..f7e2ef9049 100644
--- a/include/kernels/color_spinor_pack.cuh
+++ b/include/kernels/color_spinor_pack.cuh
@@ -160,7 +160,7 @@ namespace quda {
       cache.save(thread_max);
       cache.sync();
       real this_site_max = static_cast<real>(0);
-QUDA_UNROLL
+#pragma unroll
       for (int sc = 0; sc < color_spin_threads; sc++) {
         auto sc_max = cache.load_y(sc);
         this_site_max = this_site_max > sc_max ? this_site_max : sc_max;
@@ -187,10 +187,10 @@ QUDA_UNROLL
     real thread_max_r = 0.0;
     real thread_max_i = 0.0;
 
-QUDA_UNROLL
+#pragma unroll
     for (int spin_local=0; spin_local<Ms; spin_local++) {
       int s = spin_block + spin_local;
-QUDA_UNROLL
+#pragma unroll
       for (int color_local=0; color_local<Mc; color_local++) {
         int c = color_block + color_local;
         complex<real> z = arg.field(spinor_parity, x_cb, s, c);
@@ -261,10 +261,10 @@ QUDA_UNROLL
       int x_cb = indexFromFaceIndex(dim, dir, ghost_idx, parity, arg);
       auto max = compute_site_max<Arg>(arg, x_cb, spinor_parity, spin_block, color_block);
 
-QUDA_UNROLL
+#pragma unroll
       for (int spin_local=0; spin_local<Ms; spin_local++) {
         int s = spin_block + spin_local;
-QUDA_UNROLL
+#pragma unroll
         for (int color_local=0; color_local<Mc; color_local++) {
           int c = color_block + color_local;
           arg.field.Ghost(dim, dir, spinor_parity, ghost_idx, s, c, 0, max) = arg.field(spinor_parity, x_cb, s, c);
diff --git a/include/kernels/contraction.cuh b/include/kernels/contraction.cuh
index 7aa5ef8d25..b1f6e7f052 100644
--- a/include/kernels/contraction.cuh
+++ b/include/kernels/contraction.cuh
@@ -50,9 +50,9 @@ namespace quda
       Vector y = arg.y(x_cb, parity);
 
       Matrix<complex<real>, nSpin> A;
-QUDA_UNROLL
+#pragma unroll
       for (int mu = 0; mu < nSpin; mu++) {
-QUDA_UNROLL
+#pragma unroll
         for (int nu = 0; nu < nSpin; nu++) {
           // Color inner product: <\phi(x)_{\mu} | \phi(y)_{\nu}>
           // The Bra is conjugated
diff --git a/include/kernels/copy_clover.cuh b/include/kernels/copy_clover.cuh
index 02cd67a0ed..dc6523cac4 100644
--- a/include/kernels/copy_clover.cuh
+++ b/include/kernels/copy_clover.cuh
@@ -45,7 +45,7 @@ namespace quda {
       typename mapper<typename Arg::store_out_t>::type out[Arg::length];
       typename mapper<typename Arg::store_in_t>::type in[Arg::length];
       arg.in.load(in, x_cb, parity);
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<Arg::length; i++) out[i] = in[i];
 
       if (arg.compute_diagonal) {
@@ -73,7 +73,7 @@ QUDA_UNROLL
       typename mapper<typename Arg::store_out_t>::type out[length];
       typename mapper<typename Arg::store_in_t>::type in[length];
       arg.in.raw_load(in, x_cb, parity);
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<length; i++) out[i] = in[i];
 
       arg.out.raw_save(out, x_cb, parity);
diff --git a/include/kernels/copy_field_offset.cuh b/include/kernels/copy_field_offset.cuh
index 5e34581b5b..ea4a603bfd 100644
--- a/include/kernels/copy_field_offset.cuh
+++ b/include/kernels/copy_field_offset.cuh
@@ -117,7 +117,7 @@ namespace quda
   copy_field(int out, int in, int parity, const Arg &arg)
   {
     using Element = typename Arg::Element;
-QUDA_UNROLL
+#pragma unroll
     for (int d = 0; d < 4; d++) {
       Element element = arg.in(d, in, parity);
       arg.out(d, out, parity) = element;
@@ -159,14 +159,14 @@ QUDA_UNROLL
           // we are collecting so x_cb is the index for the input.
           idx_in = x_cb + arg.volume_4d_cb_in * s;
           getCoords5CB(coordinate, idx_in, arg.dim_in, arg.X0h_in, parity, QUDA_5D_PC);
-QUDA_UNROLL
+#pragma unroll
           for (int d = 0; d < 4; d++) { coordinate[d] += arg.offset[d]; }
           idx_out = linkIndex(coordinate, arg.dim_out) + arg.volume_4d_cb_out * coordinate[4];
         } else {
           // we are dispersing so x_cb is the index for the output.
           idx_out = x_cb + arg.volume_4d_cb_out * s;
           getCoords5CB(coordinate, idx_out, arg.dim_out, arg.X0h_out, parity, QUDA_5D_PC);
-QUDA_UNROLL
+#pragma unroll
           for (int d = 0; d < 4; d++) { coordinate[d] += arg.offset[d]; }
           idx_in = linkIndex(coordinate, arg.dim_in) + arg.volume_4d_cb_in * coordinate[4];
         }
diff --git a/include/kernels/dslash_clover_helper.cuh b/include/kernels/dslash_clover_helper.cuh
index 34eed360a4..d61cc25733 100644
--- a/include/kernels/dslash_clover_helper.cuh
+++ b/include/kernels/dslash_clover_helper.cuh
@@ -100,7 +100,7 @@ namespace quda {
 
       in.toRel(); // change to chiral basis here
 
-QUDA_UNROLL
+#pragma unroll
       for (int chirality=0; chirality<2; chirality++) {
         HMatrix<real, N> A = arg.clover(x_cb, clover_parity, chirality);
         half_fermion chi = in.chiral_project(chirality);
@@ -143,7 +143,7 @@ QUDA_UNROLL
 
       in.toRel(); // change to chiral basis here
 
-QUDA_UNROLL
+#pragma unroll
       for (int chirality=0; chirality<2; chirality++) {
         const complex<real> a(0.0, chirality == 0 ? arg.a : -arg.a);
         Mat A = arg.clover(x_cb, clover_parity, chirality);
diff --git a/include/kernels/dslash_coarse.cuh b/include/kernels/dslash_coarse.cuh
index 18491f82f2..8311ec0afd 100644
--- a/include/kernels/dslash_coarse.cuh
+++ b/include/kernels/dslash_coarse.cuh
@@ -123,7 +123,7 @@ namespace quda {
     if (!thread_dir || target::is_host()) {
 
       //Forward gather - compute fwd offset for spinor fetch
-QUDA_UNROLL
+#pragma unroll
       for(int d = thread_dim; d < Arg::nDim; d += Arg::dim_stride) // loop over dimension
       {
 	const int fwd_idx = linkIndexP1(coord, arg.dim, d);
@@ -132,13 +132,13 @@ QUDA_UNROLL
 	  if (doHalo<Arg::type>()) {
             int ghost_idx = ghostFaceIndex<1, 5>(coord, arg.dim, d, arg.nFace);
 
-QUDA_UNROLL
+#pragma unroll
 	    for(int color_local = 0; color_local < Mc; color_local++) { //Color row
 	      int c_row = color_block + color_local; // global color index
 	      int row = s_row * Arg::nColor + c_row;
-QUDA_UNROLL
+#pragma unroll
 	      for(int s_col = 0; s_col < Arg::nSpin; s_col++) { //Spin column
-QUDA_UNROLL
+#pragma unroll
 		for(int c_col = 0; c_col < Arg::nColor; c_col += Arg::color_stride) { //Color column
 		  int col = s_col * Arg::nColor + c_col + color_offset;
 		  if (!Arg::dagger)
@@ -152,13 +152,13 @@ QUDA_UNROLL
 	    }
 	  }
 	} else if (doBulk<Arg::type>()) {
-QUDA_UNROLL
+#pragma unroll
 	  for(int color_local = 0; color_local < Mc; color_local++) { //Color row
 	    int c_row = color_block + color_local; // global color index
 	    int row = s_row * Arg::nColor + c_row;
-QUDA_UNROLL
+#pragma unroll
 	    for(int s_col = 0; s_col < Arg::nSpin; s_col++) { //Spin column
-QUDA_UNROLL
+#pragma unroll
 	      for(int c_col = 0; c_col < Arg::nColor; c_col += Arg::color_stride) { //Color column
 		int col = s_col * Arg::nColor + c_col + color_offset;
 		if (!Arg::dagger)
@@ -178,7 +178,7 @@ QUDA_UNROLL
     if (thread_dir || target::is_host()) {
 
       //Backward gather - compute back offset for spinor and gauge fetch
-QUDA_UNROLL
+#pragma unroll
       for(int d = thread_dim; d < Arg::nDim; d += Arg::dim_stride)
 	{
 	const int back_idx = linkIndexM1(coord, arg.dim, d);
@@ -186,13 +186,13 @@ QUDA_UNROLL
 	if ( arg.commDim[d] && (coord[d] - arg.nFace < 0) ) {
 	  if (doHalo<Arg::type>()) {
             const int ghost_idx = ghostFaceIndex<0, 5>(coord, arg.dim, d, arg.nFace);
-QUDA_UNROLL
+#pragma unroll
 	    for (int color_local=0; color_local<Mc; color_local++) {
 	      int c_row = color_block + color_local;
 	      int row = s_row * Arg::nColor + c_row;
-QUDA_UNROLL
+#pragma unroll
 	      for (int s_col=0; s_col < Arg::nSpin; s_col++)
-QUDA_UNROLL
+#pragma unroll
 		for (int c_col=0; c_col < Arg::nColor; c_col += Arg::color_stride) {
 		  int col = s_col * Arg::nColor + c_col + color_offset;
 		  if (!Arg::dagger)
@@ -205,13 +205,13 @@ QUDA_UNROLL
 	    }
 	  }
 	} else if (doBulk<Arg::type>()) {
-QUDA_UNROLL
+#pragma unroll
 	  for(int color_local = 0; color_local < Mc; color_local++) {
 	    int c_row = color_block + color_local;
 	    int row = s_row * Arg::nColor + c_row;
-QUDA_UNROLL
+#pragma unroll
 	    for(int s_col = 0; s_col < Arg::nSpin; s_col++)
-QUDA_UNROLL
+#pragma unroll
 	      for(int c_col = 0; c_col < Arg::nColor; c_col += Arg::color_stride) {
 		int col = s_col * Arg::nColor + c_col + color_offset;
 		if (!Arg::dagger)
@@ -244,13 +244,13 @@ QUDA_UNROLL
     const int spinor_parity = (arg.nParity == 2) ? parity : 0;
 
     // M is number of colors per thread
-QUDA_UNROLL
+#pragma unroll
     for(int color_local = 0; color_local < Mc; color_local++) {//Color out
       int c = color_block + color_local; // global color index
       int row = s * Arg::nColor + c;
-QUDA_UNROLL
+#pragma unroll
       for (int s_col = 0; s_col < Arg::nSpin; s_col++) //Spin in
-QUDA_UNROLL
+#pragma unroll
 	for (int c_col = 0; c_col < Arg::nColor; c_col += Arg::color_stride) { //Color in
 	  //Factor of kappa and diagonal addition now incorporated in X
 	  int col = s_col * Arg::nColor + c_col + color_offset;
@@ -281,14 +281,14 @@ QUDA_UNROLL
 
       if (dir == 0 && dim == 0) {
         // full split over dimension and direction
-QUDA_UNROLL
+#pragma unroll
         for (int d=1; d < Arg::dim_stride; d++) { // get remaining forward gathers (if any)
           // 4-way 1,2,3  (stride = 4)
           // 2-way 1      (stride = 2)
           out += cache.load_z(target::thread_idx().z + d * 2 + 0);
         }
 
-QUDA_UNROLL
+#pragma unroll
         for (int d=0; d < Arg::dim_stride; d++) { // get all backward gathers
           out += cache.load_z(target::thread_idx().z + d * 2 + 1);
         }
@@ -348,7 +348,7 @@ QUDA_UNROLL
         // reduce down to the first group of column-split threads
         out = warp_combine<Arg::color_stride>(out);
 
-QUDA_UNROLL
+#pragma unroll
         for (int color_local=0; color_local<Mc; color_local++) {
           int c = color_block + color_local; // global color index
           if (color_offset == 0) {
diff --git a/include/kernels/dslash_mdw_fused.cuh b/include/kernels/dslash_mdw_fused.cuh
index 44527ec17c..8665b8dcc3 100644
--- a/include/kernels/dslash_mdw_fused.cuh
+++ b/include/kernels/dslash_mdw_fused.cuh
@@ -167,7 +167,7 @@ namespace quda {
     __device__ inline bool is_halo_4d(const int coordinate[4], const int dim[4], const int halo_shift[4])
     {
       bool ret = false;
-QUDA_UNROLL
+#pragma unroll
       for (int d = 0; d < 4; d++) {
         ret = ret or (coordinate[d] >= dim[d] - halo_shift[d] or coordinate[d] < halo_shift[d]);
       }
@@ -180,14 +180,14 @@ QUDA_UNROLL
       int back_x[4];
       int back_dim[4];
 
-QUDA_UNROLL
+#pragma unroll
       for (int d = 0; d < 4; d++) {
         back_x[d] = comm[d] ? x[d] - pad : x[d];
         back_dim[d] = comm[d] ? dim[d] - pad * 2 : dim[d];
       }
 
       bool is_center = true;
-QUDA_UNROLL
+#pragma unroll
       for (int d = 0; d < 4; d++) { is_center = is_center && (back_x[d] >= 0 && back_x[d] < back_dim[d]); }
 
       if (is_center) {
@@ -211,7 +211,7 @@ QUDA_UNROLL
 
       const int index_4d_cb = index_4d_cb_from_coordinate_4d(coordinate, arg.dim);
 
-QUDA_UNROLL
+#pragma unroll
       for (int d = 0; d < 4; d++) // loop over dimension
       {
         int x[4] = {coordinate[0], coordinate[1], coordinate[2], coordinate[3]};
@@ -262,7 +262,7 @@ QUDA_UNROLL
       int aux[4];
       aux[0] = shrunk_index * 2;
 
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < 3; i++) { aux[i + 1] = aux[i] / shrunk_dim[i]; }
 
       coordinate[0] = aux[0] - aux[1] * shrunk_dim[0];
@@ -275,7 +275,7 @@ QUDA_UNROLL
         += (shift[0] + shift[1] + shift[2] + shift[3] + parity + coordinate[3] + coordinate[2] + coordinate[1]) & 1;
 
 // Now go back to the extended volume.
-QUDA_UNROLL
+#pragma unroll
       for (int d = 0; d < 4; d++) { coordinate[d] += shift[d]; }
     }
 
@@ -354,7 +354,7 @@ QUDA_UNROLL
         mma::MmaOperandA op_a[Arg::reload ? 1 : tk_dim];
         mma::MmaOperandA op_a_aux[Arg::reload ? 1 : tk_dim];
         if (!Arg::reload) { // the data in registers can be resued.
-QUDA_UNROLL
+#pragma unroll
           for (int tile_k = 0; tile_k < tk_dim; tile_k++) { op_a[tile_k].template load<M_sm>(sm_a, tile_k, warp_m, wrm); }
         }
 
@@ -364,7 +364,7 @@ QUDA_UNROLL
             construct_matrix_a_m5inv<M_sm, true, Arg>(arg, sm_a); // dagger = true
             __syncthreads();
 
-QUDA_UNROLL
+#pragma unroll
             for (int tile_k = 0; tile_k < tk_dim; tile_k++) {
               op_a_aux[tile_k].template load<M_sm>(sm_a, tile_k, warp_m, wrm);
             }
diff --git a/include/kernels/dslash_staggered.cuh b/include/kernels/dslash_staggered.cuh
index ef7428d827..8f772165bf 100644
--- a/include/kernels/dslash_staggered.cuh
+++ b/include/kernels/dslash_staggered.cuh
@@ -92,7 +92,7 @@ namespace quda
     typedef Matrix<complex<real>, Arg::nColor> Link;
     const int their_spinor_parity = (arg.nParity == 2) ? 1 - parity : 0;
 
-QUDA_UNROLL
+#pragma unroll
     for (int d = 0; d < 4; d++) { // loop over dimension
 
       // standard - forward direction
diff --git a/include/kernels/dslash_twisted_clover_preconditioned.cuh b/include/kernels/dslash_twisted_clover_preconditioned.cuh
index b1fc925ed1..bc46d106f4 100644
--- a/include/kernels/dslash_twisted_clover_preconditioned.cuh
+++ b/include/kernels/dslash_twisted_clover_preconditioned.cuh
@@ -80,7 +80,7 @@ namespace quda
 
         Vector tmp;
 
-QUDA_UNROLL
+#pragma unroll
         for (int chirality = 0; chirality < 2; chirality++) {
 
           const complex<real> b(0.0, chirality == 0 ? arg.b : -arg.b);
diff --git a/include/kernels/dslash_twisted_mass_preconditioned.cuh b/include/kernels/dslash_twisted_mass_preconditioned.cuh
index 912ff95987..496e12276d 100644
--- a/include/kernels/dslash_twisted_mass_preconditioned.cuh
+++ b/include/kernels/dslash_twisted_mass_preconditioned.cuh
@@ -55,7 +55,7 @@ namespace quda
     typedef Matrix<complex<real>, Arg::nColor> Link;
     const int their_spinor_parity = nParity == 2 ? 1 - parity : 0;
 
-QUDA_UNROLL
+#pragma unroll
     for (int d = 0; d < Arg::nDim; d++) { // loop over dimension
       {                              // Forward gather - compute fwd offset for vector fetch
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
diff --git a/include/kernels/dslash_wilson.cuh b/include/kernels/dslash_wilson.cuh
index 672f590958..cd7575974a 100644
--- a/include/kernels/dslash_wilson.cuh
+++ b/include/kernels/dslash_wilson.cuh
@@ -77,7 +77,7 @@ namespace quda
     // parity for gauge field - include residual parity from 5-d => 4-d checkerboarding
     const int gauge_parity = (Arg::nDim == 5 ? (coord.x_cb / arg.dc.volume_4d_cb + parity) % 2 : parity);
 
-QUDA_UNROLL
+#pragma unroll
     for (int d = 0; d < 4; d++) { // loop over dimension - 4 and not nDim since this is used for DWF as well
       {                           // Forward gather - compute fwd offset for vector fetch
         const int fwd_idx = getNeighborIndexCB(coord, d, +1, arg.dc);
diff --git a/include/kernels/dslash_wilson_clover.cuh b/include/kernels/dslash_wilson_clover.cuh
index a51a473375..ffc8b880e8 100644
--- a/include/kernels/dslash_wilson_clover.cuh
+++ b/include/kernels/dslash_wilson_clover.cuh
@@ -68,7 +68,7 @@ namespace quda
 
         Vector tmp;
 
-QUDA_UNROLL
+#pragma unroll
         for (int chirality = 0; chirality < 2; chirality++) {
           constexpr int n = Arg::nColor * Arg::nSpin / 2;
           HMatrix<real, n> A = arg.A(coord.x_cb, parity, chirality);
diff --git a/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh b/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
index a906f6886e..e6923bf707 100644
--- a/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
+++ b/include/kernels/dslash_wilson_clover_hasenbusch_twist.cuh
@@ -68,7 +68,7 @@ namespace quda
 
         Vector tmp;
 
-QUDA_UNROLL
+#pragma unroll
         for (int chirality = 0; chirality < 2; chirality++) {
           constexpr int n = Arg::nColor * Arg::nSpin / 2;
           HMatrix<real, n> A = arg.A(coord.x_cb, parity, chirality);
diff --git a/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh b/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
index 59ef5fc056..b9a1599e94 100644
--- a/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_hasenbusch_twist_preconditioned.cuh
@@ -82,7 +82,7 @@ namespace quda
 
           Vector tmp;
 
-QUDA_UNROLL
+#pragma unroll
           for (int chirality = 0; chirality < 2; chirality++) {
 
             HMatrix<real, Arg::nColor * Arg::nSpin / 2> A_inv = arg.A_inv(coord.x_cb, parity, chirality);
@@ -110,7 +110,7 @@ QUDA_UNROLL
           Vector x = arg.x(coord.x_cb, my_spinor_parity);
           x.toRel();
           Vector tmp;
-QUDA_UNROLL
+#pragma unroll
           for (int chirality = 0; chirality < 2; chirality++) {
 
             HMatrix<real, Arg::nColor *Arg::nSpin / 2> A = arg.A(coord.x_cb, parity, chirality);
diff --git a/include/kernels/dslash_wilson_clover_preconditioned.cuh b/include/kernels/dslash_wilson_clover_preconditioned.cuh
index b9a2c985c1..9adbc64343 100644
--- a/include/kernels/dslash_wilson_clover_preconditioned.cuh
+++ b/include/kernels/dslash_wilson_clover_preconditioned.cuh
@@ -73,7 +73,7 @@ namespace quda
 
         Vector tmp;
 
-QUDA_UNROLL
+#pragma unroll
         for (int chirality = 0; chirality < 2; chirality++) {
 
           HMatrix<real, Arg::nColor * Arg::nSpin / 2> A = arg.A(coord.x_cb, parity, chirality);
diff --git a/include/kernels/gauge_det_trace.cuh b/include/kernels/gauge_det_trace.cuh
index 5ca4e1e641..bc4fc5d5fa 100644
--- a/include/kernels/gauge_det_trace.cuh
+++ b/include/kernels/gauge_det_trace.cuh
@@ -42,17 +42,17 @@ namespace quda {
     __device__ __host__ inline reduce_t operator()(reduce_t &value, int x_cb, int parity)
     {
       int X[4];
-QUDA_UNROLL
+#pragma unroll
       for(int dr=0; dr<4; ++dr) X[dr] = arg.X[dr];
 
       int x[4];
       getCoords(x, x_cb, X, parity);
-QUDA_UNROLL
+#pragma unroll
       for(int dr=0; dr<4; ++dr) {
         x[dr] += arg.border[dr];
         X[dr] += 2*arg.border[dr];
       }
-QUDA_UNROLL
+#pragma unroll
       for (int mu = 0; mu < 4; mu++) {
         Matrix<complex<typename Arg::real>, Arg::nColor> U = arg.u(mu, linkIndex(x, X), parity);
         auto local = Arg::type == compute_type::determinant ? getDeterminant(U) : getTrace(U);
diff --git a/include/kernels/gauge_fix_fft.cuh b/include/kernels/gauge_fix_fft.cuh
index 410a6cce47..06b0f89b0f 100644
--- a/include/kernels/gauge_fix_fft.cuh
+++ b/include/kernels/gauge_fix_fft.cuh
@@ -238,29 +238,29 @@ namespace quda {
     Float t1 = 0.0;
     //first normalize first row
     //sum of squares of row
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) t1 += norm(U(0,c));
     t1 = (Float)1.0 / sqrt(t1);
     //14
     //used to normalize row
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) U(0,c) *= t1;
     //6
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) t2 += conj(U(0,c)) * U(1,c);
     //24
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) U(1,c) -= t2 * U(0,c);
     //24
     //normalize second row
     //sum of squares of row
     t1 = 0.0;
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) t1 += norm(U(1,c));
     t1 = (Float)1.0 / sqrt(t1);
     //14
     //used to normalize row
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) U(1, c) *= t1;
     //6
     //Reconstruct last row
diff --git a/include/kernels/gauge_fix_ovr.cuh b/include/kernels/gauge_fix_ovr.cuh
index 65e7ad8b87..644d9c2463 100644
--- a/include/kernels/gauge_fix_ovr.cuh
+++ b/include/kernels/gauge_fix_ovr.cuh
@@ -57,12 +57,12 @@ namespace quda {
       using Link = Matrix<complex<typename Arg::real>, 3>;
 
       int X[4];
-QUDA_UNROLL
+#pragma unroll
       for (int dr = 0; dr < 4; dr++) X[dr] = arg.X[dr];
 
       int x[4];
       getCoords(x, x_cb, X, parity);
-QUDA_UNROLL
+#pragma unroll
       for (int dr = 0; dr < 4; ++dr) {
         x[dr] += arg.border[dr];
         X[dr] += 2 * arg.border[dr];
@@ -142,7 +142,7 @@ QUDA_UNROLL
       int parity = arg.parity;
 
       int X[4], x[4];
-QUDA_UNROLL
+#pragma unroll
       for (int dr = 0; dr < 4; dr++) X[dr] = arg.X[dr];
 
       if (!Arg::halo) {
@@ -157,7 +157,7 @@ QUDA_UNROLL
         x[0] = idx % X[0];
       }
 
-QUDA_UNROLL
+#pragma unroll
       for (int dr = 0; dr < 4; dr++) {
         x[dr] += arg.border[dr];
         X[dr] += 2 * arg.border[dr];
diff --git a/include/kernels/gauge_heatbath.cuh b/include/kernels/gauge_heatbath.cuh
index b9f3e3f2c2..c55ae79d72 100644
--- a/include/kernels/gauge_heatbath.cuh
+++ b/include/kernels/gauge_heatbath.cuh
@@ -614,12 +614,12 @@ namespace quda {
       auto parity = arg.parity;
 
       int X[4];
-QUDA_UNROLL
+#pragma unroll
       for (int dr = 0; dr < 4; ++dr) X[dr] = arg.X[dr];
 
       int x[4];
       getCoords(x, x_cb, X, parity);
-QUDA_UNROLL
+#pragma unroll
       for (int dr = 0; dr < 4; ++dr) {
         x[dr] += arg.border[dr];
         X[dr] += 2 * arg.border[dr];
@@ -630,7 +630,7 @@ QUDA_UNROLL
       setZero(&staple);
 
       Link U;
-QUDA_UNROLL
+#pragma unroll
       for (int nu = 0; nu < 4; nu++) if (mu != nu) {
           int dx[4] = { 0, 0, 0, 0 };
           Link link = arg.dataOr(nu, e_cb, parity);
diff --git a/include/kernels/gauge_plaq.cuh b/include/kernels/gauge_plaq.cuh
index b89e699a7b..563d674000 100644
--- a/include/kernels/gauge_plaq.cuh
+++ b/include/kernels/gauge_plaq.cuh
@@ -66,12 +66,12 @@ namespace quda {
 
       int x[4];
       getCoords(x, x_cb, arg.X, parity);
-QUDA_UNROLL
+#pragma unroll
       for (int dr=0; dr<4; ++dr) x[dr] += arg.border[dr]; // extended grid coordinates
 
-QUDA_UNROLL
+#pragma unroll
       for (int mu = 0; mu < 3; mu++) {
-QUDA_UNROLL
+#pragma unroll
         for (int nu = 0; nu < 3; nu++) {
           if (nu >= mu + 1) plaq[0] += plaquette(arg, x, parity, mu, nu);
         }
diff --git a/include/kernels/gauge_qcharge.cuh b/include/kernels/gauge_qcharge.cuh
index a7410e93b0..fadfd45321 100644
--- a/include/kernels/gauge_qcharge.cuh
+++ b/include/kernels/gauge_qcharge.cuh
@@ -54,7 +54,7 @@ namespace quda
       // first compute the field energy
       Link iden;
       setIdentity(&iden);
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<6; i++) {
         // Make traceless
         auto tmp = F[i] - n_inv * getTrace(F[i]) * iden;
@@ -68,7 +68,7 @@ QUDA_UNROLL
       double Q_idx = 0.0;
       double Qi[3] = {0.0,0.0,0.0};
       // unroll computation
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<3; i++) {
         Qi[i] = getTrace(F[i] * F[5 - i]).real();
       }
diff --git a/include/kernels/gauge_utils.cuh b/include/kernels/gauge_utils.cuh
index cb13f832d6..646d0e8e96 100644
--- a/include/kernels/gauge_utils.cuh
+++ b/include/kernels/gauge_utils.cuh
@@ -23,7 +23,7 @@ namespace quda
   {
     setZero(&staple);
     thread_array<int, 4> dx = { };
-QUDA_UNROLL
+#pragma unroll
     for (int mu = 0; mu < 4 ; mu++) {
       // Identify directions orthogonal to the link and
       // ignore the dir_ignore direction (usually the temporal dim
diff --git a/include/kernels/hisq_paths_force.cuh b/include/kernels/hisq_paths_force.cuh
index 907a84877a..893c7ea587 100644
--- a/include/kernels/hisq_paths_force.cuh
+++ b/include/kernels/hisq_paths_force.cuh
@@ -204,7 +204,7 @@ namespace quda {
       {
         int x[4];
         getCoords(x, x_cb, arg.X, parity);
-QUDA_UNROLL
+#pragma unroll
         for (int d=0; d<4; d++) x[d] += arg.border[d];
         int e_cb = linkIndex(x,arg.E);
 
@@ -611,7 +611,7 @@ QUDA_UNROLL
         for (int d=0; d<4; d++) x[d] += arg.border[d];
         int e_cb = linkIndex(x,arg.E);
 
-QUDA_UNROLL
+#pragma unroll
         for (int sig=0; sig<4; ++sig) {
           Link Uw = arg.link(sig, e_cb, parity);
           Link Ox = arg.oProd(sig, e_cb, parity);
@@ -672,7 +672,7 @@ QUDA_UNROLL
          */
 
         // compute the force for forward long links
-QUDA_UNROLL
+#pragma unroll
         for (int sig=0; sig<4; sig++) {
           int point_c = e_cb;
 
diff --git a/include/kernels/laplace.cuh b/include/kernels/laplace.cuh
index 81fdcbab24..da24bf77b7 100644
--- a/include/kernels/laplace.cuh
+++ b/include/kernels/laplace.cuh
@@ -81,7 +81,7 @@ namespace quda
     typedef Matrix<complex<real>, Arg::nColor> Link;
     const int their_spinor_parity = (arg.nParity == 2) ? 1 - parity : 0;
 
-QUDA_UNROLL
+#pragma unroll
     for (int d = 0; d < Arg::nDim; d++) { // loop over dimension
       if (d != dir) {
         {
diff --git a/include/kernels/momentum.cuh b/include/kernels/momentum.cuh
index f3c6e91ab8..be3f79bbd5 100644
--- a/include/kernels/momentum.cuh
+++ b/include/kernels/momentum.cuh
@@ -111,7 +111,7 @@ namespace quda {
       for (int d=0; d<4; d++) x[d] += arg.border[d];
       int e_cb = linkIndex(x, arg.E);
 
-QUDA_UNROLL
+#pragma unroll
       for (int d=0; d<4; d++) {
         Matrix<complex<typename Arg::Float>, Arg::nColor> m = arg.mom(d, x_cb, parity);
         Matrix<complex<typename Arg::Float>, Arg::nColor> f = arg.force(d, e_cb, parity);
diff --git a/include/kernels/multi_blas_core.cuh b/include/kernels/multi_blas_core.cuh
index 3f31acdc2c..0509681abe 100644
--- a/include/kernels/multi_blas_core.cuh
+++ b/include/kernels/multi_blas_core.cuh
@@ -96,7 +96,7 @@ namespace quda
           w = ::quda::zero<complex<typename Arg::real>, Arg::n/2>();
         }
 
-QUDA_UNROLL
+#pragma unroll
         for (int l_ = 0; l_ < Arg::NXZ; l_ += warp_split) {
           const int l = l_ + l_idx;
           if (l < Arg::NXZ || warp_split == 1) {
@@ -143,7 +143,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &, int i, int j) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int k = 0; k < x.size(); k++) y[k] += a(j, i) * x[k];
       }
 
@@ -165,7 +165,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &, int i, int j) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int k = 0; k < x.size(); k++) y[k] = cmac(a(j, i), x[k], y[k]);
       }
 
@@ -212,7 +212,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &w, int i, int j) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int k = 0; k < x.size(); k++) {
           if (j == 0) w[k] = y[k];
           w[k] = cmac(a(j, i), x[k], w[k]);
@@ -242,7 +242,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &w, int i, int) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int k = 0; k < x.size(); k++) {
           y[k] += a[i] * w[k];
           w[k] = b[i] * x[k] + c[i] * w[k];
@@ -278,7 +278,7 @@ QUDA_UNROLL
       // i loops over NYW, j loops over NXZ
       template <typename T> __device__ __host__ inline void operator()(T &x, T &y, T &, T &w, int, int j) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int k = 0; k < x.size(); k++) {
           y[k] = cmac(a[j], x[k], y[k]);
           w[k] = cmac(b[j], x[k], w[k]);
diff --git a/include/kernels/multi_reduce_core.cuh b/include/kernels/multi_reduce_core.cuh
index 875d883786..0fb689a268 100644
--- a/include/kernels/multi_reduce_core.cuh
+++ b/include/kernels/multi_reduce_core.cuh
@@ -106,7 +106,7 @@ namespace quda
         // Each NYW owns its own thread.
         // The NXZ's are all in the same thread block,
         // so they can share the same memory.
-QUDA_UNROLL
+#pragma unroll
         for (int l = 0; l < Arg::NXZ; l++) {
           if (arg.f.read.X) arg.X[l].load(x, i, parity);
           if (arg.f.read.Z) arg.Z[l].load(z, i, parity);
@@ -159,7 +159,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ inline void operator()(reduce_t &sum, T &x, T &y, T &, T &, int, int) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int k=0; k < x.size(); k++) dot_<reduce_t, real>(sum, x[k], y[k]);
       }
 
@@ -190,7 +190,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ inline void operator()(reduce_t &sum, T &x, T &y, T &, T &, int, int) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int k=0; k < x.size(); k++) cdot_<reduce_t, real>(sum, x[k], y[k]);
       }
 
@@ -208,7 +208,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ inline void operator()(reduce_t &sum, T &x, T &y, T &, T &w, int i, int j) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int k = 0; k < x.size(); k++) {
           cdot_<reduce_t, real>(sum, x[k], y[k]);
           if (i == j) w[k] = y[k];
diff --git a/include/kernels/pgauge_init.cuh b/include/kernels/pgauge_init.cuh
index 46525be8ee..583c567d6f 100644
--- a/include/kernels/pgauge_init.cuh
+++ b/include/kernels/pgauge_init.cuh
@@ -73,29 +73,29 @@ namespace quda {
     Float t1 = 0.0;
     //first normalize first row
     //sum of squares of row
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) t1 += norm(U(0,c));
     t1 = (Float)1.0 / sqrt(t1);
     //14
     //used to normalize row
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) U(0,c) *= t1;
     //6
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) t2 += conj(U(0,c)) * U(1,c);
     //24
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) U(1,c) -= t2 * U(0,c);
     //24
     //normalize second row
     //sum of squares of row
     t1 = 0.0;
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) t1 += norm(U(1,c));
     t1 = (Float)1.0 / sqrt(t1);
     //14
     //used to normalize row
-QUDA_UNROLL
+#pragma unroll
     for ( int c = 0; c < 3; c++ ) U(1, c) *= t1;
     //6
     //Reconstruct lat row
diff --git a/include/kernels/prolongator.cuh b/include/kernels/prolongator.cuh
index 91209e1bcb..6843ceef84 100644
--- a/include/kernels/prolongator.cuh
+++ b/include/kernels/prolongator.cuh
@@ -48,9 +48,9 @@ namespace quda {
     int parity_coarse = (x_coarse >= arg.in.VolumeCB()) ? 1 : 0;
     int x_coarse_cb = x_coarse - parity_coarse*arg.in.VolumeCB();
 
-QUDA_UNROLL
+#pragma unroll
     for (int s=0; s<Arg::fineSpin; s++) {
-QUDA_UNROLL
+#pragma unroll
       for (int c=0; c<Arg::coarseColor; c++) {
         out[s*Arg::coarseColor+c] = arg.in(parity_coarse, x_coarse_cb, arg.spin_map(s,parity), c);
       }
@@ -70,25 +70,25 @@ QUDA_UNROLL
 
     constexpr int color_unroll = 2;
 
-QUDA_UNROLL
+#pragma unroll
     for (int s=0; s<Arg::fineSpin; s++) {
-QUDA_UNROLL
+#pragma unroll
       for (int fine_color_local = 0; fine_color_local < fine_color_per_thread; fine_color_local++) {
         int i = fine_color_block + fine_color_local; // global fine color index
 
         complex<typename Arg::real> partial[color_unroll];
-QUDA_UNROLL
+#pragma unroll
         for (int k=0; k<color_unroll; k++) partial[k] = 0.0;
 
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<Arg::coarseColor; j+=color_unroll) {
           // V is a ColorMatrixField with internal dimensions Ns * Nc * Nvec
-QUDA_UNROLL
+#pragma unroll
           for (int k=0; k<color_unroll; k++)
             partial[k] = cmac(arg.V(v_parity, x_cb, s, i, j + k), in[s * Arg::coarseColor + j + k], partial[k]);
         }
 
-QUDA_UNROLL
+#pragma unroll
         for (int k=1; k<color_unroll; k++) partial[0] += partial[k];
         arg.out(spinor_parity, x_cb, s, i) = partial[0];
       }
diff --git a/include/kernels/reduce_core.cuh b/include/kernels/reduce_core.cuh
index c0e3fa679f..c86a11c8cd 100644
--- a/include/kernels/reduce_core.cuh
+++ b/include/kernels/reduce_core.cuh
@@ -127,7 +127,7 @@ namespace quda
       Norm1(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i=0; i < x.size(); i++) sum += norm1_<reduce_t, real>(x[i]);
       }
       constexpr int flops() const { return 2; }   //! flops per element
@@ -149,7 +149,7 @@ QUDA_UNROLL
       Norm2(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) norm2_<reduce_t, real>(sum, x[i]);
       }
       constexpr int flops() const { return 2; }   //! flops per element
@@ -172,7 +172,7 @@ QUDA_UNROLL
       Dot(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) dot_<reduce_t, real>(sum, x[i], y[i]);
       }
       constexpr int flops() const { return 2; }   //! flops per element
@@ -191,7 +191,7 @@ QUDA_UNROLL
       axpbyzNorm2(const real &a, const real &b) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           z[i] = a * x[i] + b * y[i];
           norm2_<reduce_t, real>(sum, z[i]);
@@ -212,7 +212,7 @@ QUDA_UNROLL
       AxpyReDot(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] += a * x[i];
           dot_<reduce_t, real>(sum, x[i], y[i]);
@@ -233,7 +233,7 @@ QUDA_UNROLL
       caxpyNorm2(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           norm2_<reduce_t, real>(sum, y[i]);
@@ -256,7 +256,7 @@ QUDA_UNROLL
       caxpyxmaznormx(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           x[i] = cmac(-a, z[i], x[i]);
@@ -281,7 +281,7 @@ QUDA_UNROLL
       cabxpyzaxnorm(const complex<real> &a, const complex<real> &b) : a(a.real()), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           x[i] *= a;
           z[i] = cmac(b, x[i], y[i]);
@@ -312,7 +312,7 @@ QUDA_UNROLL
       Cdot(const complex<real> &, const complex<real> &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) cdot_<reduce_t, real>(sum, x[i], y[i]);
       }
       constexpr int flops() const { return 4; }   //! flops per element
@@ -332,7 +332,7 @@ QUDA_UNROLL
       caxpydotzy(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           cdot_<reduce_t, real>(sum, z[i], y[i]);
@@ -396,7 +396,7 @@ QUDA_UNROLL
       caxpbypzYmbwcDotProductUYNormY_(const complex<real> &a, const complex<real> &b) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &v) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           y[i] = cmac(a, x[i], y[i]);
           y[i] = cmac(b, z[i], y[i]);
@@ -422,7 +422,7 @@ QUDA_UNROLL
       axpyCGNorm2(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           auto y_new = y[i] + a * x[i];
           norm2_<real_reduce_t, real>(sum[0], y_new);
@@ -459,7 +459,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ void operator()(reduce_t &, T &x, T &y, T &, T &, T &)
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           norm2_<real_reduce_t, real>(aux[0], x[i]);
           norm2_<real_reduce_t, real>(aux[1], y[i]);
@@ -504,7 +504,7 @@ QUDA_UNROLL
 
       template <typename T> __device__ __host__ void operator()(reduce_t &, T &x, T &y, T &z, T &, T &)
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           norm2_<real_reduce_t, real>(aux[0], x[i] + y[i]);
           norm2_<real_reduce_t, real>(aux[1], z[i]);
@@ -536,7 +536,7 @@ QUDA_UNROLL
       tripleCGReduction_(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           norm2_<real_reduce_t, real>(sum[0], x[i]);
           norm2_<real_reduce_t, real>(sum[1], y[i]);
@@ -561,7 +561,7 @@ QUDA_UNROLL
       quadrupleCGReduction_(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           norm2_<real_reduce_t, real>(sum[0], x[i]);
           norm2_<real_reduce_t, real>(sum[1], y[i]);
@@ -588,7 +588,7 @@ QUDA_UNROLL
       quadrupleCG3InitNorm_(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &v) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           z[i] = x[i];
           w[i] = y[i];
@@ -619,7 +619,7 @@ QUDA_UNROLL
       quadrupleCG3UpdateNorm_(const real &a, const real &b) : a(a), b(b) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &v) const
       {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           auto tmpx = x[i];
           auto tmpy = y[i];
diff --git a/include/kernels/restrictor.cuh b/include/kernels/restrictor.cuh
index 28d0779d73..817e0513a2 100644
--- a/include/kernels/restrictor.cuh
+++ b/include/kernels/restrictor.cuh
@@ -73,33 +73,33 @@ namespace quda {
     const int spinor_parity = (arg.nParity == 2) ? parity : 0;
     const int v_parity = (arg.V.Nparity() == 2) ? parity : 0;
 
-QUDA_UNROLL
+#pragma unroll
     for (int s=0; s<Arg::fineSpin; s++)
-QUDA_UNROLL
+#pragma unroll
       for (int coarse_color_local=0; coarse_color_local<coarse_color_per_thread; coarse_color_local++) {
 	out[s*coarse_color_per_thread+coarse_color_local] = 0.0;
       }
 
-QUDA_UNROLL
+#pragma unroll
     for (int coarse_color_local=0; coarse_color_local<coarse_color_per_thread; coarse_color_local++) {
       int i = coarse_color_block + coarse_color_local;
-QUDA_UNROLL
+#pragma unroll
       for (int s=0; s<Arg::fineSpin; s++) {
 
 	constexpr int color_unroll = Arg::fineColor == 3 ? 3 : 2;
 
 	complex<typename Arg::real> partial[color_unroll];
-QUDA_UNROLL
+#pragma unroll
 	for (int k=0; k<color_unroll; k++) partial[k] = 0.0;
 
-QUDA_UNROLL
+#pragma unroll
 	for (int j=0; j<Arg::fineColor; j+=color_unroll) {
-QUDA_UNROLL
+#pragma unroll
 	  for (int k=0; k<color_unroll; k++)
 	    partial[k] = cmac(conj(arg.V(v_parity, x_cb, s, j+k, i)), arg.in(spinor_parity, x_cb, s, j+k), partial[k]);
 	}
 
-QUDA_UNROLL
+#pragma unroll
 	for (int k=0; k<color_unroll; k++) out[s*coarse_color_per_thread + coarse_color_local] += partial[k];
       }
     }
@@ -137,9 +137,9 @@ QUDA_UNROLL
         rotateCoarseColor(tmp, arg, parity, x_fine_cb, coarse_color_block);
 
         // perform any local spin coarsening
-QUDA_UNROLL
+#pragma unroll
         for (int s = 0; s<Arg::fineSpin; s++) {
-QUDA_UNROLL
+#pragma unroll
           for (int v = 0; v<coarse_color_per_thread; v++) {
             reduced[arg.spin_map(s,parity) * coarse_color_per_thread + v] += tmp[s*coarse_color_per_thread + v];
           }
@@ -155,9 +155,9 @@ QUDA_UNROLL
         const int parity_coarse = x_coarse >= arg.out.VolumeCB() ? 1 : 0;
         const int x_coarse_cb = x_coarse - parity_coarse*arg.out.VolumeCB();
 
-QUDA_UNROLL
+#pragma unroll
         for (int s = 0; s < Arg::coarseSpin; s++) {
-QUDA_UNROLL
+#pragma unroll
           for (int coarse_color_local=0; coarse_color_local<coarse_color_per_thread; coarse_color_local++) {
             int v = coarse_color_thread * coarse_color_per_thread + coarse_color_local;
             arg.out(parity_coarse, x_coarse_cb, s, v) = reduced[s*coarse_color_per_thread+coarse_color_local];
diff --git a/include/kernels/staggered_coarse_op_kernel.cuh b/include/kernels/staggered_coarse_op_kernel.cuh
index 37429b1f38..7fe913bfdf 100644
--- a/include/kernels/staggered_coarse_op_kernel.cuh
+++ b/include/kernels/staggered_coarse_op_kernel.cuh
@@ -96,7 +96,7 @@ namespace quda {
       int hyperCorner = 4*(coord[3]%2)+2*(coord[2]%2)+(coord[1]%2);
       int c_row = 8*ic_f + hyperCorner;
 
-QUDA_UNROLL
+#pragma unroll
       for (int mu=0; mu < nDim; mu++) {
         // The coarse color column depends on my fine hypercube+mu corner.
         coord[mu]++; // linkIndexP1, it's fine if this wraps because we're modding by 2.
@@ -126,9 +126,9 @@ QUDA_UNROLL
 
       // lastly, add staggered mass term to diagonal
       if (ic_f == 0 && jc_f == 0 && x_cb < arg.coarseVolumeCB) {
-QUDA_UNROLL
+#pragma unroll
         for (int s = 0; s < Arg::coarseSpin; s++) {
-QUDA_UNROLL
+#pragma unroll
           for (int c = 0; c < Arg::coarseColor; c++) {
             arg.X(0,parity,x_cb,s,s,c,c) = complex(arg.two_mass, 0.0); // staggered conventions. No need to +=
           } //Color
diff --git a/include/kernels/staggered_kd_apply_xinv_kernel.cuh b/include/kernels/staggered_kd_apply_xinv_kernel.cuh
index 762350a498..bbe8b70166 100644
--- a/include/kernels/staggered_kd_apply_xinv_kernel.cuh
+++ b/include/kernels/staggered_kd_apply_xinv_kernel.cuh
@@ -72,7 +72,7 @@ namespace quda {
 
       // Get location of unit corner of hypercube
       int x_c[nDim];
-QUDA_UNROLL
+#pragma unroll
       for (int d = 0; d < nDim; d++)
         x_c[d] = 2 * (coord[d] / 2);
 
@@ -84,13 +84,13 @@ QUDA_UNROLL
 
       // Begin accumulating into the output vector
       int nbr_corner = 0;
-QUDA_UNROLL
+#pragma unroll
       for (int nbr_parity = 0; nbr_parity < 2; nbr_parity++) {
-QUDA_UNROLL
+#pragma unroll
         for (int nbr_t = 0; nbr_t < 2; nbr_t++) {
-QUDA_UNROLL
+#pragma unroll
           for (int nbr_z = 0; nbr_z < 2; nbr_z++) {
-QUDA_UNROLL
+#pragma unroll
             for (int nbr_y = 0; nbr_y < 2; nbr_y++) {
               const int offset[4] = { (nbr_parity + nbr_t + nbr_z + nbr_y) & 1, nbr_y, nbr_z, nbr_t };
               const int neighbor_idx = linkIndexShift(x_c, offset, arg.dim);
diff --git a/include/kernels/staggered_outer_product.cuh b/include/kernels/staggered_outer_product.cuh
index 534e9f3d90..e0a333652d 100644
--- a/include/kernels/staggered_outer_product.cuh
+++ b/include/kernels/staggered_outer_product.cuh
@@ -61,7 +61,7 @@ namespace quda {
       matrix result;
       const vector x = arg.inA(x_cb, 0);
 
-QUDA_UNROLL
+#pragma unroll
       for (int dim=0; dim<4; ++dim) {
         int shift[4] = {0,0,0,0};
         shift[dim] = 1;
diff --git a/include/linalg.cuh b/include/linalg.cuh
index f62eb2071a..da18b4a6cf 100644
--- a/include/linalg.cuh
+++ b/include/linalg.cuh
@@ -51,12 +51,12 @@ namespace quda {
       {
 	const Mat<T,N> &L = L_;
 
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<N; i++) {
-QUDA_UNROLL
+#pragma unroll
 	  for (int j=0; j<N; j++) if (j<i+1) {
 	    complex<T> s = 0;
-QUDA_UNROLL
+#pragma unroll
 	    for (int k=0; k<N; k++) {
 	      if (k==0) {
 		s.x  = L(i,k).real()*L(j,k).real();
@@ -101,10 +101,10 @@ QUDA_UNROLL
       __device__ __host__ inline Vector forward(const Vector &b) {
 	const Mat<T,N> &L = L_;
 	Vector x;
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<N; i++) {
 	  x(i) = b(i);
-QUDA_UNROLL
+#pragma unroll
 	  for (int j=0; j<N; j++) if (j<i) {
 	    x(i).x -= L(i,j).real()*x(j).real();
 	    x(i).x += L(i,j).imag()*x(j).imag();
@@ -127,10 +127,10 @@ QUDA_UNROLL
       __device__ __host__ inline Vector backward(const Vector &b) {
 	const Mat<T,N> &L = L_;
 	Vector x;
-QUDA_UNROLL
+#pragma unroll
 	for (int i=N-1; i>=0; i--) {
 	  x(i) = b(i);
-QUDA_UNROLL
+#pragma unroll
 	  for (int j=0; j<N; j++) if (j>=i+1) {
 	    x(i).x -= L(i,j).real()*x(j).real();
 	    x(i).x += L(i,j).imag()*x(j).imag();
@@ -157,14 +157,14 @@ QUDA_UNROLL
       {
         // copy source vector into factorization precision
 	ColorSpinor<T,1,N> b_;
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < N; i++) b_(i) = b(i);
 
         auto x_ = backward(forward(b_));
 
         // copy solution vector into desired precision
         Vector x;
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < N; i++) x(i) = x_(i);
 
         return x;
@@ -179,17 +179,17 @@ QUDA_UNROLL
 	matrix_t Ainv;
 	ColorSpinor<T,1,N> v;
 
-QUDA_UNROLL
+#pragma unroll
 	for (int k=0;k<N;k++) {
 
 	  // forward substitute
 	  if (!fast) v(k) = complex<T>(static_cast<T>(1.0)/L(k,k).real());
 	  else v(k) = L(k,k).real();
 
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<N; i++) if (i>k) {
 	    v(i) = complex<T>(0.0);
-QUDA_UNROLL
+#pragma unroll
 	    for (int j=0; j<N; j++) if (j>=k && j<i) {
 	      v(i).x -= L(i,j).real() * v(j).real();
 	      v(i).x += L(i,j).imag() * v(j).imag();
@@ -204,9 +204,9 @@ QUDA_UNROLL
 	  if (!fast) v(N-1) *= static_cast<T>(1.0) / L(N-1,N-1);
 	  else v(N-1) *= L(N-1,N-1);
 
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=N-2; i>=0; i--) if (i>=k) {
-QUDA_UNROLL
+#pragma unroll
 	    for (int j=0; j<N; j++) if (j>i) {
 	      v(i).x -= L(i,j).real() * v(j).real();
 	      v(i).x += L(i,j).imag() * v(j).imag();
@@ -220,7 +220,7 @@ QUDA_UNROLL
 	  // Overwrite column k
 	  Ainv(k,k) = v(k);
 
-QUDA_UNROLL
+#pragma unroll
 	  for(int i=0;i<N;i++) if (i>k) Ainv(i,k) = v(i);
 	}
 
diff --git a/include/matrix_tile.cuh b/include/matrix_tile.cuh
index 1cf9d0195d..b16e18b352 100644
--- a/include/matrix_tile.cuh
+++ b/include/matrix_tile.cuh
@@ -20,9 +20,9 @@ namespace quda {
 
     template <typename T_> inline __device__ __host__ void operator*=(const T_ &a)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<n; j++) {
           tile[i*n+j] *= a;
         }
@@ -37,11 +37,11 @@ QUDA_UNROLL
     template <int k, bool ghost_a, bool ghost_b>
     inline __device__ __host__ void mma_nn(const MatrixTile<T, m, k, ghost_a> &a, const MatrixTile<T, k, n, ghost_b> &b)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < n; j++) {
-QUDA_UNROLL
+#pragma unroll
           for (int l = 0; l < k; l++) {
             tile[i*n+j] = cmac(a.tile[i*k+l], b.tile[l*n+j], tile[i*n+j]);
           }
@@ -57,11 +57,11 @@ QUDA_UNROLL
     template <int k, bool ghost_a, bool ghost_b>
     inline __device__ __host__ void mma_nt(const MatrixTile<T, m, k, ghost_a> &a, const MatrixTile<T, n, k, ghost_b> &b)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < n; j++) {
-QUDA_UNROLL
+#pragma unroll
           for (int l = 0; l < k; l++) {
             tile[i*n+j] = cmac(a.tile[i*k+l], conj(b.tile[j*k+l]), tile[i*n+j]);
           }
@@ -77,11 +77,11 @@ QUDA_UNROLL
     template <int k, bool ghost_a, bool ghost_b>
     inline __device__ __host__ void mma_tn(const MatrixTile<T, k, m, ghost_a> &a, const MatrixTile<T, k, n, ghost_b> &b)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < n; j++) {
-QUDA_UNROLL
+#pragma unroll
           for (int l = 0; l < k; l++) {
             tile[i*n+j] = cmac(conj(a.tile[l*m+i]), b.tile[l*n+j], tile[i*n+j]);
           }
@@ -94,9 +94,9 @@ QUDA_UNROLL
      */
     template <typename Accessor> inline __device__ __host__ std::enable_if_t<Accessor::supports_ghost_zone, void> load(const Accessor &a, int d, int parity, int x_cb, int i0, int j0)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<n; j++) {
           if (ghost) // ideally `if constexpr`, if we had this we could avoid this whole `supports_ghost_zone` thing.
             tile[i*n+j] = a.Ghost(d, parity, x_cb, i0 + i, j0 + j);
@@ -112,9 +112,9 @@ QUDA_UNROLL
     template <typename Accessor> inline __device__ __host__ std::enable_if_t<!Accessor::supports_ghost_zone, void> load(const Accessor &a, int d, int parity, int x_cb, int i0, int j0)
     {
       static_assert(!ghost, "Cannot use ghost MatrixTile with field that doesn't support ghost zones");
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<n; j++) {
           tile[i*n+j] = a(d, parity, x_cb, i0 + i, j0 + j);
         }
@@ -123,9 +123,9 @@ QUDA_UNROLL
 
     template <typename Accessor> inline __device__ __host__ std::enable_if_t<Accessor::supports_ghost_zone, void> load(const Accessor &a, int d, int parity, int x_cb, int si, int sj, int i0, int j0)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<n; j++) {
           if (ghost)
             tile[i*n+j] = a.Ghost(d, parity, x_cb, si, sj, i0 + i, j0 + j);
@@ -138,9 +138,9 @@ QUDA_UNROLL
     template <typename Accessor> inline __device__ __host__ std::enable_if_t<!Accessor::supports_ghost_zone, void> load(const Accessor &a, int d, int parity, int x_cb, int si, int sj, int i0, int j0)
     {
       static_assert(!ghost, "Cannot use ghost MatrixTile with field that doesn't support ghost zones");
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<n; j++) {
           tile[i*n+j] = a(d, parity, x_cb, si, sj, i0 + i, j0 + j);
         }
@@ -149,9 +149,9 @@ QUDA_UNROLL
 
     template <typename Accessor> inline __device__ __host__ void save(Accessor &a, int d, int parity, int x_cb, int i0, int j0)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < n; j++) {
           if (ghost)
             a.Ghost(d, parity, x_cb, i0 + i, j0 + j) = tile[i*n+j];
@@ -163,9 +163,9 @@ QUDA_UNROLL
 
     template <typename Accessor> inline __device__ __host__ void save(Accessor &a, int d, int parity, int x_cb, int si, int sj, int i0, int j0)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < n; j++) {
           if (ghost)
             a.Ghost(d, parity, x_cb, si, sj, i0 + i, j0 + j) = tile[i*n+j];
@@ -177,9 +177,9 @@ QUDA_UNROLL
 
     template <typename Accessor> inline __device__ __host__ void loadCS(const Accessor &a, int d, int dir, int parity, int x_cb, int s, int i0, int j0)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<n; j++) {
           if (ghost)
             tile[i*n+j] = a.Ghost(d, dir, parity, x_cb, s, i0 + i, j0 + j);
@@ -191,9 +191,9 @@ QUDA_UNROLL
 
     template <typename Accessor> inline __device__ __host__ void saveCS(Accessor &a, int d, int dir, int parity, int x_cb, int s, int i0, int j0)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<n; j++) {
           if (ghost)
             a.Ghost(d, dir, parity, x_cb, s, i0 + i, j0 + j) = tile[i*n+j];
@@ -206,18 +206,18 @@ QUDA_UNROLL
     inline __device__ __host__ real abs_max()
     {
       real maxTile[m*n];
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < n; j++) {
           maxTile[i*n+j] = fmax(fabs(tile[i*n+j].real()), fabs(tile[i*n+j].imag()));
         }
       }
 
       real max = 0.0;
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < m; i++) {
-QUDA_UNROLL
+#pragma unroll
         for (int j = 0; j < n; j++) {
           max = fmax(max, maxTile[i*n+j]);
         }
diff --git a/include/quda_api.h b/include/quda_api.h
index 9382fe80d6..3dc9b1b545 100644
--- a/include/quda_api.h
+++ b/include/quda_api.h
@@ -11,10 +11,6 @@
 #include "targets/omptarget/quda_api.h"
 #endif
 
-#ifndef QUDA_UNROLL
-#define QUDA_UNROLL _Pragma("unroll")
-#endif
-
 /**
    @file quda_api.h
 
diff --git a/include/quda_matrix.h b/include/quda_matrix.h
index 8f667be395..c922fce373 100644
--- a/include/quda_matrix.h
+++ b/include/quda_matrix.h
@@ -82,13 +82,13 @@ namespace quda {
 
         template <class U> __device__ __host__ inline Matrix(const Matrix<U, N> &a)
         {
-QUDA_UNROLL
+#pragma unroll
           for (int i = 0; i < N * N; i++) data[i] = a.data[i];
         }
 
         __device__ __host__ inline Matrix(const T data_[])
         {
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<N*N; i++) data[i] = data_[i];
         }
 
@@ -116,7 +116,7 @@ QUDA_UNROLL
 
 	template<class U>
 	  __device__ __host__ inline void operator=(const Matrix<U,N> & b) {
-QUDA_UNROLL
+#pragma unroll
 	  for (int i=0; i<N*N; i++) data[i] = b.data[i];
 	}
 
@@ -139,10 +139,10 @@ QUDA_UNROLL
         */
         __device__ __host__ inline real L1() {
           real l1 = 0;
-QUDA_UNROLL
+#pragma unroll
           for (int j=0; j<N; j++) {
             real col_sum = 0;
-QUDA_UNROLL
+#pragma unroll
             for (int i=0; i<N; i++) {
               col_sum += abs(data[i*N + j]);
             }
@@ -158,9 +158,9 @@ QUDA_UNROLL
         */
         __device__ __host__ inline real L2() {
           real l2 = 0;
-QUDA_UNROLL
+#pragma unroll
           for (int j=0; j<N; j++) {
-QUDA_UNROLL
+#pragma unroll
             for (int i=0; i<N; i++) {
               l2 += norm(data[i*N + j]);
             }
@@ -175,10 +175,10 @@ QUDA_UNROLL
         */
         __device__ __host__ inline real Linf() {
           real linf = 0;
-QUDA_UNROLL
+#pragma unroll
           for (int i=0; i<N; i++) {
             real row_sum = 0;
-QUDA_UNROLL
+#pragma unroll
             for (int j=0; j<N; j++) {
               row_sum += abs(data[i*N + j]);
             }
@@ -206,12 +206,12 @@ QUDA_UNROLL
         {
           const auto identity = conj(*this) * *this;
 
-QUDA_UNROLL
+#pragma unroll
           for (int i=0; i<N; ++i){
             if( fabs(identity(i,i).real() - 1.0) > max_error ||
                 fabs(identity(i,i).imag()) > max_error) return false;
 
-QUDA_UNROLL
+#pragma unroll
             for (int j=i+1; j<N; ++j){
               if( fabs(identity(i,j).real()) > max_error ||
                   fabs(identity(i,j).imag()) > max_error ||
@@ -222,9 +222,9 @@ QUDA_UNROLL
             }
           }
 
-QUDA_UNROLL
+#pragma unroll
           for (int i=0; i<N; i++) {
-QUDA_UNROLL
+#pragma unroll
             for (int j=0; j<N; j++) {
               if (is_nan((*this)(i, j).real()) || is_nan((*this)(i, j).imag())) return false;
             }
@@ -304,7 +304,7 @@ QUDA_UNROLL
       constexpr int size() const { return N * N; }
 
       __device__ __host__ inline HMatrix() {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < N * N; i++) data[i] = (T)0.0;
       }
 
@@ -314,7 +314,7 @@ QUDA_UNROLL
       HMatrix &operator=(HMatrix<T, N> &&) = default;
 
       __device__ __host__ inline HMatrix(const T data_[]) {
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<N*N; i++) data[i] = data_[i];
       }
 
@@ -335,7 +335,7 @@ QUDA_UNROLL
 
       template<class U>
 	__device__ __host__ inline void operator=(const HMatrix<U,N> & b) {
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<N*N; i++) data[i] = b.data[i];
       }
 
@@ -352,15 +352,15 @@ QUDA_UNROLL
       __device__ __host__ inline HMatrix<T,N> square() const {
 	HMatrix<T,N> result;
 	complex<T> tmp;
-QUDA_UNROLL
+#pragma unroll
 	for (int i=0; i<N; i++) {
-QUDA_UNROLL
+#pragma unroll
 	  for (int k=0; k<N; k++) if (i<=k) { // else compiler can't handle triangular unroll
               tmp.real((*this)(i, 0).real() * (*this)(0, k).real());
               tmp.real(tmp.real() - (*this)(i, 0).imag() * (*this)(0, k).imag());
               tmp.imag((*this)(i, 0).real() * (*this)(0, k).imag());
               tmp.imag(tmp.imag() + (*this)(i, 0).imag() * (*this)(0, k).real());
-QUDA_UNROLL
+#pragma unroll
 	    for (int j=1; j<N; j++) {
               tmp.real(tmp.real() + (*this)(i, j).real() * (*this)(j, k).real());
               tmp.real(tmp.real() - (*this)(i, j).imag() * (*this)(j, k).imag());
@@ -381,7 +381,7 @@ QUDA_UNROLL
       {
         HMatrix<T, N> result;
         T max = static_cast<T>(0.0);
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < N * N; i++) max = (abs(data[i]) > max ? abs(data[i]) : max);
         return max;
       }
@@ -401,9 +401,9 @@ QUDA_UNROLL
 
   template<class T,int N>
     __device__ __host__ Matrix<T,N>::Matrix(const HMatrix<typename RealType<T>::type,N> &a) {
-QUDA_UNROLL
+#pragma unroll
     for (int i=0; i<N; i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int j=0; j<N; j++) {
 	(*this)(i,j) = a(i,j);
       }
@@ -432,7 +432,7 @@ QUDA_UNROLL
     __device__ __host__ inline Mat<T,N> operator+(const Mat<T,N> & a, const Mat<T,N> & b)
     {
       Mat<T,N> result;
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < a.size(); i++) result.data[i] = a.data[i] + b.data[i];
       return result;
     }
@@ -441,7 +441,7 @@ QUDA_UNROLL
   template< template<typename,int> class Mat, class T, int N>
     __device__ __host__ inline Mat<T,N> operator+=(Mat<T,N> & a, const Mat<T,N> & b)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < a.size(); i++) a.data[i] += b.data[i];
       return a;
     }
@@ -449,7 +449,7 @@ QUDA_UNROLL
   template< template<typename,int> class Mat, class T, int N>
     __device__ __host__ inline Mat<T,N> operator+=(Mat<T,N> & a, const T & b)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < a.rows(); i++) a(i, i) += b;
       return a;
     }
@@ -457,7 +457,7 @@ QUDA_UNROLL
   template< template<typename,int> class Mat, class T, int N>
     __device__ __host__ inline Mat<T,N> operator-=(Mat<T,N> & a, const Mat<T,N> & b)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < a.size(); i++) a.data[i] -= b.data[i];
       return a;
     }
@@ -466,7 +466,7 @@ QUDA_UNROLL
     __device__ __host__ inline Mat<T,N> operator-(const Mat<T,N> & a, const Mat<T,N> & b)
     {
       Mat<T,N> result;
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < a.size(); i++) result.data[i] = a.data[i] - b.data[i];
       return result;
     }
@@ -474,7 +474,7 @@ QUDA_UNROLL
   template< template<typename,int> class Mat, class T, int N, class S>
     __device__ __host__ inline Mat<T,N> operator*(const S & scalar, const Mat<T,N> & a){
       Mat<T,N> result;
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < a.size(); ++i) result.data[i] = scalar * a.data[i];
       return result;
     }
@@ -493,7 +493,7 @@ QUDA_UNROLL
   template< template<typename,int> class Mat, class T, int N>
     __device__ __host__ inline Mat<T,N> operator-(const Mat<T,N> & a){
       Mat<T,N> result;
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < a.size(); ++i) result.data[i] = -a.data[i];
       return result;
     }
@@ -506,12 +506,12 @@ QUDA_UNROLL
     __device__ __host__ inline Mat<T,N> operator*(const Mat<T,N> &a, const Mat<T,N> &b)
     {
       Mat<T,N> result;
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; i++) {
-QUDA_UNROLL
+#pragma unroll
 	for (int k=0; k<N; k++) {
 	  result(i,k) = a(i,0) * b(0,k);
-QUDA_UNROLL
+#pragma unroll
 	  for (int j=1; j<N; j++) {
 	    result(i,k) += a(i,j) * b(j,k);
 	  }
@@ -527,15 +527,15 @@ QUDA_UNROLL
     __device__ __host__ inline Matrix<complex<T>,N> operator*(const Matrix<complex<T>,N> &a, const Matrix<complex<T>,N> &b)
     {
       Matrix<complex<T>,N> result;
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; i++) {
-QUDA_UNROLL
+#pragma unroll
 	for (int k=0; k<N; k++) {
           result(i, k).real(a(i, 0).real() * b(0, k).real());
           result(i, k).real(result(i, k).real() - a(i, 0).imag() * b(0, k).imag());
           result(i, k).imag(a(i, 0).real() * b(0, k).imag());
           result(i, k).imag(result(i, k).imag() + a(i, 0).imag() * b(0, k).real());
-QUDA_UNROLL
+#pragma unroll
 	  for (int j=1; j<N; j++) {
             result(i, k).real(result(i, k).real() + a(i, j).real() * b(j, k).real());
             result(i, k).real(result(i, k).real() - a(i, j).imag() * b(j, k).imag());
@@ -562,12 +562,12 @@ QUDA_UNROLL
                                                                                      const Matrix<U, N> &b)
   {
     Matrix<typename PromoteTypeId<T, U>::type, N> result;
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; i++) {
-QUDA_UNROLL
+#pragma unroll
 	for (int k=0; k<N; k++) {
 	  result(i,k) = a(i,0) * b(0,k);
-QUDA_UNROLL
+#pragma unroll
 	  for (int j=1; j<N; j++) {
 	    result(i,k) += a(i,j) * b(j,k);
 	  }
@@ -593,9 +593,9 @@ QUDA_UNROLL
     __device__ __host__ inline
     Matrix<T,N> conj(const Matrix<T,N> & other){
       Matrix<T,N> result;
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<N; ++j){
           result(i,j) = conj( other(j,i) );
         }
@@ -650,10 +650,10 @@ QUDA_UNROLL
     __device__ __host__ inline
     void setIdentity(Matrix<T,N>* m){
 
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
         (*m)(i,i) = 1;
-QUDA_UNROLL
+#pragma unroll
         for (int j=i+1; j<N; ++j){
           (*m)(i,j) = (*m)(j,i) = 0;
         }
@@ -665,10 +665,10 @@ QUDA_UNROLL
     __device__ __host__ inline
     void setIdentity(Matrix<float2,N>* m){
 
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
         (*m)(i,i) = make_float2(1,0);
-QUDA_UNROLL
+#pragma unroll
         for (int j=i+1; j<N; ++j){
           (*m)(i,j) = (*m)(j,i) = make_float2(0.,0.);
         }
@@ -680,10 +680,10 @@ QUDA_UNROLL
     __device__ __host__ inline
     void setIdentity(Matrix<double2,N>* m){
 
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
         (*m)(i,i) = make_double2(1,0);
-QUDA_UNROLL
+#pragma unroll
         for (int j=i+1; j<N; ++j){
           (*m)(i,j) = (*m)(j,i) = make_double2(0.,0.);
         }
@@ -696,9 +696,9 @@ QUDA_UNROLL
     __device__ __host__ inline
     void setZero(Matrix<T,N>* m){
 
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<N; ++j){
           (*m)(i,j) = 0;
         }
@@ -710,9 +710,9 @@ QUDA_UNROLL
     __device__ __host__ inline
     void setZero(Matrix<float2,N>* m){
 
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<N; ++j){
           (*m)(i,j) = make_float2(0.,0.);
         }
@@ -724,9 +724,9 @@ QUDA_UNROLL
     __device__ __host__ inline
     void setZero(Matrix<double2,N>* m){
 
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<N; ++j){
           (*m)(i,j) = make_double2(0.,0.);
         }
@@ -742,9 +742,9 @@ QUDA_UNROLL
 
     // second make it traceless
     real imag_trace = 0.0;
-QUDA_UNROLL
+#pragma unroll
     for (int i=0; i<N; i++) imag_trace += am(i,i).y;
-QUDA_UNROLL
+#pragma unroll
     for (int i=0; i<N; i++) {
       am(i,i).y -= imag_trace/N;
     }
@@ -759,9 +759,9 @@ QUDA_UNROLL
 
     // second make it traceless
     real imag_trace = 0.0;
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < N; i++) imag_trace += am(i, i).y;
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < N; i++) { am(i, i).y -= imag_trace / N; }
     // third scale out anti hermitian part
     Complex i_2(0.0, 0.5);
@@ -799,7 +799,7 @@ QUDA_UNROLL
     __device__  __host__ inline
     void copyColumn(const Matrix<T,N>& m, int c, Array<T,N>* a)
     {
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
         (*a)[i] = m(i,c); // c is the column index
       }
@@ -809,9 +809,9 @@ QUDA_UNROLL
   // Need some print utilities
   template<class T, int N>
     std::ostream & operator << (std::ostream & os, const Matrix<T,N> & m){
-QUDA_UNROLL
+#pragma unroll
       for (int i=0; i<N; ++i){
-QUDA_UNROLL
+#pragma unroll
         for (int j=0; j<N; ++j){
           os << m(i,j) << " ";
         }
@@ -922,9 +922,9 @@ QUDA_UNROLL
       int j=0;
 
       //error = ||U^dagger U - I||_L2
-QUDA_UNROLL
+#pragma unroll
       for (i=0; i<3; ++i)
-QUDA_UNROLL
+#pragma unroll
 	for (j=0; j<3; ++j)
 	  if(i==j) {
 	    temp = identity_comp(i,j);
diff --git a/include/shared_memory_cache_helper.cuh b/include/shared_memory_cache_helper.cuh
index 6bbc0448f2..7ab7ef7a37 100644
--- a/include/shared_memory_cache_helper.cuh
+++ b/include/shared_memory_cache_helper.cuh
@@ -113,7 +113,7 @@ namespace quda
       atom_t tmp[n_element];
       memcpy(tmp, (void*)&a, sizeof(T));
       int j = (z * block.y + y) * block.x + x;
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < n_element; i++) cache<dynamic>()[i * stride + j] = tmp[i];
     }
 
@@ -121,7 +121,7 @@ QUDA_UNROLL
     {
       atom_t tmp[n_element];
       int j = (z * block.y + y) * block.x + x;
-QUDA_UNROLL
+#pragma unroll
       for (int i = 0; i < n_element; i++) tmp[i] = cache<dynamic>()[i * stride + j];
       T a;
       memcpy((void*)&a, tmp, sizeof(T));
diff --git a/include/su3_project.cuh b/include/su3_project.cuh
index 4576b3dcf8..c868ea6e0b 100644
--- a/include/su3_project.cuh
+++ b/include/su3_project.cuh
@@ -24,9 +24,9 @@ namespace quda {
   __host__ __device__ inline bool checkUnitary(const Matrix &inv, const Matrix &in, const Float tol)
   {
     // first check U - U^{-1} = 0
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < in.rows(); i++) {
-QUDA_UNROLL
+#pragma unroll
       for (int j = 0; j < in.rows(); j++) {
         if (fabs(in(i,j).real() - inv(j,i).real()) > tol ||
             fabs(in(i,j).imag() + inv(j,i).imag()) > tol) return false;
@@ -36,12 +36,12 @@ QUDA_UNROLL
     // now check 1 - U^dag * U = 0
     // this check is more expensive so delay until we have passed first check
     const Matrix identity = conj(in)*in;
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < in.rows(); i++) {
       if (fabs(identity(i,i).real() - static_cast<Float>(1.0)) > tol ||
           fabs(identity(i,i).imag()) > tol)
         return false;
-QUDA_UNROLL
+#pragma unroll
       for (int j = 0; j < in.rows(); j++) {
         if (i>j) { // off-diagonal identity check
         if (fabs(identity(i,j).real()) > tol || fabs(identity(i,j).imag()) > tol ||
diff --git a/include/targets/generic/aos.h b/include/targets/generic/aos.h
index 82236f731a..dfdcae42fc 100644
--- a/include/targets/generic/aos.h
+++ b/include/targets/generic/aos.h
@@ -12,7 +12,7 @@ namespace quda
    */
   template <typename T, int n> __host__ __device__ void block_load(T out[n], const T *in)
   {
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < n; i++) out[i] = in[i];
   }
 
@@ -25,7 +25,7 @@ QUDA_UNROLL
    */
   template <typename T, int n> __host__ __device__ void block_store(T *out, const T in[n])
   {
-QUDA_UNROLL
+#pragma unroll
     for (int i = 0; i < n; i++) out[i] = in[i];
   }
 
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index 485105dbda..f59f343bac 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -1,9 +1,5 @@
 #pragma once
 
-#ifndef QUDA_UNROLL
-#define QUDA_UNROLL
-#endif
-
 // OMPTARGET SPECIFIC workarounds
 #define __host__
 #define __device__
diff --git a/include/targets/omptarget/warp_collective.h b/include/targets/omptarget/warp_collective.h
index 993acc9d5f..a6c9c9ceaf 100644
--- a/include/targets/omptarget/warp_collective.h
+++ b/include/targets/omptarget/warp_collective.h
@@ -23,10 +23,10 @@ namespace quda
       const int warpend = (wid+1)*warp_size;
       const int split_size = warp_size / warp_split;
       if (warp_split > 1) {
-QUDA_UNROLL
+#pragma unroll
         for (int i = 0; i < x.size(); i++) {
           // reduce down to the first group of column-split threads
-QUDA_UNROLL
+#pragma unroll
           for (int offset = warp_size / 2; offset >= split_size; offset /= 2) {
             #pragma omp barrier
             storage[tid] = x[i];

From 4ff05f6b498de9ed7415337075a833bd2e3ccac2 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 21 Jul 2022 18:17:52 +0000
Subject: [PATCH 142/198] omp: remove some unnecessary QUDA_RT_CONSTS

---
 include/dslash_helper.cuh                                   | 1 -
 include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh | 1 -
 include/kernels/dslash_pack.cuh                             | 2 --
 3 files changed, 4 deletions(-)

diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index 2423fe536d..b815c3ce55 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -500,7 +500,6 @@ namespace quda
   template <KernelType kernel_type, int nParity, class D, typename Arg>
   void __device__ __forceinline__ shmem_exterior(D &dslash, const Arg &arg, int s)
   {
-    QUDA_RT_CONSTS;
     // shmem exterior kernel with grid-strided loop
     if (kernel_type == UBER_KERNEL || kernel_type == EXTERIOR_KERNEL_ALL) {
       // figure out some details on blocks
diff --git a/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh b/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
index 247056cd78..bf0b42060e 100644
--- a/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
+++ b/include/kernels/dslash_ndeg_twisted_mass_preconditioned.cuh
@@ -55,7 +55,6 @@ namespace quda
     template <KernelType mykernel_type = kernel_type>
     __device__ __host__ __forceinline__ void operator()(int idx, int flavor, int parity)
     {
-      QUDA_RT_CONSTS;
       typedef typename mapper<typename Arg::Float>::type real;
       typedef ColorSpinor<real, Arg::nColor, 4> Vector;
 
diff --git a/include/kernels/dslash_pack.cuh b/include/kernels/dslash_pack.cuh
index a42a744bc4..b4c22a4bb9 100644
--- a/include/kernels/dslash_pack.cuh
+++ b/include/kernels/dslash_pack.cuh
@@ -268,7 +268,6 @@ namespace quda
 
     template <int twist> __device__ __forceinline__ void operator()(const Arg &arg, int s, int parity)
     {
-      QUDA_RT_CONSTS;
       // (active_dims * 2 + dir) * blocks_per_dir + local_block_idx
       int local_block_idx = target::block_idx().x % arg.blocks_per_dir;
       int dim_dir = target::block_idx().x / arg.blocks_per_dir;
@@ -400,7 +399,6 @@ namespace quda
 
     __device__ __forceinline__ void operator()(const Arg &arg, int s, int parity, int = 0)
     {
-      QUDA_RT_CONSTS;
       // (active_dims * 2 + dir) * blocks_per_dir + local_block_idx
       int local_block_idx = target::block_idx().x % arg.blocks_per_dir;
       int dim_dir = target::block_idx().x / arg.blocks_per_dir;

From 24af9efa4a8faf9890ab93b4f797a1617af243f6 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 22 Jul 2022 00:07:51 +0000
Subject: [PATCH 143/198] omp: add constant_kernel_arg.h for
 dslash5_domain_wall

---
 lib/dslash5_domain_wall.cu | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/dslash5_domain_wall.cu b/lib/dslash5_domain_wall.cu
index 10e6ae8a48..bc0aa85579 100644
--- a/lib/dslash5_domain_wall.cu
+++ b/lib/dslash5_domain_wall.cu
@@ -1,6 +1,7 @@
 #include <color_spinor_field.h>
 #include <dslash_quda.h>
 #include <instantiate.h>
+#include <constant_kernel_arg.h>
 #include <tunable_nd.h>
 #include <kernels/dslash_domain_wall_m5.cuh>
 

From c3de292c6b31c5d347fd2741e8dbc5c34e43f158 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 26 Jul 2022 03:18:06 +0000
Subject: [PATCH 144/198] omp: use host pinned memory for global device var

---
 .../omptarget/block_reduction_kernel.h        |  8 ++--
 include/targets/omptarget/kernel.h            | 28 ++++++-------
 include/targets/omptarget/quda_api.h          | 22 +++++-----
 include/targets/omptarget/reduction_kernel.h  | 16 +++----
 include/targets/omptarget/target_device.h     | 18 ++++----
 lib/targets/omptarget/quda_api.cpp            | 42 ++++++++++---------
 6 files changed, 70 insertions(+), 64 deletions(-)

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index b0b4aa2ee8..a4d661ea1e 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -114,8 +114,8 @@ namespace quda
                       0) __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
@@ -145,8 +145,8 @@ namespace quda
                       0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg *argp)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 5cbc10989c..6baac495c6 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -50,8 +50,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
@@ -76,8 +76,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
@@ -133,8 +133,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
@@ -159,8 +159,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
@@ -218,8 +218,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
@@ -244,8 +244,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
@@ -272,8 +272,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool dummy = false>
   __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index f59f343bac..b66ad820a4 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -13,10 +13,10 @@
 
 #define QUDA_RT_CONSTS \
   const dim3\
-    blockDim=target::omptarget::launch_param.block,\
-    gridDim=target::omptarget::launch_param.grid,\
-    threadIdx(omp_get_thread_num()%target::omptarget::launch_param.block.x, (omp_get_thread_num()/target::omptarget::launch_param.block.x)%target::omptarget::launch_param.block.y, omp_get_thread_num()/(target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y)),\
-    blockIdx(omp_get_team_num()%target::omptarget::launch_param.grid.x, (omp_get_team_num()/target::omptarget::launch_param.grid.x)%target::omptarget::launch_param.grid.y, omp_get_team_num()/(target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y))
+    blockDim=target::omptarget::launch_param->block,\
+    gridDim=target::omptarget::launch_param->grid,\
+    threadIdx(omp_get_thread_num()%target::omptarget::launch_param->block.x, (omp_get_thread_num()/target::omptarget::launch_param->block.x)%target::omptarget::launch_param->block.y, omp_get_thread_num()/(target::omptarget::launch_param->block.x*target::omptarget::launch_param->block.y)),\
+    blockIdx(omp_get_team_num()%target::omptarget::launch_param->grid.x, (omp_get_team_num()/target::omptarget::launch_param->grid.x)%target::omptarget::launch_param->grid.y, omp_get_team_num()/(target::omptarget::launch_param->grid.x*target::omptarget::launch_param->grid.y))
 
 #include <functional>
 #include <iostream>
@@ -92,17 +92,19 @@ namespace quda {
         int num_teams;
         int cache_length;
       };
-      inline int *get_shared_cache(void)
-      {
-        extern SharedCache shared_cache;
-        return &shared_cache.addr[omp_get_team_num()*shared_cache.cache_length];
-      }
 
       struct LaunchParam{
         dim3 block;
         dim3 grid;
+        SharedCache shared_cache;
       };
-      extern LaunchParam launch_param;
+      extern LaunchParam *launch_param;
+      extern LaunchParam *launch_param_host;
+
+      inline int *get_shared_cache(void)
+      {
+        return &launch_param->shared_cache.addr[omp_get_team_num()*launch_param->shared_cache.cache_length];
+      }
     }
   }
 }
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 0721dc82ce..7df4a1a9c5 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -59,8 +59,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
@@ -85,8 +85,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
@@ -153,8 +153,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
@@ -179,8 +179,8 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param.grid;
-    const auto& block = target::omptarget::launch_param.block;
+    const auto& grid = target::omptarget::launch_param_host->grid;
+    const auto& block = target::omptarget::launch_param_host->block;
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
     #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index fc76a8f893..a0a5674edd 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -57,7 +57,7 @@ namespace quda {
     */
     __device__ __host__ inline dim3 block_dim()
     {
-      return target::omptarget::launch_param.block;
+      return target::omptarget::launch_param->block;
     }
 
     /**
@@ -67,7 +67,7 @@ namespace quda {
     */
     __device__ __host__ inline dim3 grid_dim()
     {
-      return target::omptarget::launch_param.grid;
+      return target::omptarget::launch_param->grid;
     }
 
     /**
@@ -78,7 +78,7 @@ namespace quda {
     __device__ __host__ inline dim3 block_idx()
     {
       const auto n = (unsigned int)omp_get_team_num();
-      return dim3(n%target::omptarget::launch_param.grid.x, (n/target::omptarget::launch_param.grid.x)%target::omptarget::launch_param.grid.y, n/(target::omptarget::launch_param.grid.x*target::omptarget::launch_param.grid.y));
+      return dim3(n%target::omptarget::launch_param->grid.x, (n/target::omptarget::launch_param->grid.x)%target::omptarget::launch_param->grid.y, n/(target::omptarget::launch_param->grid.x*target::omptarget::launch_param->grid.y));
     }
 
     /**
@@ -89,7 +89,7 @@ namespace quda {
     __device__ __host__ inline dim3 thread_idx()
     {
       const auto n = (unsigned int)omp_get_thread_num();
-      return dim3(n%target::omptarget::launch_param.block.x, (n/target::omptarget::launch_param.block.x)%target::omptarget::launch_param.block.y, n/(target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y));
+      return dim3(n%target::omptarget::launch_param->block.x, (n/target::omptarget::launch_param->block.x)%target::omptarget::launch_param->block.y, n/(target::omptarget::launch_param->block.x*target::omptarget::launch_param->block.y));
     }
 
     /**
@@ -99,8 +99,8 @@ namespace quda {
     {
       const auto n = (unsigned int)omp_get_thread_num();
       switch (dim) {
-      case 1: return n%target::omptarget::launch_param.block.x;
-      case 2: return n%(target::omptarget::launch_param.block.x*target::omptarget::launch_param.block.y);
+      case 1: return n%target::omptarget::launch_param->block.x;
+      case 2: return n%(target::omptarget::launch_param->block.x*target::omptarget::launch_param->block.y);
       case 3:
       default: return n;
       }
@@ -112,10 +112,10 @@ namespace quda {
     template <int dim> __device__ __host__ inline auto block_size()
     {
       switch (dim) {
-      case 1: return target::omptarget::launch_param.block.x;
-      case 2: return target::omptarget::launch_param.block.y * target::omptarget::launch_param.block.x;
+      case 1: return target::omptarget::launch_param->block.x;
+      case 2: return target::omptarget::launch_param->block.y * target::omptarget::launch_param->block.x;
       case 3:
-      default: return target::omptarget::launch_param.block.z * target::omptarget::launch_param.block.y * target::omptarget::launch_param.block.x;
+      default: return target::omptarget::launch_param->block.z * target::omptarget::launch_param->block.y * target::omptarget::launch_param->block.x;
       }
     }
 
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 667e6a1f84..fc62aab099 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -33,47 +33,51 @@ namespace quda
   namespace target {
     namespace omptarget {
       #pragma omp declare target
-      SharedCache shared_cache;
+      LaunchParam *launch_param;
       #pragma omp end declare target
 
-      LaunchParam launch_param;
-      #pragma omp declare target to(launch_param)
+      LaunchParam *launch_param_host;
 
       int qudaSetupLaunchParameter(const TuneParam &tp)
       {
-        launch_param.grid = tp.grid;
-        launch_param.block = tp.block;
-        #pragma omp target update to(launch_param)
-
         static int init = 0;
         int num_teams = tp.grid.x*tp.grid.y*tp.grid.z;
         if(!init){
-          device::constant_arg_buffer = omp_target_alloc_host(device::max_constant_size(), omp_get_default_device());
+          int dev = omp_get_default_device();
+          launch_param_host = (LaunchParam *)omp_target_alloc_host(sizeof(LaunchParam), dev);
+          if(!launch_param_host){
+            errorQuda("failed to allocate launch_param for %lu bytes host pinned memory.", sizeof(LaunchParam));
+          }
+          omp_target_memcpy(omp_get_mapped_ptr(&launch_param, dev), &launch_param_host, sizeof(LaunchParam *), 0, 0, dev, omp_get_initial_device());
+
+          device::constant_arg_buffer = omp_target_alloc_host(device::max_constant_size(), dev);
           if(!device::constant_arg_buffer){
             warningQuda("failed to allocate %lu bytes host memory for kernel arguments.", device::max_constant_size());
             return -1;
           }
-          shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), omp_get_default_device());
-          if(!shared_cache.addr){
+          launch_param_host->shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), dev);
+          if(!launch_param_host->shared_cache.addr){
             // warnQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
             return -1;
           }
           init = 1;
-          shared_cache.num_teams = num_teams;
-          shared_cache.cache_length = device::max_shared_memory_size()/sizeof(shared_cache.addr[0]);
+          launch_param_host->shared_cache.num_teams = num_teams;
+          launch_param_host->shared_cache.cache_length = device::max_shared_memory_size()/sizeof(launch_param_host->shared_cache.addr[0]);
         }
-        if(shared_cache.num_teams<num_teams){
-          omp_target_free(shared_cache.addr, omp_get_default_device());
-          shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), omp_get_default_device());
-          if(!shared_cache.addr){
+        if(launch_param_host->shared_cache.num_teams<num_teams){
+          int dev = omp_get_default_device();
+          omp_target_free(launch_param_host->shared_cache.addr, dev);
+          launch_param_host->shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), dev);
+          if(!launch_param_host->shared_cache.addr){
             // warnQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
             init = 0;
             return -1;
           }
-          shared_cache.num_teams = num_teams;
-          shared_cache.cache_length = device::max_shared_memory_size()/sizeof(shared_cache.addr[0]);
+          launch_param_host->shared_cache.num_teams = num_teams;
+          launch_param_host->shared_cache.cache_length = device::max_shared_memory_size()/sizeof(launch_param_host->shared_cache.addr[0]);
         }
-        #pragma omp target update to(shared_cache)
+        launch_param_host->grid = tp.grid;
+        launch_param_host->block = tp.block;
         return 0;
       }
 

From 5dc20af5e35a88884e73ad89e0bcd0a4183c7c90 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 26 Jul 2022 23:18:53 +0000
Subject: [PATCH 145/198] Revert "omp: use constant_kernel_arg for coarse_op"

This reverts commit 3d434c5f20900521faa66d8687f5f6f848e252a6.
---
 lib/coarse_op.cuh          | 1 -
 lib/staggered_coarse_op.cu | 1 -
 2 files changed, 2 deletions(-)

diff --git a/lib/coarse_op.cuh b/lib/coarse_op.cuh
index b69cd118f5..fbefd474a0 100644
--- a/lib/coarse_op.cuh
+++ b/lib/coarse_op.cuh
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <constant_kernel_arg.h>
 #include <kernels/coarse_op_kernel.cuh>
 #include <uint_to_char.h>
 #include <coarse_op_mma_launch.h>
diff --git a/lib/staggered_coarse_op.cu b/lib/staggered_coarse_op.cu
index 6d42d48ff2..6e7e21d635 100644
--- a/lib/staggered_coarse_op.cu
+++ b/lib/staggered_coarse_op.cu
@@ -2,7 +2,6 @@
 
 #include <transfer.h>
 #include <gauge_field.h>
-#include <constant_kernel_arg.h>
 #include <tunable_nd.h>
 #include <instantiate.h>
 

From 4b9a3ce00d3f498fab4eabb1d99bce12274bcc20 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 26 Jul 2022 23:19:29 +0000
Subject: [PATCH 146/198] Revert "omp: add constant_kernel_arg.h for
 dslash5_domain_wall"

This reverts commit 24af9efa4a8faf9890ab93b4f797a1617af243f6.
---
 lib/dslash5_domain_wall.cu | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/dslash5_domain_wall.cu b/lib/dslash5_domain_wall.cu
index bc0aa85579..10e6ae8a48 100644
--- a/lib/dslash5_domain_wall.cu
+++ b/lib/dslash5_domain_wall.cu
@@ -1,7 +1,6 @@
 #include <color_spinor_field.h>
 #include <dslash_quda.h>
 #include <instantiate.h>
-#include <constant_kernel_arg.h>
 #include <tunable_nd.h>
 #include <kernels/dslash_domain_wall_m5.cuh>
 

From 79c6c7d8926d434cf9809a9e748203cf4a6581a3 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 26 Jul 2022 23:23:30 +0000
Subject: [PATCH 147/198] Revert "omp: reduce max_kernel_arg_size to 2048"

This reverts commit f6392616c171f9e5d418156786c9ac2f393188d1.
---
 include/dslash_helper.cuh                 | 2 +-
 include/targets/omptarget/target_device.h | 2 +-
 lib/hisq_paths_force_quda.cu              | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index b815c3ce55..b8f2068565 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -10,7 +10,7 @@
 #include <dslash_shmem.h>
 #include <shmem_pack_helper.cuh>
 
-#if defined(_NVHPC_CUDA) || defined(QUDA_TARGET_OMPTARGET)
+#if defined(_NVHPC_CUDA)
 #include <constant_kernel_arg.h>
 constexpr bool use_kernel_arg = false;
 #else
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index a0a5674edd..d413f9959c 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -156,7 +156,7 @@ namespace quda {
        the kernel arguments passed to a kernel on the target
        architecture.
     */
-    constexpr size_t max_kernel_arg_size() { return 2048; }
+    constexpr size_t max_kernel_arg_size() { return 4096; }
 
     /**
        @brief Helper function that returns the bank width of the
diff --git a/lib/hisq_paths_force_quda.cu b/lib/hisq_paths_force_quda.cu
index 3574a9c4d5..5febd467b2 100644
--- a/lib/hisq_paths_force_quda.cu
+++ b/lib/hisq_paths_force_quda.cu
@@ -4,7 +4,6 @@
 #include <ks_improved_force.h>
 #include <tune_quda.h>
 #include <instantiate.h>
-#include <constant_kernel_arg.h>
 #include <tunable_nd.h>
 #include <kernels/hisq_paths_force.cuh>
 

From c98f8c2f7b6be845486e4ab4b960be00c95c1a45 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 27 Jul 2022 16:33:18 +0000
Subject: [PATCH 148/198] blas_core: explicitly static assert kernel arg for
 caxpyxmazMR_

---
 include/kernels/blas_core.cuh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/include/kernels/blas_core.cuh b/include/kernels/blas_core.cuh
index 38c3d912ba..ac0d8bae96 100644
--- a/include/kernels/blas_core.cuh
+++ b/include/kernels/blas_core.cuh
@@ -10,6 +10,7 @@ namespace quda
 
   namespace blas
   {
+    template <typename real> struct caxpyxmazMR_;
 
     /**
        Parameter struct for generic blas kernel
@@ -43,7 +44,11 @@ namespace quda
         V(v),
         f(f),
         nParity(nParity)
-      { ; }
+      {
+        if constexpr (std::is_same_v<Functor, caxpyxmazMR_<real>>){
+          static_assert(device::use_kernel_arg<BlasArg>(), "This arg must be passed as a kernel argument");
+        }
+      }
     };
 
     /**

From 0fc6bc27ba0dfcecccced27b3334cbf3ab0c5ed2 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 27 Jul 2022 16:40:01 +0000
Subject: [PATCH 149/198] kernel_param: force use_kernel_arg when larger than 1

Introduce int use_kernel_arg in BlasFunctor and caxpyxmazMR_,
and force use_kernel_arg with caxpyxmazMR_::use_kernel_arg = 2.
---
 include/kernel_helper.h              | 4 ++--
 include/kernels/blas_core.cuh        | 6 ++++--
 include/targets/cuda/target_device.h | 2 +-
 include/targets/hip/target_device.h  | 2 +-
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/include/kernel_helper.h b/include/kernel_helper.h
index f0512c6f11..226e97b96f 100644
--- a/include/kernel_helper.h
+++ b/include/kernel_helper.h
@@ -12,8 +12,8 @@ namespace quda
     kernel_t(const void *func, const char *name) : func(func), name(name) { }
   };
 
-  template <bool use_kernel_arg_ = true> struct kernel_param {
-    static constexpr bool use_kernel_arg = use_kernel_arg_;
+  template <int use_kernel_arg_ = true> struct kernel_param {
+    static constexpr int use_kernel_arg = use_kernel_arg_;
     dim3 threads;          /** number of active threads required */
     int comms_rank;        /** per process value of comm_rank() */
     int comms_rank_global; /** per process value comm_rank_global() */
diff --git a/include/kernels/blas_core.cuh b/include/kernels/blas_core.cuh
index ac0d8bae96..a7318abfd5 100644
--- a/include/kernels/blas_core.cuh
+++ b/include/kernels/blas_core.cuh
@@ -23,7 +23,7 @@ namespace quda
        @tparam Functor_ Functor used to operate on data
     */
     template <typename real_, int n_, typename store_t, int N, typename y_store_t, int Ny, typename Functor>
-    struct BlasArg : kernel_param<> {
+    struct BlasArg : kernel_param<Functor::use_kernel_arg> {
       using real = real_;
       static constexpr int n = n_;
       Spinor<store_t, N> X;
@@ -36,7 +36,7 @@ namespace quda
       const int nParity;
       BlasArg(ColorSpinorField &x, ColorSpinorField &y, ColorSpinorField &z, ColorSpinorField &w,
               ColorSpinorField &v, Functor f, int length, int nParity) :
-        kernel_param(dim3(length, nParity, 1)),
+        kernel_param<Functor::use_kernel_arg>(dim3(length, nParity, 1)),
         X(x),
         Y(y),
         Z(z),
@@ -90,6 +90,7 @@ namespace quda
        Base class from which all blas functors should derive
      */
     struct BlasFunctor {
+      static constexpr int use_kernel_arg = true;
       //! pre-computation routine before the main loop
       __device__ __host__ void init() const { ; }
     };
@@ -368,6 +369,7 @@ namespace quda
        Second performs the operator x[i] -= a*z[i]
     */
     template <typename real> struct caxpyxmazMR_ {
+      static constexpr int use_kernel_arg = 2;
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
       complex<real> a;
diff --git a/include/targets/cuda/target_device.h b/include/targets/cuda/target_device.h
index c9bfe6e4fc..e7bb50a2ff 100644
--- a/include/targets/cuda/target_device.h
+++ b/include/targets/cuda/target_device.h
@@ -211,7 +211,7 @@ namespace quda
     */
     template <typename Arg> constexpr bool use_kernel_arg()
     {
-      return (sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg);
+      return Arg::use_kernel_arg > 1 || (sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg);
     }
 
     /**
diff --git a/include/targets/hip/target_device.h b/include/targets/hip/target_device.h
index 7fd009a991..fbaef6c8e0 100644
--- a/include/targets/hip/target_device.h
+++ b/include/targets/hip/target_device.h
@@ -188,7 +188,7 @@ namespace quda
     */
     template <typename Arg> constexpr bool use_kernel_arg()
     {
-      return sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg;
+      return Arg::use_kernel_arg > 1 || sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg;
     }
 
     /**

From 1bbdf17c1c038f359cb54f5cc56a4492e3ab5fc2 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 27 Jul 2022 16:42:56 +0000
Subject: [PATCH 150/198] omp: reduce max_kernel_arg_size down to 64

---
 include/dslash_helper.cuh                  | 2 +-
 include/targets/omptarget/target_device.h  | 4 ++--
 include/targets/omptarget/tunable_kernel.h | 5 ++++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/include/dslash_helper.cuh b/include/dslash_helper.cuh
index b8f2068565..b815c3ce55 100644
--- a/include/dslash_helper.cuh
+++ b/include/dslash_helper.cuh
@@ -10,7 +10,7 @@
 #include <dslash_shmem.h>
 #include <shmem_pack_helper.cuh>
 
-#if defined(_NVHPC_CUDA)
+#if defined(_NVHPC_CUDA) || defined(QUDA_TARGET_OMPTARGET)
 #include <constant_kernel_arg.h>
 constexpr bool use_kernel_arg = false;
 #else
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index d413f9959c..6db01a4c90 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -156,7 +156,7 @@ namespace quda {
        the kernel arguments passed to a kernel on the target
        architecture.
     */
-    constexpr size_t max_kernel_arg_size() { return 4096; }
+    constexpr size_t max_kernel_arg_size() { return 64; }
 
     /**
        @brief Helper function that returns the bank width of the
@@ -178,7 +178,7 @@ namespace quda {
     */
     template <typename Arg> constexpr bool use_kernel_arg()
     {
-      return (sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg);
+      return Arg::use_kernel_arg > 1 || (sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg);
     }
 
     /**
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index 4e2b7b682d..f50c12cb0d 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <constant_kernel_arg.h>
 #include <tune_quda.h>
 #include <target_device.h>
 #include <kernel_helper.h>
@@ -26,8 +27,10 @@ namespace quda {
         (arg.threads.x%tp.block.x==0 &&
          arg.threads.y%tp.block.y==0 &&
          arg.threads.z%tp.block.z==0));
+    if(getVerbosity() >= QUDA_DEBUG_VERBOSE)
+      printfQuda("Checking threads setup for arg %d %d %d tp grid %d %d %d block %d %d %d\n",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
     if(!r && getVerbosity() >= QUDA_VERBOSE)
-      ompwip("WARNING: rejecting threads setup arg %d %d %d tp (%d %d %d)x(%d %d %d)",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
+      warningQuda("WARNING: rejecting threads setup arg %d %d %d tp grid %d %d %d block %d %d %d",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
     return r;
   }
 

From 1469d4825375c11a39193134abb3a277a566349d Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 27 Jul 2022 16:43:16 +0000
Subject: [PATCH 151/198] omp: correct setup launch param behavior

---
 lib/targets/omptarget/quda_api.cpp | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index fc62aab099..3dfa931800 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -47,17 +47,19 @@ namespace quda
           launch_param_host = (LaunchParam *)omp_target_alloc_host(sizeof(LaunchParam), dev);
           if(!launch_param_host){
             errorQuda("failed to allocate launch_param for %lu bytes host pinned memory.", sizeof(LaunchParam));
+            return -1;
           }
           omp_target_memcpy(omp_get_mapped_ptr(&launch_param, dev), &launch_param_host, sizeof(LaunchParam *), 0, 0, dev, omp_get_initial_device());
 
           device::constant_arg_buffer = omp_target_alloc_host(device::max_constant_size(), dev);
           if(!device::constant_arg_buffer){
-            warningQuda("failed to allocate %lu bytes host memory for kernel arguments.", device::max_constant_size());
+            errorQuda("failed to allocate %lu bytes host memory for kernel arguments.", device::max_constant_size());
             return -1;
           }
           launch_param_host->shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), dev);
           if(!launch_param_host->shared_cache.addr){
-            // warnQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
+            if(getVerbosity() >= QUDA_VERBOSE)
+              warningQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
             return -1;
           }
           init = 1;
@@ -66,11 +68,14 @@ namespace quda
         }
         if(launch_param_host->shared_cache.num_teams<num_teams){
           int dev = omp_get_default_device();
-          omp_target_free(launch_param_host->shared_cache.addr, dev);
+          if(launch_param_host->shared_cache.addr)
+            omp_target_free(launch_param_host->shared_cache.addr, dev);
           launch_param_host->shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), dev);
           if(!launch_param_host->shared_cache.addr){
-            // warnQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
-            init = 0;
+            if(getVerbosity() >= QUDA_VERBOSE)
+              warningQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
+            launch_param_host->shared_cache.num_teams = 0;
+            launch_param_host->shared_cache.cache_length = 0;
             return -1;
           }
           launch_param_host->shared_cache.num_teams = num_teams;

From 9bd0fda25118baff02f27c78f8a69cc4db1cf3b9 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 28 Jul 2022 04:15:45 +0000
Subject: [PATCH 152/198] cmake: use C++20 by default

---
 CMakeLists.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 537e5c565c..1be125808f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -231,10 +231,10 @@ option(QUDA_ALTERNATIVE_I_TO_F "enable using alternative integer-to-float conver
 
 option(QUDA_OPENMP "enable OpenMP" OFF)
 set(QUDA_CXX_STANDARD
-    17
-    CACHE STRING "set the CXX Standard (14 or 17)")
+    20
+    CACHE STRING "set the CXX Standard (14, 17, 20)")
 
-set_property(CACHE QUDA_CXX_STANDARD PROPERTY STRINGS 14 17)
+set_property(CACHE QUDA_CXX_STANDARD PROPERTY STRINGS 14 17 20)
 
 option(QUDA_BACKWARDS "Enable stacktrace generation using backwards-cpp")
 

From b976d80a0b9864fa4330745e650390a259c3461e Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 28 Jul 2022 04:18:08 +0000
Subject: [PATCH 153/198] array: remove default for c++20 compatibility

---
 include/array.h | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/include/array.h b/include/array.h
index acb4ee4e0a..654b7848c2 100644
--- a/include/array.h
+++ b/include/array.h
@@ -17,13 +17,6 @@ namespace quda
     constexpr T &operator[](int i) { return data[i]; }
     constexpr const T &operator[](int i) const { return data[i]; }
     constexpr int size() const { return n; }
-
-    array() = default;
-    array(const array<T, n> &) = default;
-    array(array<T, n> &&) = default;
-
-    array<T, n> &operator=(const array<T, n> &) = default;
-    array<T, n> &operator=(array<T, n> &&) = default;
   };
 
   template <typename T, int n> std::ostream &operator<<(std::ostream &output, const array<T, n> &a)

From decaacb02556f242970bbb647130320f7044abf9 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 28 Jul 2022 04:19:02 +0000
Subject: [PATCH 154/198] omp: introducing requires_threads_sync in some kernel
 args to avoid hanging threads

We can't have threads exits early in openmp and use barrier in the rest of the threads.
OpenMP requires all the threads in a team call participate in a barrier.
In order to help the kernel launcher to filter out potentially hanging kernels,
some kernel args gets a static constexpr ThreadsSync requires_threads_sync,
which tells the kernel launcher if any of the x/y/z thread blocks has to be
divisible to the total threads.
---
 include/kernel_helper.h                       | 11 +++
 include/kernels/dslash_coarse.cuh             |  9 +++
 include/kernels/gauge_fix_ovr.cuh             |  1 +
 include/kernels/multi_blas_core.cuh           |  1 +
 .../omptarget/block_reduction_kernel.h        |  1 +
 .../targets/omptarget/quda_omptarget_api.h    | 11 +++
 include/targets/omptarget/reduce_helper.h     |  1 +
 include/targets/omptarget/tunable_kernel.h    | 73 +++++++++++++------
 lib/targets/omptarget/quda_api.cpp            |  3 +-
 9 files changed, 88 insertions(+), 23 deletions(-)
 create mode 100644 include/targets/omptarget/quda_omptarget_api.h

diff --git a/include/kernel_helper.h b/include/kernel_helper.h
index 226e97b96f..7a315a55a5 100644
--- a/include/kernel_helper.h
+++ b/include/kernel_helper.h
@@ -5,6 +5,17 @@
 namespace quda
 {
 
+  enum ThreadsSync {
+    ThreadsSyncNo = 0,
+    ThreadsSyncX = 1,
+    ThreadsSyncY = 2,
+    ThreadsSyncXY = 3,
+    ThreadsSyncZ = 4,
+    ThreadsSyncXZ = 5,
+    ThreadsSyncYZ = 6,
+    ThreadsSyncAll = 7
+  };
+
   struct kernel_t {
     const void *func;
     const std::string name;
diff --git a/include/kernels/dslash_coarse.cuh b/include/kernels/dslash_coarse.cuh
index 8311ec0afd..82ce708609 100644
--- a/include/kernels/dslash_coarse.cuh
+++ b/include/kernels/dslash_coarse.cuh
@@ -20,6 +20,8 @@ namespace quda {
   template <bool dslash_, bool clover_, bool dagger_, DslashType type_, int color_stride_, int dim_stride_, typename Float,
             typename yFloat, typename ghostFloat, int nSpin_, int nColor_, QudaFieldOrder csOrder, QudaGaugeFieldOrder gOrder>
   struct DslashCoarseArg : kernel_param<> {
+    static constexpr ThreadsSync requires_threads_sync = ThreadsSyncAll;
+
     static constexpr bool dslash = dslash_;
     static constexpr bool clover = clover_;
     static constexpr bool dagger = dagger_;
@@ -342,11 +344,18 @@ namespace quda {
 
       if (doBulk<Arg::type>() && Arg::clover && dir==0 && dim==0) applyClover<Mc>(out, arg, x_cb, src_idx, parity, s, color_block, color_offset);
 
+#ifdef QUDA_TARGET_OMPTARGET
+      // reduce down to the first group of column-split threads, do it for every threads in openmp.
+      out = warp_combine<Arg::color_stride>(out);
+#endif
+
       if (dir==0 && dim==0) {
         const int my_spinor_parity = (arg.nParity == 2) ? parity : 0;
 
+#ifndef QUDA_TARGET_OMPTARGET
         // reduce down to the first group of column-split threads
         out = warp_combine<Arg::color_stride>(out);
+#endif
 
 #pragma unroll
         for (int color_local=0; color_local<Mc; color_local++) {
diff --git a/include/kernels/gauge_fix_ovr.cuh b/include/kernels/gauge_fix_ovr.cuh
index 644d9c2463..a8bda0c04b 100644
--- a/include/kernels/gauge_fix_ovr.cuh
+++ b/include/kernels/gauge_fix_ovr.cuh
@@ -102,6 +102,7 @@ namespace quda {
    */
   template <typename store_t, QudaReconstructType recon, int gauge_dir_, bool halo_, int type_>
   struct GaugeFixArg : kernel_param<> {
+    static constexpr ThreadsSync requires_threads_sync = ThreadsSyncAll;
     using real = typename mapper<store_t>::type;
     static constexpr int gauge_dir = gauge_dir_;
     static constexpr bool halo = halo_;
diff --git a/include/kernels/multi_blas_core.cuh b/include/kernels/multi_blas_core.cuh
index 0509681abe..810dfb05ae 100644
--- a/include/kernels/multi_blas_core.cuh
+++ b/include/kernels/multi_blas_core.cuh
@@ -35,6 +35,7 @@ namespace quda
     struct MultiBlasArg : kernel_param<>,
       SpinorXZ<NXZ_, store_t, N, Functor_::use_z>,
       SpinorYW<max_YW_size<NXZ_, store_t, y_store_t, Functor_>(), store_t, N, y_store_t, Ny, Functor_::use_w> {
+      static constexpr ThreadsSync requires_threads_sync = ThreadsSyncAll;
       using real = real_;
       using Functor = Functor_;
       static constexpr int warp_split = warp_split_;
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index a4d661ea1e..73b18f09d1 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -62,6 +62,7 @@ namespace quda
      @param[in] arg Kernel argument
    */
   template <unsigned int block_size_, typename Arg_> struct BlockKernelArg : Arg_ {
+    static constexpr ThreadsSync requires_threads_sync = ThreadsSyncYZ;
     using Arg = Arg_;
     static constexpr unsigned int block_size = block_size_;
     BlockKernelArg(const Arg &arg) : Arg(arg) { }
diff --git a/include/targets/omptarget/quda_omptarget_api.h b/include/targets/omptarget/quda_omptarget_api.h
new file mode 100644
index 0000000000..91b5de8f23
--- /dev/null
+++ b/include/targets/omptarget/quda_omptarget_api.h
@@ -0,0 +1,11 @@
+#pragma once
+
+namespace quda {
+  namespace target {
+    namespace omptarget {
+      int qudaSetupLaunchParameter(const TuneParam &);
+      void set_runtime_error(int error, const char *api_func, const char *func, const char *file,
+                             const char *line, bool allow_error = false);
+    }
+  }
+}
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 37d46a379e..69fc19591d 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -23,6 +23,7 @@ namespace quda
      memory
    */
   template <typename T, bool use_kernel_arg = true> struct ReduceArg : kernel_param<use_kernel_arg> {
+    static constexpr ThreadsSync requires_threads_sync = ThreadsSyncAll;
     using reduce_t = T;
 
     template <typename Reducer, typename Arg, typename I>
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index f50c12cb0d..c817ab5b63 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -5,33 +5,61 @@
 #include <target_device.h>
 #include <kernel_helper.h>
 #include <kernel.h>
+#include <quda_omptarget_api.h>
 
 namespace quda {
 
-  namespace target {
-    namespace omptarget {
-      // defined in ../../../lib/targets/omptarget/quda_api.cpp:/qudaSetupLaunchParameter
-      int qudaSetupLaunchParameter(const TuneParam &);
-      void set_runtime_error(int error, const char *api_func, const char *func, const char *file,
-                             const char *line, bool allow_error = false);
-    }
-  }
+  template <typename Arg>
+  concept announce_threads_sync = requires {
+    Arg::requires_threads_sync;
+  };
 
   template <typename Arg>
   inline bool acceptThreads(const TuneParam &tp, const Arg &arg)
   {
-    bool r = tp.block.x*tp.block.y*tp.block.z<=device::max_block_size() &&
-       (arg.threads.x<tp.block.x ||    // trivial cases where arg.threads.x == 1 or few
-        arg.threads.y<tp.block.y ||
-        arg.threads.z<tp.block.z ||
-        (arg.threads.x%tp.block.x==0 &&
-         arg.threads.y%tp.block.y==0 &&
-         arg.threads.z%tp.block.z==0));
-    if(getVerbosity() >= QUDA_DEBUG_VERBOSE)
-      printfQuda("Checking threads setup for arg %d %d %d tp grid %d %d %d block %d %d %d\n",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
-    if(!r && getVerbosity() >= QUDA_VERBOSE)
-      warningQuda("WARNING: rejecting threads setup arg %d %d %d tp grid %d %d %d block %d %d %d",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
-    return r;
+    bool fit = tp.block.x*tp.block.y*tp.block.z<=device::max_block_size();
+    bool xrem = arg.threads.x % (tp.block.x * tp.grid.x) > 0;
+    bool yrem = arg.threads.y % (tp.block.y * tp.grid.y) > 0;
+    bool zrem = arg.threads.z % (tp.block.z * tp.grid.z) > 0;
+    if (!fit) {
+      if (getVerbosity() >= QUDA_DEBUG_VERBOSE)
+        warningQuda("rejecting threads setup with a large block size with arg %d %d %d tp grid %d %d %d block %d %d %d\n",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
+      return false;
+    }
+    if (xrem || yrem || zrem) {
+      if constexpr(announce_threads_sync<Arg>){
+        if (((Arg::requires_threads_sync & ThreadsSyncX) && xrem) ||
+            ((Arg::requires_threads_sync & ThreadsSyncY) && yrem) ||
+            ((Arg::requires_threads_sync & ThreadsSyncZ) && zrem)) {
+          if(getVerbosity() >= QUDA_DEBUG_VERBOSE)
+            warningQuda("rejecting threads setup with a non-divisible block size with arg %d %d %d tp grid %d %d %d block %d %d %d\n",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
+          return false;
+        } else {
+          if(getVerbosity() >= QUDA_DEBUG_VERBOSE)
+            warningQuda("accepting threads setup with a non-divisible block size with arg %d %d %d tp grid %d %d %d block %d %d %d\n",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
+          return true;
+        }
+      } else {  // ! announce_threads_sync<Arg>
+        if (getVerbosity() >= QUDA_DEBUG_VERBOSE) {
+          bool cont = true;
+          std::string reply;
+            ompwip("threads setup with a non-divisible block size with arg %d %d %d tp grid %d %d %d block %d %d %d, yes to stop?",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
+          std::getline(std::cin, reply);
+          if (reply[0] == 'y' || reply[0] == 'Y') {
+            cont = false;
+          }
+          return cont;
+        } else {
+          if (getVerbosity() >= QUDA_VERBOSE)
+            ompwip("accepting threads setup with a non-divisible block size with arg %d %d %d tp grid %d %d %d block %d %d %d",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
+          return true;
+        }
+      }
+    } else {
+      if(getVerbosity() >= QUDA_DEBUG_VERBOSE)
+        warningQuda("threads setup with arg %d %d %d tp grid %d %d %d block %d %d %d\n",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);
+    }
+    return true;
   }
 
   class TunableKernel : public Tunable
@@ -46,6 +74,7 @@ namespace quda {
     template <template <typename> class Functor, bool grid_stride, typename Arg>
     qudaError_t launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
+      launch_error = QUDA_SUCCESS;
       if (acceptThreads(tp, arg) && 0==target::omptarget::qudaSetupLaunchParameter(tp)) {
         if constexpr (device::use_kernel_arg<Arg>()) {
           reinterpret_cast<void(*)(Arg)>(const_cast<void*>(kernel.func))(arg);
@@ -55,11 +84,11 @@ namespace quda {
           memcpy(argp, &arg, sizeof(Arg));
           reinterpret_cast<void(*)(Arg*)>(const_cast<void*>(kernel.func))(argp);
         }
-        launch_error = QUDA_SUCCESS;
+        launch_error = qudaGetLastError();
       } else {
-        target::omptarget::set_runtime_error(QUDA_ERROR, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
         launch_error = QUDA_ERROR;
       }
+      target::omptarget::set_runtime_error(launch_error, __func__, __func__, __FILE__, __STRINGIFY__(__LINE__), activeTuning());
       return launch_error;
     }
 
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 3dfa931800..58efb14d22 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -5,6 +5,7 @@
 #include <timer.h>
 #include <device.h>
 #include <target_device.h>
+#include <quda_omptarget_api.h>
 
 namespace quda
 {
@@ -178,7 +179,7 @@ namespace quda
       }
 
       void set_runtime_error(int error, const char *api_func, const char *func, const char *file,
-                             const char *line, bool allow_error = false)
+                             const char *line, bool allow_error)
       {
         if (error == 0) return;
         last_error = error == 0 ? QUDA_SUCCESS : QUDA_ERROR;

From b9f1d1ee06389ca605cbfdd2f4e1a0e3d8059f6c Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 3 Aug 2022 16:30:58 +0000
Subject: [PATCH 155/198] omp: reserve a fixed amount of global mem to emulate
 SLM with env QUDA_OMPTARGET_SLM_RESERVE

---
 lib/targets/omptarget/quda_api.cpp | 37 +++++++++++++++---------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 58efb14d22..904306b08e 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -42,6 +42,7 @@ namespace quda
       int qudaSetupLaunchParameter(const TuneParam &tp)
       {
         static int init = 0;
+        static size_t res_mem = 1ULL << 32;
         int num_teams = tp.grid.x*tp.grid.y*tp.grid.z;
         if(!init){
           int dev = omp_get_default_device();
@@ -57,30 +58,30 @@ namespace quda
             errorQuda("failed to allocate %lu bytes host memory for kernel arguments.", device::max_constant_size());
             return -1;
           }
-          launch_param_host->shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), dev);
+          char *res_mem_env = getenv("QUDA_OMPTARGET_SLM_RESERVE");
+          if(res_mem_env){
+            size_t res_mem_env_val = std::strtoull(res_mem_env, nullptr, 10);
+            if(res_mem_env_val > 0){
+              res_mem = res_mem_env_val;
+              warningQuda("using environment variable QUDA_OMPTARGET_SLM_RESERVE=%ul", res_mem);
+            }else{
+              warningQuda("cannot parse environment variable QUDA_OMPTARGET_SLM_RESERVE=%s", res_mem_env);
+            }
+          }
+          launch_param_host->shared_cache.addr = (int*)omp_target_alloc(res_mem, dev);
           if(!launch_param_host->shared_cache.addr){
-            if(getVerbosity() >= QUDA_VERBOSE)
-              warningQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
+            errorQuda("failed to allocate %lu bytes device memory for shared cache.", res_mem);
             return -1;
           }
-          init = 1;
-          launch_param_host->shared_cache.num_teams = num_teams;
+          launch_param_host->shared_cache.num_teams = res_mem/(size_t)device::max_shared_memory_size();
           launch_param_host->shared_cache.cache_length = device::max_shared_memory_size()/sizeof(launch_param_host->shared_cache.addr[0]);
+          warningQuda("allocated %lu bytes device memory for shared cache among maximum %d teams.", res_mem, launch_param_host->shared_cache.num_teams);
+          init = 1;
         }
         if(launch_param_host->shared_cache.num_teams<num_teams){
-          int dev = omp_get_default_device();
-          if(launch_param_host->shared_cache.addr)
-            omp_target_free(launch_param_host->shared_cache.addr, dev);
-          launch_param_host->shared_cache.addr = (int*)omp_target_alloc(num_teams*(size_t)device::max_shared_memory_size(), dev);
-          if(!launch_param_host->shared_cache.addr){
-            if(getVerbosity() >= QUDA_VERBOSE)
-              warningQuda("failed to allocate %lu bytes device memory for shared cache among %d teams.", num_teams*(size_t)device::max_shared_memory_size(), num_teams);
-            launch_param_host->shared_cache.num_teams = 0;
-            launch_param_host->shared_cache.cache_length = 0;
-            return -1;
-          }
-          launch_param_host->shared_cache.num_teams = num_teams;
-          launch_param_host->shared_cache.cache_length = device::max_shared_memory_size()/sizeof(launch_param_host->shared_cache.addr[0]);
+          if(getVerbosity() >= QUDA_VERBOSE)
+            warningQuda("requested %d teams, which is larger than maximum %d teams reserved for shared memory %lu bytes", num_teams, launch_param_host->shared_cache.num_teams, res_mem);
+          return -1;
         }
         launch_param_host->grid = tp.grid;
         launch_param_host->block = tp.block;

From 02ae61a6a5cb678bdf65b1e3c70cfdd7d07294dc Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 4 Aug 2022 02:20:55 +0000
Subject: [PATCH 156/198] omp: only filter out by arg.threads%tp.block

---
 include/targets/omptarget/tunable_kernel.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index c817ab5b63..f5bd5886de 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -18,9 +18,9 @@ namespace quda {
   inline bool acceptThreads(const TuneParam &tp, const Arg &arg)
   {
     bool fit = tp.block.x*tp.block.y*tp.block.z<=device::max_block_size();
-    bool xrem = arg.threads.x % (tp.block.x * tp.grid.x) > 0;
-    bool yrem = arg.threads.y % (tp.block.y * tp.grid.y) > 0;
-    bool zrem = arg.threads.z % (tp.block.z * tp.grid.z) > 0;
+    bool xrem = arg.threads.x % tp.block.x > 0;
+    bool yrem = arg.threads.y % tp.block.y > 0;
+    bool zrem = arg.threads.z % tp.block.z > 0;
     if (!fit) {
       if (getVerbosity() >= QUDA_DEBUG_VERBOSE)
         warningQuda("rejecting threads setup with a large block size with arg %d %d %d tp grid %d %d %d block %d %d %d\n",arg.threads.x,arg.threads.y,arg.threads.z,tp.grid.x,tp.grid.y,tp.grid.z,tp.block.x,tp.block.y,tp.block.z);

From f4f9499d3959b940a8d2424d7af139b2ead7e1a8 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 4 Aug 2022 02:21:25 +0000
Subject: [PATCH 157/198] omp: fix reserve mem size printf

---
 lib/targets/omptarget/quda_api.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 904306b08e..7536ffc4c3 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -63,7 +63,7 @@ namespace quda
             size_t res_mem_env_val = std::strtoull(res_mem_env, nullptr, 10);
             if(res_mem_env_val > 0){
               res_mem = res_mem_env_val;
-              warningQuda("using environment variable QUDA_OMPTARGET_SLM_RESERVE=%ul", res_mem);
+              warningQuda("using environment variable QUDA_OMPTARGET_SLM_RESERVE=%lu", res_mem);
             }else{
               warningQuda("cannot parse environment variable QUDA_OMPTARGET_SLM_RESERVE=%s", res_mem_env);
             }

From 6768d31200c6d21c2b9f0001bdd80533e83cfbb0 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 4 Aug 2022 02:23:48 +0000
Subject: [PATCH 158/198] omp: tag init_arg and rngArg with ThreadsSyncNo

---
 include/kernels/random_init.cuh | 1 +
 include/kernels/reduce_init.cuh | 1 +
 2 files changed, 2 insertions(+)

diff --git a/include/kernels/random_init.cuh b/include/kernels/random_init.cuh
index ef0eb93d47..13a37afd2e 100644
--- a/include/kernels/random_init.cuh
+++ b/include/kernels/random_init.cuh
@@ -9,6 +9,7 @@
 namespace quda {
 
   struct rngArg : kernel_param<> {
+    static constexpr ThreadsSync requires_threads_sync = ThreadsSyncNo;
     int commCoord[QUDA_MAX_DIM];
     int X[QUDA_MAX_DIM];
     int X_global[QUDA_MAX_DIM];
diff --git a/include/kernels/reduce_init.cuh b/include/kernels/reduce_init.cuh
index f5a130a385..1915e24188 100644
--- a/include/kernels/reduce_init.cuh
+++ b/include/kernels/reduce_init.cuh
@@ -6,6 +6,7 @@ namespace quda {
   namespace reducer {
 
     template <typename T_> struct init_arg : kernel_param<> {
+      static constexpr ThreadsSync requires_threads_sync = ThreadsSyncNo;
       using T = T_;
       T *count;
       init_arg(T *count, int n_reduce) :

From 18aacdba17f09baacd5af62bb220590f759a6e61 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 8 Aug 2022 21:38:54 +0000
Subject: [PATCH 159/198] omp: update reduce_helper with use_kernel_arg

---
 include/targets/omptarget/reduce_helper.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 69fc19591d..0360b4738e 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -22,7 +22,7 @@ namespace quda
      parameter struct as an explicit kernel argument or from constant
      memory
    */
-  template <typename T, bool use_kernel_arg = true> struct ReduceArg : kernel_param<use_kernel_arg> {
+  template <typename T, use_kernel_arg_p use_kernel_arg = use_kernel_arg_yes> struct ReduceArg : kernel_param<use_kernel_arg> {
     static constexpr ThreadsSync requires_threads_sync = ThreadsSyncAll;
     using reduce_t = T;
 

From 556f767e1ea579ac5e0733952412841ffb7bb1fc Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 9 Aug 2022 15:04:55 +0000
Subject: [PATCH 160/198] omp: follow use_kernel_arg changes

---
 include/targets/omptarget/reduce_helper.h | 2 +-
 include/targets/omptarget/target_device.h | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 0360b4738e..1eb183ce70 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -22,7 +22,7 @@ namespace quda
      parameter struct as an explicit kernel argument or from constant
      memory
    */
-  template <typename T, use_kernel_arg_p use_kernel_arg = use_kernel_arg_yes> struct ReduceArg : kernel_param<use_kernel_arg> {
+  template <typename T, use_kernel_arg_p use_kernel_arg = use_kernel_arg_p::TRUE> struct ReduceArg : kernel_param<use_kernel_arg> {
     static constexpr ThreadsSync requires_threads_sync = ThreadsSyncAll;
     using reduce_t = T;
 
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 6db01a4c90..aa0d566df6 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -178,7 +178,8 @@ namespace quda {
     */
     template <typename Arg> constexpr bool use_kernel_arg()
     {
-      return Arg::use_kernel_arg > 1 || (sizeof(Arg) <= device::max_kernel_arg_size() && Arg::use_kernel_arg);
+      return Arg::always_use_kernel_arg() ||
+        (Arg::default_use_kernel_arg() && sizeof(Arg) <= device::max_kernel_arg_size());
     }
 
     /**

From 30f2fdd77561311837cc2c528176fbaf65686647 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 11 Aug 2022 20:47:58 +0000
Subject: [PATCH 161/198] omp/mkl: use strided API; add zgetrf/i; use 64bit for
 large batch

---
 lib/targets/omptarget/blas_lapack_mkl.cpp    | 108 ++++++++++++-------
 lib/targets/omptarget/target_omptarget.cmake |   3 +-
 2 files changed, 72 insertions(+), 39 deletions(-)

diff --git a/lib/targets/omptarget/blas_lapack_mkl.cpp b/lib/targets/omptarget/blas_lapack_mkl.cpp
index aa6cd5cec5..141b146404 100644
--- a/lib/targets/omptarget/blas_lapack_mkl.cpp
+++ b/lib/targets/omptarget/blas_lapack_mkl.cpp
@@ -52,7 +52,7 @@ namespace quda
                                   QudaFieldLocation location)
       {
         if (getVerbosity() >= QUDA_VERBOSE)
-          printfQuda("BatchInvertMatrix (native - cuBLAS): Nc = %d, batch = %lu\n", n, batch);
+          printfQuda("BatchInvertMatrix (native - MKL): Nc = %d, batch = %lu\n", n, batch);
 
         long long flops = 0;
         timeval start, stop;
@@ -65,45 +65,39 @@ namespace quda
 
 #ifdef _DEBUG
         // Debug code: Copy original A matrix to host
-        std::complex<float> *A_h
-          = (location == QUDA_CUDA_FIELD_LOCATION ? static_cast<std::complex<float> *>(pool_pinned_malloc(size)) :
-                                                    static_cast<std::complex<float> *>(A_d));
-        if (location == QUDA_CUDA_FIELD_LOCATION) qudaMemcpy((void *)A_h, A_d, size, qudaMemcpyDeviceToHost);
+        if (prec == QUDA_SINGLE_PRECISION) {
+          std::complex<float> *A_h
+            = (location == QUDA_CUDA_FIELD_LOCATION ? static_cast<std::complex<float> *>(pool_pinned_malloc(size)) :
+                                                      static_cast<std::complex<float> *>(A_d));
+          if (location == QUDA_CUDA_FIELD_LOCATION) qudaMemcpy((void *)A_h, A_d, size, qudaMemcpyDeviceToHost);
+        } else if (prec == QUDA_DOUBLE_PRECISION) {
+          std::complex<double> *A_h
+            = (location == QUDA_CUDA_FIELD_LOCATION ? static_cast<std::complex<double> *>(pool_pinned_malloc(size)) :
+                                                      static_cast<std::complex<double> *>(A_d));
+          if (location == QUDA_CUDA_FIELD_LOCATION) qudaMemcpy((void *)A_h, A_d, size, qudaMemcpyDeviceToHost);
+        } else {
+          errorQuda("%s not implemented for precision=%d", __func__, prec);
+        }
 #endif
 
-        static_assert(sizeof(int)==sizeof(MKL_INT), "MKL_INT must be int.");
+        MKL_INT *dipiv = static_cast<MKL_INT *>(pool_device_malloc(batch * n * sizeof(MKL_INT)));
+        MKL_INT *dinfo_array = static_cast<MKL_INT *>(pool_device_malloc(batch * sizeof(MKL_INT)));
+        MKL_INT *info_array = static_cast<MKL_INT *>(pool_pinned_malloc(batch * sizeof(MKL_INT)));
+        memset(info_array, '0', batch * sizeof(MKL_INT)); // silence memcheck warnings
 
-        int *dipiv = static_cast<int *>(pool_device_malloc(batch * n * sizeof(int)));
-        int *dinfo_array = static_cast<int *>(pool_device_malloc(batch * sizeof(int)));
-        int *info_array = static_cast<int *>(pool_pinned_malloc(batch * sizeof(int)));
-        memset(info_array, '0', batch * sizeof(int)); // silence memcheck warnings
+        MKL_INT n_array = n;
+        MKL_INT stride_array = n * n;
+        MKL_INT batch_size = batch;
 
         if (prec == QUDA_SINGLE_PRECISION) {
           typedef MKL_Complex8 C;
-          C **A_array = static_cast<C **>(pool_device_malloc(2 * batch * sizeof(C *)));
-          C **Ainv_array = A_array + batch;
-          C **A_array_h = static_cast<C **>(pool_pinned_malloc(2 * batch * sizeof(C *)));
-          C **Ainv_array_h = A_array_h + batch;
-          MKL_INT **ipiv_array = static_cast<MKL_INT **>(pool_device_malloc(batch * sizeof(MKL_INT *)));
-          MKL_INT **ipiv_array_h = static_cast<MKL_INT **>(pool_pinned_malloc(batch * sizeof(MKL_INT *)));
-          for (uint64_t i = 0; i < batch; i++) {
-            A_array_h[i] = static_cast<C *>(A_d) + i * n * n;
-            Ainv_array_h[i] = static_cast<C *>(Ainv_d) + i * n * n;
-            ipiv_array_h[i] = dipiv + i * n;
-          }
-          qudaMemcpy(A_array, A_array_h, 2 * batch * sizeof(C *), qudaMemcpyHostToDevice);
-          qudaMemcpy(ipiv_array, ipiv_array_h, batch * sizeof(MKL_INT *), qudaMemcpyHostToDevice);
-
-          MKL_INT n_array = n;
-          MKL_INT group_size = batch;
-          MKL_INT group_count = (MKL_INT)1;
-          #pragma omp target variant dispatch use_device_ptr(A_array, ipiv_array, dinfo_array)
+          #pragma omp target variant dispatch use_device_ptr(A_d, dipiv, dinfo_array)
           {
-            cgetrf_batch(&n_array, &n_array, A_array, &n_array, ipiv_array, &group_count, &group_size, dinfo_array);
+            cgetrf_batch_strided(&n_array, &n_array, (C *)A_d, &n_array, &stride_array, dipiv, &n_array, &batch_size, dinfo_array);
           }
           flops += batch * FLOPS_CGETRF(n, n);
 
-          qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), qudaMemcpyDeviceToHost);
+          qudaMemcpy(info_array, dinfo_array, batch * sizeof(MKL_INT), qudaMemcpyDeviceToHost);
           for (uint64_t i = 0; i < batch; i++) {
             if (info_array[i] < 0) {
               errorQuda("%lu argument had an illegal value or another error occured, such as memory allocation failed",
@@ -113,13 +107,13 @@ namespace quda
             }
           }
 
-          #pragma omp target variant dispatch use_device_ptr(A_array, Ainv_array, ipiv_array, dinfo_array)
+          #pragma omp target variant dispatch use_device_ptr(A_d, Ainv_d, dipiv, dinfo_array)
           {
-            cgetri_oop_batch(&n_array, (const C**)A_array, &n_array, (const MKL_INT**)ipiv_array, Ainv_array, &n_array, &group_count, &group_size, dinfo_array);
+            cgetri_oop_batch_strided(&n_array, (C *)A_d, &n_array, &stride_array, dipiv, &n_array, (C *)Ainv_d, &n_array, &stride_array, &batch_size, dinfo_array);
           }
           flops += batch * FLOPS_CGETRI(n);
 
-          qudaMemcpy(info_array, dinfo_array, batch * sizeof(int), qudaMemcpyDeviceToHost);
+          qudaMemcpy(info_array, dinfo_array, batch * sizeof(MKL_INT), qudaMemcpyDeviceToHost);
 
           for (uint64_t i = 0; i < batch; i++) {
             if (info_array[i] < 0) {
@@ -130,11 +124,6 @@ namespace quda
             }
           }
 
-          pool_device_free(A_array);
-          pool_pinned_free(A_array_h);
-          pool_device_free(ipiv_array);
-          pool_pinned_free(ipiv_array_h);
-
 #ifdef _DEBUG
           // Debug code: Copy computed Ainv to host
           std::complex<float> *Ainv_h = static_cast<std::complex<float> *>(pool_pinned_malloc(size));
@@ -143,6 +132,49 @@ namespace quda
           for (uint64_t i = 0; i < batch; i++) { checkEigen<MatrixXcf, float>(A_h, Ainv_h, n, i); }
           pool_pinned_free(Ainv_h);
           pool_pinned_free(A_h);
+#endif
+        } else if (prec == QUDA_DOUBLE_PRECISION) {
+          typedef MKL_Complex16 Z;
+          #pragma omp target variant dispatch use_device_ptr(A_d, dipiv, dinfo_array)
+          {
+            zgetrf_batch_strided(&n_array, &n_array, (Z *)A_d, &n_array, &stride_array, dipiv, &n_array, &batch_size, dinfo_array);
+          }
+          flops += batch * FLOPS_ZGETRF(n, n);
+
+          qudaMemcpy(info_array, dinfo_array, batch * sizeof(MKL_INT), qudaMemcpyDeviceToHost);
+          for (uint64_t i = 0; i < batch; i++) {
+            if (info_array[i] < 0) {
+              errorQuda("%lu argument had an illegal value or another error occured, such as memory allocation failed",
+                        i);
+            } else if (info_array[i] > 0) {
+              errorQuda("%lu factorization completed but the factor U is exactly singular", i);
+            }
+          }
+
+          #pragma omp target variant dispatch use_device_ptr(A_d, Ainv_d, dipiv, dinfo_array)
+          {
+            zgetri_oop_batch_strided(&n_array, (Z *)A_d, &n_array, &stride_array, dipiv, &n_array, (Z *)Ainv_d, &n_array, &stride_array, &batch_size, dinfo_array);
+          }
+          flops += batch * FLOPS_CGETRI(n);
+
+          qudaMemcpy(info_array, dinfo_array, batch * sizeof(MKL_INT), qudaMemcpyDeviceToHost);
+          for (uint64_t i = 0; i < batch; i++) {
+            if (info_array[i] < 0) {
+              errorQuda("%lu argument had an illegal value or another error occured, such as memory allocation failed",
+                        i);
+            } else if (info_array[i] > 0) {
+              errorQuda("%lu factorization completed but the factor U is exactly singular", i);
+            }
+          }
+
+#ifdef _DEBUG
+          // Debug code: Copy computed Ainv to host
+          std::complex<double> *Ainv_h = static_cast<std::complex<double> *>(pool_pinned_malloc(size));
+          qudaMemcpy((void *)Ainv_h, Ainv_d, size, qudaMemcpyDeviceToHost);
+
+          for (uint64_t i = 0; i < batch; i++) { checkEigen<MatrixXcd, double>(A_h, Ainv_h, n, i); }
+          pool_pinned_free(Ainv_h);
+          pool_pinned_free(A_h);
 #endif
         } else {
           errorQuda("%s not implemented for precision=%d", __func__, prec);
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index e9ae16f79d..ede0af261e 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -70,9 +70,10 @@ set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES COMPILE_FLAGS "-x c++")
 # target_link_libraries(quda PUBLIC hip::hiprand roc::rocrand hip::hipcub roc::rocprim_hip)
 # target_link_libraries(quda PUBLIC roc::hipblas roc::rocblas)
 
-set(OMPTARGET_MKL_LIBRARY "-lmkl_sycl -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread")
+set(OMPTARGET_MKL_LIBRARY "-lmkl_sycl -lmkl_intel_ilp64 -lmkl_core -lmkl_intel_thread")
 
 if(${QUDA_BUILD_NATIVE_LAPACK} STREQUAL "ON")
+  target_compile_definitions(quda PRIVATE MKL_ILP64)
   target_link_libraries(quda PUBLIC ${OMPTARGET_MKL_LIBRARY})
 endif()
 

From 60eb524ab481df8861a3c35ca6a8960a430a65d4 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 26 Oct 2022 15:25:54 +0000
Subject: [PATCH 162/198] omp: add pragma once to thread_array.h

---
 include/targets/omptarget/thread_array.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/include/targets/omptarget/thread_array.h b/include/targets/omptarget/thread_array.h
index 2bd21ec0e0..434776ce93 100644
--- a/include/targets/omptarget/thread_array.h
+++ b/include/targets/omptarget/thread_array.h
@@ -1,3 +1,5 @@
+#pragma once
+
 #include "shared_memory_cache_helper.h"
 
 namespace quda

From 09b4c2dadd3bc479e00e01a5a3193d66e7700caa Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 7 Nov 2022 23:04:18 +0000
Subject: [PATCH 163/198] omp: remove our copy of mrg32k3a, use generic

---
 include/targets/omptarget/mrg32k3a.h | 207 ---------------------------
 1 file changed, 207 deletions(-)
 delete mode 100644 include/targets/omptarget/mrg32k3a.h

diff --git a/include/targets/omptarget/mrg32k3a.h b/include/targets/omptarget/mrg32k3a.h
deleted file mode 100644
index 0dacbfcf4f..0000000000
--- a/include/targets/omptarget/mrg32k3a.h
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
-   An implementation of MRG32k3a based on constexpr.
-   Original algorithm from
-      Pierre L'Ecuyer
-      Good Parameter Sets for Combined Multiple Recursive Random Number Generators
-      Operations Research, 47, 1 (1999), 159-164.
- */
-
-#pragma once
-
-#include<cmath>
-#include<cstdint>
-#include<iostream>
-#include<limits>
-
-namespace quda
-{
-  namespace target
-  {
-    namespace rng
-    {
-      template <typename E,int N>
-      struct Arr {
-        constexpr E& operator[](int i) {return d[i];}
-        constexpr const E& operator[](int i) const {return d[i];}
-        E d[N];
-      };
-
-      template <typename E>
-      struct Arr3 {
-        constexpr Arr3() {};
-        constexpr Arr3(E x,E y,E z):d{x,y,z} {};
-        constexpr E& operator[](int i) {return d[i];}
-        constexpr const E& operator[](int i) const {return d[i];}
-        constexpr bool operator==(const Arr3& x) const {return d[0]==x.d[0] && d[1]==x.d[1] && d[2]==x.d[2];}
-        E d[3];
-      };
-
-      template <typename E>
-      inline std::ostream& operator<<(std::ostream& o, const Arr3<E>& a)
-      {
-        return o<<'['<<a[0]<<' '<<a[1]<<' '<<a[2]<<']';
-      }
-
-      using State = Arr3<uint32_t>;
-      using Trans = Arr3<State>;
-
-      struct MRG32k3a {
-        State s1,s2;
-      };
-
-      inline std::ostream& operator<<(std::ostream& o, const MRG32k3a& prn)
-      {
-        return o<<"MRG32k3a("<<prn.s1<<' '<<prn.s2<<')';
-      }
-
-      inline constexpr Trans squaremod(Trans a, uint64_t m)
-      {
-        Trans x;
-        for(int i=0;i<3;++i){
-          Arr3<uint64_t> t{0u,0u,0u};
-          for(int k=0;k<3;++k){
-            uint64_t aik = (uint64_t)a[i][k];
-            t[0] += aik*(uint64_t)a[k][0] % m;
-            t[1] += aik*(uint64_t)a[k][1] % m;
-            t[2] += aik*(uint64_t)a[k][2] % m;
-          }
-          x[i][0] = (uint32_t)(t[0] % m);
-          x[i][1] = (uint32_t)(t[1] % m);
-          x[i][2] = (uint32_t)(t[2] % m);
-        }
-        return x;
-      }
-
-      template <int p>
-      inline constexpr Arr<Trans,p> squaremodarray(Trans a, uint64_t m)
-      {
-        Arr<Trans,p> x;
-        x[0] = a;
-        for(int i=1;i<p;++i)
-          x[i] = squaremod(x[i-1], m);
-        return x;
-      }
-
-      inline void matvecmod(Trans a, State& v, uint64_t m)
-      {
-        uint64_t v0 = (uint64_t)v[0];
-        uint64_t v1 = (uint64_t)v[1];
-        uint64_t v2 = (uint64_t)v[2];
-        v[0] = (uint32_t)((((uint64_t)a[0][0]*v0) % m + ((uint64_t)a[0][1]*v1) % m + ((uint64_t)a[0][2]*v2) % m) % m);
-        v[1] = (uint32_t)((((uint64_t)a[1][0]*v0) % m + ((uint64_t)a[1][1]*v1) % m + ((uint64_t)a[1][2]*v2) % m) % m);
-        v[2] = (uint32_t)((((uint64_t)a[2][0]*v0) % m + ((uint64_t)a[2][1]*v1) % m + ((uint64_t)a[2][2]*v2) % m) % m);
-      }
-
-      constexpr double norm = 2.328306549295728e-10;
-      constexpr double m1 = 4294967087.0;
-      constexpr double m2 = 4294944443.0;
-      constexpr double a12 = 1403580.0;
-      constexpr double a13n = 810728.0;
-      constexpr double a21 = 527612.0;
-      constexpr double a23n = 1370589.0;
-      constexpr uint32_t defaultSEED = 12345U;
-      constexpr int subsequenceBase = 76;
-
-      constexpr Trans a1 {
-        State {0u,                  1u,            0u},
-        State {0u,                  0u,            1u},
-        State {(uint32_t)(m1-a13n), (uint32_t)a12, 0u}};
-      constexpr Trans a2 {
-        State {0u,                  1u, 0u},
-        State {0u,                  0u, 1u},
-        State {(uint32_t)(m2-a23n), 0u, (uint32_t)a21}};
-
-      constexpr int maxpower2 = 190;
-      constexpr Arr<Trans,maxpower2> a1sq = squaremodarray<maxpower2>(a1, (uint64_t)m1);
-      constexpr Arr<Trans,maxpower2> a2sq = squaremodarray<maxpower2>(a2, (uint64_t)m2);
-
-      static_assert(
-        a1sq[76] ==
-          Trans {
-            State {82758667u,   1871391091u, 4127413238u},
-            State {3672831523u, 69195019u,   1871391091u},
-            State {3672091415u, 3528743235u, 69195019u}},
-        "a1sq[76] wrong!");
-      static_assert(
-        a2sq[76] ==
-          Trans {
-            State {1511326704u, 3759209742u, 1610795712u},
-            State {4292754251u, 1511326704u, 3889917532u},
-            State {3859662829u, 4292754251u, 3708466080u}},
-        "a2sq[76] wrong!");
-
-      inline void skip(MRG32k3a& prn, uint64_t offset, int base = 0)
-      {
-        int i = 0;
-        uint64_t s = offset;
-        while(s>0){
-          if(s&1u){
-            matvecmod(a1sq[base+i], prn.s1, (uint64_t)m1);
-            matvecmod(a2sq[base+i], prn.s2, (uint64_t)m2);
-          }
-          s >>= 1u;
-          ++i;
-        }
-      }
-
-      inline void seed(MRG32k3a& prn, uint64_t seed, uint64_t subsequence)
-      {
-        if(seed){
-          const uint64_t d1 = (uint64_t)defaultSEED * (uint64_t)((uint32_t)seed ^ 0x55555555u);
-          const uint64_t d2 = (uint64_t)defaultSEED * (uint64_t)((uint32_t)(seed >> 32u) ^ 0xAAAAAAAAu);
-          prn.s1[0] = (uint32_t)(d1 % (uint64_t)m1);
-          prn.s1[1] = (uint32_t)(d2 % (uint64_t)m1);
-          prn.s1[2] = (uint32_t)(d1 % (uint64_t)m1);
-          prn.s2[0] = (uint32_t)(d2 % (uint64_t)m2);
-          prn.s2[1] = (uint32_t)(d1 % (uint64_t)m2);
-          prn.s2[2] = (uint32_t)(d2 % (uint64_t)m2);
-        }else{
-          prn.s1[0] = defaultSEED;
-          prn.s1[1] = defaultSEED;
-          prn.s1[2] = defaultSEED;
-          prn.s2[0] = defaultSEED;
-          prn.s2[1] = defaultSEED;
-          prn.s2[2] = defaultSEED;
-        }
-        skip(prn, subsequence, subsequenceBase);
-      }
-
-      inline double uniform(MRG32k3a& prn)
-      {
-        double p1,p2;
-        p1 = a12 * (double)(prn.s1[1]) - a13n * (double)(prn.s1[0]);
-        p1 = std::fmod(p1, m1);
-        if(p1<0.0)
-          p1 += m1;
-        prn.s1[0] = prn.s1[1];
-        prn.s1[1] = prn.s1[2];
-        prn.s1[2] = (uint32_t)p1;
-
-        p2 = a21 * (double)(prn.s2[2]) - a23n * (double)(prn.s2[0]);
-        p2 = std::fmod(p2, m2);
-        if(p2<0.0)
-          p2 += m2;
-        prn.s2[0] = prn.s2[1];
-        prn.s2[1] = prn.s2[2];
-        prn.s2[2] = (uint32_t)p2;
-
-        if(p1<=p2)
-          return (p1 - p2 + m1) * norm;
-        else
-          return (p1 - p2) * norm;
-      }
-
-      template <typename R>
-      inline void gaussian(MRG32k3a& prn, R& x, R& y)
-      {
-        constexpr R TINY = std::numeric_limits<R>::min();
-        R v,p,r;
-        v = (R)uniform(prn);
-        p = (R)uniform(prn) * (R)2.0 * (R)3.141592653589793238462643383279502884;
-        r = std::sqrt((R)(-2.0) * std::log(v + TINY));
-        x = r * std::sin(p);
-        y = r * std::cos(p);
-      }
-    }
-  }
-}

From 9dac653a432b354a02b7d56cb97963c52800e303 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 7 Nov 2022 23:25:42 +0000
Subject: [PATCH 164/198] omp: mark launch_param declare target

---
 include/targets/omptarget/quda_api.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index b66ad820a4..a64bd3eea6 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -98,7 +98,9 @@ namespace quda {
         dim3 grid;
         SharedCache shared_cache;
       };
+#pragma omp declare target
       extern LaunchParam *launch_param;
+#pragma omp end declare target
       extern LaunchParam *launch_param_host;
 
       inline int *get_shared_cache(void)

From b91feea4ee49687ff2518a12af508264306e51b0 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 17 Nov 2022 16:08:49 +0000
Subject: [PATCH 165/198] omp/blas_lapack_mkl: use 5.1 standard omp dispatch
 instead of Intel extension

---
 lib/targets/omptarget/blas_lapack_mkl.cpp | 112 +++++++++-------------
 1 file changed, 44 insertions(+), 68 deletions(-)

diff --git a/lib/targets/omptarget/blas_lapack_mkl.cpp b/lib/targets/omptarget/blas_lapack_mkl.cpp
index 141b146404..6b2db44e7d 100644
--- a/lib/targets/omptarget/blas_lapack_mkl.cpp
+++ b/lib/targets/omptarget/blas_lapack_mkl.cpp
@@ -91,10 +91,8 @@ namespace quda
 
         if (prec == QUDA_SINGLE_PRECISION) {
           typedef MKL_Complex8 C;
-          #pragma omp target variant dispatch use_device_ptr(A_d, dipiv, dinfo_array)
-          {
-            cgetrf_batch_strided(&n_array, &n_array, (C *)A_d, &n_array, &stride_array, dipiv, &n_array, &batch_size, dinfo_array);
-          }
+          #pragma omp dispatch is_device_ptr(A_d, dipiv, dinfo_array)
+          cgetrf_batch_strided(&n_array, &n_array, (C *)A_d, &n_array, &stride_array, dipiv, &n_array, &batch_size, dinfo_array);
           flops += batch * FLOPS_CGETRF(n, n);
 
           qudaMemcpy(info_array, dinfo_array, batch * sizeof(MKL_INT), qudaMemcpyDeviceToHost);
@@ -107,10 +105,8 @@ namespace quda
             }
           }
 
-          #pragma omp target variant dispatch use_device_ptr(A_d, Ainv_d, dipiv, dinfo_array)
-          {
-            cgetri_oop_batch_strided(&n_array, (C *)A_d, &n_array, &stride_array, dipiv, &n_array, (C *)Ainv_d, &n_array, &stride_array, &batch_size, dinfo_array);
-          }
+          #pragma omp dispatch is_device_ptr(A_d, Ainv_d, dipiv, dinfo_array)
+          cgetri_oop_batch_strided(&n_array, (C *)A_d, &n_array, &stride_array, dipiv, &n_array, (C *)Ainv_d, &n_array, &stride_array, &batch_size, dinfo_array);
           flops += batch * FLOPS_CGETRI(n);
 
           qudaMemcpy(info_array, dinfo_array, batch * sizeof(MKL_INT), qudaMemcpyDeviceToHost);
@@ -135,10 +131,8 @@ namespace quda
 #endif
         } else if (prec == QUDA_DOUBLE_PRECISION) {
           typedef MKL_Complex16 Z;
-          #pragma omp target variant dispatch use_device_ptr(A_d, dipiv, dinfo_array)
-          {
-            zgetrf_batch_strided(&n_array, &n_array, (Z *)A_d, &n_array, &stride_array, dipiv, &n_array, &batch_size, dinfo_array);
-          }
+          #pragma omp dispatch is_device_ptr(A_d, dipiv, dinfo_array)
+          zgetrf_batch_strided(&n_array, &n_array, (Z *)A_d, &n_array, &stride_array, dipiv, &n_array, &batch_size, dinfo_array);
           flops += batch * FLOPS_ZGETRF(n, n);
 
           qudaMemcpy(info_array, dinfo_array, batch * sizeof(MKL_INT), qudaMemcpyDeviceToHost);
@@ -151,10 +145,8 @@ namespace quda
             }
           }
 
-          #pragma omp target variant dispatch use_device_ptr(A_d, Ainv_d, dipiv, dinfo_array)
-          {
-            zgetri_oop_batch_strided(&n_array, (Z *)A_d, &n_array, &stride_array, dipiv, &n_array, (Z *)Ainv_d, &n_array, &stride_array, &batch_size, dinfo_array);
-          }
+          #pragma omp dispatch is_device_ptr(A_d, Ainv_d, dipiv, dinfo_array)
+          zgetri_oop_batch_strided(&n_array, (Z *)A_d, &n_array, &stride_array, dipiv, &n_array, (Z *)Ainv_d, &n_array, &stride_array, &batch_size, dinfo_array);
           flops += batch * FLOPS_CGETRI(n);
 
           qudaMemcpy(info_array, dinfo_array, batch * sizeof(MKL_INT), qudaMemcpyDeviceToHost);
@@ -374,20 +366,16 @@ namespace quda
                                             (double)(static_cast<std::complex<double>>(blas_param.beta).imag()));
 
           if (batch > 1) {
-            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
-            {
-              cblas_zgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
-                                        &alpha, (Z *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
-                                        (Z *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
-                                        (Z *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
-            }
+            #pragma omp dispatch is_device_ptr(A_d, B_d, C_d)
+            cblas_zgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                      &alpha, (Z *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                      (Z *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
+                                      (Z *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
           } else {
-            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
-            {
-              cblas_zgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
-                          (Z *)A_d + blas_param.a_offset, blas_param.lda, (Z *)B_d + blas_param.b_offset,
-                          blas_param.ldb, &beta, (Z *)C_d + blas_param.c_offset, blas_param.ldc);
-            }
+            #pragma omp dispatch is_device_ptr(A_d, B_d, C_d)
+            cblas_zgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
+                        (Z *)A_d + blas_param.a_offset, blas_param.lda, (Z *)B_d + blas_param.b_offset,
+                        blas_param.ldb, &beta, (Z *)C_d + blas_param.c_offset, blas_param.ldc);
           }
           flops += batch * FLOPS_CGEMM(blas_param.m, blas_param.n, blas_param.k);
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_C) {
@@ -402,20 +390,16 @@ namespace quda
                                           (float)(static_cast<std::complex<double>>(blas_param.beta).imag()));
 
           if (batch > 1) {
-            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
-            {
-              cblas_cgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
-                                         &alpha, (C *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
-                                         (C *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
-                                         (C *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
-            }
+            #pragma omp dispatch is_device_ptr(A_d, B_d, C_d)
+            cblas_cgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                      &alpha, (C *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                      (C *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, &beta,
+                                      (C *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
           } else {
-            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
-            {
-              cblas_cgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
-                          (C *)A_d + blas_param.a_offset, blas_param.lda, (C *)B_d + blas_param.b_offset,
-                          blas_param.ldb, &beta, (C *)C_d + blas_param.c_offset, blas_param.ldc);
-            }
+            #pragma omp dispatch is_device_ptr(A_d, B_d, C_d)
+            cblas_cgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, &alpha,
+                        (C *)A_d + blas_param.a_offset, blas_param.lda, (C *)B_d + blas_param.b_offset,
+                        blas_param.ldb, &beta, (C *)C_d + blas_param.c_offset, blas_param.ldc);
           }
           flops += batch * FLOPS_CGEMM(blas_param.m, blas_param.n, blas_param.k);
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_D) {
@@ -426,20 +410,16 @@ namespace quda
           const D beta = (D)(static_cast<std::complex<double>>(blas_param.beta).real());
 
           if (batch > 1) {
-            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
-            {
-              cblas_dgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
-                                        alpha, (D *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
-                                        (D *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, beta,
-                                        (D *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
-            }
+            #pragma omp dispatch is_device_ptr(A_d, B_d, C_d)
+            cblas_dgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                      alpha, (D *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                      (D *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, beta,
+                                      (D *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
           } else {
-            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
-            {
-              cblas_dgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, alpha,
-                          (D *)A_d + blas_param.a_offset, blas_param.lda, (D *)B_d + blas_param.b_offset,
-                          blas_param.ldb, beta, (D *)C_d + blas_param.c_offset, blas_param.ldc);
-            }
+            #pragma omp dispatch is_device_ptr(A_d, B_d, C_d)
+            cblas_dgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, alpha,
+                        (D *)A_d + blas_param.a_offset, blas_param.lda, (D *)B_d + blas_param.b_offset,
+                        blas_param.ldb, beta, (D *)C_d + blas_param.c_offset, blas_param.ldc);
           }
           flops += batch * FLOPS_SGEMM(blas_param.m, blas_param.n, blas_param.k);
         } else if (blas_param.data_type == QUDA_BLAS_DATATYPE_S) {
@@ -450,20 +430,16 @@ namespace quda
           const S beta = (S)(static_cast<std::complex<float>>(blas_param.beta).real());
 
           if (batch > 1) {
-            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
-            {
-              cblas_sgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
-                                        alpha, (S *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
-                                        (S *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, beta,
-                                        (S *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
-            }
+            #pragma omp dispatch is_device_ptr(A_d, B_d, C_d)
+            cblas_sgemm_batch_strided(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k,
+                                      alpha, (S *)A_d + blas_param.a_offset, blas_param.lda, a_stride,
+                                      (S *)B_d + blas_param.b_offset, blas_param.ldb, b_stride, beta,
+                                      (S *)C_d + blas_param.c_offset, blas_param.ldc, c_stride, batch);
           } else {
-            #pragma omp target variant dispatch use_device_ptr(A_d, B_d, C_d)
-            {
-              cblas_sgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, alpha,
-                          (S *)A_d + blas_param.a_offset, blas_param.lda, (S *)B_d + blas_param.b_offset,
-                          blas_param.ldb, beta, (S *)C_d + blas_param.c_offset, blas_param.ldc);
-            }
+            #pragma omp dispatch is_device_ptr(A_d, B_d, C_d)
+            cblas_sgemm(CblasColMajor, trans_a, trans_b, blas_param.m, blas_param.n, blas_param.k, alpha,
+                        (S *)A_d + blas_param.a_offset, blas_param.lda, (S *)B_d + blas_param.b_offset,
+                        blas_param.ldb, beta, (S *)C_d + blas_param.c_offset, blas_param.ldc);
           }
           flops += batch * FLOPS_SGEMM(blas_param.m, blas_param.n, blas_param.k);
         } else {

From 7a9705d80aefa4405fc8038d013eff3a0dbc0df4 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 6 Dec 2022 20:48:30 +0000
Subject: [PATCH 166/198] omp: declare target some static constexpr

---
 include/kernels/blas_core.cuh         | 30 ++++++++++++++++++++
 include/kernels/multi_blas_core.cuh   | 12 ++++++++
 include/kernels/multi_reduce_core.cuh |  6 ++++
 include/kernels/reduce_core.cuh       | 40 +++++++++++++++++++++++++++
 include/targets/generic/mrg32k3a.h    |  2 ++
 5 files changed, 90 insertions(+)

diff --git a/include/kernels/blas_core.cuh b/include/kernels/blas_core.cuh
index 96f0c2b5f1..f123cfaa16 100644
--- a/include/kernels/blas_core.cuh
+++ b/include/kernels/blas_core.cuh
@@ -95,8 +95,10 @@ namespace quda
        Functor to perform the operation z = a*x + b*y
     */
     template <typename real> struct axpbyz_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 0, 0> read{ };
       static constexpr memory_access<0, 0, 0, 0, 1> write{ };
+#pragma omp end declare target
       const real a;
       const real b;
       axpbyz_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
@@ -112,8 +114,10 @@ namespace quda
        Functor to perform the operation y = a * x
     */
     template <typename real> struct axy_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 0> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       const real a;
       axy_(const real &a, const real &, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
@@ -128,8 +132,10 @@ namespace quda
        Functor to perform the operator y += a*x (complex-valued)
     */
     template <typename real> struct caxpy_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       caxpy_(const complex<real> &a, const complex<real> &, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
@@ -160,8 +166,10 @@ namespace quda
     }
 
     template <typename real> struct caxpby_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpby_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -177,8 +185,10 @@ namespace quda
        Functor performing the operation: w[i] = a*x[i] + b*y[i] + c*z[i]
     */
     template <typename real> struct axpbypczw_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1> read{ };
       static constexpr memory_access<0, 0, 0, 1> write{ };
+#pragma omp end declare target
       const real a;
       const real b;
       const real c;
@@ -197,8 +207,10 @@ namespace quda
        Functor performing the operations: y[i] = a*x[i] + y[i]; x[i] = b*z[i] + c*x[i]
     */
     template <typename real> struct axpyBzpcx_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
+#pragma omp end declare target
       const real a;
       const real b;
       const real c;
@@ -218,8 +230,10 @@ namespace quda
        Functor performing the operations: y[i] = a*x[i] + y[i]; x[i] = z[i] + b*x[i]
     */
     template <typename real> struct axpyZpbx_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
+#pragma omp end declare target
       const real a;
       const real b;
       axpyZpbx_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
@@ -238,8 +252,10 @@ namespace quda
        Functor performing the operation z[i] = x[i] + a * y[i] + b * z[i]
     */
     template <typename real> struct cxpaypbz_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<0, 0, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       cxpaypbz_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -258,8 +274,10 @@ namespace quda
        Functor performing the operations y[i] = a*x[i] + y[i] and x[i] = b*z[i] + x[i]
     */
     template <typename real> struct caxpyBzpx_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpyBzpx_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -278,8 +296,10 @@ namespace quda
        Functor performing the operations y[i] = a*x[i] + y[i] and z[i] = b*x[i] + z[i]
     */
     template <typename real> struct caxpyBxpz_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<0, 1, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpyBxpz_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -298,8 +318,10 @@ namespace quda
        Functor performing the operations z[i] = a*x[i] + b*y[i] + z[i] and y[i] -= b*w[i]
     */
     template <typename real> struct caxpbypzYmbw_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1> read{ };
       static constexpr memory_access<0, 1, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpbypzYmbw_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -319,8 +341,10 @@ namespace quda
        Functor performing the operation y[i] += a*b*x[i], x[i] *= a
     */
     template <typename real> struct cabxpyAx_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
+#pragma omp end declare target
       const real a;
       const complex<real> b;
       cabxpyAx_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a.real()), b(b) { ; }
@@ -341,8 +365,10 @@ namespace quda
        Second performs the operator x[i] -= a*z[i]
     */
     template <typename real> struct caxpyxmaz_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       caxpyxmaz_(const complex<real> &a, const complex<real> &, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
@@ -366,8 +392,10 @@ namespace quda
     */
     template <typename real> struct caxpyxmazMR_ {
       static constexpr use_kernel_arg_p use_kernel_arg = use_kernel_arg_p::ALWAYS;
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
+#pragma omp end declare target
       complex<real> a;
       double3 *Ar3;
       bool init_;
@@ -405,8 +433,10 @@ namespace quda
        Third performs the operation w[i] = z[i] + b*w[i]
     */
     template <typename real> struct tripleCGUpdate_ : public BlasFunctor {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1> read{ };
       static constexpr memory_access<0, 1, 1, 1> write{ };
+#pragma omp end declare target
       const real a;
       const real b;
       tripleCGUpdate_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
diff --git a/include/kernels/multi_blas_core.cuh b/include/kernels/multi_blas_core.cuh
index 4d951e32b1..fb2577a2a2 100644
--- a/include/kernels/multi_blas_core.cuh
+++ b/include/kernels/multi_blas_core.cuh
@@ -133,8 +133,10 @@ namespace quda
     */
     template <typename real>
     struct multiaxpy_ : public MultiBlasFunctor<real> {
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = false;
       static constexpr int NXZ_max = 0;
@@ -155,8 +157,10 @@ namespace quda
     */
     template <typename real>
     struct multicaxpy_ : public MultiBlasFunctor<complex<real>> {
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = false;
       static constexpr int NXZ_max = 0;
@@ -177,8 +181,10 @@ namespace quda
     */
     template <typename real>
     struct multiaxpyz_ : public MultiBlasFunctor<real> {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 0> read{ };
       static constexpr memory_access<0, 0, 0, 1> write{ };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       static constexpr int NXZ_max = 0;
@@ -202,8 +208,10 @@ namespace quda
     */
     template <typename real>
     struct multicaxpyz_ : public MultiBlasFunctor<complex<real>> {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 0> read{ };
       static constexpr memory_access<0, 0, 0, 1> write{ };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       static constexpr int NXZ_max = 0;
@@ -227,8 +235,10 @@ namespace quda
     */
     template <typename real>
     struct multi_axpyBzpcx_ : public MultiBlasFunctor<real, true> {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 1> read{ };
       static constexpr memory_access<0, 1, 0, 1> write{ };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       static constexpr int NXZ_max = 1; // we never have NXZ > 1 for this kernel
@@ -257,8 +267,10 @@ namespace quda
     */
     template <typename real>
     struct multi_caxpyBxpz_ : public MultiBlasFunctor<complex<real>, true> {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 1> read{ };
       static constexpr memory_access<0, 1, 0, 1> write{ };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       static constexpr int NXZ_max = 0;
diff --git a/include/kernels/multi_reduce_core.cuh b/include/kernels/multi_reduce_core.cuh
index 9fcadd14c9..2f70c91852 100644
--- a/include/kernels/multi_reduce_core.cuh
+++ b/include/kernels/multi_reduce_core.cuh
@@ -150,8 +150,10 @@ namespace quda
 
     template <typename reduce_t, typename real>
     struct multiDot : public MultiReduceFunctor<reduce_t, real> {
+#pragma omp declare target
       static constexpr memory_access<1, 1> read { };
       static constexpr memory_access< > write { };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = false;
       multiDot(int NXZ, int NYW) : MultiReduceFunctor<reduce_t, real>(NXZ, NYW) { }
@@ -181,8 +183,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct multiCdot : public MultiReduceFunctor<array<real_reduce_t, 2>, complex<real>> {
       using reduce_t = array<real_reduce_t, 2>;
+#pragma omp declare target
       static constexpr memory_access<1, 1> read { };
       static constexpr memory_access< > write { };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = false;
       multiCdot(int NXZ, int NYW) : MultiReduceFunctor<reduce_t, complex<real>>(NXZ, NYW) { }
@@ -199,8 +203,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct multiCdotCopy : public MultiReduceFunctor<typename VectorType<real_reduce_t, 2>::type, complex<real>> {
       using reduce_t = typename VectorType<real_reduce_t, 2>::type;
+#pragma omp declare target
       static constexpr memory_access<1, 1> read { };
       static constexpr memory_access<0, 0, 0, 1> write { };
+#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       multiCdotCopy(int NXZ, int NYW) : MultiReduceFunctor<reduce_t, complex<real>>(NXZ, NYW) { }
diff --git a/include/kernels/reduce_core.cuh b/include/kernels/reduce_core.cuh
index ea409ebc0e..c54183a048 100644
--- a/include/kernels/reduce_core.cuh
+++ b/include/kernels/reduce_core.cuh
@@ -117,8 +117,10 @@ namespace quda
     template <typename reduce_t, typename real>
     struct Max : public ReduceFunctor<reduce_t> {
       using reducer = maximum<reduce_t>;
+#pragma omp declare target
       static constexpr memory_access<1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       Max(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &max, T &x, T &, T &, T &, T &) const
       {
@@ -134,8 +136,10 @@ namespace quda
     template <typename reduce_t, typename real>
     struct MaxDeviation : public ReduceFunctor<reduce_t> {
       using reducer = maximum<reduce_t>;
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       MaxDeviation(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &max, T &x, T &y, T &, T &, T &) const
       {
@@ -161,8 +165,10 @@ namespace quda
 
     template <typename reduce_t, typename real>
     struct Norm1 : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       Norm1(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &, T &, T &, T &) const
       {
@@ -183,8 +189,10 @@ namespace quda
 
     template <typename reduce_t, typename real>
     struct Norm2 : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       Norm2(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &, T &, T &, T &) const
       {
@@ -206,8 +214,10 @@ namespace quda
 
     template <typename reduce_t, typename real>
     struct Dot : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1,1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       Dot(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
@@ -223,8 +233,10 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct axpbyzNorm2 : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 0> read{ };
       static constexpr memory_access<0, 0, 1> write{ };
+#pragma omp end declare target
       const real a;
       const real b;
       axpbyzNorm2(const real &a, const real &b) : a(a), b(b) { ; }
@@ -245,8 +257,10 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct AxpyReDot : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       const real a;
       AxpyReDot(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
@@ -266,8 +280,10 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct caxpyNorm2 : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       caxpyNorm2(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
@@ -289,8 +305,10 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct cabxpyzaxnorm : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 0> read{ };
       static constexpr memory_access<1, 0, 1> write{ };
+#pragma omp end declare target
       const real a;
       const complex<real> b;
       cabxpyzaxnorm(const complex<real> &a, const complex<real> &b) : a(a.real()), b(b) { ; }
@@ -322,8 +340,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct Cdot : public ReduceFunctor<array<real_reduce_t, 2>> {
       using reduce_t = array<real_reduce_t, 2>;
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       Cdot(const complex<real> &, const complex<real> &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
@@ -341,8 +361,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct caxpydotzy : public ReduceFunctor<array<real_reduce_t, 2>> {
       using reduce_t = array<real_reduce_t, 2>;
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       caxpydotzy(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
@@ -373,8 +395,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct CdotNormAB : public ReduceFunctor<array<real_reduce_t, 4>> {
       using reduce_t = array<real_reduce_t, 4>;
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       CdotNormAB(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
@@ -404,8 +428,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct caxpbypzYmbwcDotProductUYNormY_ : public ReduceFunctor<array<real_reduce_t, 3>> {
       using reduce_t = array<real_reduce_t, 3>;
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1, 1> read{ };
       static constexpr memory_access<0, 1, 1> write{ };
+#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpbypzYmbwcDotProductUYNormY_(const complex<real> &a, const complex<real> &b) : a(a), b(b) { ; }
@@ -431,8 +457,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct axpyCGNorm2 : public ReduceFunctor<array<real_reduce_t, 2>> {
       using reduce_t = array<real_reduce_t, 2>;
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
+#pragma omp end declare target
       const real a;
       axpyCGNorm2(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
@@ -463,8 +491,10 @@ namespace quda
       using reducer = plus<reduce_t>;
       static constexpr bool site_unroll = true;
 
+#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       reduce_t aux;
       HeavyQuarkResidualNorm_(const real &, const real &) : aux {} { ; }
 
@@ -510,8 +540,10 @@ namespace quda
       using reducer = plus<reduce_t>;
       static constexpr bool site_unroll = true;
 
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       reduce_t aux;
       xpyHeavyQuarkResidualNorm_(const real &, const real &) : aux {} { ; }
 
@@ -550,8 +582,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct tripleCGReduction_ : public ReduceFunctor<array<real_reduce_t, 3>> {
       using reduce_t = array<real_reduce_t, 3>;
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       tripleCGReduction_(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
@@ -575,8 +609,10 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct quadrupleCGReduction_ : public ReduceFunctor<array<real_reduce_t, 4>> {
       using reduce_t = array<real_reduce_t, 4>;
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1> read{ };
       static constexpr memory_access<> write{ };
+#pragma omp end declare target
       quadrupleCGReduction_(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &) const
       {
@@ -601,8 +637,10 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct quadrupleCG3InitNorm_ : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 0, 1> read{ };
       static constexpr memory_access<1, 1, 1, 1> write{ };
+#pragma omp end declare target
       const real a;
       quadrupleCG3InitNorm_(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &v) const
@@ -631,8 +669,10 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct quadrupleCG3UpdateNorm_ : public ReduceFunctor<reduce_t> {
+#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1, 1> read{ };
       static constexpr memory_access<1, 1, 1, 1> write{ };
+#pragma omp end declare target
       const real a;
       const real b;
       quadrupleCG3UpdateNorm_(const real &a, const real &b) : a(a), b(b) { ; }
diff --git a/include/targets/generic/mrg32k3a.h b/include/targets/generic/mrg32k3a.h
index e0dd4e2a4e..df5bb10e26 100644
--- a/include/targets/generic/mrg32k3a.h
+++ b/include/targets/generic/mrg32k3a.h
@@ -117,8 +117,10 @@ namespace quda
       constexpr auto a2sq = squaremodarray<maxpower2>(a2, static_cast<uint64_t>(m2));
 
       // device copies of a1sq and a2sq
+#pragma omp declare target
       __device__ constexpr auto a1sq_d = a1sq;
       __device__ constexpr auto a2sq_d = a2sq;
+#pragma omp end declare target
 
       // default functor for grabbing the appropriate a1sq
       template <bool is_device> struct get_a1sq {

From 5d3404417ac3491811df8d7a744d54008cd6f107 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 9 Dec 2022 23:43:15 +0000
Subject: [PATCH 167/198] color_spinor_pack: tag PackGhostArg with require
 thread sync X

---
 include/kernels/color_spinor_pack.cuh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/kernels/color_spinor_pack.cuh b/include/kernels/color_spinor_pack.cuh
index 96b7fd8c67..ff56b29b5b 100644
--- a/include/kernels/color_spinor_pack.cuh
+++ b/include/kernels/color_spinor_pack.cuh
@@ -60,6 +60,7 @@ namespace quda {
 
   template <typename store_t, typename ghost_store_t, int nSpin_, int nColor_, int nDim_, QudaFieldOrder order>
   struct PackGhostArg : kernel_param<> {
+    static constexpr ThreadsSync requires_threads_sync = ThreadsSyncX;
     static constexpr bool block_float = sizeof(store_t) == QUDA_SINGLE_PRECISION && isFixed<ghost_store_t>::value;
 
     // ensure we only compile supported block-float kernels

From fdd58d6bfebdf7324e3eea14a761b4d4dd09e32a Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 12 Jan 2023 20:36:05 +0000
Subject: [PATCH 168/198] omp: static_assert enough shared cache for
 shared_atomic VUV

---
 include/kernels/coarse_op_kernel.cuh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 6308b7ca57..55fddbaca9 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -1373,6 +1373,7 @@ namespace quda {
       __shared__ complex<storeType> Y[Arg::max_color_height_per_block][Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
 #else
       typedef complex<storeType> TyX0[Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
+      static_assert(sizeof(TyX0)*Arg::max_color_height_per_block*2 <= device::max_shared_memory_size(), "Shared cache not large enough for X and Y");
       TyX0 *X = reinterpret_cast<TyX0 *>(target::omptarget::get_shared_cache());
       TyX0 *Y = X+Arg::max_color_height_per_block;
 #endif

From 55e44c4b8560fd500240b75876e0e573d035df90 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 24 Jan 2023 22:08:07 +0000
Subject: [PATCH 169/198] omp: follow upstream changes to shared_memory_cache

---
 .../targets/omptarget/shared_memory_cache_helper.h  | 13 +++----------
 include/targets/omptarget/target_device.h           |  6 ------
 include/targets/omptarget/thread_array.h            |  2 +-
 3 files changed, 4 insertions(+), 17 deletions(-)

diff --git a/include/targets/omptarget/shared_memory_cache_helper.h b/include/targets/omptarget/shared_memory_cache_helper.h
index 997ccf5b09..ba43c82150 100644
--- a/include/targets/omptarget/shared_memory_cache_helper.h
+++ b/include/targets/omptarget/shared_memory_cache_helper.h
@@ -29,20 +29,13 @@ namespace quda
        parameters correspond to the y and z dimensions of the block,
        respectively.  The x dimension of the block will be set
        according the maximum number of threads possible, given these
-       dimensions.  To prevent shared-memory bank conflicts this width
-       is optionally padded to allow for access along the y and z dimensions.
+       dimensions.
    */
-  template <typename T, int block_size_y = 1, int block_size_z = 1, bool pad = false, bool dynamic = true>
+  template <typename T, int block_size_y = 1, int block_size_z = 1, bool dynamic = true>
   class SharedMemoryCache
   {
     /** maximum number of threads in x given the y and z block sizes */
-    static constexpr int max_block_size_x = device::max_block_size<block_size_y, block_size_z>();
-
-    /** pad in the x dimension width if requested to ensure that it isn't a multiple of the bank width */
-    static constexpr int block_size_x = !pad ?
-      max_block_size_x :
-      ((max_block_size_x + device::shared_memory_bank_width() - 1) / device::shared_memory_bank_width())
-        * device::shared_memory_bank_width();
+    static constexpr int block_size_x = device::max_block_size<block_size_y, block_size_z>();
 
     using atom_t = std::conditional_t<sizeof(T) % 16 == 0, int4, std::conditional_t<sizeof(T) % 8 == 0, int2, int>>;
     static_assert(sizeof(T) % 4 == 0, "Shared memory cache does not support sub-word size types");
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index aa0d566df6..c053d30f0a 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -158,12 +158,6 @@ namespace quda {
     */
     constexpr size_t max_kernel_arg_size() { return 64; }
 
-    /**
-       @brief Helper function that returns the bank width of the
-       shared memory bank width on the target architecture.
-    */
-    constexpr int shared_memory_bank_width() { return 32; }
-
     /**
        @brief Use a compile time fixed size for the shared local memory,
        until we can find a way to set it dynamically.
diff --git a/include/targets/omptarget/thread_array.h b/include/targets/omptarget/thread_array.h
index 434776ce93..77751adf1b 100644
--- a/include/targets/omptarget/thread_array.h
+++ b/include/targets/omptarget/thread_array.h
@@ -11,7 +11,7 @@ namespace quda
      shared memory using the SharedMemoryCache object.
    */
   template <typename T, int n> struct thread_array {
-    SharedMemoryCache<array<T, n>, 1, 1, false, false> device_array;
+    SharedMemoryCache<array<T, n>, 1, 1, false> device_array;
     int offset;
     array<T, n> host_array;
     array<T, n> &array_;

From 8baca170e465f1548879f8e3700c6eb4f880ed82 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 25 Jan 2023 05:17:51 +0000
Subject: [PATCH 170/198] omp: set pragma unroll threshold; teach cmake to
 leave it be

---
 lib/targets/omptarget/target_omptarget.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index ede0af261e..ba34895c78 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -32,7 +32,7 @@ message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}
 
 # ######################################################################################################################
 # define omptarget flags
-set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 -mllvm -vpo-paropt-simulate-get-num-threads-in-target=false)
+set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
 message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
 
 # QUDA_HASH for tunecache

From eebc07468ed16eda56e9397c59a387acf2fdd5b3 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 22 Feb 2023 02:45:37 +0000
Subject: [PATCH 171/198] omp: remove pragma-unroll-threshold from default
 options

---
 lib/targets/omptarget/target_omptarget.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index ba34895c78..5adc540315 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -32,7 +32,8 @@ message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}
 
 # ######################################################################################################################
 # define omptarget flags
-set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+#set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
 message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
 
 # QUDA_HASH for tunecache

From 983d09ecbce7d8cd07070ebd6af7b98888ed93db Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 15 Mar 2023 15:22:03 +0000
Subject: [PATCH 172/198] omp: better control of thread limit

---
 lib/targets/omptarget/device.cpp   | 8 +++++---
 lib/targets/omptarget/quda_api.cpp | 6 ++++--
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index bd81a0d4b7..5a3efd2dcc 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -35,6 +35,7 @@ struct DeviceProp{
 
 static void getDeviceProperties(DeviceProp*p,int dev)
 {
+  constexpr int max_threads = quda::device::max_block_size();
   int m = 0;
 
   if(dev>=0 && dev<=omp_get_num_devices())
@@ -43,8 +44,9 @@ static void getDeviceProperties(DeviceProp*p,int dev)
     p->name = omphostname;
 
   p->id = dev;
-  #pragma omp target device(dev) map(from:m)
-  m = omp_get_num_procs();
+  #pragma omp target teams device(dev) thread_limit(max_threads) map(from:m)
+  if(omp_get_team_num()==0)
+    m = omp_get_num_procs();
   p->num_procs = m;
 
 /*
@@ -54,7 +56,7 @@ static void getDeviceProperties(DeviceProp*p,int dev)
 */
   p->max_teams = QUDA_OMP_MAX_TEAMS;
 
-  #pragma omp target teams device(dev) map(from:m)
+  #pragma omp target teams device(dev) thread_limit(max_threads) map(from:m)
   if(omp_get_team_num()==0)
     m = omp_get_max_threads();
   p->max_threads= m;
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 7536ffc4c3..383c62d3e5 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -91,8 +91,9 @@ namespace quda
       static inline int
       ompMemset(void *p, unsigned char b, std::size_t s)
       {
+        constexpr int max_threads = device::max_block_size();
         unsigned char *c = reinterpret_cast<unsigned char *>(p);
-      #pragma omp target teams distribute parallel for simd is_device_ptr(c)
+      #pragma omp target teams distribute parallel for simd thread_limit(max_threads) is_device_ptr(c)
         for(std::size_t i=0;i<s;++i) c[i] = b;
         return 0;
       }
@@ -106,8 +107,9 @@ namespace quda
       static inline int
       ompMemset2D(void *p, size_t pitch, unsigned char b, size_t w, size_t h)
       {
+        constexpr int max_threads = device::max_block_size();
         unsigned char *c = reinterpret_cast<unsigned char *>(p);
-      #pragma omp target teams distribute parallel for simd is_device_ptr(c) collapse(2)
+      #pragma omp target teams distribute parallel for simd thread_limit(max_threads) is_device_ptr(c) collapse(2)
         for(std::size_t i=0;i<h;++i)
           for(std::size_t j=0;j<w;++j)
             c[j+i*pitch] = b;

From d4be5f6909864c19e9e4867f8a02f56436fa01c0 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 15 Mar 2023 16:05:40 +0000
Subject: [PATCH 173/198] omp: optinally build omp target with jit

---
 lib/targets/omptarget/target_omptarget.cmake | 26 ++++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index 5adc540315..996174610e 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -30,15 +30,27 @@ set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 4505
 target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
 message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")
 
+option(QUDA_OMPTARGET_JIT "Build OpenMP target backend with JIT" OFF)
+mark_as_advanced(QUDA_OMPTARGET_JIT)
+
 # ######################################################################################################################
 # define omptarget flags
-#set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
-set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+
+if(QUDA_OMPTARGET_JIT)
+    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64 -fopenmp-version=51 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+else()
+    #set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+endif()
 message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
 
 # QUDA_HASH for tunecache
 set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${QUDA_GPU_ARCH},cxx_version=${CMAKE_CXX_COMPILER_VERSION})
-set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:${QUDA_GPU_ARCH}")
+if(QUDA_OMPTARGET_JIT)
+    set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:JIT")
+else()
+    set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:${QUDA_GPU_ARCH}")
+endif()
 
 # ######################################################################################################################
 # omptarget specific compile options
@@ -63,7 +75,11 @@ target_compile_options(
 
 target_compile_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS})
 
-target_link_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS} -Xopenmp-target-backend "-device ${QUDA_GPU_ARCH}")
+if(QUDA_OMPTARGET_JIT)
+    target_link_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS})
+else()
+    target_link_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS} -Xopenmp-target-backend "-device ${QUDA_GPU_ARCH}")
+endif()
 
 set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES LANGUAGE CXX)
 set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES COMPILE_FLAGS "-x c++")
@@ -73,7 +89,7 @@ set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES COMPILE_FLAGS "-x c++")
 
 set(OMPTARGET_MKL_LIBRARY "-lmkl_sycl -lmkl_intel_ilp64 -lmkl_core -lmkl_intel_thread")
 
-if(${QUDA_BUILD_NATIVE_LAPACK} STREQUAL "ON")
+if(QUDA_BUILD_NATIVE_LAPACK)
   target_compile_definitions(quda PRIVATE MKL_ILP64)
   target_link_libraries(quda PUBLIC ${OMPTARGET_MKL_LIBRARY})
 endif()

From 07bd6bbb215de9bea10468e96b2ce03d259031a0 Mon Sep 17 00:00:00 2001
From: maddyscientist <mclark@nvidia.com>
Date: Wed, 8 Feb 2023 15:36:18 -0800
Subject: [PATCH 174/198] Fix initialization bug for test_split_grid

---
 tests/staggered_dslash_test_utils.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/staggered_dslash_test_utils.h b/tests/staggered_dslash_test_utils.h
index 85c2726876..5c6d885673 100644
--- a/tests/staggered_dslash_test_utils.h
+++ b/tests/staggered_dslash_test_utils.h
@@ -79,8 +79,8 @@ struct StaggeredDslashTestWrapper {
   char **argv_copy;
 
   // Split grid options
-  int num_src;
-  int test_split_grid;
+  bool test_split_grid = false;
+  int num_src = 1;
 
   void staggeredDslashRef()
   {
@@ -154,7 +154,6 @@ struct StaggeredDslashTestWrapper {
 
     num_src = grid_partition[0] * grid_partition[1] * grid_partition[2] * grid_partition[3];
     test_split_grid = num_src > 1;
-
     if (test_split_grid) { dtest_type = dslash_test_type::Dslash; }
 
     inv_param.dagger = dagger ? QUDA_DAG_YES : QUDA_DAG_NO;

From 2be6636c031199436ace5d753312a8a6b6ce72bf Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 23 Mar 2023 01:37:50 +0000
Subject: [PATCH 175/198] omp: update backend following upstream

---
 .../targets/omptarget/constant_kernel_arg.h   |  3 ++
 include/targets/omptarget/tunable_kernel.h    | 39 ++++++++++++++++---
 lib/targets/omptarget/device.cpp              | 12 ++++++
 lib/targets/omptarget/quda_api.cpp            | 12 ++----
 4 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/include/targets/omptarget/constant_kernel_arg.h b/include/targets/omptarget/constant_kernel_arg.h
index bd8ed7b976..76962aefb2 100644
--- a/include/targets/omptarget/constant_kernel_arg.h
+++ b/include/targets/omptarget/constant_kernel_arg.h
@@ -14,6 +14,9 @@
    instantiation correctly.
  */
 
+// set a preprocessor flag that we have included constant_kernel_arg.h
+#define QUDA_USE_CONSTANT_MEMORY
+
 namespace quda
 {
 
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index f5bd5886de..535d754eec 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -3,6 +3,7 @@
 #include <constant_kernel_arg.h>
 #include <tune_quda.h>
 #include <target_device.h>
+#include <lattice_field.h>
 #include <kernel_helper.h>
 #include <kernel.h>
 #include <quda_omptarget_api.h>
@@ -62,15 +63,25 @@ namespace quda {
     return true;
   }
 
+  /**
+     @brief This helper function indicates if the present
+     compilation unit has explicit constant memory usage enabled.
+  */
+  static bool use_constant_memory()
+  {
+#ifdef QUDA_USE_CONSTANT_MEMORY
+    return true;
+#else
+    return false;
+#endif
+  }
+
   class TunableKernel : public Tunable
   {
 
   protected:
     QudaFieldLocation location;
 
-    virtual unsigned int sharedBytesPerThread() const { return 0; }
-    virtual unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
-
     template <template <typename> class Functor, bool grid_stride, typename Arg>
     qudaError_t launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
@@ -106,14 +117,30 @@ namespace quda {
       const_cast<TunableKernel*>(this)->launch_device<Functor, grid_stride>(KERNEL(raw_kernel), tp, stream, arg);
     }
 
-    TunableKernel(QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) : location(location) { } 
+    TunableKernel(const LatticeField &field, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
+      location(location != QUDA_INVALID_FIELD_LOCATION ? location : field.Location())
+    {
+      strcpy(vol, field.VolString().c_str());
+      strcpy(aux, compile_type_str(field, location));
+      if (this->location == QUDA_CUDA_FIELD_LOCATION && use_constant_memory()) strcat(aux, "cmem,");
+      if (this->location == QUDA_CPU_FIELD_LOCATION) strcat(aux, getOmpThreadStr());
+      strcat(aux, field.AuxString().c_str());
+    }
+
+    TunableKernel(size_t n_items, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) : location(location)
+    {
+      u64toa(vol, n_items);
+      strcpy(aux, compile_type_str(location));
+      if (location == QUDA_CUDA_FIELD_LOCATION && use_constant_memory()) strcat(aux, "cmem,");
+      if (this->location == QUDA_CPU_FIELD_LOCATION) strcat(aux, getOmpThreadStr());
+    }
 
-    virtual bool advanceTuneParam(TuneParam &param) const
+    virtual bool advanceTuneParam(TuneParam &param) const override
     {
       return location == QUDA_CPU_FIELD_LOCATION ? false : Tunable::advanceTuneParam(param);
     }
 
-    TuneKey tuneKey() const { return TuneKey(vol, typeid(*this).name(), aux); }
+    TuneKey tuneKey() const override { return TuneKey(vol, typeid(*this).name(), aux); }
   };
 
 }
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 5a3efd2dcc..e1854fd897 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -106,6 +106,18 @@ namespace quda
       return device_count;
     }
 
+    void get_visible_devices_string(char device_list_string[128])
+    {
+      char *device_order_env = getenv("ZE_AFFINITY_MASK");
+      char *omptarget_devices = getenv("LIBOMPTARGET_DEVICES");
+      if(omptarget_devices && device_order_env)
+        snprintf(device_list_string, sizeof(device_list_string), "%s%s", omptarget_devices, device_order_env);
+      else if(omptarget_devices)
+        snprintf(device_list_string, sizeof(device_list_string), "%s", omptarget_devices);
+      else if(device_order_env)
+        snprintf(device_list_string, sizeof(device_list_string), "%s", device_order_env);
+    }
+
     void print_device_properties()
     {
       DeviceProp dp;
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 383c62d3e5..aab3bda927 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -210,9 +210,6 @@ namespace quda
     const char *file;
     const char *line;
 
-    unsigned int sharedBytesPerThread() const { return 0; }
-    unsigned int sharedBytesPerBlock(const TuneParam &) const { return 0; }
-
   public:
     inline QudaMem(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const qudaStream_t &stream,
                    bool async, const char *func, const char *file, const char *line) :
@@ -277,7 +274,7 @@ namespace quda
       apply(stream);
     }
 
-    inline void apply(const qudaStream_t &stream)
+    void apply(const qudaStream_t &stream) override
     {
       if (!active_tuning) tuneLaunch(*this, getTuning(), getVerbosity());
 
@@ -296,9 +293,9 @@ namespace quda
       }
     }
 
-    bool advanceTuneParam(TuneParam &) const { return false; }
+    bool advanceTuneParam(TuneParam &) const override { return false; }
 
-    TuneKey tuneKey() const
+    TuneKey tuneKey() const override
     {
       char vol[128];
       strcpy(vol, "bytes=");
@@ -306,8 +303,7 @@ namespace quda
       return TuneKey(vol, name, aux);
     }
 
-    long long flops() const { return 0; }
-    long long bytes() const { return kind == qudaMemcpyDeviceToDevice ? 2 * count : count; }
+    long long bytes() const override { return kind == qudaMemcpyDeviceToDevice ? 2 * count : count; }
   };
 
   void qudaMemcpy_(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const char *func, const char *file,

From 301bc9d7671d4f08530e45b5c259a2728dac0372 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 24 Mar 2023 03:52:20 +0000
Subject: [PATCH 176/198] omp: fix sizeof
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A declaration of a parameter as “array of type” shall
be adjusted to “qualified pointer to type”.
---
 lib/targets/omptarget/device.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index e1854fd897..7a22cc9f6f 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -108,14 +108,15 @@ namespace quda
 
     void get_visible_devices_string(char device_list_string[128])
     {
+      constexpr int len = 128;
       char *device_order_env = getenv("ZE_AFFINITY_MASK");
       char *omptarget_devices = getenv("LIBOMPTARGET_DEVICES");
       if(omptarget_devices && device_order_env)
-        snprintf(device_list_string, sizeof(device_list_string), "%s%s", omptarget_devices, device_order_env);
+        snprintf(device_list_string, len, "%s%s", omptarget_devices, device_order_env);
       else if(omptarget_devices)
-        snprintf(device_list_string, sizeof(device_list_string), "%s", omptarget_devices);
+        snprintf(device_list_string, len, "%s", omptarget_devices);
       else if(device_order_env)
-        snprintf(device_list_string, sizeof(device_list_string), "%s", device_order_env);
+        snprintf(device_list_string, len, "%s", device_order_env);
     }
 
     void print_device_properties()

From 3e8b1a2349ba0d95cf6efd1c91f8816b20f29e27 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 30 Mar 2023 22:04:19 +0000
Subject: [PATCH 177/198] Revert "omp: declare target some static constexpr"

This reverts commit 7a9705d80aefa4405fc8038d013eff3a0dbc0df4.
---
 include/kernels/blas_core.cuh         | 30 --------------------
 include/kernels/multi_blas_core.cuh   | 12 --------
 include/kernels/multi_reduce_core.cuh |  6 ----
 include/kernels/reduce_core.cuh       | 40 ---------------------------
 include/targets/generic/mrg32k3a.h    |  2 --
 5 files changed, 90 deletions(-)

diff --git a/include/kernels/blas_core.cuh b/include/kernels/blas_core.cuh
index f123cfaa16..96f0c2b5f1 100644
--- a/include/kernels/blas_core.cuh
+++ b/include/kernels/blas_core.cuh
@@ -95,10 +95,8 @@ namespace quda
        Functor to perform the operation z = a*x + b*y
     */
     template <typename real> struct axpbyz_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 0, 0> read{ };
       static constexpr memory_access<0, 0, 0, 0, 1> write{ };
-#pragma omp end declare target
       const real a;
       const real b;
       axpbyz_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
@@ -114,10 +112,8 @@ namespace quda
        Functor to perform the operation y = a * x
     */
     template <typename real> struct axy_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 0> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       const real a;
       axy_(const real &a, const real &, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
@@ -132,10 +128,8 @@ namespace quda
        Functor to perform the operator y += a*x (complex-valued)
     */
     template <typename real> struct caxpy_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       caxpy_(const complex<real> &a, const complex<real> &, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &, T &, T &) const
@@ -166,10 +160,8 @@ namespace quda
     }
 
     template <typename real> struct caxpby_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpby_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -185,10 +177,8 @@ namespace quda
        Functor performing the operation: w[i] = a*x[i] + b*y[i] + c*z[i]
     */
     template <typename real> struct axpbypczw_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1> read{ };
       static constexpr memory_access<0, 0, 0, 1> write{ };
-#pragma omp end declare target
       const real a;
       const real b;
       const real c;
@@ -207,10 +197,8 @@ namespace quda
        Functor performing the operations: y[i] = a*x[i] + y[i]; x[i] = b*z[i] + c*x[i]
     */
     template <typename real> struct axpyBzpcx_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
-#pragma omp end declare target
       const real a;
       const real b;
       const real c;
@@ -230,10 +218,8 @@ namespace quda
        Functor performing the operations: y[i] = a*x[i] + y[i]; x[i] = z[i] + b*x[i]
     */
     template <typename real> struct axpyZpbx_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
-#pragma omp end declare target
       const real a;
       const real b;
       axpyZpbx_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
@@ -252,10 +238,8 @@ namespace quda
        Functor performing the operation z[i] = x[i] + a * y[i] + b * z[i]
     */
     template <typename real> struct cxpaypbz_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<0, 0, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       cxpaypbz_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -274,10 +258,8 @@ namespace quda
        Functor performing the operations y[i] = a*x[i] + y[i] and x[i] = b*z[i] + x[i]
     */
     template <typename real> struct caxpyBzpx_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpyBzpx_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -296,10 +278,8 @@ namespace quda
        Functor performing the operations y[i] = a*x[i] + y[i] and z[i] = b*x[i] + z[i]
     */
     template <typename real> struct caxpyBxpz_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<0, 1, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpyBxpz_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -318,10 +298,8 @@ namespace quda
        Functor performing the operations z[i] = a*x[i] + b*y[i] + z[i] and y[i] -= b*w[i]
     */
     template <typename real> struct caxpbypzYmbw_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1> read{ };
       static constexpr memory_access<0, 1, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpbypzYmbw_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a), b(b) { ; }
@@ -341,10 +319,8 @@ namespace quda
        Functor performing the operation y[i] += a*b*x[i], x[i] *= a
     */
     template <typename real> struct cabxpyAx_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
-#pragma omp end declare target
       const real a;
       const complex<real> b;
       cabxpyAx_(const complex<real> &a, const complex<real> &b, const complex<real> &) : a(a.real()), b(b) { ; }
@@ -365,10 +341,8 @@ namespace quda
        Second performs the operator x[i] -= a*z[i]
     */
     template <typename real> struct caxpyxmaz_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       caxpyxmaz_(const complex<real> &a, const complex<real> &, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(T &x, T &y, T &z, T &, T &) const
@@ -392,10 +366,8 @@ namespace quda
     */
     template <typename real> struct caxpyxmazMR_ {
       static constexpr use_kernel_arg_p use_kernel_arg = use_kernel_arg_p::ALWAYS;
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<1, 1> write{ };
-#pragma omp end declare target
       complex<real> a;
       double3 *Ar3;
       bool init_;
@@ -433,10 +405,8 @@ namespace quda
        Third performs the operation w[i] = z[i] + b*w[i]
     */
     template <typename real> struct tripleCGUpdate_ : public BlasFunctor {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1> read{ };
       static constexpr memory_access<0, 1, 1, 1> write{ };
-#pragma omp end declare target
       const real a;
       const real b;
       tripleCGUpdate_(const real &a, const real &b, const real &) : a(a), b(b) { ; }
diff --git a/include/kernels/multi_blas_core.cuh b/include/kernels/multi_blas_core.cuh
index abc0f0e810..90fdc1aabe 100644
--- a/include/kernels/multi_blas_core.cuh
+++ b/include/kernels/multi_blas_core.cuh
@@ -131,10 +131,8 @@ namespace quda
     */
     template <typename real>
     struct multiaxpy_ : public MultiBlasFunctor<real> {
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = false;
       static constexpr int NXZ_max = 0;
@@ -155,10 +153,8 @@ namespace quda
     */
     template <typename real>
     struct multicaxpy_ : public MultiBlasFunctor<complex<real>> {
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = false;
       static constexpr int NXZ_max = 0;
@@ -179,10 +175,8 @@ namespace quda
     */
     template <typename real>
     struct multiaxpyz_ : public MultiBlasFunctor<real> {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 0> read{ };
       static constexpr memory_access<0, 0, 0, 1> write{ };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       static constexpr int NXZ_max = 0;
@@ -206,10 +200,8 @@ namespace quda
     */
     template <typename real>
     struct multicaxpyz_ : public MultiBlasFunctor<complex<real>> {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 0> read{ };
       static constexpr memory_access<0, 0, 0, 1> write{ };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       static constexpr int NXZ_max = 0;
@@ -233,10 +225,8 @@ namespace quda
     */
     template <typename real>
     struct multi_axpyBzpcx_ : public MultiBlasFunctor<real, true> {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 1> read{ };
       static constexpr memory_access<0, 1, 0, 1> write{ };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       static constexpr int NXZ_max = 1; // we never have NXZ > 1 for this kernel
@@ -265,10 +255,8 @@ namespace quda
     */
     template <typename real>
     struct multi_caxpyBxpz_ : public MultiBlasFunctor<complex<real>, true> {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 1> read{ };
       static constexpr memory_access<0, 1, 0, 1> write{ };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       static constexpr int NXZ_max = 0;
diff --git a/include/kernels/multi_reduce_core.cuh b/include/kernels/multi_reduce_core.cuh
index 2f70c91852..9fcadd14c9 100644
--- a/include/kernels/multi_reduce_core.cuh
+++ b/include/kernels/multi_reduce_core.cuh
@@ -150,10 +150,8 @@ namespace quda
 
     template <typename reduce_t, typename real>
     struct multiDot : public MultiReduceFunctor<reduce_t, real> {
-#pragma omp declare target
       static constexpr memory_access<1, 1> read { };
       static constexpr memory_access< > write { };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = false;
       multiDot(int NXZ, int NYW) : MultiReduceFunctor<reduce_t, real>(NXZ, NYW) { }
@@ -183,10 +181,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct multiCdot : public MultiReduceFunctor<array<real_reduce_t, 2>, complex<real>> {
       using reduce_t = array<real_reduce_t, 2>;
-#pragma omp declare target
       static constexpr memory_access<1, 1> read { };
       static constexpr memory_access< > write { };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = false;
       multiCdot(int NXZ, int NYW) : MultiReduceFunctor<reduce_t, complex<real>>(NXZ, NYW) { }
@@ -203,10 +199,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct multiCdotCopy : public MultiReduceFunctor<typename VectorType<real_reduce_t, 2>::type, complex<real>> {
       using reduce_t = typename VectorType<real_reduce_t, 2>::type;
-#pragma omp declare target
       static constexpr memory_access<1, 1> read { };
       static constexpr memory_access<0, 0, 0, 1> write { };
-#pragma omp end declare target
       static constexpr bool use_z = false;
       static constexpr bool use_w = true;
       multiCdotCopy(int NXZ, int NYW) : MultiReduceFunctor<reduce_t, complex<real>>(NXZ, NYW) { }
diff --git a/include/kernels/reduce_core.cuh b/include/kernels/reduce_core.cuh
index 2a0e7db2d9..4597f0b8cd 100644
--- a/include/kernels/reduce_core.cuh
+++ b/include/kernels/reduce_core.cuh
@@ -117,10 +117,8 @@ namespace quda
     template <typename reduce_t, typename real>
     struct Max : public ReduceFunctor<reduce_t> {
       using reducer = maximum<reduce_t>;
-#pragma omp declare target
       static constexpr memory_access<1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       Max(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &max, T &x, T &, T &, T &, T &) const
       {
@@ -137,10 +135,8 @@ namespace quda
     struct MaxDeviation : public ReduceFunctor<deviation_t<real_reduce_t>> {
       using reduce_t = deviation_t<real_reduce_t>;
       using reducer = maximum<reduce_t>;
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       MaxDeviation(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &max, T &x, T &y, T &, T &, T &) const
       {
@@ -170,10 +166,8 @@ namespace quda
 
     template <typename reduce_t, typename real>
     struct Norm1 : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       Norm1(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &, T &, T &, T &) const
       {
@@ -194,10 +188,8 @@ namespace quda
 
     template <typename reduce_t, typename real>
     struct Norm2 : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       Norm2(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &, T &, T &, T &) const
       {
@@ -219,10 +211,8 @@ namespace quda
 
     template <typename reduce_t, typename real>
     struct Dot : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1,1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       Dot(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
@@ -238,10 +228,8 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct axpbyzNorm2 : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 0> read{ };
       static constexpr memory_access<0, 0, 1> write{ };
-#pragma omp end declare target
       const real a;
       const real b;
       axpbyzNorm2(const real &a, const real &b) : a(a), b(b) { ; }
@@ -262,10 +250,8 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct AxpyReDot : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       const real a;
       AxpyReDot(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
@@ -285,10 +271,8 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct caxpyNorm2 : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       caxpyNorm2(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
@@ -310,10 +294,8 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct cabxpyzaxnorm : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 0> read{ };
       static constexpr memory_access<1, 0, 1> write{ };
-#pragma omp end declare target
       const real a;
       const complex<real> b;
       cabxpyzaxnorm(const complex<real> &a, const complex<real> &b) : a(a.real()), b(b) { ; }
@@ -345,10 +327,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct Cdot : public ReduceFunctor<array<real_reduce_t, 2>> {
       using reduce_t = array<real_reduce_t, 2>;
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       Cdot(const complex<real> &, const complex<real> &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
@@ -366,10 +346,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct caxpydotzy : public ReduceFunctor<array<real_reduce_t, 2>> {
       using reduce_t = array<real_reduce_t, 2>;
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       caxpydotzy(const complex<real> &a, const complex<real> &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
@@ -400,10 +378,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct CdotNormAB : public ReduceFunctor<array<real_reduce_t, 4>> {
       using reduce_t = array<real_reduce_t, 4>;
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       CdotNormAB(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
       {
@@ -433,10 +409,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct caxpbypzYmbwcDotProductUYNormY_ : public ReduceFunctor<array<real_reduce_t, 3>> {
       using reduce_t = array<real_reduce_t, 3>;
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1, 1> read{ };
       static constexpr memory_access<0, 1, 1> write{ };
-#pragma omp end declare target
       const complex<real> a;
       const complex<real> b;
       caxpbypzYmbwcDotProductUYNormY_(const complex<real> &a, const complex<real> &b) : a(a), b(b) { ; }
@@ -462,10 +436,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct axpyCGNorm2 : public ReduceFunctor<array<real_reduce_t, 2>> {
       using reduce_t = array<real_reduce_t, 2>;
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<0, 1> write{ };
-#pragma omp end declare target
       const real a;
       axpyCGNorm2(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &, T &, T &) const
@@ -496,10 +468,8 @@ namespace quda
       using reducer = plus<reduce_t>;
       static constexpr bool site_unroll = true;
 
-#pragma omp declare target
       static constexpr memory_access<1, 1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       reduce_t aux;
       HeavyQuarkResidualNorm_(const real &, const real &) : aux {} { ; }
 
@@ -545,10 +515,8 @@ namespace quda
       using reducer = plus<reduce_t>;
       static constexpr bool site_unroll = true;
 
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       reduce_t aux;
       xpyHeavyQuarkResidualNorm_(const real &, const real &) : aux {} { ; }
 
@@ -587,10 +555,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct tripleCGReduction_ : public ReduceFunctor<array<real_reduce_t, 3>> {
       using reduce_t = array<real_reduce_t, 3>;
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       tripleCGReduction_(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &, T &) const
       {
@@ -614,10 +580,8 @@ namespace quda
     template <typename real_reduce_t, typename real>
     struct quadrupleCGReduction_ : public ReduceFunctor<array<real_reduce_t, 4>> {
       using reduce_t = array<real_reduce_t, 4>;
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1> read{ };
       static constexpr memory_access<> write{ };
-#pragma omp end declare target
       quadrupleCGReduction_(const real &, const real &) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &) const
       {
@@ -642,10 +606,8 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct quadrupleCG3InitNorm_ : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 0, 0, 1> read{ };
       static constexpr memory_access<1, 1, 1, 1> write{ };
-#pragma omp end declare target
       const real a;
       quadrupleCG3InitNorm_(const real &a, const real &) : a(a) { ; }
       template <typename T> __device__ __host__ void operator()(reduce_t &sum, T &x, T &y, T &z, T &w, T &v) const
@@ -674,10 +636,8 @@ namespace quda
     */
     template <typename reduce_t, typename real>
     struct quadrupleCG3UpdateNorm_ : public ReduceFunctor<reduce_t> {
-#pragma omp declare target
       static constexpr memory_access<1, 1, 1, 1, 1> read{ };
       static constexpr memory_access<1, 1, 1, 1> write{ };
-#pragma omp end declare target
       const real a;
       const real b;
       quadrupleCG3UpdateNorm_(const real &a, const real &b) : a(a), b(b) { ; }
diff --git a/include/targets/generic/mrg32k3a.h b/include/targets/generic/mrg32k3a.h
index df5bb10e26..e0dd4e2a4e 100644
--- a/include/targets/generic/mrg32k3a.h
+++ b/include/targets/generic/mrg32k3a.h
@@ -117,10 +117,8 @@ namespace quda
       constexpr auto a2sq = squaremodarray<maxpower2>(a2, static_cast<uint64_t>(m2));
 
       // device copies of a1sq and a2sq
-#pragma omp declare target
       __device__ constexpr auto a1sq_d = a1sq;
       __device__ constexpr auto a2sq_d = a2sq;
-#pragma omp end declare target
 
       // default functor for grabbing the appropriate a1sq
       template <bool is_device> struct get_a1sq {

From e642e403ce6c934d4132716e5f97e183775501d2 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 26 Apr 2023 16:16:57 +0000
Subject: [PATCH 178/198] omp: make sure extra args/spaces does not break
 version string

---
 lib/targets/omptarget/target_omptarget.cmake | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index 996174610e..e68b3dd931 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -44,12 +44,14 @@ else()
 endif()
 message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
 
+string(REPLACE " " "" QUDA_GPU_ARCH_STRIP "${QUDA_GPU_ARCH}")
+
 # QUDA_HASH for tunecache
-set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${QUDA_GPU_ARCH},cxx_version=${CMAKE_CXX_COMPILER_VERSION})
+set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${QUDA_GPU_ARCH_STRIP},cxx_version=${CMAKE_CXX_COMPILER_VERSION})
 if(QUDA_OMPTARGET_JIT)
     set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:JIT")
 else()
-    set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:${QUDA_GPU_ARCH}")
+    set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:${QUDA_GPU_ARCH_STRIP}")
 endif()
 
 # ######################################################################################################################

From 5f438d44c91088fd93799675d2c172817e886be1 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 24 May 2023 00:28:51 +0000
Subject: [PATCH 179/198] omp: update target specific code to conform to
 upstream

---
 .../omptarget/shared_memory_cache_helper.h    | 84 +++++++++++++------
 lib/targets/omptarget/device.cpp              | 10 +++
 2 files changed, 69 insertions(+), 25 deletions(-)

diff --git a/include/targets/omptarget/shared_memory_cache_helper.h b/include/targets/omptarget/shared_memory_cache_helper.h
index ba43c82150..7d0c31efe7 100644
--- a/include/targets/omptarget/shared_memory_cache_helper.h
+++ b/include/targets/omptarget/shared_memory_cache_helper.h
@@ -31,9 +31,15 @@ namespace quda
        according the maximum number of threads possible, given these
        dimensions.
    */
-  template <typename T, int block_size_y = 1, int block_size_z = 1, bool dynamic = true>
-  class SharedMemoryCache
+  template <typename T, int block_size_y_ = 1, int block_size_z_ = 1, bool dynamic_ = true> class SharedMemoryCache
   {
+  public:
+    using value_type = T;
+    static constexpr int block_size_y = block_size_y_;
+    static constexpr int block_size_z = block_size_z_;
+    static constexpr bool dynamic = dynamic_;
+
+  private:
     /** maximum number of threads in x given the y and z block sizes */
     static constexpr int block_size_x = device::max_block_size<block_size_y, block_size_z>();
 
@@ -45,29 +51,38 @@ namespace quda
 
     const dim3 block;
     const int stride;
+    const unsigned int offset = 0; // dynamic offset in bytes
 
     /**
        @brief This is a dummy instantiation for the host compiler
     */
     template <bool, typename dummy = void> struct cache_dynamic {
-      atom_t *operator()()
+      atom_t *operator()(unsigned)
       {
         static atom_t *cache_;
         return reinterpret_cast<atom_t *>(cache_);
       }
     };
 
-    template <bool is_device, typename dummy = void> struct cache_static : cache_dynamic<is_device> {
-    };
-
     /**
        @brief This is the handle to the shared memory, dynamic specialization
        @return Shared memory pointer
      */
     template <typename dummy> struct cache_dynamic<true, dummy> {
-      __device__ inline atom_t *operator()()
+      __device__ inline atom_t *operator()(unsigned int offset)
       {
-        return reinterpret_cast<atom_t *>(target::omptarget::get_shared_cache());
+        return reinterpret_cast<atom_t *>(reinterpret_cast<char *>(target::omptarget::get_shared_cache()) + offset);
+      }
+    };
+
+    /**
+       @brief This is a dummy instantiation for the host compiler
+    */
+    template <bool is_device, typename dummy = void> struct cache_static {
+      atom_t *operator()()
+      {
+        static atom_t *cache_;
+        return reinterpret_cast<atom_t *>(cache_);
       }
     };
 
@@ -83,17 +98,17 @@ namespace quda
       }
     };
 
-    template <bool dynamic_shared> __device__ __host__ inline std::enable_if_t<dynamic_shared, atom_t *> cache()
+    template <bool dynamic_shared> __device__ __host__ inline std::enable_if_t<dynamic_shared, atom_t *> cache() const
     {
-      return target::dispatch<cache_dynamic>();
+      return target::dispatch<cache_dynamic>(offset);
     }
 
-    template <bool dynamic_shared> __device__ __host__ inline std::enable_if_t<!dynamic_shared, atom_t *> cache()
+    template <bool dynamic_shared> __device__ __host__ inline std::enable_if_t<!dynamic_shared, atom_t *> cache() const
     {
       return target::dispatch<cache_static>();
     }
 
-    __device__ __host__ inline void save_detail(const T &a, int x, int y, int z)
+    __device__ __host__ inline void save_detail(const T &a, int x, int y, int z) const
     {
       atom_t tmp[n_element];
       memcpy(tmp, (void *)&a, sizeof(T));
@@ -102,7 +117,7 @@ namespace quda
       for (int i = 0; i < n_element; i++) cache<dynamic>()[i * stride + j] = tmp[i];
     }
 
-    __device__ __host__ inline T load_detail(int x, int y, int z)
+    __device__ __host__ inline T load_detail(int x, int y, int z) const
     {
       atom_t tmp[n_element];
       int j = (z * block.y + y) * block.x + x;
@@ -136,16 +151,20 @@ namespace quda
        constructor.
 
        @param[in] block Block dimensions for the 3-d shared memory object
+       @param[in] thread_offset "Perceived" offset from dynamic shared
+       memory base pointer (used when we have multiple caches in
+       scope).  Need to include block size to actual offset.
     */
-    constexpr SharedMemoryCache(dim3 block = dim3(block_size_x, block_size_y, block_size_z)) :
-      block(block), stride(block.x * block.y * block.z)
+    constexpr SharedMemoryCache(dim3 block = dim3(block_size_x, block_size_y, block_size_z),
+                                unsigned int thread_offset = 0) :
+      block(block), stride(block.x * block.y * block.z), offset(stride * thread_offset)
     {
     }
 
     /**
        @brief Grab the raw base address to shared memory.
     */
-    __device__ __host__ inline T *data() { return reinterpret_cast<T *>(cache<dynamic>()); }
+    __device__ __host__ inline auto data() const { return reinterpret_cast<T *>(cache<dynamic>()); }
 
     /**
        @brief Save the value into the 3-d shared memory cache.
@@ -154,7 +173,7 @@ namespace quda
        @param[in] y The y index to use
        @param[in] z The z index to use
      */
-    __device__ __host__ inline void save(const T &a, int x = -1, int y = -1, int z = -1)
+    __device__ __host__ inline void save(const T &a, int x = -1, int y = -1, int z = -1) const
     {
       auto tid = target::thread_idx();
       x = (x == -1) ? tid.x : x;
@@ -168,7 +187,7 @@ namespace quda
        @param[in] a The value to store in the shared memory cache
        @param[in] x The x index to use
      */
-    __device__ __host__ inline void save_x(const T &a, int x = -1)
+    __device__ __host__ inline void save_x(const T &a, int x = -1) const
     {
       auto tid = target::thread_idx();
       x = (x == -1) ? tid.x : x;
@@ -180,7 +199,7 @@ namespace quda
        @param[in] a The value to store in the shared memory cache
        @param[in] y The y index to use
      */
-    __device__ __host__ inline void save_y(const T &a, int y = -1)
+    __device__ __host__ inline void save_y(const T &a, int y = -1) const
     {
       auto tid = target::thread_idx();
       y = (y == -1) ? tid.y : y;
@@ -192,7 +211,7 @@ namespace quda
        @param[in] a The value to store in the shared memory cache
        @param[in] z The z index to use
      */
-    __device__ __host__ inline void save_z(const T &a, int z = -1)
+    __device__ __host__ inline void save_z(const T &a, int z = -1) const
     {
       auto tid = target::thread_idx();
       z = (z == -1) ? tid.z : z;
@@ -206,7 +225,7 @@ namespace quda
        @param[in] z The z index to use
        @return The value at coordinates (x,y,z)
      */
-    __device__ __host__ inline T load(int x = -1, int y = -1, int z = -1)
+    __device__ __host__ inline T load(int x = -1, int y = -1, int z = -1) const
     {
       auto tid = target::thread_idx();
       x = (x == -1) ? tid.x : x;
@@ -220,7 +239,7 @@ namespace quda
        @param[in] x The x index to use
        @return The value at coordinates (x,y,z)
     */
-    __device__ __host__ inline T load_x(int x = -1)
+    __device__ __host__ inline T load_x(int x = -1) const
     {
       auto tid = target::thread_idx();
       x = (x == -1) ? tid.x : x;
@@ -232,7 +251,7 @@ namespace quda
        @param[in] y The y index to use
        @return The value at coordinates (x,y,z)
     */
-    __device__ __host__ inline T load_y(int y = -1)
+    __device__ __host__ inline T load_y(int y = -1) const
     {
       auto tid = target::thread_idx();
       y = (y == -1) ? tid.y : y;
@@ -244,7 +263,7 @@ namespace quda
        @param[in] z The z index to use
        @return The value at coordinates (x,y,z)
     */
-    __device__ __host__ inline T load_z(int z = -1)
+    __device__ __host__ inline T load_z(int z = -1) const
     {
       auto tid = target::thread_idx();
       z = (z == -1) ? tid.z : z;
@@ -254,7 +273,22 @@ namespace quda
     /**
        @brief Synchronize the cache
     */
-    __device__ __host__ void sync() { target::dispatch<sync_impl>(); }
+    __device__ __host__ void sync() const { target::dispatch<sync_impl>(); }
+
+    /**
+       @brief Cast operator to allow cache objects to be used where T
+       is expected
+     */
+    __device__ __host__ operator T() const { return load(); }
+
+    /**
+       @brief Assignment operator to allow cache objects to be used on
+       the lhs where T is otherwise expected.
+     */
+    __device__ __host__ void operator=(const T &src) const { save(src); }
   };
 
 } // namespace quda
+
+// include overloads
+#include "../generic/shared_memory_cache_helper.h"
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 7a22cc9f6f..fd48026f9b 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -72,6 +72,8 @@ namespace quda
 
     static bool initialized = false;
 
+    static int device_id = -1;
+
     void print_device(DeviceProp dp)
     {
       int dev = dp.id;
@@ -93,6 +95,14 @@ namespace quda
         print_device(deviceProp);
       }
       omp_set_default_device(dev);
+
+      device_id = dev;
+    }
+
+    void init_thread()
+    {
+      if (device_id == -1) errorQuda("No OMP device has been initialized for this process");
+      omp_set_default_device(device_id);
     }
 
     int get_device_count()

From 993788319ff74edcb577cbdfba60fd273b0e6a5c Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 24 May 2023 22:18:42 +0000
Subject: [PATCH 180/198] omp: tests/cmake WA: ignore unresolved symbols from
 device functions in tune_test

---
 tests/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 9ca912f8b3..faf26a0552 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -165,7 +165,7 @@ if(QUDA_QIO)
 endif()
 
 add_executable(tune_test tune_test.cpp)
-target_link_libraries(tune_test ${TEST_LIBS})
+target_link_libraries(tune_test ${TEST_LIBS} "-Wl,--unresolved-symbols=ignore-all")
 quda_checkbuildtest(tune_test QUDA_BUILD_ALL_TESTS)
 install(TARGETS tune_test ${QUDA_EXCLUDE_FROM_INSTALL} DESTINATION ${CMAKE_INSTALL_BINDIR})
 

From c77b2d359b32e40d6bed1d7d6dac20c2fa404369 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 2 Nov 2023 02:03:52 +0000
Subject: [PATCH 181/198] omp: update target api following upstream

---
 lib/targets/omptarget/malloc.cpp   |  1 +
 lib/targets/omptarget/quda_api.cpp | 42 ++++++++++++++++++++++++------
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/lib/targets/omptarget/malloc.cpp b/lib/targets/omptarget/malloc.cpp
index d0507a814d..b79e85741b 100644
--- a/lib/targets/omptarget/malloc.cpp
+++ b/lib/targets/omptarget/malloc.cpp
@@ -8,6 +8,7 @@
 #include <quda_internal.h>
 #include <device.h>
 #include <shmem_helper.cuh>
+#include "timer.h"
 
 #ifdef USE_QDPJIT
 #include "qdp_quda.h"
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index aab3bda927..e8822b5bbd 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -313,6 +313,14 @@ namespace quda
     QudaMem copy(dst, src, count, kind, device::get_default_stream(), false, func, file, line);
   }
 
+  void qudaMemcpy_(const quda_ptr &dst, const quda_ptr &src, size_t count, qudaMemcpyKind kind, const char *func,
+                   const char *file, const char *line)
+  {
+    if (count == 0) return;
+    QudaMem copy(dst.data(), src.data(), count, kind, device::get_default_stream(), false, func,
+                 file, line);
+  }
+
   void qudaMemcpyAsync_(void *dst, const void *src, size_t count, qudaMemcpyKind kind, const qudaStream_t &stream,
                         const char *func, const char *file, const char *line)
   {
@@ -339,6 +347,16 @@ namespace quda
     QudaMem set(ptr, value, count, device::get_default_stream(), false, func, file, line);
   }
 
+  void qudaMemset_(quda_ptr &ptr, int value, size_t count, const char *func, const char *file, const char *line)
+  {
+    if (count == 0) return;
+    if (ptr.is_device()) {
+      QudaMem set(ptr.data(), value, count, device::get_default_stream(), false, func, file, line);
+    } else {
+      memset(ptr.data(), value, count);
+    }
+  }
+
   void qudaMemsetAsync_(void *ptr, int value, size_t count, const qudaStream_t &stream, const char *func,
                         const char *file, const char *line)
   {
@@ -346,18 +364,26 @@ namespace quda
     QudaMem copy(ptr, value, count, stream, true, func, file, line);
   }
 
-  void qudaMemset2D_(void *ptr, size_t pitch, int value, size_t width, size_t height, const char *func,
-                     const char *file, const char *line)
+  void qudaMemsetAsync_(quda_ptr &ptr, int value, size_t count, const qudaStream_t &stream, const char *func,
+                        const char *file, const char *line)
   {
-    auto error = ompMemset2D(ptr, pitch, value, width, height);
-    set_runtime_error(error, __func__, func, file, line);
+    if (count == 0) return;
+    if (ptr.is_device()) {
+      QudaMem set(ptr.data(), value, count, stream, true, func, file, line);
+    } else {
+      memset(ptr.data(), value, count);
+    }
   }
 
-  void qudaMemset2DAsync_(void *ptr, size_t pitch, int value, size_t width, size_t height, const qudaStream_t &stream,
-                          const char *func, const char *file, const char *line)
+  void qudaMemset2DAsync_(quda_ptr &ptr, size_t offset, size_t pitch, int value, size_t width, size_t height,
+                          const qudaStream_t &stream, const char *func, const char *file, const char *line)
   {
-    auto error = ompMemset2DAsync(ptr, pitch, value, width, height, stream);
-    set_runtime_error(error, __func__, func, file, line);
+    if (ptr.is_device()) {
+      auto error = ompMemset2DAsync(static_cast<char *>(ptr.data()) + offset, pitch, value, width, height, stream);
+      set_runtime_error(error, __func__, func, file, line);
+    } else {
+      for (auto i = 0u; i < height; i++) memset(static_cast<char *>(ptr.data()) + offset + i * pitch, value, width);
+    }
   }
 
   void qudaMemPrefetchAsync_(void *ptr, size_t count, QudaFieldLocation mem_space, const qudaStream_t &stream,

From 36276e9346b301c4bfb019f4cf6552780512f601 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 9 Nov 2023 19:37:54 +0000
Subject: [PATCH 182/198] omp: use omp groupprivate for slm

- removed global vars in kernel
- use simple array for thread_array
---
 include/kernels/coarse_op_kernel.cuh          |  2 +-
 include/kernels/gauge_utils.cuh               |  1 +
 .../targets/omptarget/block_reduce_helper.h   | 10 +-
 .../omptarget/block_reduction_kernel.h        | 26 ++++--
 include/targets/omptarget/kernel.h            | 93 ++++++++++++-------
 include/targets/omptarget/quda_api.h          | 44 ++++-----
 include/targets/omptarget/reduce_helper.h     |  6 +-
 include/targets/omptarget/reduction_kernel.h  | 52 +++++++----
 .../omptarget/shared_memory_cache_helper.h    |  4 +-
 include/targets/omptarget/target_device.h     | 79 +++++++++++-----
 include/targets/omptarget/thread_array.h      | 37 +-------
 include/targets/omptarget/warp_collective.h   |  4 +-
 lib/targets/omptarget/quda_api.cpp            | 52 +++--------
 lib/targets/omptarget/target_omptarget.cmake  | 11 ++-
 14 files changed, 226 insertions(+), 195 deletions(-)

diff --git a/include/kernels/coarse_op_kernel.cuh b/include/kernels/coarse_op_kernel.cuh
index 8e1778c47b..18a1c705c8 100644
--- a/include/kernels/coarse_op_kernel.cuh
+++ b/include/kernels/coarse_op_kernel.cuh
@@ -1403,7 +1403,7 @@ namespace quda {
 #else
       typedef complex<storeType> TyX0[Arg::max_color_width_per_block][4][Arg::coarseSpin][Arg::coarseSpin];
       static_assert(sizeof(TyX0)*Arg::max_color_height_per_block*2 <= device::max_shared_memory_size(), "Shared cache not large enough for X and Y");
-      TyX0 *X = reinterpret_cast<TyX0 *>(target::omptarget::get_shared_cache());
+      TyX0 *X = reinterpret_cast<TyX0 *>(device::get_shared_cache());
       TyX0 *Y = X+Arg::max_color_height_per_block;
 #endif
 
diff --git a/include/kernels/gauge_utils.cuh b/include/kernels/gauge_utils.cuh
index 48c7e6c1cc..169e437601 100644
--- a/include/kernels/gauge_utils.cuh
+++ b/include/kernels/gauge_utils.cuh
@@ -1,6 +1,7 @@
 #include <gauge_field_order.h>
 #include <index_helper.cuh>
 #include <quda_matrix.h>
+#include <shared_memory_cache_helper.h>
 #include <thread_array.h>
 
 namespace quda
diff --git a/include/targets/omptarget/block_reduce_helper.h b/include/targets/omptarget/block_reduce_helper.h
index 2269057016..02134f0c2e 100644
--- a/include/targets/omptarget/block_reduce_helper.h
+++ b/include/targets/omptarget/block_reduce_helper.h
@@ -21,7 +21,7 @@ namespace quda
     constexpr bool enough_shared_mem(void)
     {
       constexpr auto max_nthr = device::max_block_size();
-      return max_nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128;  // FIXME arbitrary, the number is arbitrary, offset 128 below & in reduce_helper.h:/reduce
+      return max_nthr*sizeof(T) <= device::max_shared_memory_size()-sizeof(device::get_shared_cache()[0])*128;  // FIXME arbitrary, the number is arbitrary, offset 128 below & in reduce_helper.h:/reduce
     }
     /**
        @brief OpenMP reduction over a group of consecutive threads smaller than omp_num_threads()
@@ -30,7 +30,7 @@ namespace quda
     inline T any_reduce_impl(const reducer_t &r, const T &value_, const int batch, const int block_size, const bool all, const bool async)
     {
       static_assert(enough_shared_mem<T>(), "Shared cache not large enough for tempStorage");
-      T *storage = (T*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
+      T *storage = (T*)&device::get_shared_cache()[128];  // FIXME arbitrary
       const int tid = omp_get_thread_num();
       const auto& v0 = r.init();
 #if 1
@@ -44,8 +44,10 @@ namespace quda
         storage[tid] = value;
         const auto j = tid+offset;
         #pragma omp barrier
-        const auto& v = j<batch_end ? storage[j] : v0;
-        value = r(value, v);
+        if(j<batch_end)
+          value = r(value, storage[j]);
+        else
+          value = r(value, v0);
       }
       if(all){
         if(tid==block_size*batch)
diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index 73b18f09d1..e8e78e99d2 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -115,14 +115,17 @@ namespace quda
                       0) __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
     {
-      BlockKernel2D_impl<Functor, Arg>(arg);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        BlockKernel2D_impl<Functor, Arg>(arg);
+      }
     }
   }
 
@@ -146,14 +149,17 @@ namespace quda
                       0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg *argp)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
     {
-      BlockKernel2D_impl<Functor, Arg>(*argp);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        BlockKernel2D_impl<Functor, Arg>(*argp);
+      }
     }
   }
 
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index 6baac495c6..d2b6114ccb 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -50,14 +50,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
     {
-      Kernel1D_impl<Functor, Arg, grid_stride>(arg);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Kernel1D_impl<Functor, Arg, grid_stride>(arg);
+      }
     }
   }
 
@@ -76,14 +79,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
     {
-      Kernel1D_impl<Functor, Arg, grid_stride>(*argp);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Kernel1D_impl<Functor, Arg, grid_stride>(*argp);
+      }
     }
   }
 
@@ -133,14 +139,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
     {
-      Kernel2D_impl<Functor, Arg, grid_stride>(arg);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Kernel2D_impl<Functor, Arg, grid_stride>(arg);
+      }
     }
   }
 
@@ -159,14 +168,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
     {
-      Kernel2D_impl<Functor, Arg, grid_stride>(*argp);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Kernel2D_impl<Functor, Arg, grid_stride>(*argp);
+      }
     }
   }
 
@@ -218,14 +230,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
     {
-      Kernel3D_impl<Functor, Arg, grid_stride>(arg);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Kernel3D_impl<Functor, Arg, grid_stride>(arg);
+      }
     }
   }
 
@@ -244,14 +259,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = false>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
     {
-      Kernel3D_impl<Functor, Arg, grid_stride>(*argp);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Kernel3D_impl<Functor, Arg, grid_stride>(*argp);
+      }
     }
   }
 
@@ -272,15 +290,18 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool dummy = false>
   __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
     {
-      Functor<Arg> f(&arg);
-      f();
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Functor<Arg> f(&arg);
+        f();
+      }
     }
   }
 
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index a64bd3eea6..d8197b0a86 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -12,11 +12,11 @@
 #define CUDA_SUCCESS QUDA_SUCCESS
 
 #define QUDA_RT_CONSTS \
+  const dim3 & blockDim=target::omptarget::launch_param_kernel_block();\
+  const dim3 & gridDim=target::omptarget::launch_param_kernel_grid();\
   const dim3\
-    blockDim=target::omptarget::launch_param->block,\
-    gridDim=target::omptarget::launch_param->grid,\
-    threadIdx(omp_get_thread_num()%target::omptarget::launch_param->block.x, (omp_get_thread_num()/target::omptarget::launch_param->block.x)%target::omptarget::launch_param->block.y, omp_get_thread_num()/(target::omptarget::launch_param->block.x*target::omptarget::launch_param->block.y)),\
-    blockIdx(omp_get_team_num()%target::omptarget::launch_param->grid.x, (omp_get_team_num()/target::omptarget::launch_param->grid.x)%target::omptarget::launch_param->grid.y, omp_get_team_num()/(target::omptarget::launch_param->grid.x*target::omptarget::launch_param->grid.y))
+    threadIdx(omp_get_thread_num()%blockDim.x, (omp_get_thread_num()/blockDim.x)%blockDim.y, omp_get_thread_num()/(blockDim.x*blockDim.y)),\
+    blockIdx(omp_get_team_num()%gridDim.x, (omp_get_team_num()/gridDim.x)%gridDim.y, omp_get_team_num()/(gridDim.x*gridDim.y))
 
 #include <functional>
 #include <iostream>
@@ -87,25 +87,27 @@ using CUresult = int;  // ../lib/coarse_op.cuh:/CUresult
 namespace quda {
   namespace target {
     namespace omptarget {
-      struct SharedCache{
-        int *addr;
-        int num_teams;
-        int cache_length;
-      };
+      dim3 & launch_param_grid(void);
+      dim3 & launch_param_block(void);
 
-      struct LaunchParam{
-        dim3 block;
-        dim3 grid;
-        SharedCache shared_cache;
-      };
-#pragma omp declare target
-      extern LaunchParam *launch_param;
-#pragma omp end declare target
-      extern LaunchParam *launch_param_host;
-
-      inline int *get_shared_cache(void)
+      inline dim3 & launch_param_kernel_block(void)
+      {
+        static char block[sizeof(dim3)];
+        #pragma omp groupprivate(block)
+        return *reinterpret_cast<dim3*>(block);
+      }
+      inline dim3 & launch_param_kernel_grid(void)
+      {
+        static char grid[sizeof(dim3)];
+        #pragma omp groupprivate(grid)
+        return *reinterpret_cast<dim3*>(grid);
+      }
+      inline void launch_param_device_set(dim3 grid, dim3 block)
       {
-        return &launch_param->shared_cache.addr[omp_get_team_num()*launch_param->shared_cache.cache_length];
+        dim3 & gref = launch_param_kernel_grid();
+        dim3 & bref = launch_param_kernel_block();
+        gref = grid;
+        bref = block;
       }
     }
   }
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index 1eb183ce70..a0615bf825 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -124,9 +124,9 @@ namespace quda
     constexpr auto n_batch_block = std::min(Arg::max_n_batch_block, device::max_block_size());
     using BlockReduce = BlockReduce<T, Reducer::reduce_block_dim, n_batch_block>;
     // bool isLastBlockDone[n_batch_block];
-    static_assert(sizeof(bool)*n_batch_block <= sizeof(target::omptarget::get_shared_cache()[0])*64, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 128 is used in block_reduce_helper.h:/tempStorage/
-    bool *isLastBlockDone = (bool*)target::omptarget::get_shared_cache();
-    bool *hasLastBlockDone = (bool*)&target::omptarget::get_shared_cache()[64];
+    static_assert(sizeof(bool)*n_batch_block <= sizeof(device::get_shared_cache()[0])*64, "Shared cache not large enough for isLastBlockDone");  // FIXME arbitrary, 128 is used in block_reduce_helper.h:/tempStorage/
+    bool *isLastBlockDone = (bool*)device::get_shared_cache();
+    bool *hasLastBlockDone = (bool*)&device::get_shared_cache()[64];
     // printf("team %d thread %d isLastBlockDone %p\n", omp_get_team_num(), omp_get_thread_num(), isLastBlockDone);
 
     T aggregate = BlockReduce(target::thread_idx().z).Reduce(in, r);
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index 7df4a1a9c5..c497e6acba 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -59,14 +59,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
     {
-      Reduction2D_impl<Functor, Arg, grid_stride>(arg);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Reduction2D_impl<Functor, Arg, grid_stride>(arg);
+      }
     }
   }
 
@@ -85,14 +88,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
     {
-      Reduction2D_impl<Functor, Arg, grid_stride>(*argp);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        Reduction2D_impl<Functor, Arg, grid_stride>(*argp);
+      }
     }
   }
 
@@ -153,14 +159,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
     {
-      MultiReduction_impl<Functor, Arg, grid_stride>(arg);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        MultiReduction_impl<Functor, Arg, grid_stride>(arg);
+      }
     }
   }
 
@@ -179,14 +188,17 @@ namespace quda
   template <template <typename> class Functor, typename Arg, bool grid_stride = true>
   __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction(Arg *argp)
   {
-    const auto& grid = target::omptarget::launch_param_host->grid;
-    const auto& block = target::omptarget::launch_param_host->block;
+    const dim3 grid = target::omptarget::launch_param_grid();
+    const dim3 block = target::omptarget::launch_param_block();
     const int gd = grid.x*grid.y*grid.z;
     const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp)
-    #pragma omp parallel num_threads(ld)
+    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
     {
-      MultiReduction_impl<Functor, Arg, grid_stride>(*argp);
+      target::omptarget::launch_param_device_set(grid, block);
+      #pragma omp parallel num_threads(ld)
+      {
+        MultiReduction_impl<Functor, Arg, grid_stride>(*argp);
+      }
     }
   }
 
diff --git a/include/targets/omptarget/shared_memory_cache_helper.h b/include/targets/omptarget/shared_memory_cache_helper.h
index 7d0c31efe7..4fa5910e91 100644
--- a/include/targets/omptarget/shared_memory_cache_helper.h
+++ b/include/targets/omptarget/shared_memory_cache_helper.h
@@ -71,7 +71,7 @@ namespace quda
     template <typename dummy> struct cache_dynamic<true, dummy> {
       __device__ inline atom_t *operator()(unsigned int offset)
       {
-        return reinterpret_cast<atom_t *>(reinterpret_cast<char *>(target::omptarget::get_shared_cache()) + offset);
+        return reinterpret_cast<atom_t *>(reinterpret_cast<char *>(device::get_shared_cache()) + offset);
       }
     };
 
@@ -94,7 +94,7 @@ namespace quda
       __device__ inline atom_t *operator()()
       {
         static_assert(n_element * block_size_x * block_size_y * block_size_z * sizeof(atom_t) <= device::max_shared_memory_size(), "not enough shared memory");
-        return reinterpret_cast<atom_t *>(target::omptarget::get_shared_cache());
+        return reinterpret_cast<atom_t *>(device::get_shared_cache());
       }
     };
 
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index c053d30f0a..a2b42ec680 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -50,47 +50,69 @@ namespace quda {
     */
     __device__ __host__ inline bool is_host() { return dispatch<is_host_impl>(); }
 
+    template <bool is_device> struct block_dim_impl {
+      inline dim3 operator()() { return dim3(1, 1, 1); }
+    };
+    template <> struct block_dim_impl<true> {
+      __device__ inline dim3 operator()() { return target::omptarget::launch_param_kernel_block(); }
+    };
+
     /**
        @brief Helper function that returns the thread block
        dimensions.  On CUDA this returns the intrinsic blockDim,
        whereas on the host this returns (1, 1, 1).
     */
-    __device__ __host__ inline dim3 block_dim()
-    {
-      return target::omptarget::launch_param->block;
-    }
+    __device__ __host__ inline dim3 block_dim() { return dispatch<block_dim_impl>(); }
+
+    template <bool is_device> struct grid_dim_impl {
+      inline dim3 operator()() { return dim3(1, 1, 1); }
+    };
+    template <> struct grid_dim_impl<true> {
+      __device__ inline dim3 operator()() { return target::omptarget::launch_param_kernel_grid(); }
+    };
 
     /**
        @brief Helper function that returns the grid dimensions.  On
        CUDA this returns the intrinsic blockDim, whereas on the host
        this returns (1, 1, 1).
     */
-    __device__ __host__ inline dim3 grid_dim()
-    {
-      return target::omptarget::launch_param->grid;
-    }
+    __device__ __host__ inline dim3 grid_dim() { return dispatch<grid_dim_impl>(); }
+
+    template <bool is_device> struct block_idx_impl {
+      inline dim3 operator()() { return dim3(0, 0, 0); }
+    };
+    template <> struct block_idx_impl<true> {
+      __device__ inline dim3 operator()() {
+        const dim3 & gridDim=target::omptarget::launch_param_kernel_grid();
+        const auto n = (unsigned int)omp_get_team_num();
+        return dim3(n%gridDim.x, (n/gridDim.x)%gridDim.y, n/(gridDim.x*gridDim.y));
+      }
+    };
 
     /**
        @brief Helper function that returns the thread indices within a
        thread block.  On CUDA this returns the intrinsic
        blockIdx, whereas on the host this just returns (0, 0, 0).
     */
-    __device__ __host__ inline dim3 block_idx()
-    {
-      const auto n = (unsigned int)omp_get_team_num();
-      return dim3(n%target::omptarget::launch_param->grid.x, (n/target::omptarget::launch_param->grid.x)%target::omptarget::launch_param->grid.y, n/(target::omptarget::launch_param->grid.x*target::omptarget::launch_param->grid.y));
-    }
+    __device__ __host__ inline dim3 block_idx() { return dispatch<block_idx_impl>(); }
+
+    template <bool is_device> struct thread_idx_impl {
+      inline dim3 operator()() { return dim3(0, 0, 0); }
+    };
+    template <> struct thread_idx_impl<true> {
+      __device__ inline dim3 operator()() {
+        const dim3 & blockDim=target::omptarget::launch_param_kernel_block();
+        const auto n = (unsigned int)omp_get_thread_num();
+        return dim3(n%blockDim.x, (n/blockDim.x)%blockDim.y, n/(blockDim.x*blockDim.y));
+      }
+    };
 
     /**
        @brief Helper function that returns the thread indices within a
        thread block.  On CUDA this returns the intrinsic
        threadIdx, whereas on the host this just returns (0, 0, 0).
     */
-    __device__ __host__ inline dim3 thread_idx()
-    {
-      const auto n = (unsigned int)omp_get_thread_num();
-      return dim3(n%target::omptarget::launch_param->block.x, (n/target::omptarget::launch_param->block.x)%target::omptarget::launch_param->block.y, n/(target::omptarget::launch_param->block.x*target::omptarget::launch_param->block.y));
-    }
+    __device__ __host__ inline dim3 thread_idx() { return dispatch<thread_idx_impl>(); }
 
     /**
        @brief Helper function that returns a linear thread index within a thread block.
@@ -98,9 +120,10 @@ namespace quda {
     template <int dim> __device__ __host__ inline auto thread_idx_linear()
     {
       const auto n = (unsigned int)omp_get_thread_num();
+      const dim3 & blockDim=target::omptarget::launch_param_kernel_block();
       switch (dim) {
-      case 1: return n%target::omptarget::launch_param->block.x;
-      case 2: return n%(target::omptarget::launch_param->block.x*target::omptarget::launch_param->block.y);
+      case 1: return n%blockDim.x;
+      case 2: return n%(blockDim.x*blockDim.y);
       case 3:
       default: return n;
       }
@@ -111,11 +134,12 @@ namespace quda {
     */
     template <int dim> __device__ __host__ inline auto block_size()
     {
+      const dim3 & blockDim=target::omptarget::launch_param_kernel_block();
       switch (dim) {
-      case 1: return target::omptarget::launch_param->block.x;
-      case 2: return target::omptarget::launch_param->block.y * target::omptarget::launch_param->block.x;
+      case 1: return blockDim.x;
+      case 2: return blockDim.y * blockDim.x;
       case 3:
-      default: return target::omptarget::launch_param->block.z * target::omptarget::launch_param->block.y * target::omptarget::launch_param->block.x;
+      default: return blockDim.z * blockDim.y * blockDim.x;
       }
     }
 
@@ -184,6 +208,15 @@ namespace quda {
      */
     template <typename Arg> constexpr std::enable_if_t<use_kernel_arg<Arg>(), void *> get_constant_buffer() { return nullptr; }
 
+    /**
+       @brief Return the address of the shared local memory for the current thread group.
+     */
+    inline char *get_shared_cache(void)
+    {
+      static char s[device::max_shared_memory_size()];
+      #pragma omp groupprivate(s)
+      return s;
+    }
   }
 
 }
diff --git a/include/targets/omptarget/thread_array.h b/include/targets/omptarget/thread_array.h
index 77751adf1b..d7163af944 100644
--- a/include/targets/omptarget/thread_array.h
+++ b/include/targets/omptarget/thread_array.h
@@ -1,40 +1,13 @@
 #pragma once
 
-#include "shared_memory_cache_helper.h"
-
 namespace quda
 {
-
-  /**
-     @brief Class that provides indexable per-thread storage.  On HIP
-     this maps to using assigning each thread a unique window of
-     shared memory using the SharedMemoryCache object.
-   */
   template <typename T, int n> struct thread_array {
-    SharedMemoryCache<array<T, n>, 1, 1, false> device_array;
-    int offset;
-    array<T, n> host_array;
-    array<T, n> &array_;
-
-    __device__ __host__ constexpr thread_array() :
-      offset((target::thread_idx().z * target::block_dim().y + target::thread_idx().y) * target::block_dim().x
-             + target::thread_idx().x),
-      array_(target::is_device() ? *(device_array.data() + offset) : host_array)
-    {
-      array_ = array<T, n>(); // call default constructor
-    }
+    using value_type = T;
+    static constexpr int N = n;
+    T data[n];
 
-    template <typename... Ts>
-    __device__ __host__ constexpr thread_array(T first, const Ts... other) :
-      offset((target::thread_idx().z * target::block_dim().y + target::thread_idx().y) * target::block_dim().x
-             + target::thread_idx().x),
-      array_(target::is_device() ? *(device_array.data() + offset) : host_array)
-    {
-      array_ = array<T, n> {first, other...};
-    }
-
-    __device__ __host__ T &operator[](int i) { return array_[i]; }
-    __device__ __host__ const T &operator[](int i) const { return array_[i]; }
+    constexpr inline T &operator[](int i) { return data[i]; }
+    constexpr inline const T &operator[](int i) const { return data[i]; }
   };
-
 } // namespace quda
diff --git a/include/targets/omptarget/warp_collective.h b/include/targets/omptarget/warp_collective.h
index a6c9c9ceaf..f01453725f 100644
--- a/include/targets/omptarget/warp_collective.h
+++ b/include/targets/omptarget/warp_collective.h
@@ -14,8 +14,8 @@ namespace quda
     {
       using R = typename T::value_type;
       constexpr auto max_nthr = device::max_block_size();
-      static_assert(max_nthr*sizeof(R) <= device::max_shared_memory_size()-sizeof(target::omptarget::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
-      R *storage = (R*)&target::omptarget::get_shared_cache()[128];  // FIXME arbitrary
+      static_assert(max_nthr*sizeof(R) <= device::max_shared_memory_size()-sizeof(device::get_shared_cache()[0])*128, "Shared cache not large enough for tempStorage");  // FIXME arbitrary, the number is arbitrary, offset 128 below
+      R *storage = (R*)&device::get_shared_cache()[128];  // FIXME arbitrary
       const int tid = omp_get_thread_num();
 
       constexpr int warp_size = device::warp_size();
diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index e8822b5bbd..56970ec2ee 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -33,58 +33,34 @@ namespace quda
 
   namespace target {
     namespace omptarget {
-      #pragma omp declare target
-      LaunchParam *launch_param;
-      #pragma omp end declare target
+      struct {
+        dim3 block, grid;
+      } launch_param_host;
 
-      LaunchParam *launch_param_host;
+      dim3 & launch_param_block(void)
+      {
+        return launch_param_host.block;
+      }
+
+      dim3 & launch_param_grid(void)
+      {
+        return launch_param_host.grid;
+      }
 
       int qudaSetupLaunchParameter(const TuneParam &tp)
       {
         static int init = 0;
-        static size_t res_mem = 1ULL << 32;
-        int num_teams = tp.grid.x*tp.grid.y*tp.grid.z;
         if(!init){
           int dev = omp_get_default_device();
-          launch_param_host = (LaunchParam *)omp_target_alloc_host(sizeof(LaunchParam), dev);
-          if(!launch_param_host){
-            errorQuda("failed to allocate launch_param for %lu bytes host pinned memory.", sizeof(LaunchParam));
-            return -1;
-          }
-          omp_target_memcpy(omp_get_mapped_ptr(&launch_param, dev), &launch_param_host, sizeof(LaunchParam *), 0, 0, dev, omp_get_initial_device());
-
           device::constant_arg_buffer = omp_target_alloc_host(device::max_constant_size(), dev);
           if(!device::constant_arg_buffer){
             errorQuda("failed to allocate %lu bytes host memory for kernel arguments.", device::max_constant_size());
             return -1;
           }
-          char *res_mem_env = getenv("QUDA_OMPTARGET_SLM_RESERVE");
-          if(res_mem_env){
-            size_t res_mem_env_val = std::strtoull(res_mem_env, nullptr, 10);
-            if(res_mem_env_val > 0){
-              res_mem = res_mem_env_val;
-              warningQuda("using environment variable QUDA_OMPTARGET_SLM_RESERVE=%lu", res_mem);
-            }else{
-              warningQuda("cannot parse environment variable QUDA_OMPTARGET_SLM_RESERVE=%s", res_mem_env);
-            }
-          }
-          launch_param_host->shared_cache.addr = (int*)omp_target_alloc(res_mem, dev);
-          if(!launch_param_host->shared_cache.addr){
-            errorQuda("failed to allocate %lu bytes device memory for shared cache.", res_mem);
-            return -1;
-          }
-          launch_param_host->shared_cache.num_teams = res_mem/(size_t)device::max_shared_memory_size();
-          launch_param_host->shared_cache.cache_length = device::max_shared_memory_size()/sizeof(launch_param_host->shared_cache.addr[0]);
-          warningQuda("allocated %lu bytes device memory for shared cache among maximum %d teams.", res_mem, launch_param_host->shared_cache.num_teams);
           init = 1;
         }
-        if(launch_param_host->shared_cache.num_teams<num_teams){
-          if(getVerbosity() >= QUDA_VERBOSE)
-            warningQuda("requested %d teams, which is larger than maximum %d teams reserved for shared memory %lu bytes", num_teams, launch_param_host->shared_cache.num_teams, res_mem);
-          return -1;
-        }
-        launch_param_host->grid = tp.grid;
-        launch_param_host->block = tp.block;
+        launch_param_host.grid = tp.grid;
+        launch_param_host.block = tp.block;
         return 0;
       }
 
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index e68b3dd931..3ef22e7306 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -37,11 +37,16 @@ mark_as_advanced(QUDA_OMPTARGET_JIT)
 # define omptarget flags
 
 if(QUDA_OMPTARGET_JIT)
-    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64 -fopenmp-version=51 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64 -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
 else()
-    #set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
-    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=51 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+    #set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=60 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-device-code-split=per_kernel -fopenmp-max-parallel-link-jobs=64 -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
 endif()
+
+if(QUDA_OMPTARGET_DEBUG)
+    set(QUDA_OMPTARGET_FLAGS ${QUDA_OMPTARGET_FLAGS} -gline-tables-only)
+endif()
+
 message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
 
 string(REPLACE " " "" QUDA_GPU_ARCH_STRIP "${QUDA_GPU_ARCH}")

From 754cb56e95d2a0f0896fd6bce9ed6c95d26a1b5a Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 5 Dec 2023 23:55:12 +0000
Subject: [PATCH 183/198] omp: add option for fixed simd16

---
 lib/targets/omptarget/target_omptarget.cmake | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index 3ef22e7306..fd99d3ea6d 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -33,6 +33,9 @@ message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}
 option(QUDA_OMPTARGET_JIT "Build OpenMP target backend with JIT" OFF)
 mark_as_advanced(QUDA_OMPTARGET_JIT)
 
+option(QUDA_OMPTARGET_SIMD16 "Build OpenMP target backend with simd width 16" OFF)
+mark_as_advanced(QUDA_OMPTARGET_SIMD16)
+
 # ######################################################################################################################
 # define omptarget flags
 
@@ -47,15 +50,20 @@ if(QUDA_OMPTARGET_DEBUG)
     set(QUDA_OMPTARGET_FLAGS ${QUDA_OMPTARGET_FLAGS} -gline-tables-only)
 endif()
 
+if(QUDA_OMPTARGET_SIMD16)
+    set(QUDA_OMPTARGET_FLAGS ${QUDA_OMPTARGET_FLAGS} "SHELL:-mllvm -vpo-paropt-fixed-simd-width=16")
+endif()
+
 message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
 
 string(REPLACE " " "" QUDA_GPU_ARCH_STRIP "${QUDA_GPU_ARCH}")
 
 # QUDA_HASH for tunecache
-set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${QUDA_GPU_ARCH_STRIP},cxx_version=${CMAKE_CXX_COMPILER_VERSION})
 if(QUDA_OMPTARGET_JIT)
+    set(HASH cpu_arch=${CPU_ARCH},gpu_arch=JIT,cxx_version=${CMAKE_CXX_COMPILER_VERSION})
     set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:JIT")
 else()
+    set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${QUDA_GPU_ARCH_STRIP},cxx_version=${CMAKE_CXX_COMPILER_VERSION})
     set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:${QUDA_GPU_ARCH_STRIP}")
 endif()
 

From 1bc80f9ff6b30a900885824ca0c8bca88a14d00b Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 5 Dec 2023 23:57:08 +0000
Subject: [PATCH 184/198] omp: provide variants of kernel launching methods

---
 .../omptarget/block_reduction_kernel.h        |  30 +---
 include/targets/omptarget/kernel.h            | 155 +++---------------
 include/targets/omptarget/quda_api.h          |  20 ---
 include/targets/omptarget/reduction_kernel.h  |  89 +++-------
 include/targets/omptarget/target_device.h     |  97 +++++++++++
 include/targets/omptarget/tunable_kernel.h    |  13 --
 6 files changed, 155 insertions(+), 249 deletions(-)

diff --git a/include/targets/omptarget/block_reduction_kernel.h b/include/targets/omptarget/block_reduction_kernel.h
index e8e78e99d2..d0a8c7f0e1 100644
--- a/include/targets/omptarget/block_reduction_kernel.h
+++ b/include/targets/omptarget/block_reduction_kernel.h
@@ -115,18 +115,9 @@ namespace quda
                       0) __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg arg)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        BlockKernel2D_impl<Functor, Arg>(arg);
-      }
-    }
+    QUDA_OMPTARGET_KERNEL_BEGIN(arg)
+      BlockKernel2D_impl<Functor, Arg>(arg);
+    QUDA_OMPTARGET_KERNEL_END
   }
 
   /**
@@ -149,18 +140,9 @@ namespace quda
                       0) __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> BlockKernel2D(Arg *argp)
   {
     static_assert(!grid_stride, "grid_stride not supported for BlockKernel");
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        BlockKernel2D_impl<Functor, Arg>(*argp);
-      }
-    }
+    QUDA_OMPTARGET_KERNEL_BEGIN_PTR(argp)
+      BlockKernel2D_impl<Functor, Arg>(*argp);
+    QUDA_OMPTARGET_KERNEL_END
   }
 
 }
diff --git a/include/targets/omptarget/kernel.h b/include/targets/omptarget/kernel.h
index d2b6114ccb..070da6a4b2 100644
--- a/include/targets/omptarget/kernel.h
+++ b/include/targets/omptarget/kernel.h
@@ -3,6 +3,24 @@
 #include <target_device.h>
 #include <kernel_helper.h>
 
+#define OMP_KERNEL(kern) \
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false> \
+  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> kern(Arg arg) \
+  { \
+    QUDA_OMPTARGET_KERNEL_BEGIN(arg) \
+      kern##_impl<Functor, Arg, grid_stride>(arg); \
+    QUDA_OMPTARGET_KERNEL_END \
+  }
+
+#define OMP_KERNEL_PTR(kern) \
+  template <template <typename> class Functor, typename Arg, bool grid_stride = false> \
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> kern(Arg *argp) \
+  { \
+    QUDA_OMPTARGET_KERNEL_BEGIN_PTR(argp) \
+      kern##_impl<Functor, Arg, grid_stride>(*argp); \
+    QUDA_OMPTARGET_KERNEL_END \
+  }
+
 namespace quda
 {
 
@@ -47,22 +65,7 @@ namespace quda
      per thread.
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel1D(Arg arg)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Kernel1D_impl<Functor, Arg, grid_stride>(arg);
-      }
-    }
-  }
+  OMP_KERNEL(Kernel1D);
 
   /**
      @brief Kernel1D is the entry point of the generic 1-d kernel.
@@ -76,22 +79,7 @@ namespace quda
      per thread.
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel1D(Arg *argp)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Kernel1D_impl<Functor, Arg, grid_stride>(*argp);
-      }
-    }
-  }
+  OMP_KERNEL_PTR(Kernel1D);
 
   /**
      @brief Kernel2D_impl is the implementation of the generic 2-d
@@ -136,22 +124,7 @@ namespace quda
      per thread (in the x dimension)
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel2D(Arg arg)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Kernel2D_impl<Functor, Arg, grid_stride>(arg);
-      }
-    }
-  }
+  OMP_KERNEL(Kernel2D);
 
   /**
      @brief Kernel2D is the entry point of the generic 2-d kernel.
@@ -165,22 +138,7 @@ namespace quda
      per thread (in the x dimension)
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel2D(Arg *argp)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Kernel2D_impl<Functor, Arg, grid_stride>(*argp);
-      }
-    }
-  }
+  OMP_KERNEL_PTR(Kernel2D);
 
   /**
      @brief Kernel3D_impl is the implementation of the generic 3-d
@@ -227,22 +185,7 @@ namespace quda
      per thread (in the x dimension)
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Kernel3D(Arg arg)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Kernel3D_impl<Functor, Arg, grid_stride>(arg);
-      }
-    }
-  }
+  OMP_KERNEL(Kernel3D);
 
   /**
      @brief Kernel3D is the entry point of the generic 3-d kernel.
@@ -256,53 +199,9 @@ namespace quda
      per thread (in the x dimension)
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = false>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Kernel3D(Arg *argp)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Kernel3D_impl<Functor, Arg, grid_stride>(*argp);
-      }
-    }
-  }
-
-  /**
-     @brief raw_kernel is used for CUDA-specific kernels where we want
-     to avoid using the generic framework.  For these kernels, we
-     delegate the parallelism and bounds checking for the kernel
-     functor.
-
-     @tparam Functor Kernel functor that defines the kernel
-     @tparam Arg Kernel argument struct that set any required meta
-     data for the kernel
-     @tparam dummy unused template parameter, present to allow us to
-     utilize the generic launching framework
-
-     @param[in] arg Kernel argument
-   */
-  template <template <typename> class Functor, typename Arg, bool dummy = false>
-  __launch_bounds__(Arg::block_dim, Arg::min_blocks) __global__ void raw_kernel(Arg arg)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Functor<Arg> f(&arg);
-        f();
-      }
-    }
-  }
+  OMP_KERNEL_PTR(Kernel3D);
 
 }
+
+#undef OMP_KERNEL
+#undef OMP_KERNEL_PTR
diff --git a/include/targets/omptarget/quda_api.h b/include/targets/omptarget/quda_api.h
index d8197b0a86..9c05f2593e 100644
--- a/include/targets/omptarget/quda_api.h
+++ b/include/targets/omptarget/quda_api.h
@@ -89,26 +89,6 @@ namespace quda {
     namespace omptarget {
       dim3 & launch_param_grid(void);
       dim3 & launch_param_block(void);
-
-      inline dim3 & launch_param_kernel_block(void)
-      {
-        static char block[sizeof(dim3)];
-        #pragma omp groupprivate(block)
-        return *reinterpret_cast<dim3*>(block);
-      }
-      inline dim3 & launch_param_kernel_grid(void)
-      {
-        static char grid[sizeof(dim3)];
-        #pragma omp groupprivate(grid)
-        return *reinterpret_cast<dim3*>(grid);
-      }
-      inline void launch_param_device_set(dim3 grid, dim3 block)
-      {
-        dim3 & gref = launch_param_kernel_grid();
-        dim3 & bref = launch_param_kernel_block();
-        gref = grid;
-        bref = block;
-      }
     }
   }
 }
diff --git a/include/targets/omptarget/reduction_kernel.h b/include/targets/omptarget/reduction_kernel.h
index c497e6acba..3b4c5dc82b 100644
--- a/include/targets/omptarget/reduction_kernel.h
+++ b/include/targets/omptarget/reduction_kernel.h
@@ -3,6 +3,24 @@
 #include <target_device.h>
 #include <reduce_helper.h>
 
+#define OMP_KERNEL(kern) \
+  template <template <typename> class Functor, typename Arg, bool grid_stride = true> \
+  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> kern(Arg arg) \
+  { \
+    QUDA_OMPTARGET_KERNEL_BEGIN(arg) \
+      kern##_impl<Functor, Arg, grid_stride>(arg); \
+    QUDA_OMPTARGET_KERNEL_END \
+  }
+
+#define OMP_KERNEL_PTR(kern) \
+  template <template <typename> class Functor, typename Arg, bool grid_stride = true> \
+  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> kern(Arg *argp) \
+  { \
+    QUDA_OMPTARGET_KERNEL_BEGIN_PTR(argp) \
+      kern##_impl<Functor, Arg, grid_stride>(*argp); \
+    QUDA_OMPTARGET_KERNEL_END \
+  }
+
 namespace quda
 {
 
@@ -56,22 +74,7 @@ namespace quda
      per thread (in the x dimension)
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
-  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> Reduction2D(Arg arg)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Reduction2D_impl<Functor, Arg, grid_stride>(arg);
-      }
-    }
-  }
+  OMP_KERNEL(Reduction2D)
 
   /**
      @brief Reduction2D is the entry point of the generic 2-d
@@ -85,22 +88,7 @@ namespace quda
      per thread (in the x dimension)
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> Reduction2D(Arg *argp)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        Reduction2D_impl<Functor, Arg, grid_stride>(*argp);
-      }
-    }
-  }
+  OMP_KERNEL_PTR(Reduction2D)
 
   /**
      @brief MultiReduction_impl is the implementation of the generic
@@ -156,22 +144,7 @@ namespace quda
      per thread (in the x dimension)
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
-  __global__ std::enable_if_t<device::use_kernel_arg<Arg>(), void> MultiReduction(Arg arg)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        MultiReduction_impl<Functor, Arg, grid_stride>(arg);
-      }
-    }
-  }
+  OMP_KERNEL(MultiReduction)
 
   /**
      @brief MultiReduction is the entry point of the generic
@@ -185,21 +158,9 @@ namespace quda
      per thread (in the x dimension)
      @param[in] arg Kernel argument
    */
-  template <template <typename> class Functor, typename Arg, bool grid_stride = true>
-  __global__ std::enable_if_t<!device::use_kernel_arg<Arg>(), void> MultiReduction(Arg *argp)
-  {
-    const dim3 grid = target::omptarget::launch_param_grid();
-    const dim3 block = target::omptarget::launch_param_block();
-    const int gd = grid.x*grid.y*grid.z;
-    const int ld = block.x*block.y*block.z;
-    #pragma omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)
-    {
-      target::omptarget::launch_param_device_set(grid, block);
-      #pragma omp parallel num_threads(ld)
-      {
-        MultiReduction_impl<Functor, Arg, grid_stride>(*argp);
-      }
-    }
-  }
+  OMP_KERNEL_PTR(MultiReduction)
 
 } // namespace quda
+
+#undef OMP_KERNEL
+#undef OMP_KERNEL_PTR
diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index a2b42ec680..84a90cba3d 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -12,10 +12,107 @@
 #define QUDA_MAX_SHARED_MEMORY_SIZE 40*1024
 #endif
 
+#ifndef QUDA_OMPTARGET_PARALLEL_LAUNCH_METHOD
+/* 0: racy
+   1: if && barrier
+   2: single
+   3: teams set && parallel
+ */
+#define QUDA_OMPTARGET_PARALLEL_LAUNCH_METHOD 0
+#endif
+
+#if QUDA_OMPTARGET_PARALLEL_LAUNCH_METHOD==0
+
+  #define QUDA_OMPTARGET_PARALLEL_LAUNCH(ld,grid,block) \
+    _Pragma("omp parallel num_threads(ld)") \
+    { target::omptarget::launch_param_device_set(grid, block);
+
+#elif QUDA_OMPTARGET_PARALLEL_LAUNCH_METHOD==1
+
+  #define QUDA_OMPTARGET_PARALLEL_LAUNCH(ld,grid,block) \
+    _Pragma("omp parallel num_threads(ld)") \
+    { if(omp_get_thread_num()==0) \
+        target::omptarget::launch_param_device_set(grid, block); \
+      _Pragma("omp barrier")
+
+#elif QUDA_OMPTARGET_PARALLEL_LAUNCH_METHOD==2
+
+  #define QUDA_OMPTARGET_PARALLEL_LAUNCH(ld,grid,block) \
+    _Pragma("omp parallel num_threads(ld)") \
+    { _Pragma("omp single") \
+      target::omptarget::launch_param_device_set(grid, block);
+
+#elif QUDA_OMPTARGET_PARALLEL_LAUNCH_METHOD==3
+
+  #define QUDA_OMPTARGET_PARALLEL_LAUNCH(ld,grid,block) \
+    target::omptarget::launch_param_device_set(grid, block); \
+    _Pragma("omp parallel num_threads(ld)") \
+    {
+
+#else
+
+  #error "Allowed values for QUDA_OMPTARGET_LAUNCH_METHOD are 0, 1, 2, or 3."
+
+#endif
+
+#define QUDA_OMPTARGET_KERNEL_BEGIN(arg) \
+    const dim3 grid = target::omptarget::launch_param_grid(); \
+    const dim3 block = target::omptarget::launch_param_block(); \
+    const int gd = grid.x*grid.y*grid.z; \
+    const int ld = block.x*block.y*block.z; \
+    _Pragma("omp target teams num_teams(gd) thread_limit(ld) firstprivate(arg,grid,block)") \
+    { QUDA_OMPTARGET_PARALLEL_LAUNCH(ld,grid,block)
+
+#define QUDA_OMPTARGET_KERNEL_BEGIN_PTR(argp) \
+    const dim3 grid = target::omptarget::launch_param_grid(); \
+    const dim3 block = target::omptarget::launch_param_block(); \
+    const int gd = grid.x*grid.y*grid.z; \
+    const int ld = block.x*block.y*block.z; \
+    _Pragma("omp target teams num_teams(gd) thread_limit(ld) is_device_ptr(argp) firstprivate(grid,block)") \
+    { QUDA_OMPTARGET_PARALLEL_LAUNCH(ld,grid,block)
+
+#define QUDA_OMPTARGET_KERNEL_END }} /* closes BEGIN/BEGIN_PTR */
+
 namespace quda {
 
   namespace target {
 
+    namespace omptarget {
+      inline dim3 & launch_param_kernel_block(void)
+      {
+#if 1
+        static char block[sizeof(dim3)];
+        #pragma omp groupprivate(block)
+        return *reinterpret_cast<dim3*>(block);
+#else
+        /* omp 6 */
+        static char block[sizeof(dim3)];
+        #pragma omp threadprivate(block)
+        return *reinterpret_cast<dim3*>(block);
+#endif
+      }
+      inline dim3 & launch_param_kernel_grid(void)
+      {
+#if 1
+        static char grid[sizeof(dim3)];
+        #pragma omp groupprivate(grid)
+        return *reinterpret_cast<dim3*>(grid);
+#else
+        /* omp 6 */
+        static char grid[sizeof(dim3)];
+        #pragma omp threadprivate(grid)
+        return *reinterpret_cast<dim3*>(grid);
+#endif
+      }
+      inline void launch_param_device_set(dim3 grid, dim3 block)
+      {
+        dim3 & gref = launch_param_kernel_grid();
+        dim3 & bref = launch_param_kernel_block();
+        gref = grid;
+        bref = block;
+      }
+    } // namespace omptarget
+
 #pragma omp begin declare variant match(device={kind(host)})
     template <template <bool, typename ...> class f, typename ...Args>
       __host__ __device__ auto dispatch(Args &&... args)
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index 535d754eec..c17d092fd2 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -104,19 +104,6 @@ namespace quda {
     }
 
   public:
-    /**
-       @brief Special kernel launcher used for raw CUDA kernels with no
-       assumption made about shape of parallelism.  Kernels launched
-       using this must take responsibility of bounds checking and
-       assignment of threads.
-     */
-    template <template <typename> class Functor, typename Arg>
-    void launch_cuda(const TuneParam &tp, const qudaStream_t &stream, const Arg &arg) const
-    {
-      constexpr bool grid_stride = false;
-      const_cast<TunableKernel*>(this)->launch_device<Functor, grid_stride>(KERNEL(raw_kernel), tp, stream, arg);
-    }
-
     TunableKernel(const LatticeField &field, QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION) :
       location(location != QUDA_INVALID_FIELD_LOCATION ? location : field.Location())
     {

From fc5303b3bb8b4c8a4a4e4d0d4d4c4b6b4d87e0a4 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 6 Dec 2023 22:47:44 +0000
Subject: [PATCH 185/198] omp: update env names and compilation params

---
 include/targets/omptarget/target_device.h    |  2 +-
 lib/targets/omptarget/device.cpp             | 12 ++++++------
 lib/targets/omptarget/target_omptarget.cmake | 13 +++++++------
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/include/targets/omptarget/target_device.h b/include/targets/omptarget/target_device.h
index 84a90cba3d..e78a18c548 100644
--- a/include/targets/omptarget/target_device.h
+++ b/include/targets/omptarget/target_device.h
@@ -9,7 +9,7 @@
 #endif
 
 #ifndef QUDA_MAX_SHARED_MEMORY_SIZE
-#define QUDA_MAX_SHARED_MEMORY_SIZE 40*1024
+#define QUDA_MAX_SHARED_MEMORY_SIZE 64*1024
 #endif
 
 #ifndef QUDA_OMPTARGET_PARALLEL_LAUNCH_METHOD
diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index fd48026f9b..2125d587ae 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -8,7 +8,7 @@
 // until we have a way to query device properties correctly
 
 #ifndef QUDA_OMP_MAX_TEAMS
-#define QUDA_OMP_MAX_TEAMS 65536
+#define QUDA_OMP_MAX_TEAMS 2147483647
 #endif
 
 #ifndef QUDA_MAX_THREADS_PER_PROCESSOR
@@ -120,11 +120,11 @@ namespace quda
     {
       constexpr int len = 128;
       char *device_order_env = getenv("ZE_AFFINITY_MASK");
-      char *omptarget_devices = getenv("LIBOMPTARGET_DEVICES");
-      if(omptarget_devices && device_order_env)
-        snprintf(device_list_string, len, "%s%s", omptarget_devices, device_order_env);
-      else if(omptarget_devices)
-        snprintf(device_list_string, len, "%s", omptarget_devices);
+      char *oneapi_device_env = getenv("ONEAPI_DEVICE_SELECTOR");
+      if(oneapi_device_env && device_order_env)
+        snprintf(device_list_string, len, "%s%s", oneapi_device_env, device_order_env);
+      else if(oneapi_device_env)
+        snprintf(device_list_string, len, "%s", oneapi_device_env);
       else if(device_order_env)
         snprintf(device_list_string, len, "%s", device_order_env);
     }
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index fd99d3ea6d..72f33b892c 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -25,8 +25,8 @@ set_property(CACHE QUDA_MAX_BLOCK_SIZE PROPERTY STRINGS 256 512 768 1024)
 target_compile_definitions(quda PUBLIC QUDA_MAX_BLOCK_SIZE=${QUDA_MAX_BLOCK_SIZE})
 message(STATUS "Using maximum team size: ${QUDA_MAX_BLOCK_SIZE}")
 
-set(QUDA_MAX_SHARED_MEMORY_SIZE 40960 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
-set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 36864 40960 45056 49152 65536)
+set(QUDA_MAX_SHARED_MEMORY_SIZE 114688 CACHE STRING "OpenMP target maximum shared memory size (among threads in a team)")
+set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 32768 49152 65536 81920 98304 114688 131072)
 target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
 message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")
 
@@ -36,6 +36,8 @@ mark_as_advanced(QUDA_OMPTARGET_JIT)
 option(QUDA_OMPTARGET_SIMD16 "Build OpenMP target backend with simd width 16" OFF)
 mark_as_advanced(QUDA_OMPTARGET_SIMD16)
 
+string(REPLACE " " "" QUDA_GPU_ARCH_TAG "${QUDA_GPU_ARCH}")
+
 # ######################################################################################################################
 # define omptarget flags
 
@@ -52,19 +54,18 @@ endif()
 
 if(QUDA_OMPTARGET_SIMD16)
     set(QUDA_OMPTARGET_FLAGS ${QUDA_OMPTARGET_FLAGS} "SHELL:-mllvm -vpo-paropt-fixed-simd-width=16")
+    set(QUDA_GPU_ARCH_TAG "${QUDA_GPU_ARCH_TAG}-SIMD16")
 endif()
 
 message(STATUS "Using OpenMP target flags: ${QUDA_OMPTARGET_FLAGS}")
 
-string(REPLACE " " "" QUDA_GPU_ARCH_STRIP "${QUDA_GPU_ARCH}")
-
 # QUDA_HASH for tunecache
 if(QUDA_OMPTARGET_JIT)
     set(HASH cpu_arch=${CPU_ARCH},gpu_arch=JIT,cxx_version=${CMAKE_CXX_COMPILER_VERSION})
     set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:JIT")
 else()
-    set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${QUDA_GPU_ARCH_STRIP},cxx_version=${CMAKE_CXX_COMPILER_VERSION})
-    set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget:${QUDA_GPU_ARCH_STRIP}")
+    set(HASH cpu_arch=${CPU_ARCH},gpu_arch=${QUDA_GPU_ARCH_TAG},cxx_version=${CMAKE_CXX_COMPILER_VERSION})
+    set(GITVERSION "${PROJECT_VERSION}-${GITVERSION}-omptarget")
 endif()
 
 # ######################################################################################################################

From de9bed4368705dae945ec40048faa4fe014b6cd9 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 6 Dec 2023 22:48:24 +0000
Subject: [PATCH 186/198] omp: avoid direct char access in memset, use float
 when possible

---
 lib/targets/omptarget/quda_api.cpp | 42 ++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/lib/targets/omptarget/quda_api.cpp b/lib/targets/omptarget/quda_api.cpp
index 56970ec2ee..bb66ece86d 100644
--- a/lib/targets/omptarget/quda_api.cpp
+++ b/lib/targets/omptarget/quda_api.cpp
@@ -68,9 +68,21 @@ namespace quda
       ompMemset(void *p, unsigned char b, std::size_t s)
       {
         constexpr int max_threads = device::max_block_size();
-        unsigned char *c = reinterpret_cast<unsigned char *>(p);
-      #pragma omp target teams distribute parallel for simd thread_limit(max_threads) is_device_ptr(c)
-        for(std::size_t i=0;i<s;++i) c[i] = b;
+        constexpr size_t nb = sizeof(float);
+        if(s%nb==0){
+          float *c = reinterpret_cast<float *>(p);
+          float f;
+          unsigned char bs[nb];
+          for(std::size_t i=0; i<nb; ++i) bs[i] = b;
+          memcpy(&f, bs, nb);
+          const std::size_t sb = s/nb;
+          #pragma omp target teams distribute parallel for simd thread_limit(max_threads) is_device_ptr(c)
+          for(std::size_t i=0;i<sb;++i) c[i] = f;
+        }else{
+          unsigned char *c = reinterpret_cast<unsigned char *>(p);
+          #pragma omp target teams distribute parallel for simd thread_limit(max_threads) is_device_ptr(c)
+          for(std::size_t i=0;i<s;++i) c[i] = b;
+        }
         return 0;
       }
 
@@ -84,11 +96,25 @@ namespace quda
       ompMemset2D(void *p, size_t pitch, unsigned char b, size_t w, size_t h)
       {
         constexpr int max_threads = device::max_block_size();
-        unsigned char *c = reinterpret_cast<unsigned char *>(p);
-      #pragma omp target teams distribute parallel for simd thread_limit(max_threads) is_device_ptr(c) collapse(2)
-        for(std::size_t i=0;i<h;++i)
-          for(std::size_t j=0;j<w;++j)
-            c[j+i*pitch] = b;
+        constexpr size_t nb = sizeof(float);
+        if(w%nb==0 && pitch%nb==0){
+          float *c = reinterpret_cast<float *>(p);
+          float f;
+          unsigned char bs[nb];
+          for(std::size_t i=0; i<nb; ++i) bs[i] = b;
+          memcpy(&f, bs, nb);
+          const std::size_t wb = w/nb, pitchb = pitch/nb;
+          #pragma omp target teams distribute parallel for simd thread_limit(max_threads) is_device_ptr(c) collapse(2)
+          for(std::size_t i=0;i<h;++i)
+            for(std::size_t j=0;j<wb;++j)
+              c[j+i*pitchb] = b;
+        }else{
+          unsigned char *c = reinterpret_cast<unsigned char *>(p);
+          #pragma omp target teams distribute parallel for simd thread_limit(max_threads) is_device_ptr(c) collapse(2)
+          for(std::size_t i=0;i<h;++i)
+            for(std::size_t j=0;j<w;++j)
+              c[j+i*pitch] = b;
+        }
         return 0;
       }
 

From 426017f32609b5f893e2dec61efcaba73978182e Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 1 Feb 2024 22:28:17 +0000
Subject: [PATCH 187/198] omp: remove per_kernel compilation and add
 huge-device-code options

---
 lib/targets/omptarget/target_omptarget.cmake | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index 72f33b892c..d7a78c677a 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -45,7 +45,8 @@ if(QUDA_OMPTARGET_JIT)
     set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64 -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
 else()
     #set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=60 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
-    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-device-code-split=per_kernel -fopenmp-max-parallel-link-jobs=64 -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+    #set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-device-code-split=per_kernel -fopenmp-max-parallel-link-jobs=64 -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
 endif()
 
 if(QUDA_OMPTARGET_DEBUG)
@@ -92,9 +93,9 @@ target_compile_options(
 target_compile_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS})
 
 if(QUDA_OMPTARGET_JIT)
-    target_link_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS})
+    target_link_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS} -flink-huge-device-code)
 else()
-    target_link_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS} -Xopenmp-target-backend "-device ${QUDA_GPU_ARCH}")
+    target_link_options(quda PRIVATE ${QUDA_OMPTARGET_FLAGS} -Xopenmp-target-backend "-device ${QUDA_GPU_ARCH}" -flink-huge-device-code)
 endif()
 
 set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES LANGUAGE CXX)

From a68544f7d6897ff30670a9c3621c0ff4c3f54a12 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 29 Feb 2024 01:11:32 +0000
Subject: [PATCH 188/198] omp: update following upstream

---
 include/targets/omptarget/load_store.h        |  67 ++++
 .../omptarget/shared_memory_cache_helper.h    | 293 ------------------
 .../targets/omptarget/shared_memory_helper.h  |  88 ++++++
 include/targets/omptarget/thread_array.h      |   8 +
 .../targets/omptarget/thread_local_cache.h    |   1 +
 include/targets/omptarget/tunable_kernel.h    |   1 +
 6 files changed, 165 insertions(+), 293 deletions(-)
 create mode 100644 include/targets/omptarget/load_store.h
 create mode 100644 include/targets/omptarget/shared_memory_helper.h
 create mode 100644 include/targets/omptarget/thread_local_cache.h

diff --git a/include/targets/omptarget/load_store.h b/include/targets/omptarget/load_store.h
new file mode 100644
index 0000000000..d1bfe4a955
--- /dev/null
+++ b/include/targets/omptarget/load_store.h
@@ -0,0 +1,67 @@
+#pragma once
+
+#include <register_traits.h>
+
+namespace quda
+{
+
+  /**
+     @brief Element type used for coalesced storage.
+   */
+  template <typename T>
+  using atom_t = std::conditional_t<sizeof(T) % 16 == 0, int4, std::conditional_t<sizeof(T) % 8 == 0, int2, int>>;
+
+  // pre-declaration of vector_load that we wish to specialize
+  template <bool> struct vector_load_impl;
+
+  // CUDA specializations of the vector_load
+  template <> struct vector_load_impl<true> {
+    template <typename T> __device__ inline void operator()(T &value, const void *ptr, int idx)
+    {
+      value = reinterpret_cast<const T *>(ptr)[idx];
+    }
+
+    __device__ inline void operator()(short8 &value, const void *ptr, int idx)
+    {
+      float4 tmp;
+      operator()(tmp, ptr, idx);
+      memcpy(&value, &tmp, sizeof(float4));
+    }
+
+    __device__ inline void operator()(char8 &value, const void *ptr, int idx)
+    {
+      float2 tmp;
+      operator()(tmp, ptr, idx);
+      memcpy(&value, &tmp, sizeof(float2));
+    }
+  };
+
+  // pre-declaration of vector_store that we wish to specialize
+  template <bool> struct vector_store_impl;
+
+  // CUDA specializations of the vector_store using inline ptx
+  template <> struct vector_store_impl<true> {
+    template <typename T> __device__ inline void operator()(void *ptr, int idx, const T &value)
+    {
+      reinterpret_cast<T *>(ptr)[idx] = value;
+    }
+
+    __device__ inline void operator()(void *ptr, int idx, const short8 &value)
+    {
+      this->operator()(ptr, idx, *reinterpret_cast<const float4 *>(&value));
+    }
+
+    __device__ inline void operator()(void *ptr, int idx, const char8 &value)
+    {
+      this->operator()(ptr, idx, *reinterpret_cast<const float2 *>(&value));
+    }
+
+    __device__ inline void operator()(void *ptr, int idx, const char4 &value)
+    {
+      this->operator()(ptr, idx, *reinterpret_cast<const short2 *>(&value)); // A char4 is the same as a short2
+    }
+  };
+
+} // namespace quda
+
+#include "../generic/load_store.h"
diff --git a/include/targets/omptarget/shared_memory_cache_helper.h b/include/targets/omptarget/shared_memory_cache_helper.h
index 4fa5910e91..73be0cd01b 100644
--- a/include/targets/omptarget/shared_memory_cache_helper.h
+++ b/include/targets/omptarget/shared_memory_cache_helper.h
@@ -1,294 +1 @@
-#pragma once
-
-#include <target_device.h>
-#include <array.h>
-
-/**
-   @file shared_memory_cache_helper.h
-
-   Helper functionality for aiding the use of the shared memory for
-   sharing data between threads in a thread block.
- */
-
-namespace quda
-{
-
-  /**
-     @brief Class which wraps around a shared memory cache for type T,
-     where each thread in the thread block stores a unique value in
-     the cache which any other thread can access.
-
-     This accessor supports both explicit run-time block size and
-     compile-time sizing.
-
-     * For run-time block size, the constructor should be initialied
-       with the desired block size.
-
-     * For compile-time block size, no arguments should be passed to
-       the constructor, and then the second and third template
-       parameters correspond to the y and z dimensions of the block,
-       respectively.  The x dimension of the block will be set
-       according the maximum number of threads possible, given these
-       dimensions.
-   */
-  template <typename T, int block_size_y_ = 1, int block_size_z_ = 1, bool dynamic_ = true> class SharedMemoryCache
-  {
-  public:
-    using value_type = T;
-    static constexpr int block_size_y = block_size_y_;
-    static constexpr int block_size_z = block_size_z_;
-    static constexpr bool dynamic = dynamic_;
-
-  private:
-    /** maximum number of threads in x given the y and z block sizes */
-    static constexpr int block_size_x = device::max_block_size<block_size_y, block_size_z>();
-
-    using atom_t = std::conditional_t<sizeof(T) % 16 == 0, int4, std::conditional_t<sizeof(T) % 8 == 0, int2, int>>;
-    static_assert(sizeof(T) % 4 == 0, "Shared memory cache does not support sub-word size types");
-
-    // The number of elements of type atom_t that we break T into for optimal shared-memory access
-    static constexpr int n_element = sizeof(T) / sizeof(atom_t);
-
-    const dim3 block;
-    const int stride;
-    const unsigned int offset = 0; // dynamic offset in bytes
-
-    /**
-       @brief This is a dummy instantiation for the host compiler
-    */
-    template <bool, typename dummy = void> struct cache_dynamic {
-      atom_t *operator()(unsigned)
-      {
-        static atom_t *cache_;
-        return reinterpret_cast<atom_t *>(cache_);
-      }
-    };
-
-    /**
-       @brief This is the handle to the shared memory, dynamic specialization
-       @return Shared memory pointer
-     */
-    template <typename dummy> struct cache_dynamic<true, dummy> {
-      __device__ inline atom_t *operator()(unsigned int offset)
-      {
-        return reinterpret_cast<atom_t *>(reinterpret_cast<char *>(device::get_shared_cache()) + offset);
-      }
-    };
-
-    /**
-       @brief This is a dummy instantiation for the host compiler
-    */
-    template <bool is_device, typename dummy = void> struct cache_static {
-      atom_t *operator()()
-      {
-        static atom_t *cache_;
-        return reinterpret_cast<atom_t *>(cache_);
-      }
-    };
-
-    /**
-       @brief This is the handle to the shared memory, static specialization
-       @return Shared memory pointer
-     */
-    template <typename dummy> struct cache_static<true, dummy> {
-      __device__ inline atom_t *operator()()
-      {
-        static_assert(n_element * block_size_x * block_size_y * block_size_z * sizeof(atom_t) <= device::max_shared_memory_size(), "not enough shared memory");
-        return reinterpret_cast<atom_t *>(device::get_shared_cache());
-      }
-    };
-
-    template <bool dynamic_shared> __device__ __host__ inline std::enable_if_t<dynamic_shared, atom_t *> cache() const
-    {
-      return target::dispatch<cache_dynamic>(offset);
-    }
-
-    template <bool dynamic_shared> __device__ __host__ inline std::enable_if_t<!dynamic_shared, atom_t *> cache() const
-    {
-      return target::dispatch<cache_static>();
-    }
-
-    __device__ __host__ inline void save_detail(const T &a, int x, int y, int z) const
-    {
-      atom_t tmp[n_element];
-      memcpy(tmp, (void *)&a, sizeof(T));
-      int j = (z * block.y + y) * block.x + x;
-#pragma unroll
-      for (int i = 0; i < n_element; i++) cache<dynamic>()[i * stride + j] = tmp[i];
-    }
-
-    __device__ __host__ inline T load_detail(int x, int y, int z) const
-    {
-      atom_t tmp[n_element];
-      int j = (z * block.y + y) * block.x + x;
-#pragma unroll
-      for (int i = 0; i < n_element; i++) tmp[i] = cache<dynamic>()[i * stride + j];
-      T a;
-      memcpy((void *)&a, tmp, sizeof(T));
-      return a;
-    }
-
-    /**
-       @brief Dummy instantiation for the host compiler
-    */
-    template <bool is_device, typename dummy = void> struct sync_impl {
-      void operator()() { }
-    };
-
-    /**
-       @brief Synchronize the cache when on the device
-    */
-    template <typename dummy> struct sync_impl<true, dummy> {
-      __device__ inline void operator()() { __syncthreads(); }
-    };
-
-  public:
-    /**
-       @brief constructor for SharedMemory cache.  If no arguments are
-       pass, then the dimensions are set according to the templates
-       block_size_y and block_size_z, together with the derived
-       block_size_x.  Otherwise use the block sizes passed into the
-       constructor.
-
-       @param[in] block Block dimensions for the 3-d shared memory object
-       @param[in] thread_offset "Perceived" offset from dynamic shared
-       memory base pointer (used when we have multiple caches in
-       scope).  Need to include block size to actual offset.
-    */
-    constexpr SharedMemoryCache(dim3 block = dim3(block_size_x, block_size_y, block_size_z),
-                                unsigned int thread_offset = 0) :
-      block(block), stride(block.x * block.y * block.z), offset(stride * thread_offset)
-    {
-    }
-
-    /**
-       @brief Grab the raw base address to shared memory.
-    */
-    __device__ __host__ inline auto data() const { return reinterpret_cast<T *>(cache<dynamic>()); }
-
-    /**
-       @brief Save the value into the 3-d shared memory cache.
-       @param[in] a The value to store in the shared memory cache
-       @param[in] x The x index to use
-       @param[in] y The y index to use
-       @param[in] z The z index to use
-     */
-    __device__ __host__ inline void save(const T &a, int x = -1, int y = -1, int z = -1) const
-    {
-      auto tid = target::thread_idx();
-      x = (x == -1) ? tid.x : x;
-      y = (y == -1) ? tid.y : y;
-      z = (z == -1) ? tid.z : z;
-      save_detail(a, x, y, z);
-    }
-
-    /**
-       @brief Save the value into the 3-d shared memory cache.
-       @param[in] a The value to store in the shared memory cache
-       @param[in] x The x index to use
-     */
-    __device__ __host__ inline void save_x(const T &a, int x = -1) const
-    {
-      auto tid = target::thread_idx();
-      x = (x == -1) ? tid.x : x;
-      save_detail(a, x, tid.y, tid.z);
-    }
-
-    /**
-       @brief Save the value into the 3-d shared memory cache.
-       @param[in] a The value to store in the shared memory cache
-       @param[in] y The y index to use
-     */
-    __device__ __host__ inline void save_y(const T &a, int y = -1) const
-    {
-      auto tid = target::thread_idx();
-      y = (y == -1) ? tid.y : y;
-      save_detail(a, tid.x, y, tid.z);
-    }
-
-    /**
-       @brief Save the value into the 3-d shared memory cache.
-       @param[in] a The value to store in the shared memory cache
-       @param[in] z The z index to use
-     */
-    __device__ __host__ inline void save_z(const T &a, int z = -1) const
-    {
-      auto tid = target::thread_idx();
-      z = (z == -1) ? tid.z : z;
-      save_detail(a, tid.x, tid.y, z);
-    }
-
-    /**
-       @brief Load a value from the shared memory cache
-       @param[in] x The x index to use
-       @param[in] y The y index to use
-       @param[in] z The z index to use
-       @return The value at coordinates (x,y,z)
-     */
-    __device__ __host__ inline T load(int x = -1, int y = -1, int z = -1) const
-    {
-      auto tid = target::thread_idx();
-      x = (x == -1) ? tid.x : x;
-      y = (y == -1) ? tid.y : y;
-      z = (z == -1) ? tid.z : z;
-      return load_detail(x, y, z);
-    }
-
-    /**
-       @brief Load a vector from the shared memory cache
-       @param[in] x The x index to use
-       @return The value at coordinates (x,y,z)
-    */
-    __device__ __host__ inline T load_x(int x = -1) const
-    {
-      auto tid = target::thread_idx();
-      x = (x == -1) ? tid.x : x;
-      return load_detail(x, tid.y, tid.z);
-    }
-
-    /**
-       @brief Load a vector from the shared memory cache
-       @param[in] y The y index to use
-       @return The value at coordinates (x,y,z)
-    */
-    __device__ __host__ inline T load_y(int y = -1) const
-    {
-      auto tid = target::thread_idx();
-      y = (y == -1) ? tid.y : y;
-      return load_detail(tid.x, y, tid.z);
-    }
-
-    /**
-       @brief Load a vector from the shared memory cache
-       @param[in] z The z index to use
-       @return The value at coordinates (x,y,z)
-    */
-    __device__ __host__ inline T load_z(int z = -1) const
-    {
-      auto tid = target::thread_idx();
-      z = (z == -1) ? tid.z : z;
-      return load_detail(tid.x, tid.y, z);
-    }
-
-    /**
-       @brief Synchronize the cache
-    */
-    __device__ __host__ void sync() const { target::dispatch<sync_impl>(); }
-
-    /**
-       @brief Cast operator to allow cache objects to be used where T
-       is expected
-     */
-    __device__ __host__ operator T() const { return load(); }
-
-    /**
-       @brief Assignment operator to allow cache objects to be used on
-       the lhs where T is otherwise expected.
-     */
-    __device__ __host__ void operator=(const T &src) const { save(src); }
-  };
-
-} // namespace quda
-
-// include overloads
 #include "../generic/shared_memory_cache_helper.h"
diff --git a/include/targets/omptarget/shared_memory_helper.h b/include/targets/omptarget/shared_memory_helper.h
new file mode 100644
index 0000000000..56ec4e4162
--- /dev/null
+++ b/include/targets/omptarget/shared_memory_helper.h
@@ -0,0 +1,88 @@
+#pragma once
+
+#include <target_device.h>
+
+/**
+   @file shared_memory_helper.h
+
+   Target specific helper for allocating and accessing shared memory.
+ */
+
+namespace quda
+{
+
+  /**
+     @brief Class which is used to allocate and access shared memory.
+     The shared memory is treated as an array of type T, with the
+     number of elements given by a call to the static member
+     S::size(target::block_dim()).  The byte offset from the beginning
+     of the total shared memory block is given by the static member
+     O::shared_mem_size(target::block_dim()), or 0 if O is void.
+   */
+  template <typename T, typename S, typename O = void> class SharedMemory
+  {
+  public:
+    using value_type = T;
+
+  private:
+    T *data;
+
+    /**
+       @brief This is a dummy instantiation for the host compiler
+    */
+    template <bool, typename dummy = void> struct cache_dynamic {
+      T *operator()(unsigned int)
+      {
+        static T *cache_;
+        return cache_;
+      }
+    };
+
+    /**
+       @brief This is the handle to the dynamic shared memory
+       @return Shared memory pointer
+     */
+    template <typename dummy> struct cache_dynamic<true, dummy> {
+      __device__ inline T *operator()(unsigned int offset)
+      {
+        return reinterpret_cast<T *>(device::get_shared_cache() + offset);
+      }
+    };
+
+    __device__ __host__ inline T *cache(unsigned int offset) const { return target::dispatch<cache_dynamic>(offset); }
+
+  public:
+    /**
+       @brief Byte offset for this shared memory object.
+    */
+    static constexpr unsigned int get_offset(dim3 block)
+    {
+      unsigned int o = 0;
+      if constexpr (!std::is_same_v<O, void>) { o = O::shared_mem_size(block); }
+      return o;
+    }
+
+    /**
+       @brief Shared memory size in bytes.
+    */
+    static constexpr unsigned int shared_mem_size(dim3 block) { return get_offset(block) + S::size(block) * sizeof(T); }
+
+    /**
+       @brief Constructor for SharedMemory object.
+    */
+    constexpr SharedMemory() : data(cache(get_offset(target::block_dim()))) { }
+
+    /**
+       @brief Return this SharedMemory object.
+    */
+    constexpr auto sharedMem() const { return *this; }
+
+    /**
+       @brief Subscripting operator returning a reference to element.
+       @param[in] i The index to use.
+       @return Reference to value stored at that index.
+     */
+    __device__ __host__ T &operator[](const int i) const { return data[i]; }
+  };
+
+} // namespace quda
diff --git a/include/targets/omptarget/thread_array.h b/include/targets/omptarget/thread_array.h
index d7163af944..fb550cc782 100644
--- a/include/targets/omptarget/thread_array.h
+++ b/include/targets/omptarget/thread_array.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#ifdef QUDA_OMPTARGET_THREAD_ARRAY_SIMPLE
+
 namespace quda
 {
   template <typename T, int n> struct thread_array {
@@ -11,3 +13,9 @@ namespace quda
     constexpr inline const T &operator[](int i) const { return data[i]; }
   };
 } // namespace quda
+
+#else
+
+#include "../generic/thread_array.h"
+
+#endif
diff --git a/include/targets/omptarget/thread_local_cache.h b/include/targets/omptarget/thread_local_cache.h
new file mode 100644
index 0000000000..dd4cd863fc
--- /dev/null
+++ b/include/targets/omptarget/thread_local_cache.h
@@ -0,0 +1 @@
+#include "../generic/thread_local_cache.h"
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index c17d092fd2..ef4cff85f3 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -85,6 +85,7 @@ namespace quda {
     template <template <typename> class Functor, bool grid_stride, typename Arg>
     qudaError_t launch_device(const kernel_t &kernel, const TuneParam &tp, const qudaStream_t &stream, const Arg &arg)
     {
+      checkSharedBytes(tp);
       launch_error = QUDA_SUCCESS;
       if (acceptThreads(tp, arg) && 0==target::omptarget::qudaSetupLaunchParameter(tp)) {
         if constexpr (device::use_kernel_arg<Arg>()) {

From 629d0b494b77a8df5366c00ba37bf037a101cccc Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 29 Feb 2024 22:13:51 +0000
Subject: [PATCH 189/198] omp: provide QUDA_OMPTARGET_THREAD_ARRAY_SLM to move
 thread_array location

---
 include/kernels/gauge_stout.cuh              | 5 +++++
 include/kernels/gauge_wilson_flow.cuh        | 5 +++++
 lib/targets/omptarget/target_omptarget.cmake | 7 +++++++
 3 files changed, 17 insertions(+)

diff --git a/include/kernels/gauge_stout.cuh b/include/kernels/gauge_stout.cuh
index f9072a4ee8..5617c61950 100644
--- a/include/kernels/gauge_stout.cuh
+++ b/include/kernels/gauge_stout.cuh
@@ -139,8 +139,13 @@ namespace quda
       dir = dir + (dir >= arg.dir_ignore);
 
       Link U, Q;
+#ifdef QUDA_OMPTARGET_THREAD_ARRAY_SIMPLE
+      Link Stap;
+      Link Rect;
+#else
       ThreadLocalCache<Link, 0, computeStapleRectangleOps> Stap;
       ThreadLocalCache<Link, 0, decltype(Stap)> Rect; // offset by Stap type to ensure non-overlapping allocations
+#endif
 
       // This function gets stap = S_{mu,nu} i.e., the staple of length 3,
       // and the 1x2 and 2x1 rectangles of length 5. From the following paper:
diff --git a/include/kernels/gauge_wilson_flow.cuh b/include/kernels/gauge_wilson_flow.cuh
index 457d93beab..ed3afba30c 100644
--- a/include/kernels/gauge_wilson_flow.cuh
+++ b/include/kernels/gauge_wilson_flow.cuh
@@ -72,8 +72,13 @@ namespace quda
       // This function gets stap = S_{mu,nu} i.e., the staple of length 3,
       // and the 1x2 and 2x1 rectangles of length 5. From the following paper:
       // https://arxiv.org/abs/0801.1165
+#ifdef QUDA_OMPTARGET_THREAD_ARRAY_SIMPLE
+      Link Stap;
+      Link Rect;
+#else
       ThreadLocalCache<Link, 0, computeStapleRectangleOps> Stap;
       ThreadLocalCache<Link, 0, decltype(Stap)> Rect; // offset by Stap type to ensure non-overlapping allocations
+#endif
       computeStapleRectangle(arg, x, arg.E, parity, dir, Stap, Rect, Arg::wflow_dim);
       Z = arg.coeff1x1 * static_cast<const Link &>(Stap) + arg.coeff2x1 * static_cast<const Link &>(Rect);
       break;
diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index d7a78c677a..176c8d0440 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -30,6 +30,9 @@ set_property(CACHE QUDA_MAX_SHARED_MEMORY_SIZE PROPERTY STRINGS 32768 49152 6553
 target_compile_definitions(quda PUBLIC QUDA_MAX_SHARED_MEMORY_SIZE=${QUDA_MAX_SHARED_MEMORY_SIZE})
 message(STATUS "Using maximum shared memory size: ${QUDA_MAX_SHARED_MEMORY_SIZE}")
 
+option(QUDA_OMPTARGET_THREAD_ARRAY_SLM "Build OpenMP target backend with thread_array on SLM" ON)
+mark_as_advanced(QUDA_OMPTARGET_THREAD_ARRAY_SLM)
+
 option(QUDA_OMPTARGET_JIT "Build OpenMP target backend with JIT" OFF)
 mark_as_advanced(QUDA_OMPTARGET_JIT)
 
@@ -72,6 +75,10 @@ endif()
 # ######################################################################################################################
 # omptarget specific compile options
 
+if(NOT QUDA_OMPTARGET_THREAD_ARRAY_SLM)
+    target_compile_definitions(quda PUBLIC QUDA_OMPTARGET_THREAD_ARRAY_SIMPLE)
+endif()
+
 target_include_directories(quda PRIVATE
   ${CMAKE_SOURCE_DIR}/include/targets/omptarget)
 

From 156d81af3c21e872b65cbb17efe1300033ac3434 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Wed, 17 Apr 2024 19:37:59 +0000
Subject: [PATCH 190/198] omp: include <utility>

---
 include/quda_ptr.h         | 1 +
 lib/clover_field.cpp       | 1 +
 lib/color_spinor_field.cpp | 1 +
 lib/gauge_field.cpp        | 1 +
 lib/interface_quda.cpp     | 1 +
 lib/lattice_field.cpp      | 1 +
 6 files changed, 6 insertions(+)

diff --git a/include/quda_ptr.h b/include/quda_ptr.h
index aab76f6b89..7ee91bf717 100644
--- a/include/quda_ptr.h
+++ b/include/quda_ptr.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <ostream>
+#include <utility>
 #include "malloc_quda.h"
 
 namespace quda
diff --git a/lib/clover_field.cpp b/lib/clover_field.cpp
index ceaa2a1ab7..f23e7a0b11 100644
--- a/lib/clover_field.cpp
+++ b/lib/clover_field.cpp
@@ -3,6 +3,7 @@
 #include <string.h>
 #include <math.h>
 #include <typeinfo>
+#include <utility>
 
 #include <quda_internal.h>
 #include <clover_field.h>
diff --git a/lib/color_spinor_field.cpp b/lib/color_spinor_field.cpp
index fba73f7b62..c6d5f46fef 100644
--- a/lib/color_spinor_field.cpp
+++ b/lib/color_spinor_field.cpp
@@ -1,6 +1,7 @@
 #include <string.h>
 #include <iostream>
 #include <typeinfo>
+#include <utility>
 
 #include <color_spinor_field.h>
 #include <dslash_quda.h>
diff --git a/lib/gauge_field.cpp b/lib/gauge_field.cpp
index 069bdd7ca5..d9b34c6317 100644
--- a/lib/gauge_field.cpp
+++ b/lib/gauge_field.cpp
@@ -1,4 +1,5 @@
 #include <typeinfo>
+#include <utility>
 #include <gauge_field.h>
 #include <blas_quda.h>
 #include <timer.h>
diff --git a/lib/interface_quda.cpp b/lib/interface_quda.cpp
index 31abe62dce..5b2aecc133 100644
--- a/lib/interface_quda.cpp
+++ b/lib/interface_quda.cpp
@@ -3,6 +3,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
+#include <utility>
 #include <sys/time.h>
 #include <complex.h>
 
diff --git a/lib/lattice_field.cpp b/lib/lattice_field.cpp
index 6b4c5e669e..fc4693abc2 100644
--- a/lib/lattice_field.cpp
+++ b/lib/lattice_field.cpp
@@ -1,4 +1,5 @@
 #include <typeinfo>
+#include <utility>
 #include <quda_internal.h>
 #include <lattice_field.h>
 #include <color_spinor_field.h>

From 872c8bf04847f5f4d590f7d5e899f0adab035f10 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 26 Apr 2024 20:36:04 +0000
Subject: [PATCH 191/198] omp: QUDA_OMPTARGET_DEBUG only CPU side for now

---
 lib/targets/omptarget/target_omptarget.cmake | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/targets/omptarget/target_omptarget.cmake b/lib/targets/omptarget/target_omptarget.cmake
index 176c8d0440..e869699dea 100644
--- a/lib/targets/omptarget/target_omptarget.cmake
+++ b/lib/targets/omptarget/target_omptarget.cmake
@@ -49,11 +49,12 @@ if(QUDA_OMPTARGET_JIT)
 else()
     #set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=60 "SHELL:-mllvm -pragma-unroll-threshold=16" "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
     #set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-device-code-split=per_kernel -fopenmp-max-parallel-link-jobs=64 -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
-    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-targets=spir64_gen -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
-endif()
-
-if(QUDA_OMPTARGET_DEBUG)
-    set(QUDA_OMPTARGET_FLAGS ${QUDA_OMPTARGET_FLAGS} -gline-tables-only)
+    set(QUDA_OMPTARGET_FLAGS -fiopenmp -fopenmp-version=60 "SHELL:-mllvm -vpo-paropt-simulate-get-num-threads-in-target=false")
+    if(QUDA_OMPTARGET_DEBUG)
+        set(QUDA_OMPTARGET_FLAGS ${QUDA_OMPTARGET_FLAGS} -g -fopenmp-targets=spir64_gen=-g0)
+    else()
+        set(QUDA_OMPTARGET_FLAGS ${QUDA_OMPTARGET_FLAGS} -fopenmp-targets=spir64_gen)
+    endif()
 endif()
 
 if(QUDA_OMPTARGET_SIMD16)

From 98addc4ecb4cdc731b185231bc25054c43919542 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 23 May 2024 17:59:20 +0000
Subject: [PATCH 192/198] omp: fix omptarget after merge

---
 include/targets/omptarget/shared_memory_helper.h | 8 +++++++-
 include/targets/omptarget/tunable_kernel.h       | 1 +
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/include/targets/omptarget/shared_memory_helper.h b/include/targets/omptarget/shared_memory_helper.h
index 56ec4e4162..05205c6c84 100644
--- a/include/targets/omptarget/shared_memory_helper.h
+++ b/include/targets/omptarget/shared_memory_helper.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <target_device.h>
+#include <kernel_ops.h>
 
 /**
    @file shared_memory_helper.h
@@ -72,6 +73,11 @@ namespace quda
     */
     constexpr SharedMemory() : data(cache(get_offset(target::block_dim()))) { }
 
+    template <typename... U>
+    constexpr SharedMemory(const KernelOps<U...> &) : data(cache(get_offset(target::block_dim())))
+    {
+    }
+
     /**
        @brief Return this SharedMemory object.
     */
@@ -82,7 +88,7 @@ namespace quda
        @param[in] i The index to use.
        @return Reference to value stored at that index.
      */
-    __device__ __host__ T &operator[](const int i) const { return data[i]; }
+    __device__ __host__ T &operator[](int i) const { return data[i]; }
   };
 
 } // namespace quda
diff --git a/include/targets/omptarget/tunable_kernel.h b/include/targets/omptarget/tunable_kernel.h
index ef4cff85f3..7757954ff8 100644
--- a/include/targets/omptarget/tunable_kernel.h
+++ b/include/targets/omptarget/tunable_kernel.h
@@ -6,6 +6,7 @@
 #include <lattice_field.h>
 #include <kernel_helper.h>
 #include <kernel.h>
+#include <kernel_ops_target.h>
 #include <quda_omptarget_api.h>
 
 namespace quda {

From 16b7eab3a92b892fdafb9c3d8ee8e1c410456be4 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Thu, 13 Jun 2024 19:38:24 +0000
Subject: [PATCH 193/198] omp: work around a compiler bug

---
 lib/coarse_op.in.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/coarse_op.in.cu b/lib/coarse_op.in.cu
index c586f8f1dd..b156e25e1e 100644
--- a/lib/coarse_op.in.cu
+++ b/lib/coarse_op.in.cu
@@ -156,7 +156,7 @@ namespace quda {
     QudaFieldLocation location = checkLocation(Y, X);
 
     GaugeField *U = location == QUDA_CUDA_FIELD_LOCATION ? const_cast<GaugeField*>(&gauge) : nullptr;
-    CloverField *C = location == QUDA_CUDA_FIELD_LOCATION ? const_cast<CloverField*>(clover) : nullptr;
+    CloverField *C = location == QUDA_CUDA_FIELD_LOCATION && clover ? const_cast<CloverField*>(clover) : nullptr;
 
     if (location == QUDA_CPU_FIELD_LOCATION) {
       //First make a cpu gauge field from the cuda gauge field

From 9a5547a4e8119874b90e693121f4b530c97934a4 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Mon, 5 Aug 2024 22:16:42 +0000
Subject: [PATCH 194/198] omp: add empty device::get_state

---
 lib/targets/omptarget/device.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/targets/omptarget/device.cpp b/lib/targets/omptarget/device.cpp
index 2125d587ae..a4ee629e5b 100644
--- a/lib/targets/omptarget/device.cpp
+++ b/lib/targets/omptarget/device.cpp
@@ -105,6 +105,8 @@ namespace quda
       omp_set_default_device(device_id);
     }
 
+    state_t get_state() { return {}; }
+
     int get_device_count()
     {
       static int device_count = -1;

From f6fe04ed495468653658ba08a7625afa5bc1365c Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 9 Aug 2024 19:45:31 +0000
Subject: [PATCH 195/198] BlockKernel2D_host: create block inside omp parallel

---
 include/targets/generic/block_reduction_kernel_host.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/include/targets/generic/block_reduction_kernel_host.h b/include/targets/generic/block_reduction_kernel_host.h
index f0997a7d98..330d6d67a5 100644
--- a/include/targets/generic/block_reduction_kernel_host.h
+++ b/include/targets/generic/block_reduction_kernel_host.h
@@ -4,10 +4,11 @@ namespace quda
   template <template <typename> class Functor, typename Arg> void BlockKernel2D_host(const Arg &arg)
   {
     Functor<Arg> t(arg);
-    dim3 block(0, 0, 0);
 #pragma omp parallel for
-    for (block.y = 0; block.y < arg.grid_dim.y; block.y++) {
-      for (block.x = 0; block.x < arg.grid_dim.x; block.x++) { t(block, dim3(0, 0, 0)); }
+    for (unsigned int y = 0; y < arg.grid_dim.y; y++) {
+      for (unsigned int x = 0; x < arg.grid_dim.x; x++) {
+        t(dim3(x, y, 0), dim3(0, 0, 0));
+      }
     }
   }
 

From 71231b43513d8d7e49ccc84ab8745e17e42e0402 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 30 Aug 2024 22:02:55 +0000
Subject: [PATCH 196/198] omp: use memcpy for vector_load/store

---
 include/targets/omptarget/load_store.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/include/targets/omptarget/load_store.h b/include/targets/omptarget/load_store.h
index d1bfe4a955..175b135378 100644
--- a/include/targets/omptarget/load_store.h
+++ b/include/targets/omptarget/load_store.h
@@ -18,7 +18,7 @@ namespace quda
   template <> struct vector_load_impl<true> {
     template <typename T> __device__ inline void operator()(T &value, const void *ptr, int idx)
     {
-      value = reinterpret_cast<const T *>(ptr)[idx];
+      memcpy(&value, reinterpret_cast<const T *>(ptr) + idx, sizeof(T));
     }
 
     __device__ inline void operator()(short8 &value, const void *ptr, int idx)
@@ -43,22 +43,22 @@ namespace quda
   template <> struct vector_store_impl<true> {
     template <typename T> __device__ inline void operator()(void *ptr, int idx, const T &value)
     {
-      reinterpret_cast<T *>(ptr)[idx] = value;
+      memcpy(reinterpret_cast<T *>(ptr) + idx, &value, sizeof(T));
     }
 
     __device__ inline void operator()(void *ptr, int idx, const short8 &value)
     {
-      this->operator()(ptr, idx, *reinterpret_cast<const float4 *>(&value));
+      memcpy(reinterpret_cast<float4 *>(ptr) + idx, &value, sizeof(float4));
     }
 
     __device__ inline void operator()(void *ptr, int idx, const char8 &value)
     {
-      this->operator()(ptr, idx, *reinterpret_cast<const float2 *>(&value));
+      memcpy(reinterpret_cast<float2 *>(ptr) + idx, &value, sizeof(float2));
     }
 
     __device__ inline void operator()(void *ptr, int idx, const char4 &value)
     {
-      this->operator()(ptr, idx, *reinterpret_cast<const short2 *>(&value)); // A char4 is the same as a short2
+      memcpy(reinterpret_cast<short2 *>(ptr) + idx, &value, sizeof(short2));
     }
   };
 

From af51dd024344bf3a3780a54c185ab84a1c48b452 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Fri, 1 Nov 2024 20:48:50 +0000
Subject: [PATCH 197/198] omptarget/reduce_helper.h: atomic read the partial
 results

omp flush doesn't seem to do what sycl::atomic_fence does.
the atomic read seems to enforce the sequential consistency for the partial array
we will revisit this once we have got better omp support from vendors
---
 include/targets/omptarget/atomic_helper.h | 24 +++++++++++++++++++++++
 include/targets/omptarget/reduce_helper.h | 12 +++++++++++-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/include/targets/omptarget/atomic_helper.h b/include/targets/omptarget/atomic_helper.h
index 7da42a09a8..d7aa315b68 100644
--- a/include/targets/omptarget/atomic_helper.h
+++ b/include/targets/omptarget/atomic_helper.h
@@ -52,4 +52,28 @@ namespace quda
     if(*addr<val){*addr=val;}
   }
 
+  template <typename T>
+  inline T atomic_read(T &x)
+  {
+    T v;
+    #pragma omp atomic read
+    v = x;
+    return v;
+  }
+  template <typename T, int N>
+  inline array<T,N> atomic_read(array<T,N> &x)
+  {
+    array<T,N> v;
+    for (int i = 0; i < N; ++i)
+      v[i] = atomic_read(x[i]);
+    return v;
+  }
+  template <typename T>
+  inline deviation_t<T> atomic_read(deviation_t<T> &x)
+  {
+    deviation_t<T> v;
+    v.diff = atomic_read(x.diff);
+    v.ref = atomic_read(x.ref);
+    return v;
+  }
 } // namespace quda
diff --git a/include/targets/omptarget/reduce_helper.h b/include/targets/omptarget/reduce_helper.h
index a0615bf825..22f7f72765 100644
--- a/include/targets/omptarget/reduce_helper.h
+++ b/include/targets/omptarget/reduce_helper.h
@@ -5,6 +5,12 @@
 #include <block_reduce_helper.h>
 #include <kernel_helper.h>
 
+#define QUDA_OPENMP_ATOMIC_READ_PARTIAL_REDUCTION
+
+#ifdef QUDA_OPENMP_ATOMIC_READ_PARTIAL_REDUCTION
+#include <atomic_helper.h>
+#endif
+
 using count_t = unsigned int;
 
 namespace quda
@@ -164,7 +170,11 @@ namespace quda
       if (thisSubBlock) {
         auto i = target::thread_idx_linear<2>();
         while (i < target::grid_dim().x) {
-          sum = r(sum, const_cast<T &>(static_cast<volatile T *>(arg.partial)[idx * target::grid_dim().x + i]));
+#ifdef QUDA_OPENMP_ATOMIC_READ_PARTIAL_REDUCTION
+          sum = r(sum, atomic_read(arg.partial[idx * target::grid_dim().x + i]));
+#else
+          sum = r(sum, arg.partial[idx * target::grid_dim().x + i]);
+#endif
           // printf("team %d thread %d  sum %g\n", omp_get_team_num(), omp_get_thread_num(), *(double*)(&sum));
           i += target::block_size<2>();
         }

From d351388dc7b4666a28b53f1c3c9fb12c507caf66 Mon Sep 17 00:00:00 2001
From: Xiao-Yong Jin <xjin@anl.gov>
Date: Tue, 5 Nov 2024 17:31:35 +0000
Subject: [PATCH 198/198] omp: omptarget/atomic_helper: add atomic_read for
 complex<T>

---
 include/targets/omptarget/atomic_helper.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/include/targets/omptarget/atomic_helper.h b/include/targets/omptarget/atomic_helper.h
index d7aa315b68..94ee26ba77 100644
--- a/include/targets/omptarget/atomic_helper.h
+++ b/include/targets/omptarget/atomic_helper.h
@@ -69,6 +69,12 @@ namespace quda
     return v;
   }
   template <typename T>
+  inline complex<T> atomic_read(complex<T> &x)
+  {
+    complex<T> v (atomic_read(x.x), atomic_read(x.y));
+    return v;
+  }
+  template <typename T>
   inline deviation_t<T> atomic_read(deviation_t<T> &x)
   {
     deviation_t<T> v;