/*!
      @file    mult_Staggered_openacc-inc.h
      @brief
      @author  Hideo Matsufuru (matufuru)
      @date    $LastChangedDate: 2013-01-22 13:51:53 #$
      @version $LastChangedRevision: 2653 $
*/

// This code explicitly assumes SU(3) gauge group.

#define MULT_UV_R(u0,u1,u2,u3,u4,u5,v0,v1,v2,v3,v4,v5)   (u0*v0-u1*v1 + u2*v2-u3*v3 + u4*v4-u5*v5)
#define MULT_UV_I(u0,u1,u2,u3,u4,u5,v0,v1,v2,v3,v4,v5)   (u0*v1+u1*v0 + u2*v3+u3*v2 + u4*v5+u5*v4)

//====================================================================
void mult_staggered_D(real_t *RESTRICT v2, real_t *RESTRICT u_up,
                      real_t *RESTRICT u_dn, real_t *RESTRICT v1,
                      real_t mq, int *bc, int *Nsize, int jdag)
{
  int Nx  = Nsize[0];
  int Ny  = Nsize[1];
  int Nz  = Nsize[2];
  int Nt  = Nsize[3];
  int Nst = Nx * Ny * Nz * Nt;
  int Nst_pad = CEIL_NWP(Nst);

  int size   = NVC * Nst_pad;
  int size_u = NDF * Nst_pad * 4;

#pragma acc data present(v2[0:size], v1[0:size], \
                         u_up[0:size_u], u_dn[0:size_u]), \
                 copyin(bc[0:4], Nx, Ny, Nz, Nt, Nst, Nst_pad, mq, jdag)
#pragma acc parallel num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
 {
#pragma acc loop gang worker vector
  for(int ist = 0; ist < Nst; ++ist){
    int ix = ist % Nx;
    int iy = (ist/Nx) % Ny;
    int iz = (ist/(Nx * Ny)) % Nz;
    int it = ist/(Nx * Ny * Nz);

    int idir, nei, istg, neig;
    real_t bc2;
    real_t vt0, vt1, vt2, vt3, vt4, vt5;
    real_t ut0, ut1, ut2, ut3, ut4, ut5;
    real_t xt0, xt1, xt2, xt3, xt4, xt5;
    real_t wtr, wti;

    xt0 = 0.0;
    xt1 = 0.0;
    xt2 = 0.0;
    xt3 = 0.0;
    xt4 = 0.0;
    xt5 = 0.0;


    idir = 0;

    int iyzt = ist/Nx;
    nei  = ((ix+1) % Nx) + Nx * iyzt;
    istg = ist + Nst_pad * idir;

    bc2 = 1.0;
    if(ix == Nx-1) bc2 = bc[0];

#include "inc/mult_Staggered_uvup_openacc-inc.h"

    nei  = ix-1 + Nx * iyzt;
    if(ix == 0) nei = Nx-1 + Nx * iyzt;
    neig = nei + Nst_pad * idir;
    bc2 = 1.0;
    if(ix == 0) bc2 = bc[0];

#include "inc/mult_Staggered_uvdn_openacc-inc.h"


    idir = 1;
    int izt = ist/(Nx*Ny);
    int iy2 = (iy+1) % Ny;
    nei = ix + Nx * (iy2 + Ny * izt);
    istg = ist + Nst_pad * idir;
    bc2 = 1.0;
    if(iy == Ny-1) bc2 = bc[idir];

#include "inc/mult_Staggered_uvup_openacc-inc.h"

    iy2 = (iy-1+Ny) % Ny;
    nei = ix + Nx * (iy2 + Ny * izt);
    neig = nei + Nst_pad * idir;

    bc2 = 1.0;
    if(iy == 0) bc2 = bc[1];

#include "inc/mult_Staggered_uvdn_openacc-inc.h"


    idir = 2;

    int Nxy = Nx * Ny;
    int ixy = ix + Nx * iy;
    int iz2 = (iz+1) % Nz;
    nei = ixy + Nxy * (iz2 + Nz * it);
    istg = ist + Nst_pad * idir;
    bc2 = 1.0;
    if(iz == Nz-1) bc2 = bc[idir];

#include "inc/mult_Staggered_uvup_openacc-inc.h"

    iz2 = (iz-1+Nz) % Nz;
    nei = ixy + Nxy * (iz2 + Nz * it);
    neig = nei + Nst_pad * idir;
    bc2 = 1.0;
    if(iz == 0) bc2 = bc[idir];

#include "inc/mult_Staggered_uvdn_openacc-inc.h"


    idir = 3;
    int Nxyz = Nx * Ny * Nz;
    int ixyz = ix + Nx * (iy + Ny * iz);
    int it2 = (it+1) % Nt;
    nei = ixyz + Nxyz * it2;
    istg = ist + Nst_pad * idir;
    bc2 = 1.0;
    if(it == Nt-1) bc2 = bc[idir];

#include "inc/mult_Staggered_uvup_openacc-inc.h"

    it2 = (it-1+Nt) % Nt;
    nei = ixyz + Nxyz * it2;
    neig = nei + Nst_pad * idir;
    bc2 = 1.0;
    if(it == 0) bc2 = bc[idir];

#include "inc/mult_Staggered_uvdn_openacc-inc.h"

    real_t fac = real_t(jdag) * 0.5;
    v2[IDX2_1SP_R(0, ist)] = mq * v1[IDX2_1SP_R(0, ist)] + fac * xt0;
    v2[IDX2_1SP_I(0, ist)] = mq * v1[IDX2_1SP_I(0, ist)] + fac * xt1;
    v2[IDX2_1SP_R(1, ist)] = mq * v1[IDX2_1SP_R(1, ist)] + fac * xt2;
    v2[IDX2_1SP_I(1, ist)] = mq * v1[IDX2_1SP_I(1, ist)] + fac * xt3;
    v2[IDX2_1SP_R(2, ist)] = mq * v1[IDX2_1SP_R(2, ist)] + fac * xt4;
    v2[IDX2_1SP_I(2, ist)] = mq * v1[IDX2_1SP_I(2, ist)] + fac * xt5;

  }

 }

}

//====================================================================
void mult_staggered_1(real_t *RESTRICT buf_xp, real_t *RESTRICT buf_xm,
                      real_t *RESTRICT buf_yp, real_t *RESTRICT buf_ym,
                      real_t *RESTRICT buf_zp, real_t *RESTRICT buf_zm,
                      real_t *RESTRICT buf_tp, real_t *RESTRICT buf_tm,
                      real_t *RESTRICT u_dn, real_t *RESTRICT v1,
                      int *bc, int *Nsize, int *do_comm)
{
  int Nx   = Nsize[0];
  int Ny   = Nsize[1];
  int Nz   = Nsize[2];
  int Nt   = Nsize[3];
  int Nst  = Nx * Ny * Nz * Nt;
  int Nst_pad = CEIL_NWP(Nst);

  int size    = NVC * Nst_pad;
  int size_u  = NDF * Nst_pad * NDIM;
  int size_bx = NVC * CEIL_NWP(Ny * Nz * Nt);
  int size_by = NVC * CEIL_NWP(Nx * Nz * Nt);
  int size_bz = NVC * CEIL_NWP(Nx * Ny * Nt);
  int size_bt = NVC * CEIL_NWP(Nx * Ny * Nz);

#pragma acc data present(buf_xp[0:size_bx], buf_xm[0:size_bx], \
                         buf_yp[0:size_by], buf_ym[0:size_by], \
                         buf_zp[0:size_bz], buf_zm[0:size_bz], \
                         buf_tp[0:size_bt], buf_tm[0:size_bt], \
                         u_dn[0:size_u], v1[0:size]) \
              copyin(bc[0:4], do_comm[0:4], Nx, Ny, Nz, Nt, Nst, Nst_pad)
 {

  if(do_comm[0] > 0){

#pragma acc parallel async \
            num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
   {
    int Nyzt = Ny * Nz * Nt;

#pragma acc loop gang worker vector
    for(int iyzt = 0; iyzt < Nyzt; ++iyzt){
      int ix  = 0;
      int ist = ix + Nx * iyzt;
      real_t bc2 = bc[0];

      real_t *wt = buf_xp;

      for(int ic = 0; ic < NC; ++ic){
        wt[IDX2_1SP_R(ic, iyzt)] = bc2 * v1[IDX2_1SP_R(ic, ist)];
        wt[IDX2_1SP_I(ic, iyzt)] = bc2 * v1[IDX2_1SP_I(ic, ist)];
      }
    }

   }

#pragma acc parallel async \
  num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
   {
    int Nyzt = Ny * Nz * Nt;

#pragma acc loop gang worker vector
    for(int iyzt = 0; iyzt < Nyzt; ++iyzt){
      int ix  = Nx-1;
      int ist  = ix + Nx * iyzt;
      int istu = ist;
      real_t *wt = buf_xm;
      int ibf = iyzt;

#include "inc/mult_Staggered_uvdn1_openacc-inc.h"

    }
   }

  } // do_comm[0]


  // idir = 1;
  if(do_comm[1] > 0){

#pragma acc parallel async \
  num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
   {
    int Nzt = Nz * Nt;

#pragma acc loop gang worker vector collapse(2)
    for(int izt = 0; izt < Nzt; ++izt){
     for(int ix = 0; ix < Nx; ++ix){
       int iy   = 0;
       int ist  = ix + Nx * (iy + Ny * izt);
       int ixzt = ix + Nx * izt;
       real_t bc2 = bc[1];
       real_t *wt = buf_yp;
       for(int ic = 0; ic < NC; ++ic){
	 wt[IDX2_1SP_R(ic, ixzt)] = bc2 * v1[IDX2_1SP_R(ic, ist)];
	 wt[IDX2_1SP_I(ic, ixzt)] = bc2 * v1[IDX2_1SP_I(ic, ist)];
       }
     }
    }

   }

#pragma acc parallel async \
  num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
   {
    int Nzt = Nz * Nt;

#pragma acc loop gang worker vector collapse(2)
    for(int izt = 0; izt < Nzt; ++izt){
     for(int ix = 0; ix < Nx; ++ix){
       int iy   = Ny-1;
       int ist  = ix + Nx * (iy + Ny*izt);
       int istu = ist + Nst_pad;

       int ibf = ix + Nx * izt;
       real_t *wt = buf_ym;

#include "inc/mult_Staggered_uvdn1_openacc-inc.h"

     }
    }

   }

  } // do_comm[1]

  //  idir = 2;
  if(do_comm[2] > 0){

#pragma acc parallel async \
  num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
   {
    int Nxy = Nx * Ny;

#pragma acc loop gang worker vector collapse(2)
    for(int it = 0; it < Nt; ++it){
     for(int ixy = 0; ixy < Nxy; ++ixy){
       int iz   = 0;
       int ist  = ixy + Nxy * (iz + Nz * it);
       int ixyt = ixy + Nxy * it;

       real_t bc2 = bc[2];

       real_t *wt = buf_zp;

       for(int ic = 0; ic < NC; ++ic){
	 wt[IDX2_1SP_R(ic, ixyt)] = bc2 * v1[IDX2_1SP_R(ic, ist)];
	 wt[IDX2_1SP_I(ic, ixyt)] = bc2 * v1[IDX2_1SP_I(ic, ist)];
       }

     }
    }

   }

#pragma acc parallel async \
  num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
   {
    int Nxy = Nx * Ny;

#pragma acc loop gang worker vector collapse(2)
    for(int it = 0; it < Nt; ++it){
     for(int ixy = 0; ixy < Nxy; ++ixy){
       int iz = Nz-1;
       int ist  = ixy + Nxy * (iz + Nz * it);
       int istu = ist + Nst_pad * 2;

       real_t *wt = buf_zm;
       int ibf = ixy + Nxy * it;

#include "inc/mult_Staggered_uvdn1_openacc-inc.h"

     }
    }

   }

  } // do_comm[2]

   //  idir = 3;
  if(do_comm[3] > 0){

#pragma acc parallel async \
  num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
   {
    int Nxyz = Nx * Ny * Nz;

#pragma acc loop gang worker vector
    for(int ixyz = 0; ixyz < Nxyz; ++ixyz){
      int it = 0;
      int ist  = ixyz + Nxyz * it;
      real_t bc2 = bc[3];

      real_t *wt = buf_tp;
      for(int ic = 0; ic < NC; ++ic){
	wt[IDX2_1SP_R(ic, ixyz)] = bc2 * v1[IDX2_1SP_R(ic, ist)];
	wt[IDX2_1SP_I(ic, ixyz)] = bc2 * v1[IDX2_1SP_I(ic, ist)];
      }

    }

   }

#pragma acc parallel async \
  num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
   {
    int Nxyz = Nx * Ny * Nz;

#pragma acc loop gang worker vector
    for(int ixyz = 0; ixyz < Nxyz; ++ixyz){
      int it   = Nt-1;
      int ist  = ixyz + Nxyz * it;
      int istu = ist + Nst_pad * 3;

      int ibf = ixyz;
      real_t *wt = buf_tm;

#include "inc/mult_Staggered_uvdn1_openacc-inc.h"

    }

   }

  } // do_comm[3]

 #pragma acc wait

 } // acc data

 if(do_comm[0] > 0){
#pragma acc update async host (buf_xp[0:size_bx])
#pragma acc update async host (buf_xm[0:size_bx])
 }
 if(do_comm[1] > 0){
#pragma acc update async host (buf_yp[0:size_by])
#pragma acc update async host (buf_ym[0:size_by])
 }
 if(do_comm[2] > 0){
#pragma acc update async host (buf_zp[0:size_bz])
#pragma acc update async host (buf_zm[0:size_bz])
 }
 if(do_comm[3] > 0){
#pragma acc update async host (buf_tp[0:size_bt])
#pragma acc update async host (buf_tm[0:size_bt])
 }

 #pragma acc wait

}

//====================================================================
void mult_staggered_2(real_t *RESTRICT v2, real_t *RESTRICT u_up,
		      real_t *RESTRICT buf_xp, real_t *RESTRICT buf_xm,
		      real_t *RESTRICT buf_yp, real_t *RESTRICT buf_ym,
		      real_t *RESTRICT buf_zp, real_t *RESTRICT buf_zm,
		      real_t *RESTRICT buf_tp, real_t *RESTRICT buf_tm,
                      real_t mq, int *bc,
                      int *Nsize, int *do_comm, int jdag)
{
  int Nx  = Nsize[0];
  int Ny  = Nsize[1];
  int Nz  = Nsize[2];
  int Nt  = Nsize[3];
  int Nst = Nx * Ny * Nz * Nt;
  int Nst_pad = CEIL_NWP(Nst);

  int size    = NVC * Nst_pad;
  int size_u  = NDF * Nst_pad * NDIM;
  int size_bx = NVC * CEIL_NWP(Ny * Nz * Nt);
  int size_by = NVC * CEIL_NWP(Nx * Nz * Nt);
  int size_bz = NVC * CEIL_NWP(Nx * Ny * Nt);
  int size_bt = NVC * CEIL_NWP(Nx * Ny * Nz);

  if(do_comm[0] > 0){
#pragma acc update async device (buf_xp[0:size_bx])
#pragma acc update async device (buf_xm[0:size_bx])
  }
  if(do_comm[1] > 0){
#pragma acc update async device (buf_yp[0:size_by])
#pragma acc update async device (buf_ym[0:size_by])
  }
  if(do_comm[2] > 0){
#pragma acc update async device (buf_zp[0:size_bz])
#pragma acc update async device (buf_zm[0:size_bz])
  }
  if(do_comm[3] > 0){
#pragma acc update async device (buf_tp[0:size_bt])
#pragma acc update async device (buf_tm[0:size_bt])
  }

 #pragma acc wait


#pragma acc data present(buf_xp[0:size_bx], buf_xm[0:size_bx], \
                         buf_yp[0:size_by], buf_ym[0:size_by], \
                         buf_zp[0:size_bz], buf_zm[0:size_bz], \
                         buf_tp[0:size_bt], buf_tm[0:size_bt], \
                         v2[0:size], u_up[0:size_u]) \
                 copyin(bc[0:4], do_comm[0:4], mq, jdag, \
                        Nx, Ny, Nz, Nt, Nst, Nst_pad)
 {

#pragma acc parallel num_workers(NUM_WORKERS) vector_length(VECTOR_LENGTH)
  {

#pragma acc loop gang worker vector
   for(int ist = 0; ist < Nst; ++ist){
     int ix = ist % Nx;
     int iy = (ist/Nx) % Ny;
     int iz = (ist/(Nx*Ny)) % Nz;
     int it = ist/(Nx*Ny*Nz);

     real_t xt0, xt1, xt2, xt3, xt4, xt5;
     xt0 = 0.0;
     xt1 = 0.0;
     xt2 = 0.0;
     xt3 = 0.0;
     xt4 = 0.0;
     xt5 = 0.0;

     if(do_comm[0] > 0){

       if(ix == Nx-1){
         int iyzt = iy + Ny * (iz + Nz * it);
         int istu = ist + Nst_pad * 0;
	 int ibuf = iyzt;

	 real_t *buf = buf_xp;

#include "inc/mult_Staggered_uvup2_openacc-inc.h"

       }

       if(ix == 0){

	 int iyzt   = iy + Ny * (iz + Nz * it);
	 int ibf = iyzt;
	 real_t bc2 = bc[0];
	 xt0 -= bc2 * buf_xm[IDX2_1SP_R(0, ibf)];
	 xt1 -= bc2 * buf_xm[IDX2_1SP_I(0, ibf)];
	 xt2 -= bc2 * buf_xm[IDX2_1SP_R(1, ibf)];
	 xt3 -= bc2 * buf_xm[IDX2_1SP_I(1, ibf)];
	 xt4 -= bc2 * buf_xm[IDX2_1SP_R(2, ibf)];
	 xt5 -= bc2 * buf_xm[IDX2_1SP_I(2, ibf)];
       }

     }


     if(do_comm[1] > 0){

       if(iy == Ny-1){
         int ixzt = ix + Nx * (iz + Nz * it);
         int istu = ist + Nst_pad * 1;

	 int ibuf = ixzt;
	 real_t *buf = buf_yp;

#include "inc/mult_Staggered_uvup2_openacc-inc.h"
 
       }

       if(iy == 0){
	 int ibf = ix + Nx * (iz + Nz * it);
	 real_t bc2 = bc[1];
	 xt0 -= bc2 * buf_ym[IDX2_1SP_R(0, ibf)];
	 xt1 -= bc2 * buf_ym[IDX2_1SP_I(0, ibf)];
	 xt2 -= bc2 * buf_ym[IDX2_1SP_R(1, ibf)];
	 xt3 -= bc2 * buf_ym[IDX2_1SP_I(1, ibf)];
	 xt4 -= bc2 * buf_ym[IDX2_1SP_R(2, ibf)];
	 xt5 -= bc2 * buf_ym[IDX2_1SP_I(2, ibf)];
       }

     }


     if(do_comm[2] > 0){

       if(iz == Nz-1){
         int ixyt = ix + Nx * (iy + Ny * it);
         int ibuf = ixyt;
         int istu = ist + Nst_pad * 2;
	 real_t *buf = buf_zp;

#include "inc/mult_Staggered_uvup2_openacc-inc.h"

       }

       if(iz == 0){
	 int ibf = ix + Nx * (iy + Ny * it);
	 real_t bc2 = bc[2];
	 xt0 -= bc2 * buf_zm[IDX2_1SP_R(0, ibf)];
	 xt1 -= bc2 * buf_zm[IDX2_1SP_I(0, ibf)];
	 xt2 -= bc2 * buf_zm[IDX2_1SP_R(1, ibf)];
	 xt3 -= bc2 * buf_zm[IDX2_1SP_I(1, ibf)];
	 xt4 -= bc2 * buf_zm[IDX2_1SP_R(2, ibf)];
	 xt5 -= bc2 * buf_zm[IDX2_1SP_I(2, ibf)];
       }

     }

     if(do_comm[3] > 0){

       if(it == Nt-1){
         int ibuf = ix + Nx * (iy + Ny * iz);
         int istu = ist + Nst_pad * 3;

	 real_t *buf = buf_tp;

#include "inc/mult_Staggered_uvup2_openacc-inc.h"

       }

       if(it == 0){
	 int ibf = ix + Nx * (iy + Ny * iz);
	 real_t bc2 = bc[3];
	 xt0 -= bc2 * buf_tm[IDX2_1SP_R(0, ibf)];
	 xt1 -= bc2 * buf_tm[IDX2_1SP_I(0, ibf)];
	 xt2 -= bc2 * buf_tm[IDX2_1SP_R(1, ibf)];
	 xt3 -= bc2 * buf_tm[IDX2_1SP_I(1, ibf)];
	 xt4 -= bc2 * buf_tm[IDX2_1SP_R(2, ibf)];
	 xt5 -= bc2 * buf_tm[IDX2_1SP_I(2, ibf)];
       }

     }

     // real_t fac = real_t(jdag) * 0.5/mq;
     real_t fac = real_t(jdag) * 0.5;
     v2[IDX2_1SP_R(0, ist)] += fac * xt0;
     v2[IDX2_1SP_I(0, ist)] += fac * xt1;
     v2[IDX2_1SP_R(1, ist)] += fac * xt2;
     v2[IDX2_1SP_I(1, ist)] += fac * xt3;
     v2[IDX2_1SP_R(2, ist)] += fac * xt4;
     v2[IDX2_1SP_I(2, ist)] += fac * xt5;

   }

  }
 }

}


//============================================================END=====
