threadManager_OpenMP.cpp

Go to the documentation of this file.

#include "threadManager_OpenMP.h"

#if defined USE_OPENMP
#include <omp.h>
#include "bgqthread.h"
#endif

#include "communicator.h"
#include "bridgeIO.h"
using Bridge::vout;

//====================================================================
// initialization of static member variables.

int ThreadManager_OpenMP::m_Nthread             = 0;
Bridge::VerboseLevel ThreadManager_OpenMP::m_vl = Bridge::CRUCIAL;
valarray<double>     ThreadManager_OpenMP::m_darray_reduction(0);

//====================================================================
void ThreadManager_OpenMP::init(int nthread)
{
  m_vl = CommonParameters::Vlevel();

  vout.general(m_vl, "ThreadManager_OpenMP being setup.\n");

#if defined USE_OPENMP
  int nthread_env = 0;
#pragma omp parallel
  {
    if (omp_get_thread_num() == 0) {
      nthread_env = omp_get_num_threads();
    }
  }

  if ((nthread == nthread_env) || (nthread == 0)) {
    m_Nthread = nthread_env;
  } else {
    vout.general(m_vl, "  Number of threads(env)   = %d\n", nthread_env);
    vout.general(m_vl, "  Number of threads(input) = %d\n", nthread);
    vout.general(m_vl, "  resetting Number of threads.\n");
    omp_set_num_threads(nthread);
    m_Nthread = nthread;
  }

  vout.crucial(m_vl, "  Number of thread = %d\n", m_Nthread);

  m_darray_reduction.resize(m_Nthread);

  // setup of BG/Q thread library
  BGQThread_Init();

  /*
  int Nthread_BGQTh = BGQThread_GetNumBarrier();
  vout.general(m_vl,
      "  BGQ thread lib initialized: number of barrier objects = %d.\n",
      Nthread_BGQTh);

  if(m_Nthread > Nthread_BGQTh){
    vout.crucial(m_vl,"  Number of thread is too large.\n");
    abort();
  }
  */

#else
  // OpenMP not used

  m_Nthread = 1;
  vout.crucial(m_vl, "  Number of thread = %d\n", m_Nthread);
  vout.crucial(m_vl, "  OpenMP not used.\n");
#endif
}


//====================================================================
void ThreadManager_OpenMP::finalize()
{
  vout.general(m_vl, "Thread manager says good-bye.\n");
}


//====================================================================
int ThreadManager_OpenMP::get_num_threads()
{
#if defined USE_OPENMP
  return omp_get_num_threads();

#else
  return 1;
#endif
}


//====================================================================
int ThreadManager_OpenMP::get_thread_id()
{
#if defined USE_OPENMP
  return omp_get_thread_num();

#else
  return 0;
#endif
}


//====================================================================
void ThreadManager_OpenMP::wait()
{
#if defined USE_OPENMP
  int nth = get_num_threads();
  barrier(nth);
#endif
}


//====================================================================
void ThreadManager_OpenMP::barrier(int nth)
{
#if defined USE_OPENMP
  //#pragma omp barrier

  // using BGQ thread
  BGQThread_Barrier(0, nth);
#endif
}


//====================================================================
void ThreadManager_OpenMP::sync_barrier_all()
{
#pragma omp barrier

#pragma omp master
  {
    Communicator::sync();
  }

#pragma omp barrier
}


//====================================================================
void ThreadManager_OpenMP::reduce_sum_global(double& a,
                                             const int ith, const int nth)
{
#if defined USE_OPENMP
  // using BGQ thread

  /*
  double b = BGQThread_GatherDouble(a, 0, ith, nth);

  if(ith == 0) b = Communicator::reduce_sum(b);

  barrier(nth);

  a = BGQThread_ScatterDouble(b, 0, ith, nth);
  */

  /*
  // OpenMP native reduction
  m_darray_reduction[ith] = a;
  //  barrier(nth);
  #pragma omp barrier

  #pragma omp flush (m_darray_reduction)
  #pragma omp master
  {
    #pragma omp flush (m_darray_reduction)
    double b = 0.0;
    for(int i = 0; i < nth; ++i){
      b += m_darray_reduction[i];
    }
    //    m_darray_reduction[0] = Communicator::reduce_sum(b);
    b = Communicator::reduce_sum(b);
    m_darray_reduction[0] = b;
    #pragma omp flush (m_darray_reduction)
  }

  //  barrier(nth);
  #pragma omp barrier

  #pragma omp flush (m_darray_reduction)
  //#pragma omp flush (m_darray_reduction)
  a = m_darray_reduction[0];
  */

  // OpenMP native reduction
  m_darray_reduction[ith] = a;
  barrier(nth);
  //  #pragma omp barrier
#pragma omp flush (m_darray_reduction)

#pragma omp master
  {
#pragma omp flush (m_darray_reduction)
    double b = 0.0;
    for (int i = 0; i < nth; ++i) {
      b += m_darray_reduction[i];
    }
    //    m_darray_reduction[0] = Communicator::reduce_sum(b);
    b = Communicator::reduce_sum(b);
    m_darray_reduction[0] = b;
#pragma omp flush (m_darray_reduction)
  }

  barrier(nth);
  //  #pragma omp barrier

#pragma omp flush (m_darray_reduction)
  a = m_darray_reduction[0];

#else
  a = Communicator::reduce_sum(a);
#endif
}


//====================================================================
//============================================================END=====