fieldIO_Binary_Parallel.cpp

Go to the documentation of this file.

static const char rcsid[] = "$Id: fieldIO_Binary_Parallel.cpp 929 2013-07-12 05:43:23Z aoym $";

// this code only makes sense in MPI environment.
#ifdef USE_MPI

#include "fieldIO_Binary_Parallel.h"

//====================================================================
void FieldIO_Binary_Parallel::read_file(Field *u, string filename)
{
  static const char _function_name[] = "FieldIO_Binary_Parallel::read_file";

  initialize();

  MPI_File fh;
  int      ret;

  int nin_file = m_format->nin();
  int nex_file = m_format->nex();

//  Field::element_type *buf = new Field::element_type [m_nvol*m_nvector];
  double *buf = new double [nin_file * nex_file * m_nvol];
  if (!buf) {
    vout.general(m_vl, "%s: allocate buffer failed.\n", _function_name);
    return;
  }

  ret = MPI_File_open(Communicator_impl::world(), const_cast<char *>(filename.c_str()), MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
  if (ret) {
    vout.general(m_vl, "%s: MPI_File_open failed.\n", _function_name);
    return;
  }

  ret = MPI_File_set_view(fh, 0, m_type_vector, m_type_tiled, const_cast<char *>("native"), MPI_INFO_NULL);
  if (ret) {
    vout.general(m_vl, "%s: MPI_File_set_view failed.\n", _function_name);
    return;
  }

  ret = MPI_File_read_all(fh, (void *)buf, m_nvol * nex_file, m_type_vector, MPI_STATUS_IGNORE);
  if (ret) {
    vout.general(m_vl, "%s: MPI_File_read_all failed.\n", _function_name);
    return;
  }

  ret = MPI_File_close(&fh);
  if (ret) {
    vout.general(m_vl, "%s: MPI_File_close failed.\n", _function_name);
    return;
  }

  if (!is_bigendian()) {
//    convert_endian(buf, sizeof(Field::element_type), m_nvol*m_nvector);
    convert_endian(buf, sizeof(double), m_nvol * nin_file * nex_file);
  }

  // unpack buffer
  double *p = buf;

  for (int j = 0; j < nex_file; ++j) {
    for (int isite = 0; isite < m_nvol; ++isite) {
      for (int i = 0; i < nin_file; ++i) {
        int s, t;
        m_format->file_to_field(s, t, i, j);

        u->set(s, isite, t, *p++);
      }
    }
  }

  delete buf;
}


//====================================================================
void FieldIO_Binary_Parallel::write_file(Field *u, string filename)
{
  static const char _function_name[] = "FieldIO_Binary_Parallel::write_file";

  initialize();

  int nin_file = m_format->nin();
  int nex_file = m_format->nex();

  //  Field::element_type *buf = new Field::element_type [m_nvol*m_nvector];
  double *buf = new double [nin_file * nex_file * m_nvol];
  if (!buf) {
    vout.general(m_vl, "%s: allocate buffer failed.\n", _function_name);
    return;
  }

  // pack buffer
  double *p = buf;

  for (int j = 0; j < nex_file; ++j) {
    for (int isite = 0; isite < m_nvol; ++isite) {
      for (int i = 0; i < nin_file; ++i) {
        int s, t;
        m_format->file_to_field(s, t, i, j);

        *p++ = u->cmp(s, isite, t);
      }
    }
  }

  if (!is_bigendian()) {
    //    convert_endian(buf, sizeof(Field::element_type), m_nvol*m_nvector);
    convert_endian(buf, sizeof(double), nin_file * nex_file * m_nvol);
  }

  MPI_File fh;
  int      ret;

  ret = MPI_File_open(Communicator_impl::world(), const_cast<char *>(filename.c_str()), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
  if (ret) {
    vout.general(m_vl, "%s: MPI_File_open failed.\n", _function_name);
    return;
  }

  ret = MPI_File_set_view(fh, 0, m_type_vector, m_type_tiled, const_cast<char *>("native"), MPI_INFO_NULL);
  if (ret) {
    vout.general(m_vl, "%s: MPI_File_set_view failed.\n", _function_name);
    return;
  }

  ret = MPI_File_write_all(fh, (void *)buf, m_nvol * nex_file, m_type_vector, MPI_STATUS_IGNORE);
  if (ret) {
    vout.general(m_vl, "%s: MPI_File_write_all failed.\n", _function_name);
    return;
  }

  ret = MPI_File_close(&fh);
  if (ret) {
    vout.general(m_vl, "%s: MPI_File_close failed.\n", _function_name);
    return;
  }

  delete buf;
}


//====================================================================
int FieldIO_Binary_Parallel::initialize()
{
  static const char _function_name[] = "FieldIO_Binary_Parallel::initialize";

  if (m_is_initialized) return EXIT_SUCCESS;

  int nin_file = m_format->nin();
  int nex_file = m_format->nex();

  const int ndim = CommonParameters::Ndim();

  int *global_dims = new int[ndim];
  global_dims[0] = CommonParameters::Lx();
  global_dims[1] = CommonParameters::Ly();
  global_dims[2] = CommonParameters::Lz();
  global_dims[3] = CommonParameters::Lt();

  int *local_dims = new int[ndim];
  local_dims[0] = CommonParameters::Nx();
  local_dims[1] = CommonParameters::Ny();
  local_dims[2] = CommonParameters::Nz();
  local_dims[3] = CommonParameters::Nt();

  m_nvol = 1;
  for (int i = 0; i < ndim; ++i) {
    m_nvol *= local_dims[i];
  }

  int *grid_pos = new int[ndim];
  for (int i = 0; i < ndim; ++i) {
    grid_pos[i] = Communicator::ipe(i);
  }

  int *starts = new int[ndim];
  for (int i = 0; i < ndim; ++i) {
    starts[i] = local_dims[i] * grid_pos[i];
  }

  int ret = 0;

//  MPI_Datatype m_type_vector;
//  ret = MPI_Type_contiguous(sizeof(Field::element_type)*nin_file, MPI_BYTE, &m_type_vector);
  ret = MPI_Type_contiguous(sizeof(double) * nin_file, MPI_BYTE, &m_type_vector);
  if (ret) {
    vout.general(m_vl, "%s: MPI_Type_Contiguous failed.\n", _function_name);
    return EXIT_FAILURE;
  }

  ret = MPI_Type_commit(&m_type_vector);
  if (ret) {
    vout.general(m_vl, "%s: MPI_Type_commit failed.\n", _function_name);
    return EXIT_FAILURE;
  }

//  MPI_Datatype m_type_tiled;
  ret = MPI_Type_create_subarray(ndim, global_dims, local_dims, starts, MPI_ORDER_FORTRAN, m_type_vector, &m_type_tiled);
  if (ret) {
    vout.general(m_vl, "%s: MPI_Type_create_subarray failed.\n", _function_name);
    return EXIT_FAILURE;
  }

  ret = MPI_Type_commit(&m_type_tiled);
  if (ret) {
    vout.general(m_vl, "%s: MPI_Type_commit failed.\n", _function_name);
    return EXIT_FAILURE;
  }

  m_is_initialized = true;

  delete [] starts;
  delete [] grid_pos;
  delete [] local_dims;
  delete [] global_dims;

  vout.detailed(m_vl, "FieldIO_Binary_Parallel via MPI I/O initialize done.\n");

  return EXIT_SUCCESS;
}
#endif