18 static int pe_logical_layout(
const int ndim,
const int *dims,
int nproc,
int *npe);
19 static int find_primes(
const int n,
int *p);
52 return m_grid_dims[idir];
59 for (
int i = 0; i < m_ndim; ++i) {
60 dims[i] = m_grid_dims[i];
70 return 2 * m_ndim * rank + idir + ((ipm > 0) ? 0 : m_ndim);
98 m_dims =
new int [m_ndim];
106 m_grid_dims =
new int [m_ndim];
114 int *local_dims =
new int [m_ndim];
121 for (
int i = 0; i < m_ndim; ++i) {
122 if (local_dims[i] != 0) {
123 if (m_grid_dims[i] != 0) {
124 if (m_grid_dims[i] * local_dims[i] != m_dims[i]) {
125 fprintf(stderr,
"layout mismatch.\n");
131 if (m_dims[i] % local_dims[i] != 0) {
132 fprintf(stderr,
"layout mismatch. lattice undivisable by local volume.\n");
135 m_grid_dims[i] = m_dims[i] / local_dims[i];
141 delete [] local_dims;
145 int retv = pe_logical_layout(m_ndim, m_dims,
m_grid_size, m_grid_dims);
147 if (retv != EXIT_SUCCESS) {
148 fprintf(stderr,
"layout failed.\n");
153 physical_map_setup();
156 m_grid_coord =
new int [m_ndim];
160 m_ipe_up =
new int [m_ndim];
161 m_ipe_dn =
new int [m_ndim];
163 int *coord =
new int [m_ndim];
164 for (
int i = 0; i < m_ndim; ++i) {
165 for (
int j = 0; j < m_ndim; ++j) {
166 coord[j] = m_grid_coord[j];
170 coord[i] = (m_grid_coord[i] + 1) % m_grid_dims[i];
171 grid_rank(&m_ipe_up[i], coord);
174 coord[i] = (m_grid_coord[i] - 1 + m_grid_dims[i]) % m_grid_dims[i];
175 grid_rank(&m_ipe_dn[i], coord);
181 printf(
"rank %d: up=(%d,%d,%d,%d), dn=(%d,%d,%d,%d)\n",
183 m_ipe_up[0], m_ipe_up[1], m_ipe_up[2], m_ipe_up[3],
184 m_ipe_dn[0], m_ipe_dn[1], m_ipe_dn[2], m_ipe_dn[3]);
199 physical_map_delete();
202 delete [] m_grid_dims;
203 delete [] m_grid_coord;
220 m_sub_comm =
new MPI_Comm [Nmask];
222 for (
int imask = 0; imask < Nmask; ++imask) {
224 for (
int i = 0; i < m_ndim; ++i) {
225 coord[i] = m_grid_coord[i];
228 for (
int i = 0; i < m_ndim; ++i) {
229 bool mask = ((imask >> i) & 1) == 1 ?
true :
false;
230 if (!mask) coord[i] = 0;
234 grid_rank(&rank, coord);
244 MPI_Comm_split(
m_comm, rank, 0 , &m_sub_comm[imask]);
254 delete [] m_sub_comm;
263 static const int prime_table[] =
265 2, 3, 5, 7, 11, 13, 17, 19,
266 23, 29, 31, 37, 41, 43, 47, 53,
267 59, 61, 67, 71, 73, 79, 83, 89,
268 97, 101, 103, 107, 109, 113, 127, 131,
269 137, 139, 149, 151, 157, 163, 167, 173,
270 179, 181, 191, 193, 197, 199, 211, 223,
271 227, 229, 233, 239, 241, 251, 257, 263,
272 269, 271, 277, 281, 283, 293, 307, 311,
275 static const int nprime =
sizeof(prime_table) /
sizeof(
int);
280 static int pe_logical_layout(
const int ndim,
const int *dims,
int nproc,
int *npe)
282 int retv = EXIT_SUCCESS;
285 int nfreeproc = nproc;
287 for (
int i = 0; i < ndim; ++i) {
292 fprintf(stderr,
"illegal value: npe[%d]=%d.\n", i, npe[i]);
294 }
else if (nproc % npe[i] != 0) {
295 fprintf(stderr,
"illegal value: npe[%d]=%d does not divide NPE=%d.\n", i, npe[i], nproc);
297 }
else if (nfreeproc % npe[i] != 0) {
298 fprintf(stderr,
"illegal value: NPE=%d is not divisable by %d.\n", nproc, nproc / nfreeproc * npe[i]);
300 }
else if (dims[i] % npe[i] != 0) {
301 fprintf(stderr,
"illegal value: npe[%d]=%d does not divide L=%d.\n", i, npe[i], dims[i]);
310 fprintf(stderr,
"impossible layout.\n");
312 }
else if (nfreeproc == 1) {
313 for (
int i = 0; i < ndim; ++i) {
314 if (npe[i] == 0) npe[i] = 1;
319 fprintf(stderr,
"impossible layout. no room to divide.\n");
327 int *subdims =
new int [ndim];
329 for (
int i = 0; i < ndim; ++i) {
330 if (npe[i] == 0) subdims[nf++] = dims[i]; }
332 int *count =
new int [nprime];
333 for (
int i = 0; i < nprime; ++i) {
337 for (
int i = 0; i < nprime; ++i) {
338 int p = prime_table[i];
339 while (np > 1 && np % p == 0)
354 fprintf(stderr,
"insufficient prime table.\n");
365 for (
int i = nprime - 1; i >= 0; --i) {
366 if (count[i] == 0)
continue;
368 int p = prime_table[i];
370 for (
int j = 0; j < count[i]; ++j) {
373 for (
int k = 0; k < nf; ++k) {
374 if ((subdims[k] >= maxsubdim) && (subdims[k] % p == 0)) {
375 maxsubdim = subdims[k];
382 fprintf(stderr,
"not divisable. %d\n", p);
399 for (
int i = 0, k = 0; i < ndim; ++i) {
401 npe[i] = dims[i] / subdims[k];
415 static int find_primes(
const int n,
int *p)
417 if (n < 1)
return EXIT_FAILURE;
427 for (j = 0; j < i; ++j) {
428 if (k % p[j] == 0)
break;
433 if (i >= n)
return EXIT_FAILURE;
static int * m_grid_coord
grid coordinate.
static int layout_setup()
layout_setup() – setup logical layout
static int self()
rank within small world.
static int * m_dims
lattice extent (Lx, Ly, Lz, Lt)
static int * m_physical_to_logical
map between physical and logical grid
static int m_ndim
number of dimensions.
static int subgrid_setup()
static int subgrid_delete()
static int npe(const int idir)
static int grid_dims(int *gdims)
static int m_comm
instead of MPI_Comm m_comm;
static int * m_grid_dims
grid dimensions in directions.
static char m_map_grid[16]
static int * m_ipe_up
rank of upward neighbour in directions.
static int ipe(const int idir)
static int tag(int rank, int idir, int ipm)
static int layout_delete()
static int * m_ipe_dn
rank of downward neighbour in directions.