Bridge++  Version 1.5.4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
layout.cpp
Go to the documentation of this file.
1 
14 #include "layout.h"
15 
16 // prototype declaration
17 namespace {
18  static int pe_logical_layout(const int ndim, const int *dims, int nproc, int *npe);
19  static int find_primes(const int n, int *p);
20 }
21 
22 // static members
24 int *Communicator_impl::Layout::m_dims = 0; //< lattice extent
25 
26 // logical layout
29 
32 
33 // physical mapping
35 int *Communicator_impl::Layout::m_physical_to_logical = 0; //< map between physical and logical grid
36 
37 // subdimensional-slices
39 
40 
41 //====================================================================
42 // communicator_impl methods
43 int Communicator_impl::Layout::ipe(const int idir)
44 {
45  return m_grid_coord[idir];
46 }
47 
48 
49 //====================================================================
50 int Communicator_impl::Layout::npe(const int idir)
51 {
52  return m_grid_dims[idir];
53 }
54 
55 
56 //====================================================================
58 {
59  for (int i = 0; i < m_ndim; ++i) {
60  dims[i] = m_grid_dims[i];
61  }
62  return EXIT_SUCCESS;
63 }
64 
65 
66 //====================================================================
67 // tag() : identify transfer channel by source rank and direction (incl fw/bw)
68 int Communicator_impl::Layout::tag(int rank, int idir, int ipm)
69 {
70  return 2 * m_ndim * rank + idir + ((ipm > 0) ? 0 : m_ndim);
71 }
72 
73 
74 //====================================================================
76 
94 {
95  // fetch parameter values from CommonParameters
96 
97  m_ndim = CommonParameters::Ndim();
98 
99  // lattice size
100  m_dims = new int [m_ndim];
101 
102  m_dims[0] = CommonParameters::Lx();
103  m_dims[1] = CommonParameters::Ly();
104  m_dims[2] = CommonParameters::Lz();
105  m_dims[3] = CommonParameters::Lt();
106 
107  // suggested logical layout
108  m_grid_dims = new int [m_ndim];
109 
110  m_grid_dims[0] = CommonParameters::NPEx();
111  m_grid_dims[1] = CommonParameters::NPEy();
112  m_grid_dims[2] = CommonParameters::NPEz();
113  m_grid_dims[3] = CommonParameters::NPEt();
114 
115  // suggested local volume
116  int *local_dims = new int [m_ndim];
117 
118  local_dims[0] = CommonParameters::Nx();
119  local_dims[1] = CommonParameters::Ny();
120  local_dims[2] = CommonParameters::Nz();
121  local_dims[3] = CommonParameters::Nt();
122 
123  for (int i = 0; i < m_ndim; ++i) {
124  if (local_dims[i] != 0) { // there is a suggestion of local extent.
125  if (m_grid_dims[i] != 0) {
126  if (m_grid_dims[i] * local_dims[i] != m_dims[i]) {
127  fprintf(stderr, "layout mismatch.\n");
128  exit(EXIT_FAILURE);
129  } else {
130  // ok. m_grid_dims[i] is as specified.
131  }
132  } else { // use specified local size and find number of divisions.
133  if (m_dims[i] % local_dims[i] != 0) {
134  fprintf(stderr, "layout mismatch. lattice undivisable by local volume.\n");
135  exit(EXIT_FAILURE);
136  } else {
137  m_grid_dims[i] = m_dims[i] / local_dims[i];
138  }
139  }
140  }
141  }
142 
143  delete [] local_dims;
144 
145 
146  std::vector<int> lattice_size(m_dims, m_dims + 4);
147  std::vector<int> grid_size(m_grid_dims, m_grid_dims + 4);
148 
149  // continue setup
150  return layout_setup(lattice_size, grid_size);
151 }
152 
153 
155  const std::vector<int>& lattice_size,
156  std::vector<int>& grid_size)
157 {
158  m_ndim = lattice_size.size();
159 
160  // lattice size
161  m_dims = new int [m_ndim];
162 
163  for (int i = 0; i < m_ndim; ++i) {
164  m_dims[i] = lattice_size[i];
165  }
166 
167  // suggested logical layout
168  m_grid_dims = new int [m_ndim];
169 
170  for (int i = 0; i < m_ndim; ++i) {
171  m_grid_dims[i] = 0;
172  }
173 
174  for (int i = 0; i < grid_size.size(); ++i) {
175  m_grid_dims[i] = grid_size[i];
176  }
177 
178 
179  // find logical layout
180  int retv = pe_logical_layout(m_ndim, m_dims, m_grid_size, m_grid_dims);
181 
182  // note: Communicator_impl::m_grid_size = size of local communicator m_comm.
183 
184  if (retv != EXIT_SUCCESS) {
185  fprintf(stderr, "layout failed.\n");
186  exit(EXIT_FAILURE);
187  }
188 
189 
190  // store back to grid_size
191  grid_size.resize(m_ndim);
192  for (int i = 0; i < m_ndim; ++i) {
193  grid_size[i] = m_grid_dims[i];
194  }
195 
196 
197  // physical to logical map
198  physical_map_setup();
199 
200  // find logical coordinate of my rank
201  m_grid_coord = new int [m_ndim];
202  grid_coord(m_grid_coord, m_grid_rank);
203 
204  // find neighbour rank
205  m_ipe_up = new int [m_ndim];
206  m_ipe_dn = new int [m_ndim];
207 
208  int *coord = new int [m_ndim];
209  for (int i = 0; i < m_ndim; ++i) {
210  for (int j = 0; j < m_ndim; ++j) {
211  coord[j] = m_grid_coord[j];
212  }
213 
214  // upward
215  coord[i] = (m_grid_coord[i] + 1) % m_grid_dims[i];
216  grid_rank(&m_ipe_up[i], coord);
217 
218  // downward
219  coord[i] = (m_grid_coord[i] - 1 + m_grid_dims[i]) % m_grid_dims[i];
220  grid_rank(&m_ipe_dn[i], coord);
221  }
222  delete [] coord;
223 
224 
225 #ifdef DEBUG
226  printf("rank %d: up=(%d,%d,%d,%d), dn=(%d,%d,%d,%d)\n",
228  m_ipe_up[0], m_ipe_up[1], m_ipe_up[2], m_ipe_up[3],
229  m_ipe_dn[0], m_ipe_dn[1], m_ipe_dn[2], m_ipe_dn[3]);
230 #endif
231 
232  // subgrid of reduced dimension.
233  subgrid_setup();
234 
235  return EXIT_SUCCESS;
236 }
237 
238 
239 //====================================================================
241 {
242  subgrid_delete();
243 
244  physical_map_delete();
245 
246  delete [] m_dims;
247  delete [] m_grid_dims;
248  delete [] m_grid_coord;
249  delete [] m_ipe_up;
250  delete [] m_ipe_dn;
251 
252  return EXIT_SUCCESS;
253 }
254 
255 
256 //====================================================================
257 // subgrid for communication in reduced dimensional subsets
259 {
260  // int Nmask = (1 << m_ndim);
261  int Nmask = 1;
262 
263  // temporary modified to avoid error on BG/Q. [23 May 2014 H.M.]
264 
265  m_sub_comm = new MPI_Comm [Nmask];
266 
267  for (int imask = 0; imask < Nmask; ++imask) {
268  int coord[m_ndim];
269  for (int i = 0; i < m_ndim; ++i) {
270  coord[i] = m_grid_coord[i];
271  }
272 
273  for (int i = 0; i < m_ndim; ++i) {
274  bool mask = ((imask >> i) & 1) == 1 ? true : false;
275  if (!mask) coord[i] = 0;
276  }
277 
278  int rank = 0;
279  grid_rank(&rank, coord);
280 
281 /*
282  printf("split %2d: rank %2d: (%d,%d,%d,%d) -> (%d,%d,%d,%d), %2d\n",
283  imask,
284  m_grid_rank,
285  m_grid_coord[0], m_grid_coord[1], m_grid_coord[2], m_grid_coord[3],
286  coord[0], coord[1], coord[2], coord[3],
287  rank);
288 */
289  MPI_Comm_split(m_comm, rank, 0 /* key */, &m_sub_comm[imask]);
290  }
291 
292  return EXIT_SUCCESS;
293 }
294 
295 
296 //====================================================================
298 {
299  delete [] m_sub_comm;
300 
301  return EXIT_SUCCESS;
302 }
303 
304 
305 //====================================================================
306 namespace { // anonymous namespace
307 // layout
308  static const int prime_table[] =
309  {
310  2, 3, 5, 7, 11, 13, 17, 19,
311  23, 29, 31, 37, 41, 43, 47, 53,
312  59, 61, 67, 71, 73, 79, 83, 89,
313  97, 101, 103, 107, 109, 113, 127, 131,
314  137, 139, 149, 151, 157, 163, 167, 173,
315  179, 181, 191, 193, 197, 199, 211, 223,
316  227, 229, 233, 239, 241, 251, 257, 263,
317  269, 271, 277, 281, 283, 293, 307, 311,
318  };
319 
320  static const int nprime = sizeof(prime_table) / sizeof(int);
321 
322 //* find logical layout
323 //* divide dims_mu into npe_mu x local_mu where prod npe_mu = nproc.
324 //* npe_mu != 0 is kept intact.
325  static int pe_logical_layout(const int ndim, const int *dims, int nproc, int *npe)
326  {
327  int retv = EXIT_SUCCESS;
328 
329  int nfreedim = 0;
330  int nfreeproc = nproc;
331 
332  for (int i = 0; i < ndim; ++i) {
333  if (npe[i] == 0) {
334  ++nfreedim;
335  } else {
336  if (npe[i] < 0) {
337  fprintf(stderr, "illegal value: npe[%d]=%d.\n", i, npe[i]);
338  return EXIT_FAILURE;
339  } else if (nproc % npe[i] != 0) {
340  fprintf(stderr, "illegal value: npe[%d]=%d does not divide NPE=%d.\n", i, npe[i], nproc);
341  return EXIT_FAILURE;
342  } else if (nfreeproc % npe[i] != 0) {
343  fprintf(stderr, "illegal value: NPE=%d is not divisable by %d.\n", nproc, nproc / nfreeproc * npe[i]);
344  return EXIT_FAILURE;
345  } else if (dims[i] % npe[i] != 0) {
346  fprintf(stderr, "illegal value: npe[%d]=%d does not divide L=%d.\n", i, npe[i], dims[i]);
347  return EXIT_FAILURE;
348  } else {
349  nfreeproc /= npe[i];
350  }
351  }
352  }
353 
354  if (nfreeproc < 1) {
355  fprintf(stderr, "impossible layout.\n");
356  return EXIT_FAILURE;
357  } else if (nfreeproc == 1) {
358  for (int i = 0; i < ndim; ++i) {
359  if (npe[i] == 0) npe[i] = 1;
360  }
361  return EXIT_SUCCESS; // complete.
362  } else {
363  if (nfreedim == 0) {
364  fprintf(stderr, "impossible layout. no room to divide.\n");
365  return EXIT_FAILURE;
366  }
367  }
368 
369  // divide nfreeproc to nfreedim npe's.
370  int np = nfreeproc;
371 
372  int *subdims = new int [ndim];
373  int nf = 0;
374  for (int i = 0; i < ndim; ++i) {
375  if (npe[i] == 0) subdims[nf++] = dims[i]; }
376 
377  int *count = new int [nprime];
378  for (int i = 0; i < nprime; ++i) {
379  count[i] = 0;
380  }
381 
382  for (int i = 0; i < nprime; ++i) {
383  int p = prime_table[i];
384  while (np > 1 && np % p == 0)
385  {
386  ++count[i];
387  np /= p;
388  }
389  if (np == 1) break;
390  }
391 
392 // printf("%d = (", nfreeproc);
393 // for (int i=0; i<nprime; ++i) {
394 // printf("%d, ", count[i]);
395 // }
396 // printf(")\n");
397 
398  if (np > 1) {
399  fprintf(stderr, "insufficient prime table.\n");
400  retv = EXIT_FAILURE;
401  goto finalize;
402  }
403 
404 // printf("subdims=(");
405 // for (int i=0; i<nf; ++i) {
406 // printf("%d, ", subdims[i]);
407 // }
408 // printf(")\n");
409 
410  for (int i = nprime - 1; i >= 0; --i) {
411  if (count[i] == 0) continue;
412 
413  int p = prime_table[i];
414 
415  for (int j = 0; j < count[i]; ++j) {
416  int maxsubdim = 0;
417  int maxk = -1;
418  for (int k = 0; k < nf; ++k) {
419  if ((subdims[k] >= maxsubdim) && (subdims[k] % p == 0)) {
420  maxsubdim = subdims[k];
421  maxk = k;
422  }
423  }
424 // printf("prime=%d, maxk=%d, maxsubdim=%d\n", p, maxk, maxsubdim);
425 
426  if (maxk == -1) {
427  fprintf(stderr, "not divisable. %d\n", p);
428  retv = EXIT_FAILURE;
429  goto finalize;
430  }
431 
432  subdims[maxk] /= p;
433  }
434  }
435 
436 /*
437  printf("subdims=(");
438  for (int i=0; i<nf; ++i) {
439  printf("%d, ", subdims[i]);
440  }
441  printf(")\n");
442 */
443  // store
444  for (int i = 0, k = 0; i < ndim; ++i) {
445  if (npe[i] == 0) {
446  npe[i] = dims[i] / subdims[k];
447  ++k;
448  }
449  }
450 
451 finalize:
452  delete [] subdims;
453  delete [] count;
454 
455  return retv;
456  }
457 
458 
459  //* find n primes.
460  static int find_primes(const int n, int *p)
461  {
462  if (n < 1) return EXIT_FAILURE;
463 
464  int i = 0;
465  int k = 2;
466 
467  p[i++] = k++;
468 
469  while (1)
470  {
471  int j;
472  for (j = 0; j < i; ++j) {
473  if (k % p[j] == 0) break;
474  }
475 
476  if (j >= i) { // not divisable by p[0]..p[i-1]. found new prime.
477  p[i++] = k;
478  if (i >= n) return EXIT_FAILURE;
479  }
480 
481  ++k;
482  }
483 
484  return EXIT_SUCCESS;
485  }
486 } // anonymous namespace
487 
488 //====================================================================
489 //============================================================END=====
static int NPEy()
static int layout_setup()
initialise layout.
Definition: layout.cpp:93
static int self()
rank within small world.
static char m_map_grid[16]
Definition: layout.h:57
static int * m_ipe_up
rank of upward neighbour in directions.
Definition: layout.h:53
static int subgrid_setup()
Definition: layout.cpp:258
static int * m_ipe_dn
rank of downward neighbour in directions.
Definition: layout.h:54
static int * m_dims
lattice extent (Lx, Ly, Lz, Lt)
Definition: layout.h:47
static int subgrid_delete()
Definition: layout.cpp:297
static int * m_grid_dims
grid dimensions in directions.
Definition: layout.h:50
static int npe(const int idir)
grid size in idir direction.
Definition: layout.cpp:50
static int grid_dims(int *gdims)
find grid size.
Definition: layout.cpp:57
static MPI_Comm * m_sub_comm
subgrid
Definition: layout.h:64
static int * m_grid_coord
grid coordinate.
Definition: layout.h:51
static MPI_Comm m_comm
static int NPEx()
static int ipe(const int idir)
grid coordinate along idir direction.
Definition: layout.cpp:43
static int NPEz()
static int tag(int rank, int idir, int ipm)
generate tag for communication.
Definition: layout.cpp:68
static int layout_delete()
destroy layout and clean up.
Definition: layout.cpp:240
static int * m_physical_to_logical
map between physical and logical grid
Definition: layout.h:58
static int m_ndim
number of dimensions.
Definition: layout.h:46