Skip to content

Commit f9730fe

Browse files
urbachurbach
committed
modified to work also with 1, 2 and 3-dimensional parallelisation
1 parent fedc328 commit f9730fe

File tree

4 files changed

+164
-222
lines changed

4 files changed

+164
-222
lines changed

‎DirectPut.c

+95-100
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
***********************************************************************/
1919

2020
#ifdef HAVE_CONFIG_H
21-
# include<config.h>
21+
# include<config.h>
2222
#endif
2323
#include <stdlib.h>
2424
#include <stdio.h>
@@ -33,8 +33,10 @@
3333
#include "global.h"
3434
#include "DirectPut.h"
3535

36+
// actual number of directions
37+
unsigned int spi_num_dirs = NUM_DIRS;
3638
// total Message Size
37-
uint64_t totalMessageSize;
39+
// in bytes!
3840
uint64_t totalMessageSize;
3941
// Allocate static memory for descriptors
4042
char SPIDescriptorsMemory[ NUM_DIRS * sizeof(MUHWI_Descriptor_t) + 64 ];
@@ -63,9 +65,6 @@ struct {
6365
uint8_t hintsE;
6466
} nb2dest[NUM_DIRS];
6567

66-
// in bytes
67-
uint64_t totalMessageSize;
68-
6968
// receive counter
7069
volatile uint64_t recvCounter;
7170

@@ -170,7 +169,8 @@ void setup_mregions_bats_counters(const int bufferSize) {
170169
}
171170

172171

173-
void create_descriptors(MUHWI_Descriptor_t * descriptors, uint64_t * messageSizes, uint64_t * soffsets, uint64_t * roffsets) {
172+
void create_descriptors(MUHWI_Descriptor_t * descriptors, uint64_t * messageSizes, uint64_t * soffsets,
173+
uint64_t * roffsets, const unsigned int num_dirs) {
174174
uint64_t anyFifoMap =
175175
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AM |
176176
MUHWI_DESCRIPTOR_TORUS_FIFO_MAP_AP |
@@ -188,7 +188,7 @@ void create_descriptors(MUHWI_Descriptor_t * descriptors, uint64_t * messageSize
188188

189189
// loop over directions
190190
// CHECK offset needs to be adjusted for QCD case
191-
for(int i = 0; i < 8; i++) {
191+
for(unsigned int i = 0; i < num_dirs; i++) {
192192
// Injection Direct Put Descriptor Information Structure
193193
MUSPI_Pt2PtDirectPutDescriptorInfo_t dinfo;
194194

@@ -244,14 +244,14 @@ void create_descriptors(MUHWI_Descriptor_t * descriptors, uint64_t * messageSize
244244
fprintf(stderr, "MUSPI_CreatePt2PtDirectPutDescriptor failed with rc=%d\n",rc);
245245
exit(1);
246246
}
247-
248247
}
249248
}
250249

251250

252251
int get_destinations(int * mypers) {
253252

254253
int tmp[6];
254+
#if (defined PARALLELT || defined PARALLELXT || defined PARALLELXYT || defined PARALLELXYZT)
255255
MPI_Status mstatus;
256256
MPI_Sendrecv((void*)mypers, 6, MPI_INT, g_nb_t_up, 0,
257257
(void*)tmp, 6, MPI_INT, g_nb_t_dn, 0,
@@ -261,7 +261,8 @@ int get_destinations(int * mypers) {
261261
(void*)tmp, 6, MPI_INT, g_nb_t_up, 1,
262262
g_cart_grid, &mstatus);
263263
MUSPI_SetUpDestination( &nb2dest[0].dest, tmp[0], tmp[1], tmp[2], tmp[3], tmp[4] );
264-
264+
#endif
265+
#if (defined PARALLELXT || defined PARALLELXYT || defined PARALLELXYZT)
265266
MPI_Sendrecv((void*)mypers, 6, MPI_INT, g_nb_x_up, 2,
266267
(void*)tmp, 6, MPI_INT, g_nb_x_dn, 2,
267268
g_cart_grid, &mstatus);
@@ -270,7 +271,8 @@ int get_destinations(int * mypers) {
270271
(void*)tmp, 6, MPI_INT, g_nb_x_up, 3,
271272
g_cart_grid, &mstatus);
272273
MUSPI_SetUpDestination( &nb2dest[2].dest, tmp[0], tmp[1], tmp[2], tmp[3], tmp[4] );
273-
274+
#endif
275+
#if (defined PARALLELXYT || defined PARALLELXYZT)
274276
MPI_Sendrecv((void*)mypers, 6, MPI_INT, g_nb_y_up, 4,
275277
(void*)tmp, 6, MPI_INT, g_nb_y_dn, 4,
276278
g_cart_grid, &mstatus);
@@ -279,7 +281,8 @@ int get_destinations(int * mypers) {
279281
(void*)tmp, 6, MPI_INT, g_nb_y_up, 5,
280282
g_cart_grid, &mstatus);
281283
MUSPI_SetUpDestination( &nb2dest[4].dest, tmp[0], tmp[1], tmp[2], tmp[3], tmp[4] );
282-
284+
#endif
285+
#if (defined PARALLELXYZT)
283286
MPI_Sendrecv((void*)mypers, 6, MPI_INT, g_nb_z_up, 6,
284287
(void*)tmp, 6, MPI_INT, g_nb_z_dn, 6,
285288
g_cart_grid, &mstatus);
@@ -288,7 +291,7 @@ int get_destinations(int * mypers) {
288291
(void*)tmp, 6, MPI_INT, g_nb_z_up, 7,
289292
g_cart_grid, &mstatus);
290293
MUSPI_SetUpDestination( &nb2dest[6].dest, tmp[0], tmp[1], tmp[2], tmp[3], tmp[4] );
291-
294+
#endif
292295
return(0);
293296
}
294297

@@ -309,8 +312,7 @@ typedef struct msg_InjFifoInfo
309312

310313
uint64_t msg_InjFifoInject ( msg_InjFifoHandle_t injFifoHandle,
311314
uint32_t relativeFifoId,
312-
MUHWI_Descriptor_t *descPtr )
313-
{
315+
MUHWI_Descriptor_t *descPtr ) {
314316
msg_InjFifoInfo_t *info = (msg_InjFifoInfo_t*)injFifoHandle.pOpaqueObject;
315317

316318
uint32_t globalFifoId = (info->startingSubgroupId * BGQ_MU_NUM_INJ_FIFOS_PER_SUBGROUP) +
@@ -320,11 +322,10 @@ uint64_t msg_InjFifoInject ( msg_InjFifoHandle_t injFifoHandle,
320322
uint64_t rc = MUSPI_InjFifoInject (MUSPI_IdToInjFifo( globalFifoId % BGQ_MU_NUM_INJ_FIFOS_PER_SUBGROUP,
321323
&info->subgroup[subgroupId] ),
322324
descPtr);
323-
return rc;
325+
return rc;
324326
}
325327

326-
void msg_InjFifoTerm ( msg_InjFifoHandle_t injFifoHandle )
327-
{
328+
void msg_InjFifoTerm ( msg_InjFifoHandle_t injFifoHandle ) {
328329
return; /*Simple library do nothing! */
329330
}
330331

@@ -333,8 +334,8 @@ int msg_InjFifoInit ( msg_InjFifoHandle_t *injFifoHandlePtr,
333334
uint32_t startingFifoId,
334335
uint32_t numFifos,
335336
size_t fifoSize,
336-
Kernel_InjFifoAttributes_t *injFifoAttrs )
337-
{
337+
Kernel_InjFifoAttributes_t *injFifoAttrs ) {
338+
338339
void *buffer = NULL;
339340
uint32_t endingFifoId; // Relative to a subgroup
340341
uint32_t numFifosInSubgroup;
@@ -343,16 +344,17 @@ int msg_InjFifoInit ( msg_InjFifoHandle_t *injFifoHandlePtr,
343344
uint32_t fifoIds[BGQ_MU_NUM_INJ_FIFOS_PER_SUBGROUP];
344345
Kernel_InjFifoAttributes_t attrs[BGQ_MU_NUM_INJ_FIFOS_PER_SUBGROUP];
345346
Kernel_InjFifoAttributes_t defaultAttrs;
346-
unsigned int i;
347347
uint64_t lock_cache;
348348

349349
memset ( &defaultAttrs, 0x00, sizeof(defaultAttrs) );
350-
if ( injFifoAttrs == NULL ) injFifoAttrs = &defaultAttrs;
350+
if(injFifoAttrs == NULL) {
351+
injFifoAttrs = &defaultAttrs;
352+
}
351353

352354
// Malloc space for the info structure
353355
msg_InjFifoInfo_t *info;
354356
info = (msg_InjFifoInfo_t *) memalign(32, sizeof(msg_InjFifoInfo_t));
355-
if ( !info ) return -1;
357+
if( !info ) return -1;
356358

357359
// Initialize the info structure
358360
info->startingSubgroupId = startingSubgroupId;
@@ -361,94 +363,87 @@ int msg_InjFifoInit ( msg_InjFifoHandle_t *injFifoHandlePtr,
361363
info->numSubgroups = 0;
362364

363365
// Malloc space for the injection fifos. They are 64-byte aligned.
364-
for (i=0; i<numFifos; i++)
365-
{
366-
info->fifoPtr[i] = (uint64_t*)memalign(64, fifoSize);
367-
if ( !info->fifoPtr[i] ) return -1;
368-
}
366+
for (unsigned int i = 0; i < numFifos; i++) {
367+
info->fifoPtr[i] = (uint64_t*)memalign(64, fifoSize);
368+
if ( !info->fifoPtr[i] ) return -1;
369+
}
369370

370371
// Process one subgroup at a time.
371372
// - Allocate the fifos.
372373
// - Init the MU MMIO for the fifos.
373374
// - Activate the fifos.
374-
while ( numFifos > 0 )
375-
{
376-
info->numSubgroups++;
377-
378-
// startingFifoId is the starting fifo number relative to the
379-
// subgroup we are working on.
380-
// Determine endingFifoId, the ending fifo number relative to
381-
// the subgroup we are working on.
382-
endingFifoId = startingFifoId + numFifos-1;
383-
if ( endingFifoId > (BGQ_MU_NUM_INJ_FIFOS_PER_SUBGROUP-1) )
384-
endingFifoId = BGQ_MU_NUM_INJ_FIFOS_PER_SUBGROUP-1;
385-
numFifosInSubgroup = endingFifoId - startingFifoId + 1;
386-
info->numFifosInSubgroup[subgroupId] = numFifosInSubgroup;
387-
388-
// Init structures for allocating the fifos...
389-
// - fifo Ids
390-
// - attributes
391-
for (i=0; i<numFifosInSubgroup; i++)
392-
{
393-
fifoIds[i] = startingFifoId + i;
394-
memcpy(&attrs[i],injFifoAttrs,sizeof(attrs[i]));
395-
/* printf("Attrs[%u] = 0x%x\n",i,*((uint32_t*)&attrs[i])); */
396-
/* printf("InjFifoInit: fifoIds[%u]=%u\n",i,fifoIds[i]); */
397-
}
398-
399-
// Allocate the fifos
400-
rc = Kernel_AllocateInjFifos (subgroupId,
401-
&info->subgroup[subgroupId],
402-
numFifosInSubgroup,
403-
fifoIds,
404-
attrs);
375+
while ( numFifos > 0 ) {
376+
info->numSubgroups++;
377+
378+
// startingFifoId is the starting fifo number relative to the
379+
// subgroup we are working on.
380+
// Determine endingFifoId, the ending fifo number relative to
381+
// the subgroup we are working on.
382+
endingFifoId = startingFifoId + numFifos-1;
383+
if ( endingFifoId > (BGQ_MU_NUM_INJ_FIFOS_PER_SUBGROUP-1) ) {
384+
endingFifoId = BGQ_MU_NUM_INJ_FIFOS_PER_SUBGROUP-1;
385+
}
386+
numFifosInSubgroup = endingFifoId - startingFifoId + 1;
387+
info->numFifosInSubgroup[subgroupId] = numFifosInSubgroup;
388+
389+
// Init structures for allocating the fifos...
390+
// - fifo Ids
391+
// - attributes
392+
for (unsigned int i = 0; i < numFifosInSubgroup; i++) {
393+
fifoIds[i] = startingFifoId + i;
394+
memcpy(&attrs[i], injFifoAttrs, sizeof(attrs[i]));
395+
}
396+
397+
// Allocate the fifos
398+
rc = Kernel_AllocateInjFifos (subgroupId,
399+
&info->subgroup[subgroupId],
400+
numFifosInSubgroup,
401+
fifoIds,
402+
attrs);
403+
if ( rc ) {
404+
printf("msg_InjFifoInit: Kernel_AllocateInjFifos failed with rc=%d\n",rc);
405+
return rc;
406+
}
407+
408+
// Init the MU MMIO for the fifos.
409+
for (unsigned int i = 0; i < numFifosInSubgroup; i++) {
410+
Kernel_MemoryRegion_t memRegion;
411+
rc = Kernel_CreateMemoryRegion ( &memRegion,
412+
info->fifoPtr[numFifos-i-1],
413+
fifoSize );
405414
if ( rc ) {
406-
printf("msg_InjFifoInit: Kernel_AllocateInjFifos failed with rc=%d\n",rc);
407-
return rc;
415+
printf("msg_InjFifoInit: Kernel_CreateMemoryRegion failed with rc=%d\n",rc);
416+
return rc;
408417
}
409-
410-
// Init the MU MMIO for the fifos.
411-
for (i=0; i<numFifosInSubgroup; i++)
412-
{
413-
Kernel_MemoryRegion_t memRegion;
414-
rc = Kernel_CreateMemoryRegion ( &memRegion,
415-
info->fifoPtr[numFifos-i-1],
416-
fifoSize );
417-
if ( rc ) {
418-
printf("msg_InjFifoInit: Kernel_CreateMemoryRegion failed with rc=%d\n",rc);
419-
return rc;
420-
}
421-
422-
rc = Kernel_InjFifoInit (&info->subgroup[subgroupId],
423-
fifoIds[i],
424-
&memRegion,
425-
(uint64_t)info->fifoPtr[numFifos-i-1] -
426-
(uint64_t)memRegion.BaseVa,
427-
fifoSize-1);
428-
if ( rc ) {
429-
printf("msg_InjFifoInit: Kernel_InjFifoInit failed with rc=%d\n",rc);
430-
return rc;
431-
}
432-
433-
/* TRACE(("HW freespace=%lx\n", MUSPI_getHwFreeSpace (MUSPI_IdToInjFifo (fifoIds[i],&info->subgroup[subgroupId]))))
434-
; */
435-
}
436-
437-
// Activate the fifos.
438-
rc = Kernel_InjFifoActivate (&info->subgroup[subgroupId],
439-
numFifosInSubgroup,
440-
fifoIds,
441-
KERNEL_INJ_FIFO_ACTIVATE);
418+
419+
// initialise the Fifos
420+
rc = Kernel_InjFifoInit (&info->subgroup[subgroupId],
421+
fifoIds[i],
422+
&memRegion,
423+
(uint64_t)info->fifoPtr[numFifos-i-1] -
424+
(uint64_t)memRegion.BaseVa,
425+
fifoSize-1);
442426
if ( rc ) {
443-
printf("msg_InjFifoInit: Kernel_InjFifoActivate failed with rc=%d\n",rc);
444-
return rc;
427+
printf("msg_InjFifoInit: Kernel_InjFifoInit failed with rc=%d\n",rc);
428+
return rc;
445429
}
446-
447-
startingFifoId = 0; // Next subgroup will start at fifo 0.
448-
449-
subgroupId++; // Next subgroup.
450-
numFifos -= numFifosInSubgroup;
451430
}
431+
432+
// Activate the fifos.
433+
rc = Kernel_InjFifoActivate (&info->subgroup[subgroupId],
434+
numFifosInSubgroup,
435+
fifoIds,
436+
KERNEL_INJ_FIFO_ACTIVATE);
437+
if ( rc ) {
438+
printf("msg_InjFifoInit: Kernel_InjFifoActivate failed with rc=%d\n",rc);
439+
return rc;
440+
}
441+
442+
startingFifoId = 0; // Next subgroup will start at fifo 0.
443+
444+
subgroupId++; // Next subgroup.
445+
numFifos -= numFifosInSubgroup;
446+
}
452447

453448
injFifoHandlePtr->pOpaqueObject = (void *)info;
454449
return 0;

0 commit comments

Comments
 (0)