Skip to content

Commit 4b8f3e1

Browse files
author
David Eberius
committed
This is an update to the SPC code to add two major features:
bin counters and the mmap interface. Bin counters are designed to allow for more context in the counters so that one counter can have several sub-counters attached to it as bins with breakpoints to denote when each bin gets incremented. These are used extensively for collective algorithm counters which count the number of times each collective algorithm is called and under which circumstances small/large communicator/ message. The mmap interface creates a shared file using mmap so tools can attach to this and read SPC values directly using an XML file to point out where the data for each counter resides. This update also includes some bug fixes and quality of life improvements to the code. Signed-off-by: David Eberius <deberius@vols.utk.edu>
1 parent 69bd54c commit 4b8f3e1

24 files changed

Lines changed: 1214 additions & 101 deletions

‎examples/spc_example.c‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <stdio.h>
1010
#include <stdlib.h>
1111
#include <string.h>
12+
#include <unistd.h>
1213

1314
#include "mpi.h"
1415

@@ -47,15 +48,16 @@ int main(int argc, char **argv)
4748
message_size = atoi(argv[2]);
4849
}
4950

50-
int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index;
51+
int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index, xml_index;
5152
MPI_Datatype datatype;
5253
MPI_T_enum enumtype;
5354
MPI_Comm comm;
5455
char name[256], description[256];
5556

5657
/* Counter names to be read by ranks 0 and 1 */
57-
char *counter_names[] = {"runtime_spc_OMPI_BYTES_SENT_USER",
58-
"runtime_spc_OMPI_BYTES_RECEIVED_USER" };
58+
char *counter_names[] = {"runtime_spc_OMPI_SPC_BYTES_SENT_USER",
59+
"runtime_spc_OMPI_SPC_BYTES_RECEIVED_USER" };
60+
char *xml_counter = "runtime_spc_OMPI_SPC_XML_FILE";
5961

6062
MPI_Init(NULL, NULL);
6163
MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
@@ -68,27 +70,34 @@ int main(int argc, char **argv)
6870
}
6971

7072
/* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECIEVED_USER SPCs */
71-
index = -1;
73+
index = xml_index = -1;
7274
MPI_T_pvar_get_num(&num);
7375
for(i = 0; i < num; i++) {
7476
name_len = desc_len = 256;
7577
PMPI_T_pvar_get_info(i, name, &name_len, &verbosity,
7678
&var_class, &datatype, &enumtype, description, &desc_len, &bind,
7779
&readonly, &continuous, &atomic);
80+
7881
if(strcmp(name, counter_names[rank]) == 0) {
7982
index = i;
8083
printf("[%d] %s -> %s\n", rank, name, description);
8184
}
85+
if(strcmp(name, xml_counter) == 0) {
86+
xml_index = i;
87+
printf("[%d] %s -> %s (index -> %d)\n", rank, name, description, xml_index);
88+
}
8289
}
8390

8491
/* Make sure we found the counters */
85-
if(index == -1) {
92+
if(index == -1 || xml_index == -1) {
8693
fprintf(stderr, "ERROR: Couldn't find the appropriate SPC counter in the MPI_T pvars.\n");
8794
MPI_Abort(MPI_COMM_WORLD, -1);
8895
}
8996

90-
int ret;
97+
int ret, xml_count;
9198
long long value;
99+
char *xml_filename = (char*)malloc(64 * sizeof(char));
100+
sprintf(xml_filename, "this_is_a_test");
92101

93102
MPI_T_pvar_session session;
94103
MPI_T_pvar_handle handle;
@@ -97,13 +106,29 @@ int main(int argc, char **argv)
97106
ret = MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count);
98107
ret = MPI_T_pvar_start(session, handle);
99108

109+
MPI_T_pvar_session xml_session;
110+
MPI_T_pvar_handle xml_handle;
111+
if(xml_index >= 0) {
112+
ret = MPI_T_pvar_session_create(&xml_session);
113+
ret = MPI_T_pvar_handle_alloc(xml_session, xml_index, NULL, &xml_handle, &xml_count);
114+
printf("xml_count: %d\n", xml_count);
115+
ret = MPI_T_pvar_start(xml_session, xml_handle);
116+
}
117+
100118
message_exchange(num_messages, message_size);
101119

102120
ret = MPI_T_pvar_read(session, handle, &value);
121+
if(xml_index >= 0) {
122+
ret = MPI_T_pvar_read(xml_session, xml_handle, &xml_filename);
123+
}
124+
103125
/* Print the counter values in order by rank */
104126
for(i = 0; i < 2; i++) {
105127
if(i == rank) {
106128
printf("[%d] Value Read: %lld\n", rank, value);
129+
if(xml_index >= 0) {
130+
printf("[%d] Value Read: %s\n", rank, xml_filename);
131+
}
107132
fflush(stdout);
108133
}
109134
MPI_Barrier(MPI_COMM_WORLD);

‎ompi/mca/coll/base/coll_base_allgather.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include "ompi/mca/coll/coll.h"
3232
#include "ompi/mca/coll/base/coll_tags.h"
3333
#include "ompi/mca/coll/base/coll_base_functions.h"
34+
#include "ompi/runtime/ompi_spc.h"
3435
#include "coll_base_topo.h"
3536
#include "coll_base_util.h"
3637

@@ -99,6 +100,8 @@ int ompi_coll_base_allgather_intra_bruck(const void *sbuf, int scount,
99100
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
100101
"coll:base:allgather_intra_bruck rank %d", rank));
101102

103+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLGATHER_BRUCK, scount * sdtype->super.size, size);
104+
102105
err = ompi_datatype_get_extent (rdtype, &rlb, &rext);
103106
if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
104107

@@ -286,6 +289,8 @@ ompi_coll_base_allgather_intra_recursivedoubling(const void *sbuf, int scount,
286289
"coll:base:allgather_intra_recursivedoubling rank %d, size %d",
287290
rank, size));
288291

292+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLGATHER_RECURSIVE_DOUBLING, scount * sdtype->super.size, size);
293+
289294
err = ompi_datatype_get_extent (rdtype, &rlb, &rext);
290295
if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
291296

@@ -372,6 +377,8 @@ int ompi_coll_base_allgather_intra_ring(const void *sbuf, int scount,
372377
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
373378
"coll:base:allgather_intra_ring rank %d", rank));
374379

380+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLGATHER_RING, scount * sdtype->super.size, size);
381+
375382
err = ompi_datatype_get_extent (rdtype, &rlb, &rext);
376383
if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
377384

@@ -508,6 +515,8 @@ ompi_coll_base_allgather_intra_neighborexchange(const void *sbuf, int scount,
508515
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
509516
"coll:base:allgather_intra_neighborexchange rank %d", rank));
510517

518+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLGATHER_NEIGHBOR_EXCHANGE, scount * sdtype->super.size, size);
519+
511520
err = ompi_datatype_get_extent (rdtype, &rlb, &rext);
512521
if (MPI_SUCCESS != err) { line = __LINE__; goto err_hndl; }
513522

@@ -611,6 +620,8 @@ int ompi_coll_base_allgather_intra_two_procs(const void *sbuf, int scount,
611620
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
612621
"ompi_coll_base_allgather_intra_two_procs rank %d", rank));
613622

623+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLGATHER_TWO_PROCS, scount * sdtype->super.size, 2);
624+
614625
if (2 != ompi_comm_size(comm)) {
615626
return MPI_ERR_UNSUPPORTED_OPERATION;
616627
}
@@ -689,6 +700,8 @@ ompi_coll_base_allgather_intra_basic_linear(const void *sbuf, int scount,
689700
int err;
690701
ptrdiff_t lb, extent;
691702

703+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLGATHER_LINEAR, scount * sdtype->super.size, ompi_comm_size(comm));
704+
692705
/* Handle MPI_IN_PLACE (see explanantion in reduce.c for how to
693706
allocate temp buffer) -- note that rank 0 can use IN_PLACE
694707
natively, and we can just alias the right position in rbuf

‎ompi/mca/coll/base/coll_base_allreduce.c‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
#include "ompi/mca/pml/pml.h"
3737
#include "ompi/op/op.h"
3838
#include "ompi/mca/coll/base/coll_base_functions.h"
39+
#include "ompi/runtime/ompi_spc.h"
3940
#include "coll_base_topo.h"
4041
#include "coll_base_util.h"
4142

@@ -63,6 +64,8 @@ ompi_coll_base_allreduce_intra_nonoverlapping(const void *sbuf, void *rbuf, int
6364

6465
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:allreduce_intra_nonoverlapping rank %d", rank));
6566

67+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLREDUCE_NONOVERLAPPING, count * dtype->super.size, ompi_comm_size(comm));
68+
6669
/* Reduce to 0 and broadcast. */
6770

6871
if (MPI_IN_PLACE == sbuf) {
@@ -145,6 +148,8 @@ ompi_coll_base_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf,
145148
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
146149
"coll:base:allreduce_intra_recursivedoubling rank %d", rank));
147150

151+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLREDUCE_RECURSIVE_DOUBLING, count * dtype->super.size, size);
152+
148153
/* Special case for size == 1 */
149154
if (1 == size) {
150155
if (MPI_IN_PLACE != sbuf) {
@@ -358,6 +363,8 @@ ompi_coll_base_allreduce_intra_ring(const void *sbuf, void *rbuf, int count,
358363
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
359364
"coll:base:allreduce_intra_ring rank %d, count %d", rank, count));
360365

366+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLREDUCE_RING, count * dtype->super.size, size);
367+
361368
/* Special case for size == 1 */
362369
if (1 == size) {
363370
if (MPI_IN_PLACE != sbuf) {
@@ -637,6 +644,8 @@ ompi_coll_base_allreduce_intra_ring_segmented(const void *sbuf, void *rbuf, int
637644
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
638645
"coll:base:allreduce_intra_ring_segmented rank %d, count %d", rank, count));
639646

647+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLREDUCE_RING_SEGMENTED, count * dtype->super.size, size);
648+
640649
/* Special case for size == 1 */
641650
if (1 == size) {
642651
if (MPI_IN_PLACE != sbuf) {
@@ -890,6 +899,8 @@ ompi_coll_base_allreduce_intra_basic_linear(const void *sbuf, void *rbuf, int co
890899

891900
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:allreduce_intra_basic_linear rank %d", rank));
892901

902+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLREDUCE_LINEAR, count * dtype->super.size, ompi_comm_size(comm));
903+
893904
/* Reduce to 0 and broadcast. */
894905

895906
if (MPI_IN_PLACE == sbuf) {

‎ompi/mca/coll/base/coll_base_alltoall.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "ompi/mca/coll/base/coll_tags.h"
3333
#include "ompi/mca/pml/pml.h"
3434
#include "ompi/mca/coll/base/coll_base_functions.h"
35+
#include "ompi/runtime/ompi_spc.h"
3536
#include "coll_base_topo.h"
3637
#include "coll_base_util.h"
3738

@@ -53,6 +54,8 @@ mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, int rcount,
5354
size = ompi_comm_size(comm);
5455
rank = ompi_comm_rank(comm);
5556

57+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLTOALL_INPLACE, rcount * rdtype->super.size, size);
58+
5659
/* If only one process, we're done. */
5760
if (1 == size) {
5861
return MPI_SUCCESS;
@@ -151,6 +154,8 @@ int ompi_coll_base_alltoall_intra_pairwise(const void *sbuf, int scount,
151154
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
152155
"coll:base:alltoall_intra_pairwise rank %d", rank));
153156

157+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLTOALL_PAIRWISE, rcount * rdtype->super.size, size);
158+
154159
err = ompi_datatype_get_extent (sdtype, &lb, &sext);
155160
if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; }
156161
err = ompi_datatype_get_extent (rdtype, &lb, &rext);
@@ -212,6 +217,8 @@ int ompi_coll_base_alltoall_intra_bruck(const void *sbuf, int scount,
212217
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
213218
"coll:base:alltoall_intra_bruck rank %d", rank));
214219

220+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLTOALL_BRUCK, rcount * rdtype->super.size, size);
221+
215222
err = ompi_datatype_type_extent (sdtype, &sext);
216223
if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; }
217224

@@ -358,6 +365,8 @@ int ompi_coll_base_alltoall_intra_linear_sync(const void *sbuf, int scount,
358365
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
359366
"ompi_coll_base_alltoall_intra_linear_sync rank %d", rank));
360367

368+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLTOALL_LINEAR_SYNC, rcount * rdtype->super.size, size);
369+
361370
error = ompi_datatype_get_extent(sdtype, &slb, &sext);
362371
if (OMPI_SUCCESS != error) {
363372
return error;
@@ -512,6 +521,8 @@ int ompi_coll_base_alltoall_intra_two_procs(const void *sbuf, int scount,
512521
return MPI_ERR_UNSUPPORTED_OPERATION;
513522
}
514523

524+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLTOALL_TWO_PROCS, rcount * rdtype->super.size, 2);
525+
515526
err = ompi_datatype_get_extent (sdtype, &lb, &sext);
516527
if (err != MPI_SUCCESS) { line = __LINE__; goto err_hndl; }
517528

@@ -594,6 +605,8 @@ int ompi_coll_base_alltoall_intra_basic_linear(const void *sbuf, int scount,
594605
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
595606
"ompi_coll_base_alltoall_intra_basic_linear rank %d", rank));
596607

608+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_ALLTOALL_LINEAR, rcount * rdtype->super.size, size);
609+
597610
err = ompi_datatype_get_extent(sdtype, &lb, &sndinc);
598611
if (OMPI_SUCCESS != err) {
599612
return err;

‎ompi/mca/coll/base/coll_base_barrier.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "ompi/mca/coll/base/coll_tags.h"
3434
#include "ompi/mca/pml/pml.h"
3535
#include "ompi/mca/coll/base/coll_base_functions.h"
36+
#include "ompi/runtime/ompi_spc.h"
3637
#include "coll_base_topo.h"
3738
#include "coll_base_util.h"
3839

@@ -109,6 +110,8 @@ int ompi_coll_base_barrier_intra_doublering(struct ompi_communicator_t *comm,
109110

110111
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"ompi_coll_base_barrier_intra_doublering rank %d", rank));
111112

113+
SPC_RECORD(OMPI_SPC_BASE_BARRIER_DOUBLE_RING, 1);
114+
112115
left = ((rank-1)%size);
113116
right = ((rank+1)%size);
114117

@@ -182,6 +185,8 @@ int ompi_coll_base_barrier_intra_recursivedoubling(struct ompi_communicator_t *c
182185
"ompi_coll_base_barrier_intra_recursivedoubling rank %d",
183186
rank));
184187

188+
SPC_RECORD(OMPI_SPC_BASE_BARRIER_RECURSIVE_DOUBLING, 1);
189+
185190
/* do nearest power of 2 less than size calc */
186191
adjsize = opal_next_poweroftwo(size);
187192
adjsize >>= 1;
@@ -262,6 +267,8 @@ int ompi_coll_base_barrier_intra_bruck(struct ompi_communicator_t *comm,
262267
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
263268
"ompi_coll_base_barrier_intra_bruck rank %d", rank));
264269

270+
SPC_RECORD(OMPI_SPC_BASE_BARRIER_BRUCK, 1);
271+
265272
/* exchange data with rank-2^k and rank+2^k */
266273
for (distance = 1; distance < size; distance <<= 1) {
267274
from = (rank + size - distance) % size;
@@ -304,6 +311,8 @@ int ompi_coll_base_barrier_intra_two_procs(struct ompi_communicator_t *comm,
304311
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,
305312
"ompi_coll_base_barrier_intra_two_procs rank %d", remote));
306313

314+
SPC_RECORD(OMPI_SPC_BASE_BARRIER_TWO_PROCS, 1);
315+
307316
remote = (remote + 1) & 0x1;
308317

309318
err = ompi_coll_base_sendrecv_zero(remote, MCA_COLL_BASE_TAG_BARRIER,
@@ -338,6 +347,8 @@ int ompi_coll_base_barrier_intra_basic_linear(struct ompi_communicator_t *comm,
338347
return MPI_SUCCESS;
339348
rank = ompi_comm_rank(comm);
340349

350+
SPC_RECORD(OMPI_SPC_BASE_BARRIER_LINEAR, 1);
351+
341352
/* All non-root send & receive zero-length message. */
342353
if (rank > 0) {
343354
err = MCA_PML_CALL(send (NULL, 0, MPI_BYTE, 0,
@@ -414,6 +425,8 @@ int ompi_coll_base_barrier_intra_tree(struct ompi_communicator_t *comm,
414425
"ompi_coll_base_barrier_intra_tree %d",
415426
rank));
416427

428+
SPC_RECORD(OMPI_SPC_BASE_BARRIER_TREE, 1);
429+
417430
/* Find the nearest power of 2 of the communicator size. */
418431
depth = opal_next_poweroftwo_inclusive(size);
419432

‎ompi/mca/coll/base/coll_base_bcast.c‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include "ompi/mca/coll/base/coll_tags.h"
3232
#include "ompi/mca/pml/pml.h"
3333
#include "ompi/mca/coll/base/coll_base_functions.h"
34+
#include "ompi/runtime/ompi_spc.h"
3435
#include "coll_base_topo.h"
3536
#include "coll_base_util.h"
3637

@@ -265,6 +266,8 @@ ompi_coll_base_bcast_intra_bintree ( void* buffer,
265266
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_binary rank %d ss %5d typelng %lu segcount %d",
266267
ompi_comm_rank(comm), segsize, (unsigned long)typelng, segcount));
267268

269+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_BCAST_BINTREE, count * datatype->super.size, ompi_comm_size(comm));
270+
268271
return ompi_coll_base_bcast_intra_generic( buffer, count, datatype, root, comm, module,
269272
segcount, data->cached_bintree );
270273
}
@@ -293,6 +296,8 @@ ompi_coll_base_bcast_intra_pipeline( void* buffer,
293296
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_pipeline rank %d ss %5d typelng %lu segcount %d",
294297
ompi_comm_rank(comm), segsize, (unsigned long)typelng, segcount));
295298

299+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_BCAST_PIPELINE, count*typelng, ompi_comm_size(comm));
300+
296301
return ompi_coll_base_bcast_intra_generic( buffer, count, datatype, root, comm, module,
297302
segcount, data->cached_pipeline );
298303
}
@@ -321,6 +326,8 @@ ompi_coll_base_bcast_intra_chain( void* buffer,
321326
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_chain rank %d fo %d ss %5d typelng %lu segcount %d",
322327
ompi_comm_rank(comm), chains, segsize, (unsigned long)typelng, segcount));
323328

329+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_BCAST_CHAIN, count*typelng, ompi_comm_size(comm));
330+
324331
return ompi_coll_base_bcast_intra_generic( buffer, count, datatype, root, comm, module,
325332
segcount, data->cached_chain );
326333
}
@@ -349,6 +356,8 @@ ompi_coll_base_bcast_intra_binomial( void* buffer,
349356
OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"coll:base:bcast_intra_binomial rank %d ss %5d typelng %lu segcount %d",
350357
ompi_comm_rank(comm), segsize, (unsigned long)typelng, segcount));
351358

359+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_BCAST_BINOMIAL, count*typelng, ompi_comm_size(comm));
360+
352361
return ompi_coll_base_bcast_intra_generic( buffer, count, datatype, root, comm, module,
353362
segcount, data->cached_bmtree );
354363
}
@@ -388,6 +397,8 @@ ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
388397

389398
err = ompi_datatype_type_size( datatype, &type_size );
390399

400+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_BCAST_SPLIT_BINTREE, count*type_size, ompi_comm_size(comm));
401+
391402
/* Determine number of segments and number of elements per segment */
392403
counts[0] = count/2;
393404
if (count % 2 != 0) counts[0]++;
@@ -685,6 +696,11 @@ ompi_coll_base_bcast_intra_basic_linear(void *buff, int count,
685696
ompi_coll_base_free_reqs(reqs, i);
686697
}
687698

699+
size_t typelng;
700+
ompi_datatype_type_size( datatype, &typelng );
701+
702+
SPC_COLL_BIN_RECORD(OMPI_SPC_BASE_BCAST_LINEAR, count*typelng, size);
703+
688704
/* All done */
689705
return err;
690706
}

0 commit comments

Comments
 (0)