diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..bfaf303b3db56fd2119b3b42c65f4d464adcfa1d
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,5 @@
+mpistalls: mpistalls.c mpiuse.c atomic.h cycle.h
+ $(CC) -g -O0 -o mpistalls mpistalls.c mpiuse.c -I/usr/include/mpi -lmpi -lpthread
+
+clean:
+ rm mpistalls
diff --git a/atomic.h b/atomic.h
new file mode 100644
index 0000000000000000000000000000000000000000..1f6ccdf9bc2d81f1f2d566fcdabf6d8777be13de
--- /dev/null
+++ b/atomic.h
@@ -0,0 +1,31 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_ATOMIC_H
+#define SWIFT_ATOMIC_H
+
+#define atomic_add(v, i) __sync_fetch_and_add(v, i)
+#define atomic_sub(v, i) __sync_fetch_and_sub(v, i)
+#define atomic_or(v, i) __sync_fetch_and_or(v, i)
+#define atomic_and(v, i) __sync_fetch_and_and(v, i)
+#define atomic_inc(v) atomic_add(v, 1)
+#define atomic_dec(v) atomic_sub(v, 1)
+#define atomic_cas(v, o, n) __sync_val_compare_and_swap(v, o, n)
+#define atomic_swap(v, n) __sync_lock_test_and_set(v, n)
+
+#endif
diff --git a/error.h b/error.h
new file mode 100644
index 0000000000000000000000000000000000000000..e1433b663aa6e6c0d08dcf3d10c600b08576e611
--- /dev/null
+++ b/error.h
@@ -0,0 +1,16 @@
+#include <mpi.h>
+
+/* Exit in error macro. */
+#define error(s, ...) \
+ ({ \
+ fflush(stdout); \
+ fprintf(stderr, "%s:%s():%i: " s "\n", __FILE__, __FUNCTION__, \
+ __LINE__, ##__VA_ARGS__); \
+ MPI_Abort(MPI_COMM_WORLD, -1); \
+ })
+
+/* Print a message */
+#define message(s, ...) \
+ ({ \
+ printf("%s: " s "\n", __FUNCTION__, ##__VA_ARGS__); \
+ })
diff --git a/inject.c b/inject.c
new file mode 100644
index 0000000000000000000000000000000000000000..b819e0f3bfbf726d87910c382af5483d35a5534e
--- /dev/null
+++ b/inject.c
@@ -0,0 +1,13 @@
+#include <stdio.h>
+
+
+
+
+
+
+/* Injection of MPI_Isends and MPI_Irecvs into the queues. */
+void inject_one(void) {
+}
+
+
+
diff --git a/mpistalls.c b/mpistalls.c
index 73ecafa51e2b27d5c4718830ba4983e550ebeeff..95312a8d8069981e831d71e1e5266e5727c7c28f 100644
--- a/mpistalls.c
+++ b/mpistalls.c
@@ -10,35 +10,36 @@
#include <mpi.h>
#include <pthread.h>
-/* Exit in error macro. */
-#define error(s, ...) \
- ({ \
- fflush(stdout); \
- fprintf(stderr, "%s:%s():%i: " s "\n", __FILE__, __FUNCTION__, \
- __LINE__, ##__VA_ARGS__); \
- MPI_Abort(MPI_COMM_WORLD, -1); \
- })
+#include "error.h"
+#include "mpiuse.h"
+
/* Injection thread, initiates MPI_Isend and MPI_Irecv requests at various
* times. */
void *inject_thread(void *arg) {
- fprintf(stderr, "%d: injection thread starts\n", *((int *)arg));
+ message("%d: injection thread starts", *((int *)arg));
- while (1) {};
+ while (1) {
+ //inject_one();
+ }
}
/* Send thread, checks if MPI_Isend requests have completed. */
void *send_thread(void *arg) {
- fprintf(stderr, "%d: send thread starts\n", *((int *)arg));
+ message("%d: send thread starts", *((int *)arg));
- while (1) {};
+ while (1) {
+ //send_test();
+ }
}
/* Recv thread, checks if MPI_Irecv requests have completed. */
void *recv_thread(void *arg) {
- fprintf(stderr, "%d: recv thread starts\n", *((int *)arg));
+ message("%d: recv thread starts", *((int *)arg));
- while (1) {};
+ while (1) {
+ //recv_test();
+ }
}
@@ -60,7 +61,7 @@ int main(int argc, char *argv[]) {
if (res != MPI_SUCCESS)
error("Call to MPI_Comm_rank failed with error %i.", res);
- printf("Hello from rank: %d\n", myrank);
+ message("Hello from rank: %d", myrank);
/* Make three threads, one for injecting tasks and two to check for
* completions of the sends and recv independently. */
@@ -74,6 +75,11 @@ int main(int argc, char *argv[]) {
if (pthread_create(&recvthread, NULL, &recv_thread, &myrank) != 0)
error("Failed to create recv thread.");
+
+ /* Read the MPI logger output that defines the communcations we will
+ * undertake and the time differences between injections into the queues. */
+ mpiuse_log_restore("testdata/mpiuse_report-step2.dat");
+
/* Wait until all threads have exited and all MPI requests have completed. */
pthread_join(injectthread, NULL);
pthread_join(sendthread, NULL);
diff --git a/mpiuse.c b/mpiuse.c
new file mode 100644
index 0000000000000000000000000000000000000000..eb6aef116f53c8ebbc75208a60991196675222b3
--- /dev/null
+++ b/mpiuse.c
@@ -0,0 +1,201 @@
+/* This file is part of SWIFT.
+ * Copyright (c) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/**
+ * @file mpiuse.c
+ * @brief file of routines to report about MPI tasks used in SWIFT.
+ */
+
+/* Standard includes. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Local defines. */
+#include "mpiuse.h"
+
+/* Local includes. */
+#include "atomic.h"
+#include "cycle.h"
+#include "error.h"
+
+/* The initial size and increment of the log entries buffer. */
+#define MPIUSE_INITLOG 1000000
+
+/* Entry for logger of MPI send and recv requests in a step. */
+struct mpiuse_log_entry {
+
+ /* Rank of entry. */
+ int rank;
+
+ /* Type and subtype of MPI task. */
+ int type;
+ int subtype;
+
+ /* Step of action. */
+ int step;
+
+ /* Whether an activation, send or recv, or if handoff completed. Not the
+ * same as delivered, need to match across ranks to see that. */
+ int activation;
+
+ /* Memory of the request. */
+ size_t size;
+
+ /* Pointer to the request associated with the call. Needs to be
+ * unique and match to the successful */
+ union {
+ void *ptr;
+ uint8_t vptr[sizeof(uintptr_t)]; /* For rnode keys. */
+ };
+
+ /* Ticks at time of this action. */
+ ticks tic;
+
+ /* Rank of otherside of communication. */
+ int otherrank;
+
+ /* The tag. */
+ int tag;
+};
+
+/* The log of activations and handoffs. All volatile as accessed from threads
+ * that use the value to synchronise. */
+static struct mpiuse_log_entry *volatile mpiuse_log = NULL;
+static volatile size_t mpiuse_log_size = 0;
+static volatile size_t mpiuse_log_count = 0;
+static volatile size_t mpiuse_log_done = 0;
+
+/**
+ * @brief reallocate the entries log if space is needed.
+ */
+static void mpiuse_log_reallocate(size_t ind) {
+
+ if (ind == 0) {
+
+ /* Need to perform initialization. Be generous. */
+ if ((mpiuse_log = (struct mpiuse_log_entry *)malloc(
+ sizeof(struct mpiuse_log_entry) * MPIUSE_INITLOG)) == NULL)
+ error("Failed to allocate MPI use log.");
+
+ /* Last action. */
+ mpiuse_log_size = MPIUSE_INITLOG;
+
+ } else {
+ struct mpiuse_log_entry *new_log;
+ if ((new_log = (struct mpiuse_log_entry *)malloc(
+ sizeof(struct mpiuse_log_entry) *
+ (mpiuse_log_size + MPIUSE_INITLOG))) == NULL)
+ error("Failed to re-allocate MPI use log.");
+
+ /* Wait for all writes to the old buffer to complete. */
+ while (mpiuse_log_done < mpiuse_log_size)
+ ;
+
+ /* Copy to new buffer. */
+ memcpy(new_log, mpiuse_log,
+ sizeof(struct mpiuse_log_entry) * mpiuse_log_size);
+ free(mpiuse_log);
+ mpiuse_log = new_log;
+
+ /* Last action, releases waiting threads. */
+ atomic_add(&mpiuse_log_size, MPIUSE_INITLOG);
+ }
+}
+
+/**
+ * @brief Log an MPI request or handoff.
+ *
+ * @param rank the rank
+ * @param step the step
+ * @param tic the ticks at time of log, will be relative.
+ * @param type the task type (send or recv).
+ * @param subtype the task subtype.
+ * @param activation if not is a successful MPI_Test, not MPI_Isend or
+ * MPI_Irecv.
+ * @param size the size in bytes of memory to be transfered or received.
+ * 0 for a deactivation.
+ * @param otherrank other rank associated with the transfer.
+ * @param tag the MPI tag.
+ */
+void mpiuse_log_allocation(int rank, int step, size_t tic, int type,
+ int subtype, int activation, size_t size,
+ int otherrank, int tag) {
+
+ size_t ind = atomic_inc(&mpiuse_log_count);
+
+ /* If we are at the current size we need more space. */
+ if (ind == mpiuse_log_size) mpiuse_log_reallocate(ind);
+
+ /* Other threads wait for space. */
+ while (ind > mpiuse_log_size)
+ ;
+
+ /* Record the log. */
+ mpiuse_log[ind].rank = rank;
+ mpiuse_log[ind].step = step;
+ mpiuse_log[ind].type = type;
+ mpiuse_log[ind].subtype = subtype;
+ mpiuse_log[ind].activation = activation;
+ mpiuse_log[ind].size = size;
+ mpiuse_log[ind].ptr = NULL;
+ mpiuse_log[ind].otherrank = otherrank;
+ mpiuse_log[ind].tag = tag;
+ mpiuse_log[ind].tic = tic;
+
+ atomic_inc(&mpiuse_log_done);
+}
+
+/**
+ * @brief restore the log from a dump.
+ *
+ * @param filename name of file with the previus dump in.
+ */
+void mpiuse_log_restore(const char *filename) {
+
+ /* Open the input file. */
+ FILE *fd;
+ if ((fd = fopen(filename, "r")) == NULL) {
+ message("Failed to open the MPI use log file '%s'.",
+ filename);
+ return;
+ }
+
+
+ /* Read until the end of the file is reached.*/
+ char line[132];
+ size_t stic, etic, dtic, size, sum;
+ int step, rank, otherrank, itype, isubtype, activation, tag;
+ char type[32], subtype[32];
+
+ while (!feof(fd)) {
+ if (fgets(line, 132, fd) != NULL) {
+ if (line[0] != '#') {
+ sscanf(line, "%zd %zd %zd %d %d %d %s %d %s %d %d %d %zd %zd",
+ &stic, &etic, &dtic, &step, &rank, &otherrank, type, &itype,
+ subtype, &isubtype, &activation, &tag, &size, &sum);
+
+ mpiuse_log_allocation(rank, step, stic, itype, isubtype, activation,
+ size, otherrank, tag);
+
+ }
+ }
+ }
+ fclose(fd);
+}
+
diff --git a/mpiuse.h b/mpiuse.h
new file mode 100644
index 0000000000000000000000000000000000000000..cd5fff043a7f5c513ebafb08292d5a92a050bbd9
--- /dev/null
+++ b/mpiuse.h
@@ -0,0 +1,28 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_MPIUSE_H
+#define SWIFT_MPIUSE_H
+
+/* API. */
+void mpiuse_log_allocation(int rank, int step, size_t tic, int type,
+ int subtype, int activation, size_t size,
+ int otherrank, int tag);
+
+void mpiuse_log_restore(const char *filename);
+#endif /* SWIFT_MPIUSE_H */
diff --git a/readlog.c b/readlog.c
new file mode 100644
index 0000000000000000000000000000000000000000..376357214c0397501ac164803824bc283dc0826e
--- /dev/null
+++ b/readlog.c
@@ -0,0 +1,12 @@
+#include <stdio.h>
+
+
+/**
+ * @brief read the output from a run of SWIFT with the MPI communication
+ * logger enabled.
+ *
+void mpistalls_readlog(void) {
+
+
+
+}
diff --git a/testdata/mpiuse_report-rank0-step2.dat b/testdata/mpiuse_report-rank0-step2.dat
new file mode 100644
index 0000000000000000000000000000000000000000..51043d68d599e8c7f0d52bec0a9740fd0ec1f0b1
--- /dev/null
+++ b/testdata/mpiuse_report-rank0-step2.dat
@@ -0,0 +1,91 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+608817 21543633709 0 2 0 2 recv 23 xv 11 1 445 142720 142720
+610727 21543635619 0 2 0 3 recv 23 xv 11 1 453 311296 454016
+612045 21543636937 0 2 0 2 recv 23 xv 11 1 446 219648 673664
+613550 21543638442 0 2 0 3 recv 23 xv 11 1 38 449024 1122688
+615327 21543640219 0 2 0 3 recv 23 xv 11 1 62 282624 1405312
+616285 21543641177 0 2 0 1 recv 23 xv 11 1 177 379648 1784960
+618295 21543643187 0 2 0 3 recv 23 xv 11 1 50 400128 2185088
+619247 21543644139 0 2 0 2 recv 23 xv 11 1 160 94208 2279296
+620661 21543645553 0 2 0 3 recv 23 xv 11 1 49 318720 2598016
+623155 21543648047 0 2 0 2 recv 23 xv 11 1 158 131072 2729088
+625506 21543650398 0 2 0 3 recv 23 xv 11 1 40 416512 3145600
+627165 21543652057 0 2 0 2 recv 23 xv 11 1 156 111360 3256960
+629619 21543654511 0 2 0 3 recv 23 xv 11 1 67 312448 3569408
+631490 21543656382 0 2 0 3 recv 23 xv 11 1 65 292096 3861504
+632856 21543657748 0 2 0 3 recv 23 xv 11 1 465 77696 3939200
+635292 21543660184 0 2 0 2 recv 23 xv 11 1 3 583168 4522368
+636746 21543661638 0 2 0 2 recv 23 xv 11 1 2 107136 4629504
+639251 21543664143 0 2 0 3 recv 23 xv 11 1 130 711424 5340928
+640726 21543665618 0 2 0 1 recv 23 xv 11 1 118 553984 5894912
+642551 21543667443 0 2 0 1 recv 23 xv 11 1 45 328576 6223488
+644510 21543669402 0 2 0 3 recv 23 xv 11 1 192 345984 6569472
+646036 21543670928 0 2 0 1 recv 23 xv 11 1 44 1056768 7626240
+647586 21543672478 0 2 0 3 recv 23 xv 11 1 230 255232 7881472
+649599 21543674491 0 2 0 1 recv 23 xv 11 1 194 143872 8025344
+651388 21543676280 0 2 0 1 recv 23 xv 11 1 202 360832 8386176
+653752 21543678644 0 2 0 1 recv 23 xv 11 1 192 1066112 9452288
+655523 21543680415 0 2 0 2 recv 23 xv 11 1 249 28544 9480832
+657245 21543682137 0 2 0 2 recv 23 xv 11 1 245 83840 9564672
+659152 21543684044 0 2 0 2 recv 23 xv 11 1 244 78720 9643392
+661120 21543686012 0 2 0 1 recv 23 xv 11 1 205 547968 10191360
+664106 21543688998 0 2 0 3 recv 23 xv 11 1 200 368000 10559360
+666953 21543691845 0 2 0 1 recv 23 xv 11 1 137 212608 10771968
+668385 21543693277 0 2 0 3 recv 23 xv 11 1 137 418304 11190272
+669960 21543694852 0 2 0 3 recv 23 xv 11 1 136 199808 11390080
+671970 21543696862 0 2 0 1 recv 23 xv 11 1 57 332160 11722240
+6447305 21549472197 5791782 2 0 2 recv 23 xv 11 0 249 -28544 11693696
+11419335 21554444227 10796180 2 0 2 recv 23 xv 11 0 158 -131072 11562624
+11758915 21554783807 11111329 2 0 3 recv 23 xv 11 0 230 -255232 11307392
+11799763 21554824655 11172598 2 0 2 recv 23 xv 11 0 156 -111360 11196032
+11838133 21554863025 11222806 2 0 3 recv 23 xv 11 0 62 -282624 10913408
+11910866 21554935758 11253621 2 0 2 recv 23 xv 11 0 245 -83840 10829568
+13014189 21556039081 12405372 2 0 2 recv 23 xv 11 0 445 -142720 10686848
+13215819 21556240711 12580527 2 0 2 recv 23 xv 11 0 3 -583168 10103680
+13730903 21556755795 13111656 2 0 2 recv 23 xv 11 0 160 -94208 10009472
+14435788 21557460680 13776636 2 0 2 recv 23 xv 11 0 244 -78720 9930752
+15491583 21558516475 14823198 2 0 3 recv 23 xv 11 0 137 -418304 9512448
+15614696 21558639588 14981840 2 0 3 recv 23 xv 11 0 465 -77696 9434752
+15728921 21558753813 15116876 2 0 2 recv 23 xv 11 0 446 -219648 9215104
+16170362 21559195254 15533616 2 0 2 recv 23 xv 11 0 2 -107136 9107968
+16572275 21559597167 15961548 2 0 3 recv 23 xv 11 0 453 -311296 8796672
+16928290 21559953182 16289039 2 0 3 recv 23 xv 11 0 130 -711424 8085248
+28452779 21571477671 27827273 2 0 3 recv 23 xv 11 0 40 -416512 7668736
+28707936 21571732828 28054184 2 0 1 recv 23 xv 11 0 192 -1066112 6602624
+28977681 21572002573 28364131 2 0 3 recv 23 xv 11 0 38 -449024 6153600
+30058812 21573083704 29440517 2 0 3 recv 23 xv 11 0 50 -400128 5753472
+30160189 21573185081 29539528 2 0 3 recv 23 xv 11 0 49 -318720 5434752
+30309729 21573334621 29639769 2 0 3 recv 23 xv 11 0 136 -199808 5234944
+30514052 21573538944 29849946 2 0 3 recv 23 xv 11 0 200 -368000 4866944
+31055012 21574079904 30410502 2 0 3 recv 23 xv 11 0 192 -345984 4520960
+38056623 21581081515 37427004 2 0 3 recv 23 xv 11 0 67 -312448 4208512
+38085547 21581110439 37454057 2 0 3 recv 23 xv 11 0 65 -292096 3916416
+38835374 21581860266 38189338 2 0 1 recv 23 xv 11 0 44 -1056768 2859648
+40237431 21583262323 39586043 2 0 1 recv 23 xv 11 0 202 -360832 2498816
+40553084 21583577976 39891964 2 0 1 recv 23 xv 11 0 205 -547968 1950848
+40777829 21583802721 40161544 2 0 1 recv 23 xv 11 0 177 -379648 1571200
+40836656 21583861548 40164686 2 0 1 recv 23 xv 11 0 57 -332160 1239040
+40873449 21583898341 40206496 2 0 1 recv 23 xv 11 0 137 -212608 1026432
+41885432 21584910324 0 2 0 2 send 22 tend_part 7 1 38 2328 1028760
+41896300 21584921192 0 2 0 3 send 22 tend_part 7 1 38 2328 1031088
+41942824 21584967716 57392 2 0 2 send 22 tend_part 7 0 38 -2328 1028760
+41950298 21584975190 53998 2 0 3 send 22 tend_part 7 0 38 -2328 1026432
+43257053 21586281945 42607454 2 0 1 recv 23 xv 11 0 194 -143872 882560
+43529999 21586554891 42887448 2 0 1 recv 23 xv 11 0 45 -328576 553984
+43591111 21586616003 42950385 2 0 1 recv 23 xv 11 0 118 -553984 0
+48599962 21591624854 0 2 0 2 send 22 tend_part 7 1 138 2712 2712
+48619658 21591644550 0 2 0 1 send 22 tend_part 7 1 138 2712 5424
+48642246 21591667138 0 2 0 3 send 22 tend_part 7 1 138 2712 8136
+48695836 21591720728 53590 2 0 3 send 22 tend_part 7 0 138 -2712 5424
+48704250 21591729142 104288 2 0 2 send 22 tend_part 7 0 138 -2712 2712
+48924980 21591949872 305322 2 0 1 send 22 tend_part 7 0 138 -2712 0
+48963599 21591988491 0 2 0 1 send 22 tend_part 7 1 192 6936 6936
+48968850 21591993742 5251 2 0 1 send 22 tend_part 7 0 192 -6936 0
+48990086 21592014978 0 2 0 3 send 22 tend_part 7 1 192 6936 6936
+48995836 21592020728 5750 2 0 3 send 22 tend_part 7 0 192 -6936 0
+##
+## Number of requests: 42
+## Maximum request size: 1.0167 (MB)
+## Sum of all requests: 11.2046 (MB)
+## Mean of all requests: 0.2668 (MB)
+##
diff --git a/testdata/mpiuse_report-rank1-step2.dat b/testdata/mpiuse_report-rank1-step2.dat
new file mode 100644
index 0000000000000000000000000000000000000000..964543c8084bab9769faae1aec76523160064cc7
--- /dev/null
+++ b/testdata/mpiuse_report-rank1-step2.dat
@@ -0,0 +1,57 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+12953800 21558567216 0 2 1 3 recv 23 tend_part 7 1 32 1368 1368
+12964804 21558578220 0 2 1 0 recv 23 tend_part 7 1 138 2712 4080
+12970634 21558584050 0 2 1 0 recv 23 tend_part 7 1 192 6936 11016
+12983064 21558596480 0 2 1 2 recv 23 xv 11 1 321 310784 321800
+12985183 21558598599 0 2 1 3 recv 23 xv 11 1 440 876032 1197832
+12986818 21558600234 0 2 1 3 recv 23 xv 11 1 403 171136 1368968
+12989387 21558602803 0 2 1 2 recv 23 xv 11 1 395 487040 1856008
+12993612 21558607028 0 2 1 2 recv 23 xv 11 1 383 146048 2002056
+12998091 21558611507 0 2 1 2 recv 23 xv 11 1 322 235904 2237960
+18459228 21564072644 0 2 1 0 send 22 xv 11 1 192 1066112 3304072
+23740217 21569353633 0 2 1 0 send 22 xv 11 1 44 1056768 4360840
+25940939 21571554355 0 2 1 0 send 22 xv 11 1 205 547968 4908808
+28186716 21573800132 0 2 1 0 send 22 xv 11 1 177 379648 5288456
+29239247 21574852663 0 2 1 3 send 22 xv 11 1 453 282752 5571208
+30275585 21575889001 17281973 2 1 2 recv 23 xv 11 0 383 -146048 5425160
+30291458 21575904874 0 2 1 0 send 22 xv 11 1 202 360832 5785992
+33012165 21578625581 0 2 1 0 send 22 xv 11 1 57 332160 6118152
+33756463 21579369879 20771280 2 1 3 recv 23 xv 11 0 440 -876032 5242120
+33892572 21579505988 20903185 2 1 2 recv 23 xv 11 0 395 -487040 4755080
+33925103 21579538519 0 2 1 0 send 22 xv 11 1 137 212608 4967688
+34725100 21580338516 0 2 1 0 send 22 xv 11 1 194 143872 5111560
+36049973 21581663389 0 2 1 0 send 22 xv 11 1 45 328576 5440136
+36265488 21581878904 17806260 2 1 0 send 22 xv 11 0 192 -1066112 4374024
+38193925 21583807341 0 2 1 3 send 22 xv 11 1 56 472064 4846088
+38822832 21584436248 15082615 2 1 0 send 22 xv 11 0 44 -1056768 3789320
+40220268 21585833684 12033552 2 1 0 send 22 xv 11 0 177 -379648 3409672
+40540832 21586154248 7528667 2 1 0 send 22 xv 11 0 57 -332160 3077512
+40705244 21586318660 6780141 2 1 0 send 22 xv 11 0 137 -212608 2864904
+40722144 21586335560 10430686 2 1 0 send 22 xv 11 0 202 -360832 2504072
+40735977 21586349393 14795038 2 1 0 send 22 xv 11 0 205 -547968 1956104
+42881032 21588494448 0 2 1 0 send 22 xv 11 1 118 553984 2510088
+43243043 21588856459 5049118 2 1 3 send 22 xv 11 0 56 -472064 2038024
+43254760 21588868176 14015513 2 1 3 send 22 xv 11 0 453 -282752 1755272
+43431687 21589045103 7381714 2 1 0 send 22 xv 11 0 45 -328576 1426696
+43441434 21589054850 8716334 2 1 0 send 22 xv 11 0 194 -143872 1282824
+43447599 21589061015 566567 2 1 0 send 22 xv 11 0 118 -553984 728840
+44195173 21589808589 0 2 1 3 send 22 xv 11 1 59 183552 912392
+44432889 21590046305 0 2 1 3 send 22 xv 11 1 47 46720 959112
+44593620 21590207036 160731 2 1 3 send 22 xv 11 0 47 -46720 912392
+44602907 21590216323 407734 2 1 3 send 22 xv 11 0 59 -183552 728840
+44639981 21590253397 31656917 2 1 2 recv 23 xv 11 0 321 -310784 418056
+44960058 21590573474 31961967 2 1 2 recv 23 xv 11 0 322 -235904 182152
+48461460 21594074876 35474642 2 1 3 recv 23 xv 11 0 403 -171136 11016
+48919281 21594532697 35954477 2 1 0 recv 23 tend_part 7 0 138 -2712 8304
+48984643 21594598059 36030843 2 1 3 recv 23 tend_part 7 0 32 -1368 6936
+49409002 21595022418 36438368 2 1 0 recv 23 tend_part 7 0 192 -6936 0
+56492967 21602106383 0 2 1 2 send 22 tend_part 7 1 359 2520 2520
+56507501 21602120917 14534 2 1 2 send 22 tend_part 7 0 359 -2520 0
+56512908 21602126324 0 2 1 3 send 22 tend_part 7 1 359 2520 2520
+56518112 21602131528 5204 2 1 3 send 22 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 25
+## Maximum request size: 1.0167 (MB)
+## Sum of all requests: 7.8303 (MB)
+## Mean of all requests: 0.3132 (MB)
+##
diff --git a/testdata/mpiuse_report-rank2-step2.dat b/testdata/mpiuse_report-rank2-step2.dat
new file mode 100644
index 0000000000000000000000000000000000000000..b5c0e6e44f823aa741d22eacd87fddb78870fcfd
--- /dev/null
+++ b/testdata/mpiuse_report-rank2-step2.dat
@@ -0,0 +1,53 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+987251 21546628055 0 2 2 3 recv 23 tend_part 7 1 32 1368 1368
+1025615 21546666419 0 2 2 0 recv 23 tend_part 7 1 38 2328 3696
+1029326 21546670130 0 2 2 0 recv 23 tend_part 7 1 138 2712 6408
+1038964 21546679768 0 2 2 1 recv 23 tend_part 7 1 359 2520 8928
+1304769 21546945573 0 2 2 3 send 22 xv 11 1 444 98816 107744
+1404115 21547044919 0 2 2 3 send 22 xv 11 1 2 107136 214880
+1406430 21547047234 0 2 2 0 send 22 xv 11 1 2 107136 322016
+1892700 21547533504 0 2 2 0 send 22 xv 11 1 446 219648 541664
+3170516 21548811320 0 2 2 1 send 22 xv 11 1 395 487040 1028704
+3551697 21549192501 0 2 2 0 send 22 xv 11 1 3 583168 1611872
+3608675 21549249479 0 2 2 1 send 22 xv 11 1 383 146048 1757920
+3722238 21549363042 0 2 2 0 send 22 xv 11 1 445 142720 1900640
+4474542 21550115346 0 2 2 1 send 22 xv 11 1 322 235904 2136544
+4660091 21550300895 0 2 2 1 send 22 xv 11 1 321 310784 2447328
+4802628 21550443432 0 2 2 0 send 22 xv 11 1 249 28544 2475872
+4947390 21550588194 144762 2 2 0 send 22 xv 11 0 249 -28544 2447328
+7943938 21553584742 6537508 2 2 0 send 22 xv 11 0 2 -107136 2340192
+7961657 21553602461 6068957 2 2 0 send 22 xv 11 0 446 -219648 2120544
+7971824 21553612628 4420127 2 2 0 send 22 xv 11 0 3 -583168 1537376
+8524640 21554165444 0 2 2 0 send 22 xv 11 1 158 131072 1668448
+8974356 21554615160 0 2 2 0 send 22 xv 11 1 156 111360 1779808
+9378345 21555019149 0 2 2 0 send 22 xv 11 1 160 94208 1874016
+9846859 21555487663 0 2 2 0 send 22 xv 11 1 245 83840 1957856
+10082623 21555723427 6360385 2 2 0 send 22 xv 11 0 445 -142720 1815136
+10425140 21556065944 0 2 2 3 send 22 xv 11 1 1 64512 1879648
+11258279 21556899083 1411420 2 2 0 send 22 xv 11 0 245 -83840 1795808
+12594280 21558235084 2169140 2 2 3 send 22 xv 11 0 1 -64512 1731296
+12624810 21558265614 11220695 2 2 3 send 22 xv 11 0 2 -107136 1624160
+12806376 21558447180 3428031 2 2 0 send 22 xv 11 0 160 -94208 1529952
+12864222 21558505026 3889866 2 2 0 send 22 xv 11 0 156 -111360 1418592
+12923342 21558564146 11618573 2 2 3 send 22 xv 11 0 444 -98816 1319776
+13069781 21558710585 4545141 2 2 0 send 22 xv 11 0 158 -131072 1188704
+13360446 21559001250 0 2 2 3 send 22 xv 11 1 30 79232 1267936
+13833681 21559474485 0 2 2 3 send 22 xv 11 1 44 64128 1332064
+13842742 21559483546 9061 2 2 3 send 22 xv 11 0 44 -64128 1267936
+14347042 21559987846 0 2 2 0 send 22 xv 11 1 244 78720 1346656
+14447458 21560088262 100416 2 2 0 send 22 xv 11 0 244 -78720 1267936
+14549561 21560190365 1189115 2 2 3 send 22 xv 11 0 30 -79232 1188704
+29619186 21575259990 26448670 2 2 1 send 22 xv 11 0 395 -487040 701664
+29973620 21575614424 26364945 2 2 1 send 22 xv 11 0 383 -146048 555616
+30232105 21575872909 25757563 2 2 1 send 22 xv 11 0 322 -235904 319712
+30884935 21576525739 26224844 2 2 1 send 22 xv 11 0 321 -310784 8928
+41905408 21587546212 40879793 2 2 0 recv 23 tend_part 7 0 38 -2328 6600
+48456241 21594097045 47468990 2 2 3 recv 23 tend_part 7 0 32 -1368 5232
+48605647 21594246451 47576321 2 2 0 recv 23 tend_part 7 0 138 -2712 2520
+56499592 21602140396 55460628 2 2 1 recv 23 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 23
+## Maximum request size: 0.5562 (MB)
+## Sum of all requests: 3.0355 (MB)
+## Mean of all requests: 0.1320 (MB)
+##
diff --git a/testdata/mpiuse_report-rank3-step2.dat b/testdata/mpiuse_report-rank3-step2.dat
new file mode 100644
index 0000000000000000000000000000000000000000..2c17b6d4bcea8e7ef7184f1772c65390d8a86e46
--- /dev/null
+++ b/testdata/mpiuse_report-rank3-step2.dat
@@ -0,0 +1,71 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+8154566 21553806876 0 2 3 0 recv 23 tend_part 7 1 38 2328 2328
+8241482 21553893792 0 2 3 0 recv 23 tend_part 7 1 138 2712 5040
+8252462 21553904772 0 2 3 1 recv 23 xv 11 1 59 183552 188592
+8255236 21553907546 0 2 3 2 recv 23 xv 11 1 44 64128 252720
+8265527 21553917837 0 2 3 0 recv 23 tend_part 7 1 192 6936 259656
+8267192 21553919502 0 2 3 1 recv 23 xv 11 1 47 46720 306376
+8281049 21553933359 0 2 3 1 recv 23 tend_part 7 1 359 2520 308896
+8285576 21553937886 0 2 3 1 recv 23 xv 11 1 56 472064 780960
+8290638 21553942948 0 2 3 1 recv 23 xv 11 1 453 282752 1063712
+8312534 21553964844 0 2 3 2 recv 23 xv 11 1 444 98816 1162528
+8314496 21553966806 0 2 3 2 recv 23 xv 11 1 30 79232 1241760
+8318186 21553970496 0 2 3 2 recv 23 xv 11 1 2 107136 1348896
+8321900 21553974210 0 2 3 2 recv 23 xv 11 1 1 64512 1413408
+8551050 21554203360 0 2 3 0 send 22 xv 11 1 465 77696 1491104
+9240343 21554892653 0 2 3 0 send 22 xv 11 1 62 282624 1773728
+9516048 21555168358 0 2 3 0 send 22 xv 11 1 230 255232 2028960
+10222671 21555874981 1910137 2 3 2 recv 23 xv 11 0 444 -98816 1930144
+11396780 21557049090 3074880 2 3 2 recv 23 xv 11 0 1 -64512 1865632
+11919637 21557571947 0 2 3 0 send 22 xv 11 1 130 711424 2577056
+12416538 21558068848 496901 2 3 0 send 22 xv 11 0 130 -711424 1865632
+12449098 21558101408 0 2 3 1 send 22 xv 11 1 403 171136 2036768
+12525243 21558177553 4207057 2 3 2 recv 23 xv 11 0 2 -107136 1929632
+13886887 21559539197 0 2 3 0 send 22 xv 11 1 453 311296 2240928
+13993363 21559645673 5738127 2 3 2 recv 23 xv 11 0 44 -64128 2176800
+14353159 21560005469 6038663 2 3 2 recv 23 xv 11 0 30 -79232 2097568
+14361689 21560013999 0 2 3 0 send 22 xv 11 1 137 418304 2515872
+15260362 21560912672 898673 2 3 0 send 22 xv 11 0 137 -418304 2097568
+16761542 21562413852 0 2 3 0 send 22 xv 11 1 200 368000 2465568
+17561805 21563214115 0 2 3 0 send 22 xv 11 1 136 199808 2665376
+17768823 21563421133 0 2 3 1 send 22 xv 11 1 440 876032 3541408
+18876067 21564528377 0 2 3 0 send 22 xv 11 1 192 345984 3887392
+19483556 21565135866 0 2 3 0 send 22 xv 11 1 40 416512 4303904
+20583795 21566236105 0 2 3 0 send 22 xv 11 1 38 449024 4752928
+21166982 21566819292 0 2 3 0 send 22 xv 11 1 49 318720 5071648
+21367099 21567019409 7480212 2 3 0 send 22 xv 11 0 453 -311296 4760352
+21428196 21567080506 11912148 2 3 0 send 22 xv 11 0 230 -255232 4505120
+21489677 21567141987 12249334 2 3 0 send 22 xv 11 0 62 -282624 4222496
+21545174 21567197484 12994124 2 3 0 send 22 xv 11 0 465 -77696 4144800
+27653028 21573305338 10891486 2 3 0 send 22 xv 11 0 200 -368000 3776800
+27725141 21573377451 10163336 2 3 0 send 22 xv 11 0 136 -199808 3576992
+27785208 21573437518 8909141 2 3 0 send 22 xv 11 0 192 -345984 3231008
+27844800 21573497110 8361244 2 3 0 send 22 xv 11 0 40 -416512 2814496
+27904150 21573556460 7320355 2 3 0 send 22 xv 11 0 38 -449024 2365472
+29544767 21575197077 0 2 3 0 send 22 xv 11 1 50 400128 2765600
+29899938 21575552248 355171 2 3 0 send 22 xv 11 0 50 -400128 2365472
+29964381 21575616691 8797399 2 3 0 send 22 xv 11 0 49 -318720 2046752
+31237607 21576889917 0 2 3 0 send 22 xv 11 1 67 312448 2359200
+32076277 21577728587 23785639 2 3 1 recv 23 xv 11 0 453 -282752 2076448
+32997616 21578649926 20548518 2 3 1 send 22 xv 11 0 403 -171136 1905312
+33045540 21578697850 15276717 2 3 1 send 22 xv 11 0 440 -876032 1029280
+35555962 21581208272 0 2 3 0 send 22 xv 11 1 65 292096 1321376
+37724703 21583377013 6487096 2 3 0 send 22 xv 11 0 67 -312448 1008928
+38038978 21583691288 2483016 2 3 0 send 22 xv 11 0 65 -292096 716832
+39312145 21584964455 31026569 2 3 1 recv 23 xv 11 0 56 -472064 244768
+41913009 21587565319 33758443 2 3 0 recv 23 tend_part 7 0 38 -2328 242440
+44653176 21590305486 36385984 2 3 1 recv 23 xv 11 0 47 -46720 195720
+44672181 21590324491 36419719 2 3 1 recv 23 xv 11 0 59 -183552 12168
+48439936 21594092246 0 2 3 2 send 22 tend_part 7 1 32 1368 13536
+48448809 21594101119 0 2 3 1 send 22 tend_part 7 1 32 1368 14904
+48521474 21594173784 81538 2 3 2 send 22 tend_part 7 0 32 -1368 13536
+48529891 21594182201 81082 2 3 1 send 22 tend_part 7 0 32 -1368 12168
+48659245 21594311555 40417763 2 3 0 recv 23 tend_part 7 0 138 -2712 9456
+48993631 21594645941 40728104 2 3 0 recv 23 tend_part 7 0 192 -6936 2520
+56516622 21602168932 48235573 2 3 1 recv 23 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 32
+## Maximum request size: 0.8354 (MB)
+## Sum of all requests: 7.2695 (MB)
+## Mean of all requests: 0.2272 (MB)
+##
diff --git a/testdata/mpiuse_report-step2.dat b/testdata/mpiuse_report-step2.dat
new file mode 100644
index 0000000000000000000000000000000000000000..9fe6228ddf043d8348555bdb6daa1fe5322cf66a
--- /dev/null
+++ b/testdata/mpiuse_report-step2.dat
@@ -0,0 +1,272 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+608817 21543633709 0 2 0 2 recv 23 xv 11 1 445 142720 142720
+610727 21543635619 0 2 0 3 recv 23 xv 11 1 453 311296 454016
+612045 21543636937 0 2 0 2 recv 23 xv 11 1 446 219648 673664
+613550 21543638442 0 2 0 3 recv 23 xv 11 1 38 449024 1122688
+615327 21543640219 0 2 0 3 recv 23 xv 11 1 62 282624 1405312
+616285 21543641177 0 2 0 1 recv 23 xv 11 1 177 379648 1784960
+618295 21543643187 0 2 0 3 recv 23 xv 11 1 50 400128 2185088
+619247 21543644139 0 2 0 2 recv 23 xv 11 1 160 94208 2279296
+620661 21543645553 0 2 0 3 recv 23 xv 11 1 49 318720 2598016
+623155 21543648047 0 2 0 2 recv 23 xv 11 1 158 131072 2729088
+625506 21543650398 0 2 0 3 recv 23 xv 11 1 40 416512 3145600
+627165 21543652057 0 2 0 2 recv 23 xv 11 1 156 111360 3256960
+629619 21543654511 0 2 0 3 recv 23 xv 11 1 67 312448 3569408
+631490 21543656382 0 2 0 3 recv 23 xv 11 1 65 292096 3861504
+632856 21543657748 0 2 0 3 recv 23 xv 11 1 465 77696 3939200
+635292 21543660184 0 2 0 2 recv 23 xv 11 1 3 583168 4522368
+636746 21543661638 0 2 0 2 recv 23 xv 11 1 2 107136 4629504
+639251 21543664143 0 2 0 3 recv 23 xv 11 1 130 711424 5340928
+640726 21543665618 0 2 0 1 recv 23 xv 11 1 118 553984 5894912
+642551 21543667443 0 2 0 1 recv 23 xv 11 1 45 328576 6223488
+644510 21543669402 0 2 0 3 recv 23 xv 11 1 192 345984 6569472
+646036 21543670928 0 2 0 1 recv 23 xv 11 1 44 1056768 7626240
+647586 21543672478 0 2 0 3 recv 23 xv 11 1 230 255232 7881472
+649599 21543674491 0 2 0 1 recv 23 xv 11 1 194 143872 8025344
+651388 21543676280 0 2 0 1 recv 23 xv 11 1 202 360832 8386176
+653752 21543678644 0 2 0 1 recv 23 xv 11 1 192 1066112 9452288
+655523 21543680415 0 2 0 2 recv 23 xv 11 1 249 28544 9480832
+657245 21543682137 0 2 0 2 recv 23 xv 11 1 245 83840 9564672
+659152 21543684044 0 2 0 2 recv 23 xv 11 1 244 78720 9643392
+661120 21543686012 0 2 0 1 recv 23 xv 11 1 205 547968 10191360
+664106 21543688998 0 2 0 3 recv 23 xv 11 1 200 368000 10559360
+666953 21543691845 0 2 0 1 recv 23 xv 11 1 137 212608 10771968
+668385 21543693277 0 2 0 3 recv 23 xv 11 1 137 418304 11190272
+669960 21543694852 0 2 0 3 recv 23 xv 11 1 136 199808 11390080
+671970 21543696862 0 2 0 1 recv 23 xv 11 1 57 332160 11722240
+6447305 21549472197 5791782 2 0 2 recv 23 xv 11 0 249 -28544 11693696
+11419335 21554444227 10796180 2 0 2 recv 23 xv 11 0 158 -131072 11562624
+11758915 21554783807 11111329 2 0 3 recv 23 xv 11 0 230 -255232 11307392
+11799763 21554824655 11172598 2 0 2 recv 23 xv 11 0 156 -111360 11196032
+11838133 21554863025 11222806 2 0 3 recv 23 xv 11 0 62 -282624 10913408
+11910866 21554935758 11253621 2 0 2 recv 23 xv 11 0 245 -83840 10829568
+13014189 21556039081 12405372 2 0 2 recv 23 xv 11 0 445 -142720 10686848
+13215819 21556240711 12580527 2 0 2 recv 23 xv 11 0 3 -583168 10103680
+13730903 21556755795 13111656 2 0 2 recv 23 xv 11 0 160 -94208 10009472
+14435788 21557460680 13776636 2 0 2 recv 23 xv 11 0 244 -78720 9930752
+15491583 21558516475 14823198 2 0 3 recv 23 xv 11 0 137 -418304 9512448
+15614696 21558639588 14981840 2 0 3 recv 23 xv 11 0 465 -77696 9434752
+15728921 21558753813 15116876 2 0 2 recv 23 xv 11 0 446 -219648 9215104
+16170362 21559195254 15533616 2 0 2 recv 23 xv 11 0 2 -107136 9107968
+16572275 21559597167 15961548 2 0 3 recv 23 xv 11 0 453 -311296 8796672
+16928290 21559953182 16289039 2 0 3 recv 23 xv 11 0 130 -711424 8085248
+28452779 21571477671 27827273 2 0 3 recv 23 xv 11 0 40 -416512 7668736
+28707936 21571732828 28054184 2 0 1 recv 23 xv 11 0 192 -1066112 6602624
+28977681 21572002573 28364131 2 0 3 recv 23 xv 11 0 38 -449024 6153600
+30058812 21573083704 29440517 2 0 3 recv 23 xv 11 0 50 -400128 5753472
+30160189 21573185081 29539528 2 0 3 recv 23 xv 11 0 49 -318720 5434752
+30309729 21573334621 29639769 2 0 3 recv 23 xv 11 0 136 -199808 5234944
+30514052 21573538944 29849946 2 0 3 recv 23 xv 11 0 200 -368000 4866944
+31055012 21574079904 30410502 2 0 3 recv 23 xv 11 0 192 -345984 4520960
+38056623 21581081515 37427004 2 0 3 recv 23 xv 11 0 67 -312448 4208512
+38085547 21581110439 37454057 2 0 3 recv 23 xv 11 0 65 -292096 3916416
+38835374 21581860266 38189338 2 0 1 recv 23 xv 11 0 44 -1056768 2859648
+40237431 21583262323 39586043 2 0 1 recv 23 xv 11 0 202 -360832 2498816
+40553084 21583577976 39891964 2 0 1 recv 23 xv 11 0 205 -547968 1950848
+40777829 21583802721 40161544 2 0 1 recv 23 xv 11 0 177 -379648 1571200
+40836656 21583861548 40164686 2 0 1 recv 23 xv 11 0 57 -332160 1239040
+40873449 21583898341 40206496 2 0 1 recv 23 xv 11 0 137 -212608 1026432
+41885432 21584910324 0 2 0 2 send 22 tend_part 7 1 38 2328 1028760
+41896300 21584921192 0 2 0 3 send 22 tend_part 7 1 38 2328 1031088
+41942824 21584967716 57392 2 0 2 send 22 tend_part 7 0 38 -2328 1028760
+41950298 21584975190 53998 2 0 3 send 22 tend_part 7 0 38 -2328 1026432
+43257053 21586281945 42607454 2 0 1 recv 23 xv 11 0 194 -143872 882560
+43529999 21586554891 42887448 2 0 1 recv 23 xv 11 0 45 -328576 553984
+43591111 21586616003 42950385 2 0 1 recv 23 xv 11 0 118 -553984 0
+48599962 21591624854 0 2 0 2 send 22 tend_part 7 1 138 2712 2712
+48619658 21591644550 0 2 0 1 send 22 tend_part 7 1 138 2712 5424
+48642246 21591667138 0 2 0 3 send 22 tend_part 7 1 138 2712 8136
+48695836 21591720728 53590 2 0 3 send 22 tend_part 7 0 138 -2712 5424
+48704250 21591729142 104288 2 0 2 send 22 tend_part 7 0 138 -2712 2712
+48924980 21591949872 305322 2 0 1 send 22 tend_part 7 0 138 -2712 0
+48963599 21591988491 0 2 0 1 send 22 tend_part 7 1 192 6936 6936
+48968850 21591993742 5251 2 0 1 send 22 tend_part 7 0 192 -6936 0
+48990086 21592014978 0 2 0 3 send 22 tend_part 7 1 192 6936 6936
+48995836 21592020728 5750 2 0 3 send 22 tend_part 7 0 192 -6936 0
+##
+## Number of requests: 42
+## Maximum request size: 1.0167 (MB)
+## Sum of all requests: 11.2046 (MB)
+## Mean of all requests: 0.2668 (MB)
+##
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+12953800 21558567216 0 2 1 3 recv 23 tend_part 7 1 32 1368 1368
+12964804 21558578220 0 2 1 0 recv 23 tend_part 7 1 138 2712 4080
+12970634 21558584050 0 2 1 0 recv 23 tend_part 7 1 192 6936 11016
+12983064 21558596480 0 2 1 2 recv 23 xv 11 1 321 310784 321800
+12985183 21558598599 0 2 1 3 recv 23 xv 11 1 440 876032 1197832
+12986818 21558600234 0 2 1 3 recv 23 xv 11 1 403 171136 1368968
+12989387 21558602803 0 2 1 2 recv 23 xv 11 1 395 487040 1856008
+12993612 21558607028 0 2 1 2 recv 23 xv 11 1 383 146048 2002056
+12998091 21558611507 0 2 1 2 recv 23 xv 11 1 322 235904 2237960
+18459228 21564072644 0 2 1 0 send 22 xv 11 1 192 1066112 3304072
+23740217 21569353633 0 2 1 0 send 22 xv 11 1 44 1056768 4360840
+25940939 21571554355 0 2 1 0 send 22 xv 11 1 205 547968 4908808
+28186716 21573800132 0 2 1 0 send 22 xv 11 1 177 379648 5288456
+29239247 21574852663 0 2 1 3 send 22 xv 11 1 453 282752 5571208
+30275585 21575889001 17281973 2 1 2 recv 23 xv 11 0 383 -146048 5425160
+30291458 21575904874 0 2 1 0 send 22 xv 11 1 202 360832 5785992
+33012165 21578625581 0 2 1 0 send 22 xv 11 1 57 332160 6118152
+33756463 21579369879 20771280 2 1 3 recv 23 xv 11 0 440 -876032 5242120
+33892572 21579505988 20903185 2 1 2 recv 23 xv 11 0 395 -487040 4755080
+33925103 21579538519 0 2 1 0 send 22 xv 11 1 137 212608 4967688
+34725100 21580338516 0 2 1 0 send 22 xv 11 1 194 143872 5111560
+36049973 21581663389 0 2 1 0 send 22 xv 11 1 45 328576 5440136
+36265488 21581878904 17806260 2 1 0 send 22 xv 11 0 192 -1066112 4374024
+38193925 21583807341 0 2 1 3 send 22 xv 11 1 56 472064 4846088
+38822832 21584436248 15082615 2 1 0 send 22 xv 11 0 44 -1056768 3789320
+40220268 21585833684 12033552 2 1 0 send 22 xv 11 0 177 -379648 3409672
+40540832 21586154248 7528667 2 1 0 send 22 xv 11 0 57 -332160 3077512
+40705244 21586318660 6780141 2 1 0 send 22 xv 11 0 137 -212608 2864904
+40722144 21586335560 10430686 2 1 0 send 22 xv 11 0 202 -360832 2504072
+40735977 21586349393 14795038 2 1 0 send 22 xv 11 0 205 -547968 1956104
+42881032 21588494448 0 2 1 0 send 22 xv 11 1 118 553984 2510088
+43243043 21588856459 5049118 2 1 3 send 22 xv 11 0 56 -472064 2038024
+43254760 21588868176 14015513 2 1 3 send 22 xv 11 0 453 -282752 1755272
+43431687 21589045103 7381714 2 1 0 send 22 xv 11 0 45 -328576 1426696
+43441434 21589054850 8716334 2 1 0 send 22 xv 11 0 194 -143872 1282824
+43447599 21589061015 566567 2 1 0 send 22 xv 11 0 118 -553984 728840
+44195173 21589808589 0 2 1 3 send 22 xv 11 1 59 183552 912392
+44432889 21590046305 0 2 1 3 send 22 xv 11 1 47 46720 959112
+44593620 21590207036 160731 2 1 3 send 22 xv 11 0 47 -46720 912392
+44602907 21590216323 407734 2 1 3 send 22 xv 11 0 59 -183552 728840
+44639981 21590253397 31656917 2 1 2 recv 23 xv 11 0 321 -310784 418056
+44960058 21590573474 31961967 2 1 2 recv 23 xv 11 0 322 -235904 182152
+48461460 21594074876 35474642 2 1 3 recv 23 xv 11 0 403 -171136 11016
+48919281 21594532697 35954477 2 1 0 recv 23 tend_part 7 0 138 -2712 8304
+48984643 21594598059 36030843 2 1 3 recv 23 tend_part 7 0 32 -1368 6936
+49409002 21595022418 36438368 2 1 0 recv 23 tend_part 7 0 192 -6936 0
+56492967 21602106383 0 2 1 2 send 22 tend_part 7 1 359 2520 2520
+56507501 21602120917 14534 2 1 2 send 22 tend_part 7 0 359 -2520 0
+56512908 21602126324 0 2 1 3 send 22 tend_part 7 1 359 2520 2520
+56518112 21602131528 5204 2 1 3 send 22 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 25
+## Maximum request size: 1.0167 (MB)
+## Sum of all requests: 7.8303 (MB)
+## Mean of all requests: 0.3132 (MB)
+##
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+987251 21546628055 0 2 2 3 recv 23 tend_part 7 1 32 1368 1368
+1025615 21546666419 0 2 2 0 recv 23 tend_part 7 1 38 2328 3696
+1029326 21546670130 0 2 2 0 recv 23 tend_part 7 1 138 2712 6408
+1038964 21546679768 0 2 2 1 recv 23 tend_part 7 1 359 2520 8928
+1304769 21546945573 0 2 2 3 send 22 xv 11 1 444 98816 107744
+1404115 21547044919 0 2 2 3 send 22 xv 11 1 2 107136 214880
+1406430 21547047234 0 2 2 0 send 22 xv 11 1 2 107136 322016
+1892700 21547533504 0 2 2 0 send 22 xv 11 1 446 219648 541664
+3170516 21548811320 0 2 2 1 send 22 xv 11 1 395 487040 1028704
+3551697 21549192501 0 2 2 0 send 22 xv 11 1 3 583168 1611872
+3608675 21549249479 0 2 2 1 send 22 xv 11 1 383 146048 1757920
+3722238 21549363042 0 2 2 0 send 22 xv 11 1 445 142720 1900640
+4474542 21550115346 0 2 2 1 send 22 xv 11 1 322 235904 2136544
+4660091 21550300895 0 2 2 1 send 22 xv 11 1 321 310784 2447328
+4802628 21550443432 0 2 2 0 send 22 xv 11 1 249 28544 2475872
+4947390 21550588194 144762 2 2 0 send 22 xv 11 0 249 -28544 2447328
+7943938 21553584742 6537508 2 2 0 send 22 xv 11 0 2 -107136 2340192
+7961657 21553602461 6068957 2 2 0 send 22 xv 11 0 446 -219648 2120544
+7971824 21553612628 4420127 2 2 0 send 22 xv 11 0 3 -583168 1537376
+8524640 21554165444 0 2 2 0 send 22 xv 11 1 158 131072 1668448
+8974356 21554615160 0 2 2 0 send 22 xv 11 1 156 111360 1779808
+9378345 21555019149 0 2 2 0 send 22 xv 11 1 160 94208 1874016
+9846859 21555487663 0 2 2 0 send 22 xv 11 1 245 83840 1957856
+10082623 21555723427 6360385 2 2 0 send 22 xv 11 0 445 -142720 1815136
+10425140 21556065944 0 2 2 3 send 22 xv 11 1 1 64512 1879648
+11258279 21556899083 1411420 2 2 0 send 22 xv 11 0 245 -83840 1795808
+12594280 21558235084 2169140 2 2 3 send 22 xv 11 0 1 -64512 1731296
+12624810 21558265614 11220695 2 2 3 send 22 xv 11 0 2 -107136 1624160
+12806376 21558447180 3428031 2 2 0 send 22 xv 11 0 160 -94208 1529952
+12864222 21558505026 3889866 2 2 0 send 22 xv 11 0 156 -111360 1418592
+12923342 21558564146 11618573 2 2 3 send 22 xv 11 0 444 -98816 1319776
+13069781 21558710585 4545141 2 2 0 send 22 xv 11 0 158 -131072 1188704
+13360446 21559001250 0 2 2 3 send 22 xv 11 1 30 79232 1267936
+13833681 21559474485 0 2 2 3 send 22 xv 11 1 44 64128 1332064
+13842742 21559483546 9061 2 2 3 send 22 xv 11 0 44 -64128 1267936
+14347042 21559987846 0 2 2 0 send 22 xv 11 1 244 78720 1346656
+14447458 21560088262 100416 2 2 0 send 22 xv 11 0 244 -78720 1267936
+14549561 21560190365 1189115 2 2 3 send 22 xv 11 0 30 -79232 1188704
+29619186 21575259990 26448670 2 2 1 send 22 xv 11 0 395 -487040 701664
+29973620 21575614424 26364945 2 2 1 send 22 xv 11 0 383 -146048 555616
+30232105 21575872909 25757563 2 2 1 send 22 xv 11 0 322 -235904 319712
+30884935 21576525739 26224844 2 2 1 send 22 xv 11 0 321 -310784 8928
+41905408 21587546212 40879793 2 2 0 recv 23 tend_part 7 0 38 -2328 6600
+48456241 21594097045 47468990 2 2 3 recv 23 tend_part 7 0 32 -1368 5232
+48605647 21594246451 47576321 2 2 0 recv 23 tend_part 7 0 138 -2712 2520
+56499592 21602140396 55460628 2 2 1 recv 23 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 23
+## Maximum request size: 0.5562 (MB)
+## Sum of all requests: 3.0355 (MB)
+## Mean of all requests: 0.1320 (MB)
+##
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+8154566 21553806876 0 2 3 0 recv 23 tend_part 7 1 38 2328 2328
+8241482 21553893792 0 2 3 0 recv 23 tend_part 7 1 138 2712 5040
+8252462 21553904772 0 2 3 1 recv 23 xv 11 1 59 183552 188592
+8255236 21553907546 0 2 3 2 recv 23 xv 11 1 44 64128 252720
+8265527 21553917837 0 2 3 0 recv 23 tend_part 7 1 192 6936 259656
+8267192 21553919502 0 2 3 1 recv 23 xv 11 1 47 46720 306376
+8281049 21553933359 0 2 3 1 recv 23 tend_part 7 1 359 2520 308896
+8285576 21553937886 0 2 3 1 recv 23 xv 11 1 56 472064 780960
+8290638 21553942948 0 2 3 1 recv 23 xv 11 1 453 282752 1063712
+8312534 21553964844 0 2 3 2 recv 23 xv 11 1 444 98816 1162528
+8314496 21553966806 0 2 3 2 recv 23 xv 11 1 30 79232 1241760
+8318186 21553970496 0 2 3 2 recv 23 xv 11 1 2 107136 1348896
+8321900 21553974210 0 2 3 2 recv 23 xv 11 1 1 64512 1413408
+8551050 21554203360 0 2 3 0 send 22 xv 11 1 465 77696 1491104
+9240343 21554892653 0 2 3 0 send 22 xv 11 1 62 282624 1773728
+9516048 21555168358 0 2 3 0 send 22 xv 11 1 230 255232 2028960
+10222671 21555874981 1910137 2 3 2 recv 23 xv 11 0 444 -98816 1930144
+11396780 21557049090 3074880 2 3 2 recv 23 xv 11 0 1 -64512 1865632
+11919637 21557571947 0 2 3 0 send 22 xv 11 1 130 711424 2577056
+12416538 21558068848 496901 2 3 0 send 22 xv 11 0 130 -711424 1865632
+12449098 21558101408 0 2 3 1 send 22 xv 11 1 403 171136 2036768
+12525243 21558177553 4207057 2 3 2 recv 23 xv 11 0 2 -107136 1929632
+13886887 21559539197 0 2 3 0 send 22 xv 11 1 453 311296 2240928
+13993363 21559645673 5738127 2 3 2 recv 23 xv 11 0 44 -64128 2176800
+14353159 21560005469 6038663 2 3 2 recv 23 xv 11 0 30 -79232 2097568
+14361689 21560013999 0 2 3 0 send 22 xv 11 1 137 418304 2515872
+15260362 21560912672 898673 2 3 0 send 22 xv 11 0 137 -418304 2097568
+16761542 21562413852 0 2 3 0 send 22 xv 11 1 200 368000 2465568
+17561805 21563214115 0 2 3 0 send 22 xv 11 1 136 199808 2665376
+17768823 21563421133 0 2 3 1 send 22 xv 11 1 440 876032 3541408
+18876067 21564528377 0 2 3 0 send 22 xv 11 1 192 345984 3887392
+19483556 21565135866 0 2 3 0 send 22 xv 11 1 40 416512 4303904
+20583795 21566236105 0 2 3 0 send 22 xv 11 1 38 449024 4752928
+21166982 21566819292 0 2 3 0 send 22 xv 11 1 49 318720 5071648
+21367099 21567019409 7480212 2 3 0 send 22 xv 11 0 453 -311296 4760352
+21428196 21567080506 11912148 2 3 0 send 22 xv 11 0 230 -255232 4505120
+21489677 21567141987 12249334 2 3 0 send 22 xv 11 0 62 -282624 4222496
+21545174 21567197484 12994124 2 3 0 send 22 xv 11 0 465 -77696 4144800
+27653028 21573305338 10891486 2 3 0 send 22 xv 11 0 200 -368000 3776800
+27725141 21573377451 10163336 2 3 0 send 22 xv 11 0 136 -199808 3576992
+27785208 21573437518 8909141 2 3 0 send 22 xv 11 0 192 -345984 3231008
+27844800 21573497110 8361244 2 3 0 send 22 xv 11 0 40 -416512 2814496
+27904150 21573556460 7320355 2 3 0 send 22 xv 11 0 38 -449024 2365472
+29544767 21575197077 0 2 3 0 send 22 xv 11 1 50 400128 2765600
+29899938 21575552248 355171 2 3 0 send 22 xv 11 0 50 -400128 2365472
+29964381 21575616691 8797399 2 3 0 send 22 xv 11 0 49 -318720 2046752
+31237607 21576889917 0 2 3 0 send 22 xv 11 1 67 312448 2359200
+32076277 21577728587 23785639 2 3 1 recv 23 xv 11 0 453 -282752 2076448
+32997616 21578649926 20548518 2 3 1 send 22 xv 11 0 403 -171136 1905312
+33045540 21578697850 15276717 2 3 1 send 22 xv 11 0 440 -876032 1029280
+35555962 21581208272 0 2 3 0 send 22 xv 11 1 65 292096 1321376
+37724703 21583377013 6487096 2 3 0 send 22 xv 11 0 67 -312448 1008928
+38038978 21583691288 2483016 2 3 0 send 22 xv 11 0 65 -292096 716832
+39312145 21584964455 31026569 2 3 1 recv 23 xv 11 0 56 -472064 244768
+41913009 21587565319 33758443 2 3 0 recv 23 tend_part 7 0 38 -2328 242440
+44653176 21590305486 36385984 2 3 1 recv 23 xv 11 0 47 -46720 195720
+44672181 21590324491 36419719 2 3 1 recv 23 xv 11 0 59 -183552 12168
+48439936 21594092246 0 2 3 2 send 22 tend_part 7 1 32 1368 13536
+48448809 21594101119 0 2 3 1 send 22 tend_part 7 1 32 1368 14904
+48521474 21594173784 81538 2 3 2 send 22 tend_part 7 0 32 -1368 13536
+48529891 21594182201 81082 2 3 1 send 22 tend_part 7 0 32 -1368 12168
+48659245 21594311555 40417763 2 3 0 recv 23 tend_part 7 0 138 -2712 9456
+48993631 21594645941 40728104 2 3 0 recv 23 tend_part 7 0 192 -6936 2520
+56516622 21602168932 48235573 2 3 1 recv 23 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 32
+## Maximum request size: 0.8354 (MB)
+## Sum of all requests: 7.2695 (MB)
+## Mean of all requests: 0.2272 (MB)
+##