From 95f1f2c83ec5ec12b34ce0ad8a39277a2e7c784a Mon Sep 17 00:00:00 2001
From: "Peter W. Draper" <p.w.draper@durham.ac.uk>
Date: Fri, 13 Sep 2019 18:30:30 +0100
Subject: [PATCH] Capture progress, start to read back an MPI log
---
Makefile | 5 +
atomic.h | 31 +++
error.h | 16 ++
inject.c | 13 ++
mpistalls.c | 36 ++--
mpiuse.c | 201 ++++++++++++++++++
mpiuse.h | 28 +++
readlog.c | 12 ++
testdata/mpiuse_report-rank0-step2.dat | 91 +++++++++
testdata/mpiuse_report-rank1-step2.dat | 57 ++++++
testdata/mpiuse_report-rank2-step2.dat | 53 +++++
testdata/mpiuse_report-rank3-step2.dat | 71 +++++++
testdata/mpiuse_report-step2.dat | 272 +++++++++++++++++++++++++
13 files changed, 871 insertions(+), 15 deletions(-)
create mode 100644 Makefile
create mode 100644 atomic.h
create mode 100644 error.h
create mode 100644 inject.c
create mode 100644 mpiuse.c
create mode 100644 mpiuse.h
create mode 100644 readlog.c
create mode 100644 testdata/mpiuse_report-rank0-step2.dat
create mode 100644 testdata/mpiuse_report-rank1-step2.dat
create mode 100644 testdata/mpiuse_report-rank2-step2.dat
create mode 100644 testdata/mpiuse_report-rank3-step2.dat
create mode 100644 testdata/mpiuse_report-step2.dat
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..bfaf303
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,5 @@
+mpistalls: mpistalls.c mpiuse.c atomic.h cycle.h
+ $(CC) -g -O0 -o mpistalls mpistalls.c mpiuse.c -I/usr/include/mpi -lmpi -lpthread
+
+clean:
+ rm mpistalls
diff --git a/atomic.h b/atomic.h
new file mode 100644
index 0000000..1f6ccdf
--- /dev/null
+++ b/atomic.h
@@ -0,0 +1,31 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_ATOMIC_H
+#define SWIFT_ATOMIC_H
+
+#define atomic_add(v, i) __sync_fetch_and_add(v, i)
+#define atomic_sub(v, i) __sync_fetch_and_sub(v, i)
+#define atomic_or(v, i) __sync_fetch_and_or(v, i)
+#define atomic_and(v, i) __sync_fetch_and_and(v, i)
+#define atomic_inc(v) atomic_add(v, 1)
+#define atomic_dec(v) atomic_sub(v, 1)
+#define atomic_cas(v, o, n) __sync_val_compare_and_swap(v, o, n)
+#define atomic_swap(v, n) __sync_lock_test_and_set(v, n)
+
+#endif
diff --git a/error.h b/error.h
new file mode 100644
index 0000000..e1433b6
--- /dev/null
+++ b/error.h
@@ -0,0 +1,16 @@
+#include <mpi.h>
+
+/* Exit in error macro. */
+#define error(s, ...) \
+ ({ \
+ fflush(stdout); \
+ fprintf(stderr, "%s:%s():%i: " s "\n", __FILE__, __FUNCTION__, \
+ __LINE__, ##__VA_ARGS__); \
+ MPI_Abort(MPI_COMM_WORLD, -1); \
+ })
+
+/* Print a message */
+#define message(s, ...) \
+ ({ \
+ printf("%s: " s "\n", __FUNCTION__, ##__VA_ARGS__); \
+ })
diff --git a/inject.c b/inject.c
new file mode 100644
index 0000000..b819e0f
--- /dev/null
+++ b/inject.c
@@ -0,0 +1,13 @@
+#include <stdio.h>
+
+
+
+
+
+
+/* Injection of MPI_Isends and MPI_Irecvs into the queues. */
+void inject_one(void) {
+}
+
+
+
diff --git a/mpistalls.c b/mpistalls.c
index 73ecafa..95312a8 100644
--- a/mpistalls.c
+++ b/mpistalls.c
@@ -10,35 +10,36 @@
#include <mpi.h>
#include <pthread.h>
-/* Exit in error macro. */
-#define error(s, ...) \
- ({ \
- fflush(stdout); \
- fprintf(stderr, "%s:%s():%i: " s "\n", __FILE__, __FUNCTION__, \
- __LINE__, ##__VA_ARGS__); \
- MPI_Abort(MPI_COMM_WORLD, -1); \
- })
+#include "error.h"
+#include "mpiuse.h"
+
/* Injection thread, initiates MPI_Isend and MPI_Irecv requests at various
* times. */
void *inject_thread(void *arg) {
- fprintf(stderr, "%d: injection thread starts\n", *((int *)arg));
+ message("%d: injection thread starts", *((int *)arg));
- while (1) {};
+ while (1) {
+ //inject_one();
+ }
}
/* Send thread, checks if MPI_Isend requests have completed. */
void *send_thread(void *arg) {
- fprintf(stderr, "%d: send thread starts\n", *((int *)arg));
+ message("%d: send thread starts", *((int *)arg));
- while (1) {};
+ while (1) {
+ //send_test();
+ }
}
/* Recv thread, checks if MPI_Irecv requests have completed. */
void *recv_thread(void *arg) {
- fprintf(stderr, "%d: recv thread starts\n", *((int *)arg));
+ message("%d: recv thread starts", *((int *)arg));
- while (1) {};
+ while (1) {
+ //recv_test();
+ }
}
@@ -60,7 +61,7 @@ int main(int argc, char *argv[]) {
if (res != MPI_SUCCESS)
error("Call to MPI_Comm_rank failed with error %i.", res);
- printf("Hello from rank: %d\n", myrank);
+ message("Hello from rank: %d", myrank);
/* Make three threads, one for injecting tasks and two to check for
* completions of the sends and recv independently. */
@@ -74,6 +75,11 @@ int main(int argc, char *argv[]) {
if (pthread_create(&recvthread, NULL, &recv_thread, &myrank) != 0)
error("Failed to create recv thread.");
+
+ /* Read the MPI logger output that defines the communcations we will
+ * undertake and the time differences between injections into the queues. */
+ mpiuse_log_restore("testdata/mpiuse_report-step2.dat");
+
/* Wait until all threads have exited and all MPI requests have completed. */
pthread_join(injectthread, NULL);
pthread_join(sendthread, NULL);
diff --git a/mpiuse.c b/mpiuse.c
new file mode 100644
index 0000000..eb6aef1
--- /dev/null
+++ b/mpiuse.c
@@ -0,0 +1,201 @@
+/* This file is part of SWIFT.
+ * Copyright (c) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/**
+ * @file mpiuse.c
+ * @brief file of routines to report about MPI tasks used in SWIFT.
+ */
+
+/* Standard includes. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Local defines. */
+#include "mpiuse.h"
+
+/* Local includes. */
+#include "atomic.h"
+#include "cycle.h"
+#include "error.h"
+
+/* The initial size and increment of the log entries buffer. */
+#define MPIUSE_INITLOG 1000000
+
+/* Entry for logger of MPI send and recv requests in a step. */
+struct mpiuse_log_entry {
+
+ /* Rank of entry. */
+ int rank;
+
+ /* Type and subtype of MPI task. */
+ int type;
+ int subtype;
+
+ /* Step of action. */
+ int step;
+
+ /* Whether an activation, send or recv, or if handoff completed. Not the
+ * same as delivered, need to match across ranks to see that. */
+ int activation;
+
+ /* Memory of the request. */
+ size_t size;
+
+ /* Pointer to the request associated with the call. Needs to be
+ * unique and match to the successful */
+ union {
+ void *ptr;
+ uint8_t vptr[sizeof(uintptr_t)]; /* For rnode keys. */
+ };
+
+ /* Ticks at time of this action. */
+ ticks tic;
+
+ /* Rank of otherside of communication. */
+ int otherrank;
+
+ /* The tag. */
+ int tag;
+};
+
+/* The log of activations and handoffs. All volatile as accessed from threads
+ * that use the value to synchronise. */
+static struct mpiuse_log_entry *volatile mpiuse_log = NULL;
+static volatile size_t mpiuse_log_size = 0;
+static volatile size_t mpiuse_log_count = 0;
+static volatile size_t mpiuse_log_done = 0;
+
+/**
+ * @brief reallocate the entries log if space is needed.
+ */
+static void mpiuse_log_reallocate(size_t ind) {
+
+ if (ind == 0) {
+
+ /* Need to perform initialization. Be generous. */
+ if ((mpiuse_log = (struct mpiuse_log_entry *)malloc(
+ sizeof(struct mpiuse_log_entry) * MPIUSE_INITLOG)) == NULL)
+ error("Failed to allocate MPI use log.");
+
+ /* Last action. */
+ mpiuse_log_size = MPIUSE_INITLOG;
+
+ } else {
+ struct mpiuse_log_entry *new_log;
+ if ((new_log = (struct mpiuse_log_entry *)malloc(
+ sizeof(struct mpiuse_log_entry) *
+ (mpiuse_log_size + MPIUSE_INITLOG))) == NULL)
+ error("Failed to re-allocate MPI use log.");
+
+ /* Wait for all writes to the old buffer to complete. */
+ while (mpiuse_log_done < mpiuse_log_size)
+ ;
+
+ /* Copy to new buffer. */
+ memcpy(new_log, mpiuse_log,
+ sizeof(struct mpiuse_log_entry) * mpiuse_log_size);
+ free(mpiuse_log);
+ mpiuse_log = new_log;
+
+ /* Last action, releases waiting threads. */
+ atomic_add(&mpiuse_log_size, MPIUSE_INITLOG);
+ }
+}
+
+/**
+ * @brief Log an MPI request or handoff.
+ *
+ * @param rank the rank
+ * @param step the step
+ * @param tic the ticks at time of log, will be relative.
+ * @param type the task type (send or recv).
+ * @param subtype the task subtype.
+ * @param activation if not is a successful MPI_Test, not MPI_Isend or
+ * MPI_Irecv.
+ * @param size the size in bytes of memory to be transfered or received.
+ * 0 for a deactivation.
+ * @param otherrank other rank associated with the transfer.
+ * @param tag the MPI tag.
+ */
+void mpiuse_log_allocation(int rank, int step, size_t tic, int type,
+ int subtype, int activation, size_t size,
+ int otherrank, int tag) {
+
+ size_t ind = atomic_inc(&mpiuse_log_count);
+
+ /* If we are at the current size we need more space. */
+ if (ind == mpiuse_log_size) mpiuse_log_reallocate(ind);
+
+ /* Other threads wait for space. */
+ while (ind > mpiuse_log_size)
+ ;
+
+ /* Record the log. */
+ mpiuse_log[ind].rank = rank;
+ mpiuse_log[ind].step = step;
+ mpiuse_log[ind].type = type;
+ mpiuse_log[ind].subtype = subtype;
+ mpiuse_log[ind].activation = activation;
+ mpiuse_log[ind].size = size;
+ mpiuse_log[ind].ptr = NULL;
+ mpiuse_log[ind].otherrank = otherrank;
+ mpiuse_log[ind].tag = tag;
+ mpiuse_log[ind].tic = tic;
+
+ atomic_inc(&mpiuse_log_done);
+}
+
+/**
+ * @brief restore the log from a dump.
+ *
+ * @param filename name of file with the previus dump in.
+ */
+void mpiuse_log_restore(const char *filename) {
+
+ /* Open the input file. */
+ FILE *fd;
+ if ((fd = fopen(filename, "r")) == NULL) {
+ message("Failed to open the MPI use log file '%s'.",
+ filename);
+ return;
+ }
+
+
+ /* Read until the end of the file is reached.*/
+ char line[132];
+ size_t stic, etic, dtic, size, sum;
+ int step, rank, otherrank, itype, isubtype, activation, tag;
+ char type[32], subtype[32];
+
+ while (!feof(fd)) {
+ if (fgets(line, 132, fd) != NULL) {
+ if (line[0] != '#') {
+ sscanf(line, "%zd %zd %zd %d %d %d %s %d %s %d %d %d %zd %zd",
+ &stic, &etic, &dtic, &step, &rank, &otherrank, type, &itype,
+ subtype, &isubtype, &activation, &tag, &size, &sum);
+
+ mpiuse_log_allocation(rank, step, stic, itype, isubtype, activation,
+ size, otherrank, tag);
+
+ }
+ }
+ }
+ fclose(fd);
+}
+
diff --git a/mpiuse.h b/mpiuse.h
new file mode 100644
index 0000000..cd5fff0
--- /dev/null
+++ b/mpiuse.h
@@ -0,0 +1,28 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_MPIUSE_H
+#define SWIFT_MPIUSE_H
+
+/* API. */
+void mpiuse_log_allocation(int rank, int step, size_t tic, int type,
+ int subtype, int activation, size_t size,
+ int otherrank, int tag);
+
+void mpiuse_log_restore(const char *filename);
+#endif /* SWIFT_MPIUSE_H */
diff --git a/readlog.c b/readlog.c
new file mode 100644
index 0000000..3763572
--- /dev/null
+++ b/readlog.c
@@ -0,0 +1,12 @@
+#include <stdio.h>
+
+
+/**
+ * @brief read the output from a run of SWIFT with the MPI communication
+ * logger enabled.
+ *
+void mpistalls_readlog(void) {
+
+
+
+}
diff --git a/testdata/mpiuse_report-rank0-step2.dat b/testdata/mpiuse_report-rank0-step2.dat
new file mode 100644
index 0000000..51043d6
--- /dev/null
+++ b/testdata/mpiuse_report-rank0-step2.dat
@@ -0,0 +1,91 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+608817 21543633709 0 2 0 2 recv 23 xv 11 1 445 142720 142720
+610727 21543635619 0 2 0 3 recv 23 xv 11 1 453 311296 454016
+612045 21543636937 0 2 0 2 recv 23 xv 11 1 446 219648 673664
+613550 21543638442 0 2 0 3 recv 23 xv 11 1 38 449024 1122688
+615327 21543640219 0 2 0 3 recv 23 xv 11 1 62 282624 1405312
+616285 21543641177 0 2 0 1 recv 23 xv 11 1 177 379648 1784960
+618295 21543643187 0 2 0 3 recv 23 xv 11 1 50 400128 2185088
+619247 21543644139 0 2 0 2 recv 23 xv 11 1 160 94208 2279296
+620661 21543645553 0 2 0 3 recv 23 xv 11 1 49 318720 2598016
+623155 21543648047 0 2 0 2 recv 23 xv 11 1 158 131072 2729088
+625506 21543650398 0 2 0 3 recv 23 xv 11 1 40 416512 3145600
+627165 21543652057 0 2 0 2 recv 23 xv 11 1 156 111360 3256960
+629619 21543654511 0 2 0 3 recv 23 xv 11 1 67 312448 3569408
+631490 21543656382 0 2 0 3 recv 23 xv 11 1 65 292096 3861504
+632856 21543657748 0 2 0 3 recv 23 xv 11 1 465 77696 3939200
+635292 21543660184 0 2 0 2 recv 23 xv 11 1 3 583168 4522368
+636746 21543661638 0 2 0 2 recv 23 xv 11 1 2 107136 4629504
+639251 21543664143 0 2 0 3 recv 23 xv 11 1 130 711424 5340928
+640726 21543665618 0 2 0 1 recv 23 xv 11 1 118 553984 5894912
+642551 21543667443 0 2 0 1 recv 23 xv 11 1 45 328576 6223488
+644510 21543669402 0 2 0 3 recv 23 xv 11 1 192 345984 6569472
+646036 21543670928 0 2 0 1 recv 23 xv 11 1 44 1056768 7626240
+647586 21543672478 0 2 0 3 recv 23 xv 11 1 230 255232 7881472
+649599 21543674491 0 2 0 1 recv 23 xv 11 1 194 143872 8025344
+651388 21543676280 0 2 0 1 recv 23 xv 11 1 202 360832 8386176
+653752 21543678644 0 2 0 1 recv 23 xv 11 1 192 1066112 9452288
+655523 21543680415 0 2 0 2 recv 23 xv 11 1 249 28544 9480832
+657245 21543682137 0 2 0 2 recv 23 xv 11 1 245 83840 9564672
+659152 21543684044 0 2 0 2 recv 23 xv 11 1 244 78720 9643392
+661120 21543686012 0 2 0 1 recv 23 xv 11 1 205 547968 10191360
+664106 21543688998 0 2 0 3 recv 23 xv 11 1 200 368000 10559360
+666953 21543691845 0 2 0 1 recv 23 xv 11 1 137 212608 10771968
+668385 21543693277 0 2 0 3 recv 23 xv 11 1 137 418304 11190272
+669960 21543694852 0 2 0 3 recv 23 xv 11 1 136 199808 11390080
+671970 21543696862 0 2 0 1 recv 23 xv 11 1 57 332160 11722240
+6447305 21549472197 5791782 2 0 2 recv 23 xv 11 0 249 -28544 11693696
+11419335 21554444227 10796180 2 0 2 recv 23 xv 11 0 158 -131072 11562624
+11758915 21554783807 11111329 2 0 3 recv 23 xv 11 0 230 -255232 11307392
+11799763 21554824655 11172598 2 0 2 recv 23 xv 11 0 156 -111360 11196032
+11838133 21554863025 11222806 2 0 3 recv 23 xv 11 0 62 -282624 10913408
+11910866 21554935758 11253621 2 0 2 recv 23 xv 11 0 245 -83840 10829568
+13014189 21556039081 12405372 2 0 2 recv 23 xv 11 0 445 -142720 10686848
+13215819 21556240711 12580527 2 0 2 recv 23 xv 11 0 3 -583168 10103680
+13730903 21556755795 13111656 2 0 2 recv 23 xv 11 0 160 -94208 10009472
+14435788 21557460680 13776636 2 0 2 recv 23 xv 11 0 244 -78720 9930752
+15491583 21558516475 14823198 2 0 3 recv 23 xv 11 0 137 -418304 9512448
+15614696 21558639588 14981840 2 0 3 recv 23 xv 11 0 465 -77696 9434752
+15728921 21558753813 15116876 2 0 2 recv 23 xv 11 0 446 -219648 9215104
+16170362 21559195254 15533616 2 0 2 recv 23 xv 11 0 2 -107136 9107968
+16572275 21559597167 15961548 2 0 3 recv 23 xv 11 0 453 -311296 8796672
+16928290 21559953182 16289039 2 0 3 recv 23 xv 11 0 130 -711424 8085248
+28452779 21571477671 27827273 2 0 3 recv 23 xv 11 0 40 -416512 7668736
+28707936 21571732828 28054184 2 0 1 recv 23 xv 11 0 192 -1066112 6602624
+28977681 21572002573 28364131 2 0 3 recv 23 xv 11 0 38 -449024 6153600
+30058812 21573083704 29440517 2 0 3 recv 23 xv 11 0 50 -400128 5753472
+30160189 21573185081 29539528 2 0 3 recv 23 xv 11 0 49 -318720 5434752
+30309729 21573334621 29639769 2 0 3 recv 23 xv 11 0 136 -199808 5234944
+30514052 21573538944 29849946 2 0 3 recv 23 xv 11 0 200 -368000 4866944
+31055012 21574079904 30410502 2 0 3 recv 23 xv 11 0 192 -345984 4520960
+38056623 21581081515 37427004 2 0 3 recv 23 xv 11 0 67 -312448 4208512
+38085547 21581110439 37454057 2 0 3 recv 23 xv 11 0 65 -292096 3916416
+38835374 21581860266 38189338 2 0 1 recv 23 xv 11 0 44 -1056768 2859648
+40237431 21583262323 39586043 2 0 1 recv 23 xv 11 0 202 -360832 2498816
+40553084 21583577976 39891964 2 0 1 recv 23 xv 11 0 205 -547968 1950848
+40777829 21583802721 40161544 2 0 1 recv 23 xv 11 0 177 -379648 1571200
+40836656 21583861548 40164686 2 0 1 recv 23 xv 11 0 57 -332160 1239040
+40873449 21583898341 40206496 2 0 1 recv 23 xv 11 0 137 -212608 1026432
+41885432 21584910324 0 2 0 2 send 22 tend_part 7 1 38 2328 1028760
+41896300 21584921192 0 2 0 3 send 22 tend_part 7 1 38 2328 1031088
+41942824 21584967716 57392 2 0 2 send 22 tend_part 7 0 38 -2328 1028760
+41950298 21584975190 53998 2 0 3 send 22 tend_part 7 0 38 -2328 1026432
+43257053 21586281945 42607454 2 0 1 recv 23 xv 11 0 194 -143872 882560
+43529999 21586554891 42887448 2 0 1 recv 23 xv 11 0 45 -328576 553984
+43591111 21586616003 42950385 2 0 1 recv 23 xv 11 0 118 -553984 0
+48599962 21591624854 0 2 0 2 send 22 tend_part 7 1 138 2712 2712
+48619658 21591644550 0 2 0 1 send 22 tend_part 7 1 138 2712 5424
+48642246 21591667138 0 2 0 3 send 22 tend_part 7 1 138 2712 8136
+48695836 21591720728 53590 2 0 3 send 22 tend_part 7 0 138 -2712 5424
+48704250 21591729142 104288 2 0 2 send 22 tend_part 7 0 138 -2712 2712
+48924980 21591949872 305322 2 0 1 send 22 tend_part 7 0 138 -2712 0
+48963599 21591988491 0 2 0 1 send 22 tend_part 7 1 192 6936 6936
+48968850 21591993742 5251 2 0 1 send 22 tend_part 7 0 192 -6936 0
+48990086 21592014978 0 2 0 3 send 22 tend_part 7 1 192 6936 6936
+48995836 21592020728 5750 2 0 3 send 22 tend_part 7 0 192 -6936 0
+##
+## Number of requests: 42
+## Maximum request size: 1.0167 (MB)
+## Sum of all requests: 11.2046 (MB)
+## Mean of all requests: 0.2668 (MB)
+##
diff --git a/testdata/mpiuse_report-rank1-step2.dat b/testdata/mpiuse_report-rank1-step2.dat
new file mode 100644
index 0000000..964543c
--- /dev/null
+++ b/testdata/mpiuse_report-rank1-step2.dat
@@ -0,0 +1,57 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+12953800 21558567216 0 2 1 3 recv 23 tend_part 7 1 32 1368 1368
+12964804 21558578220 0 2 1 0 recv 23 tend_part 7 1 138 2712 4080
+12970634 21558584050 0 2 1 0 recv 23 tend_part 7 1 192 6936 11016
+12983064 21558596480 0 2 1 2 recv 23 xv 11 1 321 310784 321800
+12985183 21558598599 0 2 1 3 recv 23 xv 11 1 440 876032 1197832
+12986818 21558600234 0 2 1 3 recv 23 xv 11 1 403 171136 1368968
+12989387 21558602803 0 2 1 2 recv 23 xv 11 1 395 487040 1856008
+12993612 21558607028 0 2 1 2 recv 23 xv 11 1 383 146048 2002056
+12998091 21558611507 0 2 1 2 recv 23 xv 11 1 322 235904 2237960
+18459228 21564072644 0 2 1 0 send 22 xv 11 1 192 1066112 3304072
+23740217 21569353633 0 2 1 0 send 22 xv 11 1 44 1056768 4360840
+25940939 21571554355 0 2 1 0 send 22 xv 11 1 205 547968 4908808
+28186716 21573800132 0 2 1 0 send 22 xv 11 1 177 379648 5288456
+29239247 21574852663 0 2 1 3 send 22 xv 11 1 453 282752 5571208
+30275585 21575889001 17281973 2 1 2 recv 23 xv 11 0 383 -146048 5425160
+30291458 21575904874 0 2 1 0 send 22 xv 11 1 202 360832 5785992
+33012165 21578625581 0 2 1 0 send 22 xv 11 1 57 332160 6118152
+33756463 21579369879 20771280 2 1 3 recv 23 xv 11 0 440 -876032 5242120
+33892572 21579505988 20903185 2 1 2 recv 23 xv 11 0 395 -487040 4755080
+33925103 21579538519 0 2 1 0 send 22 xv 11 1 137 212608 4967688
+34725100 21580338516 0 2 1 0 send 22 xv 11 1 194 143872 5111560
+36049973 21581663389 0 2 1 0 send 22 xv 11 1 45 328576 5440136
+36265488 21581878904 17806260 2 1 0 send 22 xv 11 0 192 -1066112 4374024
+38193925 21583807341 0 2 1 3 send 22 xv 11 1 56 472064 4846088
+38822832 21584436248 15082615 2 1 0 send 22 xv 11 0 44 -1056768 3789320
+40220268 21585833684 12033552 2 1 0 send 22 xv 11 0 177 -379648 3409672
+40540832 21586154248 7528667 2 1 0 send 22 xv 11 0 57 -332160 3077512
+40705244 21586318660 6780141 2 1 0 send 22 xv 11 0 137 -212608 2864904
+40722144 21586335560 10430686 2 1 0 send 22 xv 11 0 202 -360832 2504072
+40735977 21586349393 14795038 2 1 0 send 22 xv 11 0 205 -547968 1956104
+42881032 21588494448 0 2 1 0 send 22 xv 11 1 118 553984 2510088
+43243043 21588856459 5049118 2 1 3 send 22 xv 11 0 56 -472064 2038024
+43254760 21588868176 14015513 2 1 3 send 22 xv 11 0 453 -282752 1755272
+43431687 21589045103 7381714 2 1 0 send 22 xv 11 0 45 -328576 1426696
+43441434 21589054850 8716334 2 1 0 send 22 xv 11 0 194 -143872 1282824
+43447599 21589061015 566567 2 1 0 send 22 xv 11 0 118 -553984 728840
+44195173 21589808589 0 2 1 3 send 22 xv 11 1 59 183552 912392
+44432889 21590046305 0 2 1 3 send 22 xv 11 1 47 46720 959112
+44593620 21590207036 160731 2 1 3 send 22 xv 11 0 47 -46720 912392
+44602907 21590216323 407734 2 1 3 send 22 xv 11 0 59 -183552 728840
+44639981 21590253397 31656917 2 1 2 recv 23 xv 11 0 321 -310784 418056
+44960058 21590573474 31961967 2 1 2 recv 23 xv 11 0 322 -235904 182152
+48461460 21594074876 35474642 2 1 3 recv 23 xv 11 0 403 -171136 11016
+48919281 21594532697 35954477 2 1 0 recv 23 tend_part 7 0 138 -2712 8304
+48984643 21594598059 36030843 2 1 3 recv 23 tend_part 7 0 32 -1368 6936
+49409002 21595022418 36438368 2 1 0 recv 23 tend_part 7 0 192 -6936 0
+56492967 21602106383 0 2 1 2 send 22 tend_part 7 1 359 2520 2520
+56507501 21602120917 14534 2 1 2 send 22 tend_part 7 0 359 -2520 0
+56512908 21602126324 0 2 1 3 send 22 tend_part 7 1 359 2520 2520
+56518112 21602131528 5204 2 1 3 send 22 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 25
+## Maximum request size: 1.0167 (MB)
+## Sum of all requests: 7.8303 (MB)
+## Mean of all requests: 0.3132 (MB)
+##
diff --git a/testdata/mpiuse_report-rank2-step2.dat b/testdata/mpiuse_report-rank2-step2.dat
new file mode 100644
index 0000000..b5c0e6e
--- /dev/null
+++ b/testdata/mpiuse_report-rank2-step2.dat
@@ -0,0 +1,53 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+987251 21546628055 0 2 2 3 recv 23 tend_part 7 1 32 1368 1368
+1025615 21546666419 0 2 2 0 recv 23 tend_part 7 1 38 2328 3696
+1029326 21546670130 0 2 2 0 recv 23 tend_part 7 1 138 2712 6408
+1038964 21546679768 0 2 2 1 recv 23 tend_part 7 1 359 2520 8928
+1304769 21546945573 0 2 2 3 send 22 xv 11 1 444 98816 107744
+1404115 21547044919 0 2 2 3 send 22 xv 11 1 2 107136 214880
+1406430 21547047234 0 2 2 0 send 22 xv 11 1 2 107136 322016
+1892700 21547533504 0 2 2 0 send 22 xv 11 1 446 219648 541664
+3170516 21548811320 0 2 2 1 send 22 xv 11 1 395 487040 1028704
+3551697 21549192501 0 2 2 0 send 22 xv 11 1 3 583168 1611872
+3608675 21549249479 0 2 2 1 send 22 xv 11 1 383 146048 1757920
+3722238 21549363042 0 2 2 0 send 22 xv 11 1 445 142720 1900640
+4474542 21550115346 0 2 2 1 send 22 xv 11 1 322 235904 2136544
+4660091 21550300895 0 2 2 1 send 22 xv 11 1 321 310784 2447328
+4802628 21550443432 0 2 2 0 send 22 xv 11 1 249 28544 2475872
+4947390 21550588194 144762 2 2 0 send 22 xv 11 0 249 -28544 2447328
+7943938 21553584742 6537508 2 2 0 send 22 xv 11 0 2 -107136 2340192
+7961657 21553602461 6068957 2 2 0 send 22 xv 11 0 446 -219648 2120544
+7971824 21553612628 4420127 2 2 0 send 22 xv 11 0 3 -583168 1537376
+8524640 21554165444 0 2 2 0 send 22 xv 11 1 158 131072 1668448
+8974356 21554615160 0 2 2 0 send 22 xv 11 1 156 111360 1779808
+9378345 21555019149 0 2 2 0 send 22 xv 11 1 160 94208 1874016
+9846859 21555487663 0 2 2 0 send 22 xv 11 1 245 83840 1957856
+10082623 21555723427 6360385 2 2 0 send 22 xv 11 0 445 -142720 1815136
+10425140 21556065944 0 2 2 3 send 22 xv 11 1 1 64512 1879648
+11258279 21556899083 1411420 2 2 0 send 22 xv 11 0 245 -83840 1795808
+12594280 21558235084 2169140 2 2 3 send 22 xv 11 0 1 -64512 1731296
+12624810 21558265614 11220695 2 2 3 send 22 xv 11 0 2 -107136 1624160
+12806376 21558447180 3428031 2 2 0 send 22 xv 11 0 160 -94208 1529952
+12864222 21558505026 3889866 2 2 0 send 22 xv 11 0 156 -111360 1418592
+12923342 21558564146 11618573 2 2 3 send 22 xv 11 0 444 -98816 1319776
+13069781 21558710585 4545141 2 2 0 send 22 xv 11 0 158 -131072 1188704
+13360446 21559001250 0 2 2 3 send 22 xv 11 1 30 79232 1267936
+13833681 21559474485 0 2 2 3 send 22 xv 11 1 44 64128 1332064
+13842742 21559483546 9061 2 2 3 send 22 xv 11 0 44 -64128 1267936
+14347042 21559987846 0 2 2 0 send 22 xv 11 1 244 78720 1346656
+14447458 21560088262 100416 2 2 0 send 22 xv 11 0 244 -78720 1267936
+14549561 21560190365 1189115 2 2 3 send 22 xv 11 0 30 -79232 1188704
+29619186 21575259990 26448670 2 2 1 send 22 xv 11 0 395 -487040 701664
+29973620 21575614424 26364945 2 2 1 send 22 xv 11 0 383 -146048 555616
+30232105 21575872909 25757563 2 2 1 send 22 xv 11 0 322 -235904 319712
+30884935 21576525739 26224844 2 2 1 send 22 xv 11 0 321 -310784 8928
+41905408 21587546212 40879793 2 2 0 recv 23 tend_part 7 0 38 -2328 6600
+48456241 21594097045 47468990 2 2 3 recv 23 tend_part 7 0 32 -1368 5232
+48605647 21594246451 47576321 2 2 0 recv 23 tend_part 7 0 138 -2712 2520
+56499592 21602140396 55460628 2 2 1 recv 23 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 23
+## Maximum request size: 0.5562 (MB)
+## Sum of all requests: 3.0355 (MB)
+## Mean of all requests: 0.1320 (MB)
+##
diff --git a/testdata/mpiuse_report-rank3-step2.dat b/testdata/mpiuse_report-rank3-step2.dat
new file mode 100644
index 0000000..2c17b6d
--- /dev/null
+++ b/testdata/mpiuse_report-rank3-step2.dat
@@ -0,0 +1,71 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+8154566 21553806876 0 2 3 0 recv 23 tend_part 7 1 38 2328 2328
+8241482 21553893792 0 2 3 0 recv 23 tend_part 7 1 138 2712 5040
+8252462 21553904772 0 2 3 1 recv 23 xv 11 1 59 183552 188592
+8255236 21553907546 0 2 3 2 recv 23 xv 11 1 44 64128 252720
+8265527 21553917837 0 2 3 0 recv 23 tend_part 7 1 192 6936 259656
+8267192 21553919502 0 2 3 1 recv 23 xv 11 1 47 46720 306376
+8281049 21553933359 0 2 3 1 recv 23 tend_part 7 1 359 2520 308896
+8285576 21553937886 0 2 3 1 recv 23 xv 11 1 56 472064 780960
+8290638 21553942948 0 2 3 1 recv 23 xv 11 1 453 282752 1063712
+8312534 21553964844 0 2 3 2 recv 23 xv 11 1 444 98816 1162528
+8314496 21553966806 0 2 3 2 recv 23 xv 11 1 30 79232 1241760
+8318186 21553970496 0 2 3 2 recv 23 xv 11 1 2 107136 1348896
+8321900 21553974210 0 2 3 2 recv 23 xv 11 1 1 64512 1413408
+8551050 21554203360 0 2 3 0 send 22 xv 11 1 465 77696 1491104
+9240343 21554892653 0 2 3 0 send 22 xv 11 1 62 282624 1773728
+9516048 21555168358 0 2 3 0 send 22 xv 11 1 230 255232 2028960
+10222671 21555874981 1910137 2 3 2 recv 23 xv 11 0 444 -98816 1930144
+11396780 21557049090 3074880 2 3 2 recv 23 xv 11 0 1 -64512 1865632
+11919637 21557571947 0 2 3 0 send 22 xv 11 1 130 711424 2577056
+12416538 21558068848 496901 2 3 0 send 22 xv 11 0 130 -711424 1865632
+12449098 21558101408 0 2 3 1 send 22 xv 11 1 403 171136 2036768
+12525243 21558177553 4207057 2 3 2 recv 23 xv 11 0 2 -107136 1929632
+13886887 21559539197 0 2 3 0 send 22 xv 11 1 453 311296 2240928
+13993363 21559645673 5738127 2 3 2 recv 23 xv 11 0 44 -64128 2176800
+14353159 21560005469 6038663 2 3 2 recv 23 xv 11 0 30 -79232 2097568
+14361689 21560013999 0 2 3 0 send 22 xv 11 1 137 418304 2515872
+15260362 21560912672 898673 2 3 0 send 22 xv 11 0 137 -418304 2097568
+16761542 21562413852 0 2 3 0 send 22 xv 11 1 200 368000 2465568
+17561805 21563214115 0 2 3 0 send 22 xv 11 1 136 199808 2665376
+17768823 21563421133 0 2 3 1 send 22 xv 11 1 440 876032 3541408
+18876067 21564528377 0 2 3 0 send 22 xv 11 1 192 345984 3887392
+19483556 21565135866 0 2 3 0 send 22 xv 11 1 40 416512 4303904
+20583795 21566236105 0 2 3 0 send 22 xv 11 1 38 449024 4752928
+21166982 21566819292 0 2 3 0 send 22 xv 11 1 49 318720 5071648
+21367099 21567019409 7480212 2 3 0 send 22 xv 11 0 453 -311296 4760352
+21428196 21567080506 11912148 2 3 0 send 22 xv 11 0 230 -255232 4505120
+21489677 21567141987 12249334 2 3 0 send 22 xv 11 0 62 -282624 4222496
+21545174 21567197484 12994124 2 3 0 send 22 xv 11 0 465 -77696 4144800
+27653028 21573305338 10891486 2 3 0 send 22 xv 11 0 200 -368000 3776800
+27725141 21573377451 10163336 2 3 0 send 22 xv 11 0 136 -199808 3576992
+27785208 21573437518 8909141 2 3 0 send 22 xv 11 0 192 -345984 3231008
+27844800 21573497110 8361244 2 3 0 send 22 xv 11 0 40 -416512 2814496
+27904150 21573556460 7320355 2 3 0 send 22 xv 11 0 38 -449024 2365472
+29544767 21575197077 0 2 3 0 send 22 xv 11 1 50 400128 2765600
+29899938 21575552248 355171 2 3 0 send 22 xv 11 0 50 -400128 2365472
+29964381 21575616691 8797399 2 3 0 send 22 xv 11 0 49 -318720 2046752
+31237607 21576889917 0 2 3 0 send 22 xv 11 1 67 312448 2359200
+32076277 21577728587 23785639 2 3 1 recv 23 xv 11 0 453 -282752 2076448
+32997616 21578649926 20548518 2 3 1 send 22 xv 11 0 403 -171136 1905312
+33045540 21578697850 15276717 2 3 1 send 22 xv 11 0 440 -876032 1029280
+35555962 21581208272 0 2 3 0 send 22 xv 11 1 65 292096 1321376
+37724703 21583377013 6487096 2 3 0 send 22 xv 11 0 67 -312448 1008928
+38038978 21583691288 2483016 2 3 0 send 22 xv 11 0 65 -292096 716832
+39312145 21584964455 31026569 2 3 1 recv 23 xv 11 0 56 -472064 244768
+41913009 21587565319 33758443 2 3 0 recv 23 tend_part 7 0 38 -2328 242440
+44653176 21590305486 36385984 2 3 1 recv 23 xv 11 0 47 -46720 195720
+44672181 21590324491 36419719 2 3 1 recv 23 xv 11 0 59 -183552 12168
+48439936 21594092246 0 2 3 2 send 22 tend_part 7 1 32 1368 13536
+48448809 21594101119 0 2 3 1 send 22 tend_part 7 1 32 1368 14904
+48521474 21594173784 81538 2 3 2 send 22 tend_part 7 0 32 -1368 13536
+48529891 21594182201 81082 2 3 1 send 22 tend_part 7 0 32 -1368 12168
+48659245 21594311555 40417763 2 3 0 recv 23 tend_part 7 0 138 -2712 9456
+48993631 21594645941 40728104 2 3 0 recv 23 tend_part 7 0 192 -6936 2520
+56516622 21602168932 48235573 2 3 1 recv 23 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 32
+## Maximum request size: 0.8354 (MB)
+## Sum of all requests: 7.2695 (MB)
+## Mean of all requests: 0.2272 (MB)
+##
diff --git a/testdata/mpiuse_report-step2.dat b/testdata/mpiuse_report-step2.dat
new file mode 100644
index 0000000..9fe6228
--- /dev/null
+++ b/testdata/mpiuse_report-step2.dat
@@ -0,0 +1,272 @@
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+608817 21543633709 0 2 0 2 recv 23 xv 11 1 445 142720 142720
+610727 21543635619 0 2 0 3 recv 23 xv 11 1 453 311296 454016
+612045 21543636937 0 2 0 2 recv 23 xv 11 1 446 219648 673664
+613550 21543638442 0 2 0 3 recv 23 xv 11 1 38 449024 1122688
+615327 21543640219 0 2 0 3 recv 23 xv 11 1 62 282624 1405312
+616285 21543641177 0 2 0 1 recv 23 xv 11 1 177 379648 1784960
+618295 21543643187 0 2 0 3 recv 23 xv 11 1 50 400128 2185088
+619247 21543644139 0 2 0 2 recv 23 xv 11 1 160 94208 2279296
+620661 21543645553 0 2 0 3 recv 23 xv 11 1 49 318720 2598016
+623155 21543648047 0 2 0 2 recv 23 xv 11 1 158 131072 2729088
+625506 21543650398 0 2 0 3 recv 23 xv 11 1 40 416512 3145600
+627165 21543652057 0 2 0 2 recv 23 xv 11 1 156 111360 3256960
+629619 21543654511 0 2 0 3 recv 23 xv 11 1 67 312448 3569408
+631490 21543656382 0 2 0 3 recv 23 xv 11 1 65 292096 3861504
+632856 21543657748 0 2 0 3 recv 23 xv 11 1 465 77696 3939200
+635292 21543660184 0 2 0 2 recv 23 xv 11 1 3 583168 4522368
+636746 21543661638 0 2 0 2 recv 23 xv 11 1 2 107136 4629504
+639251 21543664143 0 2 0 3 recv 23 xv 11 1 130 711424 5340928
+640726 21543665618 0 2 0 1 recv 23 xv 11 1 118 553984 5894912
+642551 21543667443 0 2 0 1 recv 23 xv 11 1 45 328576 6223488
+644510 21543669402 0 2 0 3 recv 23 xv 11 1 192 345984 6569472
+646036 21543670928 0 2 0 1 recv 23 xv 11 1 44 1056768 7626240
+647586 21543672478 0 2 0 3 recv 23 xv 11 1 230 255232 7881472
+649599 21543674491 0 2 0 1 recv 23 xv 11 1 194 143872 8025344
+651388 21543676280 0 2 0 1 recv 23 xv 11 1 202 360832 8386176
+653752 21543678644 0 2 0 1 recv 23 xv 11 1 192 1066112 9452288
+655523 21543680415 0 2 0 2 recv 23 xv 11 1 249 28544 9480832
+657245 21543682137 0 2 0 2 recv 23 xv 11 1 245 83840 9564672
+659152 21543684044 0 2 0 2 recv 23 xv 11 1 244 78720 9643392
+661120 21543686012 0 2 0 1 recv 23 xv 11 1 205 547968 10191360
+664106 21543688998 0 2 0 3 recv 23 xv 11 1 200 368000 10559360
+666953 21543691845 0 2 0 1 recv 23 xv 11 1 137 212608 10771968
+668385 21543693277 0 2 0 3 recv 23 xv 11 1 137 418304 11190272
+669960 21543694852 0 2 0 3 recv 23 xv 11 1 136 199808 11390080
+671970 21543696862 0 2 0 1 recv 23 xv 11 1 57 332160 11722240
+6447305 21549472197 5791782 2 0 2 recv 23 xv 11 0 249 -28544 11693696
+11419335 21554444227 10796180 2 0 2 recv 23 xv 11 0 158 -131072 11562624
+11758915 21554783807 11111329 2 0 3 recv 23 xv 11 0 230 -255232 11307392
+11799763 21554824655 11172598 2 0 2 recv 23 xv 11 0 156 -111360 11196032
+11838133 21554863025 11222806 2 0 3 recv 23 xv 11 0 62 -282624 10913408
+11910866 21554935758 11253621 2 0 2 recv 23 xv 11 0 245 -83840 10829568
+13014189 21556039081 12405372 2 0 2 recv 23 xv 11 0 445 -142720 10686848
+13215819 21556240711 12580527 2 0 2 recv 23 xv 11 0 3 -583168 10103680
+13730903 21556755795 13111656 2 0 2 recv 23 xv 11 0 160 -94208 10009472
+14435788 21557460680 13776636 2 0 2 recv 23 xv 11 0 244 -78720 9930752
+15491583 21558516475 14823198 2 0 3 recv 23 xv 11 0 137 -418304 9512448
+15614696 21558639588 14981840 2 0 3 recv 23 xv 11 0 465 -77696 9434752
+15728921 21558753813 15116876 2 0 2 recv 23 xv 11 0 446 -219648 9215104
+16170362 21559195254 15533616 2 0 2 recv 23 xv 11 0 2 -107136 9107968
+16572275 21559597167 15961548 2 0 3 recv 23 xv 11 0 453 -311296 8796672
+16928290 21559953182 16289039 2 0 3 recv 23 xv 11 0 130 -711424 8085248
+28452779 21571477671 27827273 2 0 3 recv 23 xv 11 0 40 -416512 7668736
+28707936 21571732828 28054184 2 0 1 recv 23 xv 11 0 192 -1066112 6602624
+28977681 21572002573 28364131 2 0 3 recv 23 xv 11 0 38 -449024 6153600
+30058812 21573083704 29440517 2 0 3 recv 23 xv 11 0 50 -400128 5753472
+30160189 21573185081 29539528 2 0 3 recv 23 xv 11 0 49 -318720 5434752
+30309729 21573334621 29639769 2 0 3 recv 23 xv 11 0 136 -199808 5234944
+30514052 21573538944 29849946 2 0 3 recv 23 xv 11 0 200 -368000 4866944
+31055012 21574079904 30410502 2 0 3 recv 23 xv 11 0 192 -345984 4520960
+38056623 21581081515 37427004 2 0 3 recv 23 xv 11 0 67 -312448 4208512
+38085547 21581110439 37454057 2 0 3 recv 23 xv 11 0 65 -292096 3916416
+38835374 21581860266 38189338 2 0 1 recv 23 xv 11 0 44 -1056768 2859648
+40237431 21583262323 39586043 2 0 1 recv 23 xv 11 0 202 -360832 2498816
+40553084 21583577976 39891964 2 0 1 recv 23 xv 11 0 205 -547968 1950848
+40777829 21583802721 40161544 2 0 1 recv 23 xv 11 0 177 -379648 1571200
+40836656 21583861548 40164686 2 0 1 recv 23 xv 11 0 57 -332160 1239040
+40873449 21583898341 40206496 2 0 1 recv 23 xv 11 0 137 -212608 1026432
+41885432 21584910324 0 2 0 2 send 22 tend_part 7 1 38 2328 1028760
+41896300 21584921192 0 2 0 3 send 22 tend_part 7 1 38 2328 1031088
+41942824 21584967716 57392 2 0 2 send 22 tend_part 7 0 38 -2328 1028760
+41950298 21584975190 53998 2 0 3 send 22 tend_part 7 0 38 -2328 1026432
+43257053 21586281945 42607454 2 0 1 recv 23 xv 11 0 194 -143872 882560
+43529999 21586554891 42887448 2 0 1 recv 23 xv 11 0 45 -328576 553984
+43591111 21586616003 42950385 2 0 1 recv 23 xv 11 0 118 -553984 0
+48599962 21591624854 0 2 0 2 send 22 tend_part 7 1 138 2712 2712
+48619658 21591644550 0 2 0 1 send 22 tend_part 7 1 138 2712 5424
+48642246 21591667138 0 2 0 3 send 22 tend_part 7 1 138 2712 8136
+48695836 21591720728 53590 2 0 3 send 22 tend_part 7 0 138 -2712 5424
+48704250 21591729142 104288 2 0 2 send 22 tend_part 7 0 138 -2712 2712
+48924980 21591949872 305322 2 0 1 send 22 tend_part 7 0 138 -2712 0
+48963599 21591988491 0 2 0 1 send 22 tend_part 7 1 192 6936 6936
+48968850 21591993742 5251 2 0 1 send 22 tend_part 7 0 192 -6936 0
+48990086 21592014978 0 2 0 3 send 22 tend_part 7 1 192 6936 6936
+48995836 21592020728 5750 2 0 3 send 22 tend_part 7 0 192 -6936 0
+##
+## Number of requests: 42
+## Maximum request size: 1.0167 (MB)
+## Sum of all requests: 11.2046 (MB)
+## Mean of all requests: 0.2668 (MB)
+##
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+12953800 21558567216 0 2 1 3 recv 23 tend_part 7 1 32 1368 1368
+12964804 21558578220 0 2 1 0 recv 23 tend_part 7 1 138 2712 4080
+12970634 21558584050 0 2 1 0 recv 23 tend_part 7 1 192 6936 11016
+12983064 21558596480 0 2 1 2 recv 23 xv 11 1 321 310784 321800
+12985183 21558598599 0 2 1 3 recv 23 xv 11 1 440 876032 1197832
+12986818 21558600234 0 2 1 3 recv 23 xv 11 1 403 171136 1368968
+12989387 21558602803 0 2 1 2 recv 23 xv 11 1 395 487040 1856008
+12993612 21558607028 0 2 1 2 recv 23 xv 11 1 383 146048 2002056
+12998091 21558611507 0 2 1 2 recv 23 xv 11 1 322 235904 2237960
+18459228 21564072644 0 2 1 0 send 22 xv 11 1 192 1066112 3304072
+23740217 21569353633 0 2 1 0 send 22 xv 11 1 44 1056768 4360840
+25940939 21571554355 0 2 1 0 send 22 xv 11 1 205 547968 4908808
+28186716 21573800132 0 2 1 0 send 22 xv 11 1 177 379648 5288456
+29239247 21574852663 0 2 1 3 send 22 xv 11 1 453 282752 5571208
+30275585 21575889001 17281973 2 1 2 recv 23 xv 11 0 383 -146048 5425160
+30291458 21575904874 0 2 1 0 send 22 xv 11 1 202 360832 5785992
+33012165 21578625581 0 2 1 0 send 22 xv 11 1 57 332160 6118152
+33756463 21579369879 20771280 2 1 3 recv 23 xv 11 0 440 -876032 5242120
+33892572 21579505988 20903185 2 1 2 recv 23 xv 11 0 395 -487040 4755080
+33925103 21579538519 0 2 1 0 send 22 xv 11 1 137 212608 4967688
+34725100 21580338516 0 2 1 0 send 22 xv 11 1 194 143872 5111560
+36049973 21581663389 0 2 1 0 send 22 xv 11 1 45 328576 5440136
+36265488 21581878904 17806260 2 1 0 send 22 xv 11 0 192 -1066112 4374024
+38193925 21583807341 0 2 1 3 send 22 xv 11 1 56 472064 4846088
+38822832 21584436248 15082615 2 1 0 send 22 xv 11 0 44 -1056768 3789320
+40220268 21585833684 12033552 2 1 0 send 22 xv 11 0 177 -379648 3409672
+40540832 21586154248 7528667 2 1 0 send 22 xv 11 0 57 -332160 3077512
+40705244 21586318660 6780141 2 1 0 send 22 xv 11 0 137 -212608 2864904
+40722144 21586335560 10430686 2 1 0 send 22 xv 11 0 202 -360832 2504072
+40735977 21586349393 14795038 2 1 0 send 22 xv 11 0 205 -547968 1956104
+42881032 21588494448 0 2 1 0 send 22 xv 11 1 118 553984 2510088
+43243043 21588856459 5049118 2 1 3 send 22 xv 11 0 56 -472064 2038024
+43254760 21588868176 14015513 2 1 3 send 22 xv 11 0 453 -282752 1755272
+43431687 21589045103 7381714 2 1 0 send 22 xv 11 0 45 -328576 1426696
+43441434 21589054850 8716334 2 1 0 send 22 xv 11 0 194 -143872 1282824
+43447599 21589061015 566567 2 1 0 send 22 xv 11 0 118 -553984 728840
+44195173 21589808589 0 2 1 3 send 22 xv 11 1 59 183552 912392
+44432889 21590046305 0 2 1 3 send 22 xv 11 1 47 46720 959112
+44593620 21590207036 160731 2 1 3 send 22 xv 11 0 47 -46720 912392
+44602907 21590216323 407734 2 1 3 send 22 xv 11 0 59 -183552 728840
+44639981 21590253397 31656917 2 1 2 recv 23 xv 11 0 321 -310784 418056
+44960058 21590573474 31961967 2 1 2 recv 23 xv 11 0 322 -235904 182152
+48461460 21594074876 35474642 2 1 3 recv 23 xv 11 0 403 -171136 11016
+48919281 21594532697 35954477 2 1 0 recv 23 tend_part 7 0 138 -2712 8304
+48984643 21594598059 36030843 2 1 3 recv 23 tend_part 7 0 32 -1368 6936
+49409002 21595022418 36438368 2 1 0 recv 23 tend_part 7 0 192 -6936 0
+56492967 21602106383 0 2 1 2 send 22 tend_part 7 1 359 2520 2520
+56507501 21602120917 14534 2 1 2 send 22 tend_part 7 0 359 -2520 0
+56512908 21602126324 0 2 1 3 send 22 tend_part 7 1 359 2520 2520
+56518112 21602131528 5204 2 1 3 send 22 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 25
+## Maximum request size: 1.0167 (MB)
+## Sum of all requests: 7.8303 (MB)
+## Mean of all requests: 0.3132 (MB)
+##
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+987251 21546628055 0 2 2 3 recv 23 tend_part 7 1 32 1368 1368
+1025615 21546666419 0 2 2 0 recv 23 tend_part 7 1 38 2328 3696
+1029326 21546670130 0 2 2 0 recv 23 tend_part 7 1 138 2712 6408
+1038964 21546679768 0 2 2 1 recv 23 tend_part 7 1 359 2520 8928
+1304769 21546945573 0 2 2 3 send 22 xv 11 1 444 98816 107744
+1404115 21547044919 0 2 2 3 send 22 xv 11 1 2 107136 214880
+1406430 21547047234 0 2 2 0 send 22 xv 11 1 2 107136 322016
+1892700 21547533504 0 2 2 0 send 22 xv 11 1 446 219648 541664
+3170516 21548811320 0 2 2 1 send 22 xv 11 1 395 487040 1028704
+3551697 21549192501 0 2 2 0 send 22 xv 11 1 3 583168 1611872
+3608675 21549249479 0 2 2 1 send 22 xv 11 1 383 146048 1757920
+3722238 21549363042 0 2 2 0 send 22 xv 11 1 445 142720 1900640
+4474542 21550115346 0 2 2 1 send 22 xv 11 1 322 235904 2136544
+4660091 21550300895 0 2 2 1 send 22 xv 11 1 321 310784 2447328
+4802628 21550443432 0 2 2 0 send 22 xv 11 1 249 28544 2475872
+4947390 21550588194 144762 2 2 0 send 22 xv 11 0 249 -28544 2447328
+7943938 21553584742 6537508 2 2 0 send 22 xv 11 0 2 -107136 2340192
+7961657 21553602461 6068957 2 2 0 send 22 xv 11 0 446 -219648 2120544
+7971824 21553612628 4420127 2 2 0 send 22 xv 11 0 3 -583168 1537376
+8524640 21554165444 0 2 2 0 send 22 xv 11 1 158 131072 1668448
+8974356 21554615160 0 2 2 0 send 22 xv 11 1 156 111360 1779808
+9378345 21555019149 0 2 2 0 send 22 xv 11 1 160 94208 1874016
+9846859 21555487663 0 2 2 0 send 22 xv 11 1 245 83840 1957856
+10082623 21555723427 6360385 2 2 0 send 22 xv 11 0 445 -142720 1815136
+10425140 21556065944 0 2 2 3 send 22 xv 11 1 1 64512 1879648
+11258279 21556899083 1411420 2 2 0 send 22 xv 11 0 245 -83840 1795808
+12594280 21558235084 2169140 2 2 3 send 22 xv 11 0 1 -64512 1731296
+12624810 21558265614 11220695 2 2 3 send 22 xv 11 0 2 -107136 1624160
+12806376 21558447180 3428031 2 2 0 send 22 xv 11 0 160 -94208 1529952
+12864222 21558505026 3889866 2 2 0 send 22 xv 11 0 156 -111360 1418592
+12923342 21558564146 11618573 2 2 3 send 22 xv 11 0 444 -98816 1319776
+13069781 21558710585 4545141 2 2 0 send 22 xv 11 0 158 -131072 1188704
+13360446 21559001250 0 2 2 3 send 22 xv 11 1 30 79232 1267936
+13833681 21559474485 0 2 2 3 send 22 xv 11 1 44 64128 1332064
+13842742 21559483546 9061 2 2 3 send 22 xv 11 0 44 -64128 1267936
+14347042 21559987846 0 2 2 0 send 22 xv 11 1 244 78720 1346656
+14447458 21560088262 100416 2 2 0 send 22 xv 11 0 244 -78720 1267936
+14549561 21560190365 1189115 2 2 3 send 22 xv 11 0 30 -79232 1188704
+29619186 21575259990 26448670 2 2 1 send 22 xv 11 0 395 -487040 701664
+29973620 21575614424 26364945 2 2 1 send 22 xv 11 0 383 -146048 555616
+30232105 21575872909 25757563 2 2 1 send 22 xv 11 0 322 -235904 319712
+30884935 21576525739 26224844 2 2 1 send 22 xv 11 0 321 -310784 8928
+41905408 21587546212 40879793 2 2 0 recv 23 tend_part 7 0 38 -2328 6600
+48456241 21594097045 47468990 2 2 3 recv 23 tend_part 7 0 32 -1368 5232
+48605647 21594246451 47576321 2 2 0 recv 23 tend_part 7 0 138 -2712 2520
+56499592 21602140396 55460628 2 2 1 recv 23 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 23
+## Maximum request size: 0.5562 (MB)
+## Sum of all requests: 3.0355 (MB)
+## Mean of all requests: 0.1320 (MB)
+##
+# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
+8154566 21553806876 0 2 3 0 recv 23 tend_part 7 1 38 2328 2328
+8241482 21553893792 0 2 3 0 recv 23 tend_part 7 1 138 2712 5040
+8252462 21553904772 0 2 3 1 recv 23 xv 11 1 59 183552 188592
+8255236 21553907546 0 2 3 2 recv 23 xv 11 1 44 64128 252720
+8265527 21553917837 0 2 3 0 recv 23 tend_part 7 1 192 6936 259656
+8267192 21553919502 0 2 3 1 recv 23 xv 11 1 47 46720 306376
+8281049 21553933359 0 2 3 1 recv 23 tend_part 7 1 359 2520 308896
+8285576 21553937886 0 2 3 1 recv 23 xv 11 1 56 472064 780960
+8290638 21553942948 0 2 3 1 recv 23 xv 11 1 453 282752 1063712
+8312534 21553964844 0 2 3 2 recv 23 xv 11 1 444 98816 1162528
+8314496 21553966806 0 2 3 2 recv 23 xv 11 1 30 79232 1241760
+8318186 21553970496 0 2 3 2 recv 23 xv 11 1 2 107136 1348896
+8321900 21553974210 0 2 3 2 recv 23 xv 11 1 1 64512 1413408
+8551050 21554203360 0 2 3 0 send 22 xv 11 1 465 77696 1491104
+9240343 21554892653 0 2 3 0 send 22 xv 11 1 62 282624 1773728
+9516048 21555168358 0 2 3 0 send 22 xv 11 1 230 255232 2028960
+10222671 21555874981 1910137 2 3 2 recv 23 xv 11 0 444 -98816 1930144
+11396780 21557049090 3074880 2 3 2 recv 23 xv 11 0 1 -64512 1865632
+11919637 21557571947 0 2 3 0 send 22 xv 11 1 130 711424 2577056
+12416538 21558068848 496901 2 3 0 send 22 xv 11 0 130 -711424 1865632
+12449098 21558101408 0 2 3 1 send 22 xv 11 1 403 171136 2036768
+12525243 21558177553 4207057 2 3 2 recv 23 xv 11 0 2 -107136 1929632
+13886887 21559539197 0 2 3 0 send 22 xv 11 1 453 311296 2240928
+13993363 21559645673 5738127 2 3 2 recv 23 xv 11 0 44 -64128 2176800
+14353159 21560005469 6038663 2 3 2 recv 23 xv 11 0 30 -79232 2097568
+14361689 21560013999 0 2 3 0 send 22 xv 11 1 137 418304 2515872
+15260362 21560912672 898673 2 3 0 send 22 xv 11 0 137 -418304 2097568
+16761542 21562413852 0 2 3 0 send 22 xv 11 1 200 368000 2465568
+17561805 21563214115 0 2 3 0 send 22 xv 11 1 136 199808 2665376
+17768823 21563421133 0 2 3 1 send 22 xv 11 1 440 876032 3541408
+18876067 21564528377 0 2 3 0 send 22 xv 11 1 192 345984 3887392
+19483556 21565135866 0 2 3 0 send 22 xv 11 1 40 416512 4303904
+20583795 21566236105 0 2 3 0 send 22 xv 11 1 38 449024 4752928
+21166982 21566819292 0 2 3 0 send 22 xv 11 1 49 318720 5071648
+21367099 21567019409 7480212 2 3 0 send 22 xv 11 0 453 -311296 4760352
+21428196 21567080506 11912148 2 3 0 send 22 xv 11 0 230 -255232 4505120
+21489677 21567141987 12249334 2 3 0 send 22 xv 11 0 62 -282624 4222496
+21545174 21567197484 12994124 2 3 0 send 22 xv 11 0 465 -77696 4144800
+27653028 21573305338 10891486 2 3 0 send 22 xv 11 0 200 -368000 3776800
+27725141 21573377451 10163336 2 3 0 send 22 xv 11 0 136 -199808 3576992
+27785208 21573437518 8909141 2 3 0 send 22 xv 11 0 192 -345984 3231008
+27844800 21573497110 8361244 2 3 0 send 22 xv 11 0 40 -416512 2814496
+27904150 21573556460 7320355 2 3 0 send 22 xv 11 0 38 -449024 2365472
+29544767 21575197077 0 2 3 0 send 22 xv 11 1 50 400128 2765600
+29899938 21575552248 355171 2 3 0 send 22 xv 11 0 50 -400128 2365472
+29964381 21575616691 8797399 2 3 0 send 22 xv 11 0 49 -318720 2046752
+31237607 21576889917 0 2 3 0 send 22 xv 11 1 67 312448 2359200
+32076277 21577728587 23785639 2 3 1 recv 23 xv 11 0 453 -282752 2076448
+32997616 21578649926 20548518 2 3 1 send 22 xv 11 0 403 -171136 1905312
+33045540 21578697850 15276717 2 3 1 send 22 xv 11 0 440 -876032 1029280
+35555962 21581208272 0 2 3 0 send 22 xv 11 1 65 292096 1321376
+37724703 21583377013 6487096 2 3 0 send 22 xv 11 0 67 -312448 1008928
+38038978 21583691288 2483016 2 3 0 send 22 xv 11 0 65 -292096 716832
+39312145 21584964455 31026569 2 3 1 recv 23 xv 11 0 56 -472064 244768
+41913009 21587565319 33758443 2 3 0 recv 23 tend_part 7 0 38 -2328 242440
+44653176 21590305486 36385984 2 3 1 recv 23 xv 11 0 47 -46720 195720
+44672181 21590324491 36419719 2 3 1 recv 23 xv 11 0 59 -183552 12168
+48439936 21594092246 0 2 3 2 send 22 tend_part 7 1 32 1368 13536
+48448809 21594101119 0 2 3 1 send 22 tend_part 7 1 32 1368 14904
+48521474 21594173784 81538 2 3 2 send 22 tend_part 7 0 32 -1368 13536
+48529891 21594182201 81082 2 3 1 send 22 tend_part 7 0 32 -1368 12168
+48659245 21594311555 40417763 2 3 0 recv 23 tend_part 7 0 138 -2712 9456
+48993631 21594645941 40728104 2 3 0 recv 23 tend_part 7 0 192 -6936 2520
+56516622 21602168932 48235573 2 3 1 recv 23 tend_part 7 0 359 -2520 0
+##
+## Number of requests: 32
+## Maximum request size: 0.8354 (MB)
+## Sum of all requests: 7.2695 (MB)
+## Mean of all requests: 0.2272 (MB)
+##
--
GitLab