diff --git a/.gitignore b/.gitignore index afa445084e6ff202c6d10f1a42d9af832100a8b4..753ecb8407375c93ae44d9d3d0c26c836a5068e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ swiftmpistepsim +swiftmpifakestepsim *.o *~ diff --git a/Makefile b/Makefile index 75a2e49df9978b6383cf26aa2ad88be12d501164..7756f83a1db34e27f3bf06278d902837b88d268a 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,16 @@ CFLAGS = -g -O0 -Wall -all: swiftmpistepsim +all: swiftmpistepsim swiftmpifakestepsim -swiftmpistepsim: swiftmpistepsim.c mpiuse.c mpiuse.h atomic.h cycle.h clocks.h clocks.c - mpicc $(CFLAGS) -o swiftmpistepsim swiftmpistepsim.c mpiuse.c clocks.c +SRC = mpiuse.c clocks.c histogram.c +INC = mpiuse.h atomic.h cycle.h clocks.h histogram.h + +swiftmpistepsim: swiftmpistepsim.c $(SRC) $(INC) + mpicc $(CFLAGS) -o swiftmpistepsim swiftmpistepsim.c $(SRC) -lpthread -lm + +swiftmpifakestepsim: swiftmpifakestepsim.c $(SRC) $(INC) + mpicc $(CFLAGS) -o swiftmpifakestepsim swiftmpifakestepsim.c $(SRC) -lpthread -lm clean: rm -f swiftmpistepsim + rm -f swiftmpifakestepsim diff --git a/README.fakedata.md b/README.fakedata.md new file mode 100644 index 0000000000000000000000000000000000000000..30a8f12b5c9f7cc3611371465a2867e9dfca161d --- /dev/null +++ b/README.fakedata.md @@ -0,0 +1,50 @@ + +swiftmpifakestepsim +=================== + +The swiftmpifakestepsim program works much like swiftmpistepsim, except that +it generates fake sizes messages and can be scaled to run on any number of MPI +ranks. + +``` +Usage: ./swiftmpifakestepsim [options] nr_messages logfile.dat + options: -v verbose, -d data check, -s size (bytes/scale), + -f <1|2> randomize injection order, 1 == just sends, 2 == sends and recvs + [-r uniform random from 1 to size, | + -r -g half gaussian random from 1 with 2.5 sigma size., | + -r -c <file> use cdf from file, size is a scale factor., | + -r -o <file> use occurence sample of values in a file, size is a scale factor.,] + -x random seed +``` + +To use a fixed message size just use `-s`. + +To use a uniform distribution in the range 1 to N: +``` + -s N -r +``` + +To use a half gaussian (so biased towards smaller packets) use: +``` + -r -s <scale> -g +``` + +The cdf option reads a simple text file with a sampling of a cumulative +distribution function, where each line has three values, the minimum and +maximum range of the current bin and the value. Note that the value column +should be normalized into the range 0 to 1. +``` + -r -s <scale> -c <cdf_file> +``` + +The occurrence file has just one value per line, these should present the +sizes of the packets, this is used to form a cdf: +``` + -r -s <scale> -o <occurrence_file> +``` + +Other options are useful to make sure that the randoms are different, `-x` and +that they run in different order `-f <1|2>`. + +Peter W. Draper 24 Apr 2023 +--------------------------- diff --git a/README.md b/README.md index 2601f6ecf09cb6a2eca4b1422afd8ce5834ef078..712d27e46c35b6297be317f1cf4439f1b1c82eaf 100644 --- a/README.md +++ b/README.md @@ -64,3 +64,14 @@ eager exchanges are working and what effect the size of the packets has. --------------------------- Peter W. Draper 24 Sep 2019. + +Continuing from the notes above, there are also RDMA based versions of +swiftmpistepsim, which are found in the various branches of the +repository. That work and a re-working of SWIFT to use RDMA revealed that the +main driver for the delays wasn't the scatter in MPI completion, but the time +taken to copy data into and out of registered memory, the best solution to +which was to make better use of the memory bandwidth by copying using multiple +threads. + +----------------------------- +Peter W. Draper 24 April 2023. diff --git a/error.h b/error.h index d5e331486620f77cbf16b15ad673a9c43ad7876b..bda6a62106d387cfa62076d9e5299b8a11d44f3d 100644 --- a/error.h +++ b/error.h @@ -1,4 +1,5 @@ #include <mpi.h> + #include "clocks.h" extern int myrank; diff --git a/format.sh b/format.sh index 91346334c9b2eaf9fbb343aba44f8a02d866d1ef..db5e7ab01a1603b987c3fa7e83a8736ed7faea31 100755 --- a/format.sh +++ b/format.sh @@ -1,8 +1,8 @@ #!/bin/bash # Clang format command, can be overridden using CLANG_FORMAT_CMD. -# We currrently use version 5.0 so any overrides should provide that. -clang=${CLANG_FORMAT_CMD:="clang-format-5.0"} +# We currrently use version 13.0 so any overrides should provide that. +clang=${CLANG_FORMAT_CMD:="clang-format-13.0"} # Formatting command cmd="$clang -style=file $(git ls-files | grep '\.[ch]$')" diff --git a/histogram.c b/histogram.c new file mode 100644 index 0000000000000000000000000000000000000000..d9b1609db8e22abcbe5076e8c061538dc09d4212 --- /dev/null +++ b/histogram.c @@ -0,0 +1,138 @@ +#include "histogram.h" + +#include <float.h> +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* + * Simple histogram routines. +int main(int argc, char *argv[]) { + double *values; + int nvalues; + if (histread(argv[1], &values, &nvalues)) { + printf("## Read %d values from %s\n", nvalues, argv[1]); + struct histogram *h = calloc(1, sizeof(struct histogram)); + histmake(nvalues, values, h); + printf("## Created cumulative histogram with %d values:\n", h->nvalues); + printf("# value sum\n"); + for (int k = 0; k < h->nvalues; k++) { + printf("%f %24.17g\n", h->values[k], h->sums[k]); + } + free(h); + free(values); + return 0; + } + return 1; +} +*/ + +/** + * Create a histogram by binning a number of data values. + * + * The histogram is returned in a histogram struct, which includes the basic + * histogram (with NHIST values) and a normalized cumulative version (with + * h->nvals values, as the zero bins are coalesced). + * + * @param nvalues number of values. + * @param values the data values to bin into a histogram. + * @param h the #histogram. + */ +void histmake(int nvalues, double *values, struct histogram *h) { + + /* Find the minimum and maximum values. */ + double dmin = DBL_MAX; + double dmax = -DBL_MAX; + for (int i = 0; i < nvalues; i++) { + dmin = fmin(dmin, values[i]); + dmax = fmax(dmax, values[i]); + } + + /* Form the fixed width bin histogram. */ + double scale = (double)NHIST / (dmax - dmin); + h->width = 1.0 / scale; + h->zero = dmin - h->width / 2.0; + double sum = 0.0; + for (int i = 0; i < nvalues; i++) { + int idiff = (int)round(scale * ((double)values[i] - dmin)); + h->hist[idiff] += 1; + sum++; + } + + double norm = 1.0 / sum; + + /* Form cumulative sums and count used bins. */ + sum = 0.0; + int lastbin = 0; + int usedbins = 0; + for (int i = 0; i < NHIST; i++) { + + /* Zero bins are folded into a range. */ + if (h->hist[i] > 0) { + + /* Value is mid of bin. */ + h->values[usedbins] = 0.5 * ((lastbin * h->width + h->zero) + + ((i + 1) * h->width + h->zero)); + sum += h->hist[i] * norm; + h->sums[usedbins] = sum; + usedbins++; + lastbin = i + 1; + } + } + h->nvalues = usedbins; +} + +/** + * Read in data to histogram. Assumes a simple text file with one value per + * line. + * + * @param file the name of the file to read. + * @param values the extracted values. Free when not needed. + * @param nvalues the number of values. + * + * @result 1 for successful, 0 otherwise. + */ +int histread(const char *file, double **values, int *nvalues) { + + *values = NULL; + *nvalues = 0; + + FILE *infile = fopen(file, "r"); + if (infile == NULL) { + printf("Failed to open sizes file: %s\n", file); + return 0; + } + + /* Initial space for data. */ + int nvals = 0; + int maxvals = 1024; + double *vals = malloc(maxvals * sizeof(double)); + + char line[132]; + while (!feof(infile)) { + if (fgets(line, 132, infile) != NULL) { + if (line[0] != '#') { + int n = sscanf(line, "%lf", &vals[nvals]); + if (n != 1) { + printf("Incorrect no. of values %s\n", line); + fclose(infile); + free(vals); + return 0; + } + nvals++; + + /* Make more space. */ + if (nvals > maxvals) { + maxvals += 1024; + vals = realloc(vals, maxvals * sizeof(double)); + } + } + } + } + fclose(infile); + + *values = vals; + *nvalues = nvals; + return 1; +} diff --git a/histogram.h b/histogram.h new file mode 100644 index 0000000000000000000000000000000000000000..1ddd42544eb168b51590171df46c8648ed8b6a08 --- /dev/null +++ b/histogram.h @@ -0,0 +1,44 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2021 Peter W. Draper (p.w.draper@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_HISTOGRAM_H +#define SWIFT_HISTOGRAM_H + +/* Bins in a histogram. Note this must be an even number. */ +#define NHIST 65536 + +/** Histogram structure. */ +struct histogram { + /* Raw histogram. NHIST counts in hist and the value associated with a bin + * is index*width+zero. */ + double width; + double zero; + int hist[NHIST]; + + /* Normalized cumulative histogram. Empty bins are joined into a larger one, + * so values are the bin centre, sums the sum to that bin and nvalues the + * number of bins that have been populated, can be less than NHIST. */ + double values[NHIST]; + double sums[NHIST]; + int nvalues; +}; + +int histread(const char *file, double **values, int *nvalues); +void histmake(int nvalues, double *values, struct histogram *h); + +#endif diff --git a/mpiuse.c b/mpiuse.c index 6bd7b3116e0910b152e49112be86057bad80a0f6..e25981defe39914f325f895eb382fd2d7c1e3827 100644 --- a/mpiuse.c +++ b/mpiuse.c @@ -22,6 +22,7 @@ */ /* Standard includes. */ +#include <math.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -34,6 +35,7 @@ #include "clocks.h" #include "cycle.h" #include "error.h" +#include "histogram.h" /* Our rank. */ extern int myrank; @@ -257,3 +259,173 @@ struct mpiuse_log_entry *mpiuse_get_log(int ind) { if (ind < mpiuse_log_count && ind >= 0) return &mpiuse_log[ind]; return NULL; } + +/** + * @brief return random number from a upper part of gaussian distribution. + * + * @result the random. + */ +static double gauss_rand_upper(void) { + + double V1, V2, S; + do { + double U1 = drand48(); + double U2 = drand48(); + + V1 = U1 - 1.0; + V2 = U2 - 1.0; + S = V1 * V1 + V2 * V2; + } while (S >= 1.0 || S == 0.0); + + return fabs(V1 * sqrt(-2.0 * log(S) / S)); +} + +/** + * @brief generate a list of fake exchanges as mpiuse logs. + * + * @param nr_nodes the number of ranks that will be used. + * @param nr_logs the number of logs to generate per rank. + * @param size bytes per message, unless random when this is the maximum + * and the minimum is 1 for uniform, if using a gaussian + * distribution the value is a 2.5 sigma, for a CDF based + * selection this is just a scale factor of the values. + * @param random whether to use random sizes. + * @param seed the random seed, use same for fixed sequences. + * @param uniform whether to use a uniform distribution or gaussian, unless + * cdf is defined, in which case this parameter is ignored. + * @param cdf text file containing a normalized CDF to use as a basis for + * inverse transform sampling of the randoms. NULL for no file. + * @param odata text file containing a values representing occurences of the + * expected distribution -- converted into a normalised CDF to + * use as a basis for inverse transform sampling of the + * randoms. NULL for no file. Not used if cdf is not NULL. + */ +void mpiuse_log_generate(int nr_nodes, int nr_logs, int size, int random, + long int seed, int uniform, const char *cdf, + const char *odata) { + + /* Only used for CDFs, may need to increase these. */ + int nvals = 0; + double imin[NHIST], imax[NHIST], value[NHIST]; + + /* Note that each rank exchanges messages with all the others and each "log" + * has the same size. */ + /* Set seed. */ + if (random) srand48(seed); + + /* Check for CDF. This should be based on some real distribution, the format + * is same as output from TOPCAT, i.e. bin-low, bin-high, value space + * separated values. Note the value column should be normalised into the + * range 0 to 1 so that it maps into a uniform random distribution. */ + if (cdf != NULL) { + FILE *infile = fopen(cdf, "r"); + if (infile == NULL) error("Failed to open CDF file: %s", cdf); + char line[132]; + while (!feof(infile)) { + if (fgets(line, 132, infile) != NULL) { + if (line[0] != '#') { + int nread = sscanf(line, "%lf %lf %lf", &imin[nvals], &imax[nvals], + &value[nvals]); + if (nread == 3) nvals++; + } + } + } + fclose(infile); + } else if (odata != NULL) { + double *values; + int nvalues; + if (histread(odata, &values, &nvalues)) { + // printf("## Read %d occurence values from %s\n", nvalues, odata); + struct histogram *h = calloc(1, sizeof(struct histogram)); + histmake(nvalues, values, h); + // printf("## Created cumulative histogram with %d values:\n", + // h->nvalues); printf("# value sum\n"); + imin[0] = 0.0; + imax[0] = h->values[0]; + value[0] = h->sums[0]; + for (int k = 1; k < h->nvalues; k++) { + imin[k] = h->values[k - 1]; + imax[k] = h->values[k]; + value[k] = h->sums[k]; + // printf("%f %24.17g\n", h->values[k], h->sums[k]); + } + nvals = h->nvalues; + + free(h); + free(values); + } else { + error("Failed to read occurrence data from file: %s", odata); + } + } + + /* Message tags increment with across rank logs. */ + int tag = 1; + for (int k = 0; k < nr_logs; k++) { + + /* Set size for this messages. */ + double logsize = size; + if (random) { + if (cdf || odata) { + /* CDF based randoms. */ + double rand = drand48(); + + /* Binary search for containing bin for this rand. */ + unsigned int lower = 0; + unsigned int upper = nvals; + unsigned int middle = 0; + while (lower < upper) { + middle = (upper + lower) / 2; + if (rand > value[middle]) + lower = middle + 1; + else + upper = middle; + } + logsize = 0.5 * (imax[middle] + imin[middle]); + + } else if (uniform) { + /* Uniform randoms in the range 0 to 1 */ + logsize = (drand48() * (double)size) + 1; + } else { + // Gaussian randoms so no maximum, assume size is 2.5 sigma. + logsize = (gauss_rand_upper() * (double)size * 0.25) + 1; + } + } + + /* Cannot send more than 2^31-1 bytes at a time, so truncate. */ + if (logsize > 2147483647.0) { + message("CDF size too large : %f, truncating", logsize); + logsize = 2147483647.0; + } + + for (int i = 0; i < nr_nodes; i++) { + for (int j = 0; j < nr_nodes; j++) { + if (i != j) { + mpiuse_log_allocation(i, 1, k, SEND_TYPE, NO_SUBTYPE, 1, + (size_t)logsize, j, tag); + mpiuse_log_allocation(j, 1, k, RECV_TYPE, NO_SUBTYPE, 1, + (size_t)logsize, i, tag); + } + } + } + tag++; + } +} + +/** + * Shuffle log pointers randomizing the order. + * + * Note assumes dran48() has been seeded. + * + * @param logs the log pointers to shuffle. + * @param nlogs the number of logs. + */ +void mpiuse_shuffle_logs(struct mpiuse_log_entry **logs, int nlogs) { + + struct mpiuse_log_entry tmp; + for (int k = nlogs - 1; k > 0; k--) { + unsigned int j = (unsigned int)(drand48() * (k + 1)); + memcpy(&tmp, &logs[j], sizeof(struct mpiuse_log_entry *)); + memcpy(&logs[j], &logs[k], sizeof(struct mpiuse_log_entry *)); + memcpy(&logs[k], &tmp, sizeof(struct mpiuse_log_entry *)); + } +} diff --git a/mpiuse.h b/mpiuse.h index 707ab27547b14fc701fcfd53254d29d603007891..71d8982f0cbb020dd6a563d2f8cb756ccacf3dd5 100644 --- a/mpiuse.h +++ b/mpiuse.h @@ -84,6 +84,13 @@ struct mpiuse_log_entry { ticks tmin; }; +/* Flags for the types of request when generating fakes. */ +#ifndef SEND_TYPE +#define SEND_TYPE 25 +#define RECV_TYPE 26 +#define NO_SUBTYPE 0 +#endif + /* API. */ void mpiuse_log_allocation(int rank, int step, size_t tic, int type, int subtype, int activation, size_t size, @@ -94,4 +101,9 @@ int mpiuse_nr_logs(void); int mpiuse_nr_ranks(void); void mpiuse_dump_logs(int nranks, const char *logfile); +void mpiuse_log_generate(int nr_nodes, int nr_logs, int size, int random, + long int seed, int uniform, const char *cdf, + const char *odata); +void mpiuse_shuffle_logs(struct mpiuse_log_entry **logs, int nlogs); + #endif /* SWIFT_MPIUSE_H */ diff --git a/swiftmpifakestepsim.c b/swiftmpifakestepsim.c new file mode 100644 index 0000000000000000000000000000000000000000..d14d8538de4f2cd51d599881b6dcd4e9ef6678c8 --- /dev/null +++ b/swiftmpifakestepsim.c @@ -0,0 +1,572 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2021 Peter W. Draper + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ + +#include <limits.h> +#include <mpi.h> +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include "atomic.h" +#include "clocks.h" +#include "error.h" +#include "mpiuse.h" + +/* Global: Our rank for all to see. */ +int myrank = -1; + +/* Are we verbose. */ +static int verbose = 0; + +/* Set a data pattern and check we get this back, slow... */ +static int datacheck = 0; + +/* Default seed for pseudorandoms. */ +static long int default_seed = 1987654321; + +/* MPI communicator for each rank. XXX static XXX. */ +static MPI_Comm node_comms[512]; + +/* The local queues. */ +static struct mpiuse_log_entry **volatile reqs_queue; +static int volatile ind_req = 0; +static int volatile nr_reqs = 0; +static int volatile injecting = 1; +static struct mpiuse_log_entry **volatile recvs_queue; +static int volatile nr_recvs = 0; +static int volatile ind_recv = 0; +static int volatile todo_recv = 0; +static struct mpiuse_log_entry **volatile sends_queue; +static int volatile nr_sends = 0; +static int volatile ind_send = 0; +static int volatile todo_send = 0; + +/** + * @brief fill a data area with a pattern that can be checked for changes. + * + * @param size size of data in bytes. + * @param data the data to fill. + */ +static void datacheck_fill(size_t size, void *data) { + unsigned char *p = (unsigned char *)data; + for (size_t i = 0; i < size; i++) { + p[i] = 170; /* 10101010 in bits. */ + } +} + +/** + * @brief test a filled data area for our pattern. + * + * @param size size of data in bytes. + * @param data the data to fill. + * + * @result 1 on success, 0 otherwise. + */ +static int datacheck_test(size_t size, void *data) { + unsigned char *p = (unsigned char *)data; + for (size_t i = 0; i < size; i++) { + if (p[i] != 170) return 0; + } + return 1; +} + +/** + * @brief Injection thread, initiates MPI_Isend and MPI_Irecv requests. + */ +static void *inject_thread(void *arg) { + + if (verbose) message("%d: injection thread starts", *((int *)arg)); + ticks starttics = getticks(); + + while (ind_req < nr_reqs) { + struct mpiuse_log_entry *log = reqs_queue[ind_req]; + + /* Initialise new log elements. */ + log->done = 0; + log->nr_tests = 0; + log->tsum = 0.0; + log->tmax = 0; + log->tmin = INT_MAX; + log->endtic = 0; + log->injtic = getticks(); + + /* Differences to SWIFT: MPI_BYTE not the MPI_Type. */ + int err = 0; + if (log->type == SEND_TYPE) { + log->data = calloc(log->size, 1); + + /* Fill data with pattern. */ + if (datacheck) datacheck_fill(log->size, log->data); + + /* And send. */ + err = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank, log->tag, + node_comms[log->rank], &log->req); + + /* Add a new send request. */ + int ind = atomic_inc(&nr_sends); + sends_queue[ind] = log; + atomic_inc(&todo_send); + + } else { + + /* Ready to receive. */ + log->data = calloc(log->size, 1); + err = MPI_Irecv(log->data, log->size, MPI_BYTE, log->otherrank, log->tag, + node_comms[log->otherrank], &log->req); + + /* Add a new recv request. */ + int ind = atomic_inc(&nr_recvs); + recvs_queue[ind] = log; + atomic_inc(&todo_recv); + } + if (err != MPI_SUCCESS) error("Failed to activate send or recv"); + + ind_req++; + } + + /* All done, thread exiting. */ + if (verbose) { + message("%d injections completed, sends = %d, recvs = %d", ind_req, + nr_sends, nr_recvs); + message("remaining sends = %d, recvs = %d", todo_send, todo_recv); + } + message("took %.3f %s.", clocks_from_ticks(getticks() - starttics), + clocks_getunit()); + atomic_dec(&injecting); + return NULL; +} + +/** + * @brief main loop to run over a queue of MPI requests and test for when they + * complete. Returns the total amount of time spent in calls to MPI_Test and + * the number of times it was called. + * + * @param logs the list of logs pointing to requests. + * @param nr_logs pointer to the variable containing the current number of + * logs. + * @param todos pointer to the variable containing the number of requests that + * are still active. + * @param sum the total number of ticks spent in calls to MPI_Test. + * @param ncalls the total number of calls to MPI_Test. + * @param mint the minimum ticks an MPI_Test call took. + * @param maxt the maximum ticks an MPI_Test call took. + */ +static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs, + int volatile *todos, double *sum, int *ncalls, + ticks *mint, ticks *maxt) { + + /* Global MPI_Test statistics. */ + int lncalls = 0; + double lsum = 0.0; + ticks lmint = INT_MAX; + ticks lmaxt = 0; + + /* We loop while new requests are being injected and we still have requests + * to complete. */ + while (injecting || (!injecting && *todos > 0)) { + int nlogs = *nr_logs; + for (int k = 0; k < nlogs; k++) { + struct mpiuse_log_entry *log = logs[k]; + if (log != NULL && !log->done) { + ticks tics = getticks(); + int res; + MPI_Status stat; + int err = MPI_Test(&log->req, &res, &stat); + if (err != MPI_SUCCESS) { + error("MPI_Test call failed"); + } + + /* Increment etc. of statistics about time in MPI_Test. */ + ticks dt = getticks() - tics; + log->tsum += (double)dt; + lsum += (double)dt; + + log->nr_tests++; + lncalls++; + + if (dt < log->tmin) log->tmin = dt; + if (dt > log->tmax) log->tmax = dt; + if (dt < lmint) lmint = dt; + if (dt > lmaxt) lmaxt = dt; + + if (res) { + /* Check data sent data is unchanged and received data is as + * expected. */ + if (datacheck && !datacheck_test(log->size, log->data)) { + error("Data mismatch on completion"); + } + + /* Done, clean up. */ + log->done = 1; + log->endtic = getticks(); + free(log->data); + atomic_dec(todos); + } + } + } + } + + /* All done. */ + *sum = lsum; + *ncalls = lncalls; + *mint = lmint; + *maxt = lmaxt; + return; +} + +/** + * @brief Send thread, checks if MPI_Isend requests have completed. + */ +static void *send_thread(void *arg) { + + if (verbose) message("%d: send thread starts (%d)", *((int *)arg), injecting); + ticks starttics = getticks(); + + int ncalls; + double sum; + ticks mint; + ticks maxt; + queue_runner(sends_queue, &nr_sends, &todo_send, &sum, &ncalls, &mint, &maxt); + + message( + "%d MPI_Test calls took: %.3f, mean time %.3f, min time %.3f, max time " + "%.3f (%s)", + ncalls, clocks_from_ticks(sum), clocks_from_ticks(sum / ncalls), + clocks_from_ticks(mint), clocks_from_ticks(maxt), clocks_getunit()); + if (verbose) + message("took %.3f %s.", clocks_from_ticks(getticks() - starttics), + clocks_getunit()); + + /* Thread exits. */ + return NULL; +} + +/** + * @brief Recv thread, checks if MPI_Irecv requests have completed. + */ +static void *recv_thread(void *arg) { + + if (verbose) message("%d: recv thread starts", *((int *)arg)); + ticks starttics = getticks(); + + int ncalls; + double sum; + ticks mint; + ticks maxt; + queue_runner(recvs_queue, &nr_recvs, &todo_recv, &sum, &ncalls, &mint, &maxt); + + message( + "%d MPI_Test calls took: %.3f, mean time %.3f, min time %.3f, max time " + "%.3f (%s)", + ncalls, clocks_from_ticks(sum), clocks_from_ticks(sum / ncalls), + clocks_from_ticks(mint), clocks_from_ticks(maxt), clocks_getunit()); + if (verbose) + message("took %.3f %s.", clocks_from_ticks(getticks() - starttics), + clocks_getunit()); + + /* Thread exits. */ + return NULL; +} + +/** + * @brief Comparison function for logged times. + */ +static int cmp_logs(const void *p1, const void *p2) { + struct mpiuse_log_entry *l1 = *(struct mpiuse_log_entry **)p1; + struct mpiuse_log_entry *l2 = *(struct mpiuse_log_entry **)p2; + + /* Large unsigned values, so take care. */ + if (l1->tic > l2->tic) return 1; + if (l1->tic < l2->tic) return -1; + return 0; +} + +/** + * @brief Pick out the relevant logging data for our rank, i.e. all + * activations of sends and recvs. We ignore the original completions. + * The final list is sorted into increasing time of activation if required, + * otherwise the order is randomized. + * + * @param random randomize injection order of sends or sends and recvs, + * otherwise use order of the original logs. Just sends + * are randomized when 1, 2 randomizes both, 0 neither. + */ +static void pick_logs(int random) { + size_t nlogs = mpiuse_nr_logs(); + + /* Duplicate of logs. */ + reqs_queue = (struct mpiuse_log_entry **)calloc( + nlogs, sizeof(struct mpiuse_log_entry *)); + nr_reqs = 0; + sends_queue = (struct mpiuse_log_entry **)calloc( + nlogs, sizeof(struct mpiuse_log_entry *)); + nr_sends = 0; + recvs_queue = (struct mpiuse_log_entry **)calloc( + nlogs, sizeof(struct mpiuse_log_entry *)); + nr_recvs = 0; + + if (random == 0 || random == 2) { + for (int k = 0; k < nlogs; k++) { + struct mpiuse_log_entry *log = mpiuse_get_log(k); + if (log->rank == myrank && log->activation) { + log->data = NULL; + reqs_queue[nr_reqs] = log; + nr_reqs++; + } + } + + if (random == 0) { + /* Sort into increasing time. */ + qsort(reqs_queue, nr_reqs, sizeof(struct mpiuse_log_entry *), cmp_logs); + } else { + /* Randomize the order, so ranks do not all work in sequence. */ + mpiuse_shuffle_logs(reqs_queue, nr_reqs); + } + + /* Check. */ + if (random == 0) { + for (int k = 0; k < nr_reqs - 1; k++) { + if (reqs_queue[k]->tic > reqs_queue[k + 1]->tic) + message("reqs_queue: %lld > %lld", reqs_queue[k]->tic, + reqs_queue[k + 1]->tic); + } + } + } else { + + /* Randomizing the sends, but injecting the recvs first. */ + + /* Get recvs. */ + int nrecv = 0; + for (int k = 0; k < nlogs; k++) { + struct mpiuse_log_entry *log = mpiuse_get_log(k); + if (log->rank == myrank && log->activation && log->type == RECV_TYPE) { + log->data = NULL; + reqs_queue[nr_reqs] = log; + nr_reqs++; + nrecv++; + } + } + + /* These are sorted into log time order. */ + qsort(reqs_queue, nrecv, sizeof(struct mpiuse_log_entry *), cmp_logs); + + /* Now the sends. */ + int nsend = 0; + for (int k = 0; k < nlogs; k++) { + struct mpiuse_log_entry *log = mpiuse_get_log(k); + if (log->rank == myrank && log->activation && log->type == SEND_TYPE) { + log->data = NULL; + reqs_queue[nr_reqs] = log; + nr_reqs++; + nsend++; + } + } + + /* These are randomized. */ + mpiuse_shuffle_logs(&reqs_queue[nrecv], nsend); + } +} + +/** + * @brief usage help. + */ +static void usage(char *argv[]) { + fprintf(stderr, "Usage: %s [options] nr_messages logfile.dat\n", argv[0]); + fprintf(stderr, + " options: -v verbose, -d data check, -s size (bytes/scale), \n" + "\t -f <1|2> randomize injection order, 1 == just sends, " + "2 == sends and recvs\n" + "\t[-r uniform random from 1 to size, | \n" + "\t-r -g half gaussian random from 1 with 2.5 sigma size., | \n" + "\t-r -c <file> use cdf from file, size is a scale factor., |\n" + "\t-r -o <file> use occurence sample of values in a file, size is a " + "scale factor.,] \n" + "\t-x random seed\n"); + fflush(stderr); +} + +/** + * @brief main function. + */ +int main(int argc, char *argv[]) { + + /* Initiate MPI. */ + int prov = 0; + int res = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &prov); + if (res != MPI_SUCCESS) + error("Call to MPI_Init_thread failed with error %i.", res); + + int nr_nodes = 0; + res = MPI_Comm_size(MPI_COMM_WORLD, &nr_nodes); + if (res != MPI_SUCCESS) error("MPI_Comm_size failed with error %i.", res); + + res = MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + if (res != MPI_SUCCESS) + error("Call to MPI_Comm_rank failed with error %i.", res); + + /* Handle the command-line, we expect the number of messages to exchange per + * rank an output log and some options, the interesting ones are a size and + * whether to use a random selections of various kinds. */ + int size = 1024; + int random = 0; + int randomorder = 0; + int uniform = 1; + char *cdf = NULL; + char *odata = NULL; + int opt; + unsigned int seed = default_seed; + while ((opt = getopt(argc, argv, "vds:rgx:c:o:f:")) != -1) { + switch (opt) { + case 'd': + datacheck = 1; + break; + case 'c': + cdf = optarg; + break; + case 'f': + randomorder = atoi(optarg); + break; + case 'g': + uniform = 0; + break; + case 's': + size = atoi(optarg); + break; + case 'r': + random = 1; + break; + case 'o': + odata = optarg; + break; + case 'v': + verbose = 1; + break; + case 'x': + seed = atol(optarg); + break; + default: + if (myrank == 0) usage(argv); + return 1; + } + } + if (optind >= argc - 1) { + if (myrank == 0) usage(argv); + return 1; + } + if (cdf != NULL && odata != NULL) + error("Cannot use -c and -o options together"); + + int nr_logs = atoi(argv[optind]); + if (nr_logs == 0) + error("Expected number of messages to exchange, got: %s", argv[optind]); + char *logfile = argv[optind + 1]; + + /* Generate the fake logs for the exchanges. */ + if (myrank == 0) { + if (random) { + if (cdf != NULL) { + message( + "Generating %d fake logs for %d ranks with randoms" + " based on cdf %s scaled by factor %d", + nr_logs, nr_nodes, cdf, size); + } else if (odata != NULL) { + message( + "Generating %d fake logs for %d ranks with randoms" + " based on occurence data %s scaled by factor %d", + nr_logs, nr_nodes, cdf, size); + + } else if (uniform) { + message( + "Generating %d fake logs for %d ranks with random distribution" + " using size %d", + nr_logs, nr_nodes, size); + } else { + message( + "Generating %d fake logs for %d ranks with gaussian random " + "distribution using size %d as 2.5 sigma", + nr_logs, nr_nodes, size); + } + } else { + message("Generating %d fake logs for %d ranks of size %d", nr_logs, + nr_nodes, size); + } + } + mpiuse_log_generate(nr_nodes, nr_logs, size, random, seed, uniform, cdf, + odata); + int nranks = mpiuse_nr_ranks(); + + /* Create communicators for each MPI rank. */ + for (int i = 0; i < nr_nodes; i++) { + MPI_Comm_dup(MPI_COMM_WORLD, &node_comms[i]); + } + + /* Each rank requires its own queue, so extract them. */ + if (verbose && myrank == 0) { + if (randomorder == 0) message("Message order same as generated"); + if (randomorder == 1) message("Message send order randomized"); + if (randomorder == 2) message("Message send and recv order randomized"); + } + + pick_logs(randomorder); + + /* Time to start time. Try to make it synchronous across the ranks. */ + MPI_Barrier(MPI_COMM_WORLD); + clocks_set_cpufreq(0); + if (myrank == 0) { + message("Start of MPI tests"); + message("=================="); + if (verbose) { + if (datacheck) + message("checking data pattern on send and recv completion"); + } + } + + /* Make three threads, one for injecting tasks and two to check for + * completions of the sends and recv independently. */ + pthread_t injectthread; + if (pthread_create(&injectthread, NULL, &inject_thread, &myrank) != 0) + error("Failed to create injection thread."); + pthread_t sendthread; + if (pthread_create(&sendthread, NULL, &send_thread, &myrank) != 0) + error("Failed to create send thread."); + pthread_t recvthread; + if (pthread_create(&recvthread, NULL, &recv_thread, &myrank) != 0) + error("Failed to create recv thread."); + + /* Wait until all threads have exited and all MPI requests have completed. */ + pthread_join(injectthread, NULL); + pthread_join(sendthread, NULL); + pthread_join(recvthread, NULL); + + /* Dump the updated MPI logs. */ + MPI_Barrier(MPI_COMM_WORLD); + fflush(stdout); + if (myrank == 0) message("Dumping updated log"); + mpiuse_dump_logs(nranks, logfile); + + /* Shutdown MPI. */ + res = MPI_Finalize(); + if (res != MPI_SUCCESS) + error("call to MPI_Finalize failed with error %i.", res); + + if (myrank == 0) message("Bye"); + + return 0; +} diff --git a/swiftmpistepsim.c b/swiftmpistepsim.c index 71e3ea28410fbebc50a3cdca10c43911e7c1b2e3..c0bf1d72f6c8dda58929653016920f3743498dad 100644 --- a/swiftmpistepsim.c +++ b/swiftmpistepsim.c @@ -41,16 +41,20 @@ static int usetics = 1; /* Scale to apply to the size of the messages we send. */ static float messagescale = 1.0; +/* Size of the messages we send. This overrides the logged values when not + * zero . */ +static size_t messagesize = 0; + /* Set a data pattern and check we get this back, slow... */ static int datacheck = 0; /* Integer types of send and recv tasks, must match log. */ -static const int task_type_send = 22; -static const int task_type_recv = 23; +static const int task_type_send = 25; +static const int task_type_recv = 26; /* Global communicators for each of the subtypes. */ -static const int task_subtype_count = 30; // Just some upper limit on subtype. -static MPI_Comm subtypeMPI_comms[30]; +static const int task_subtype_count = 34; // Just some upper limit on subtype. +static MPI_Comm subtypeMPI_comms[task_subtype_count]; /* The local queues. */ static struct mpiuse_log_entry **volatile reqs_queue; @@ -79,7 +83,7 @@ static double log_clocks_cpufreq = 2194844448.0; static void datacheck_fill(size_t size, void *data) { unsigned char *p = (unsigned char *)data; for (size_t i = 0; i < size; i++) { - p[i] = 170; /* 10101010 in bits. */ + p[i] = 170; /* 10101010 in bits. */ } } @@ -386,12 +390,12 @@ static void pick_logs(void) { if (log->rank == myrank && log->activation) { if (log->type == task_type_send || log->type == task_type_recv) { + /* Override size if needed. */ + if (messagesize > 0) log->size = messagesize; + /* Scale size. */ log->size *= messagescale ; - /* Allocate space for data. */ - log->data = calloc(log->size, 1); - /* And keep this log. */ log->data = NULL; reqs_queue[nr_reqs] = log; @@ -418,10 +422,12 @@ static void pick_logs(void) { * @brief usage help. */ static void usage(char *argv[]) { - fprintf(stderr, "Usage: %s [-vf] SWIFT_mpiuse-log-file.dat logfile.dat\n", + fprintf(stderr, "Usage: %s [-vfdc:s:] SWIFT_mpiuse-log-file.dat logfile.dat\n", argv[0]); fprintf(stderr, " options: -v verbose, -f fast injections, " - "-s message size (bytes)\n"); + "-d fill messages and check values on receive, " + "-s <value> use fixed message of this size (bytes), " + "-c <value> scale factor for all messages\n"); fflush(stderr); } @@ -447,7 +453,7 @@ int main(int argc, char *argv[]) { /* Handle the command-line, we expect a mpiuse data file to read and various * options. */ int opt; - while ((opt = getopt(argc, argv, "vfdc:")) != -1) { + while ((opt = getopt(argc, argv, "vfdc:s:")) != -1) { switch (opt) { case 'd': datacheck = 1; @@ -460,6 +466,8 @@ int main(int argc, char *argv[]) { break; case 'c': messagescale = atof(optarg); + case 's': + messagesize = atoll(optarg); break; default: if (myrank == 0) usage(argv); @@ -498,6 +506,10 @@ int main(int argc, char *argv[]) { clocks_set_cpufreq(0); if (myrank == 0) { message("Start of MPI tests"); + if (messagesize > 0) { + message(" "); + message(" Using fixed message size of %zd", messagesize); + } message("=================="); if (messagescale != 1.0f) { message(" "); @@ -505,7 +517,8 @@ int main(int argc, char *argv[]) { } if (verbose) { if (!usetics) message("using fast untimed injections"); - if (datacheck) message("checking data pattern on send and recv completion"); + if (datacheck) + message("checking data pattern on send and recv completion"); } } diff --git a/testdata/EAGLE_25-mpiuse_report-step18302-16ranks-inter-step.dat.gz b/testdata/EAGLE_25-mpiuse_report-step18302-16ranks-inter-step.dat.gz index e73fbb7c19801e2c30ec387d669a5df33db747b8..cdc12c1155c307218f0c8f6a37d6d683b1b9686f 100644 Binary files a/testdata/EAGLE_25-mpiuse_report-step18302-16ranks-inter-step.dat.gz and b/testdata/EAGLE_25-mpiuse_report-step18302-16ranks-inter-step.dat.gz differ diff --git a/testdata/EAGLE_6-mpiuse_report-step2-4ranks-small-step.dat b/testdata/EAGLE_6-mpiuse_report-step2-4ranks-small-step.dat index 9fe6228ddf043d8348555bdb6daa1fe5322cf66a..dd7b7a14268ef6a159bdf686d426830545d7c29f 100644 --- a/testdata/EAGLE_6-mpiuse_report-step2-4ranks-small-step.dat +++ b/testdata/EAGLE_6-mpiuse_report-step2-4ranks-small-step.dat @@ -1,88 +1,88 @@ # stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum -608817 21543633709 0 2 0 2 recv 23 xv 11 1 445 142720 142720 -610727 21543635619 0 2 0 3 recv 23 xv 11 1 453 311296 454016 -612045 21543636937 0 2 0 2 recv 23 xv 11 1 446 219648 673664 -613550 21543638442 0 2 0 3 recv 23 xv 11 1 38 449024 1122688 -615327 21543640219 0 2 0 3 recv 23 xv 11 1 62 282624 1405312 -616285 21543641177 0 2 0 1 recv 23 xv 11 1 177 379648 1784960 -618295 21543643187 0 2 0 3 recv 23 xv 11 1 50 400128 2185088 -619247 21543644139 0 2 0 2 recv 23 xv 11 1 160 94208 2279296 -620661 21543645553 0 2 0 3 recv 23 xv 11 1 49 318720 2598016 -623155 21543648047 0 2 0 2 recv 23 xv 11 1 158 131072 2729088 -625506 21543650398 0 2 0 3 recv 23 xv 11 1 40 416512 3145600 -627165 21543652057 0 2 0 2 recv 23 xv 11 1 156 111360 3256960 -629619 21543654511 0 2 0 3 recv 23 xv 11 1 67 312448 3569408 -631490 21543656382 0 2 0 3 recv 23 xv 11 1 65 292096 3861504 -632856 21543657748 0 2 0 3 recv 23 xv 11 1 465 77696 3939200 -635292 21543660184 0 2 0 2 recv 23 xv 11 1 3 583168 4522368 -636746 21543661638 0 2 0 2 recv 23 xv 11 1 2 107136 4629504 -639251 21543664143 0 2 0 3 recv 23 xv 11 1 130 711424 5340928 -640726 21543665618 0 2 0 1 recv 23 xv 11 1 118 553984 5894912 -642551 21543667443 0 2 0 1 recv 23 xv 11 1 45 328576 6223488 -644510 21543669402 0 2 0 3 recv 23 xv 11 1 192 345984 6569472 -646036 21543670928 0 2 0 1 recv 23 xv 11 1 44 1056768 7626240 -647586 21543672478 0 2 0 3 recv 23 xv 11 1 230 255232 7881472 -649599 21543674491 0 2 0 1 recv 23 xv 11 1 194 143872 8025344 -651388 21543676280 0 2 0 1 recv 23 xv 11 1 202 360832 8386176 -653752 21543678644 0 2 0 1 recv 23 xv 11 1 192 1066112 9452288 -655523 21543680415 0 2 0 2 recv 23 xv 11 1 249 28544 9480832 -657245 21543682137 0 2 0 2 recv 23 xv 11 1 245 83840 9564672 -659152 21543684044 0 2 0 2 recv 23 xv 11 1 244 78720 9643392 -661120 21543686012 0 2 0 1 recv 23 xv 11 1 205 547968 10191360 -664106 21543688998 0 2 0 3 recv 23 xv 11 1 200 368000 10559360 -666953 21543691845 0 2 0 1 recv 23 xv 11 1 137 212608 10771968 -668385 21543693277 0 2 0 3 recv 23 xv 11 1 137 418304 11190272 -669960 21543694852 0 2 0 3 recv 23 xv 11 1 136 199808 11390080 -671970 21543696862 0 2 0 1 recv 23 xv 11 1 57 332160 11722240 -6447305 21549472197 5791782 2 0 2 recv 23 xv 11 0 249 -28544 11693696 -11419335 21554444227 10796180 2 0 2 recv 23 xv 11 0 158 -131072 11562624 -11758915 21554783807 11111329 2 0 3 recv 23 xv 11 0 230 -255232 11307392 -11799763 21554824655 11172598 2 0 2 recv 23 xv 11 0 156 -111360 11196032 -11838133 21554863025 11222806 2 0 3 recv 23 xv 11 0 62 -282624 10913408 -11910866 21554935758 11253621 2 0 2 recv 23 xv 11 0 245 -83840 10829568 -13014189 21556039081 12405372 2 0 2 recv 23 xv 11 0 445 -142720 10686848 -13215819 21556240711 12580527 2 0 2 recv 23 xv 11 0 3 -583168 10103680 -13730903 21556755795 13111656 2 0 2 recv 23 xv 11 0 160 -94208 10009472 -14435788 21557460680 13776636 2 0 2 recv 23 xv 11 0 244 -78720 9930752 -15491583 21558516475 14823198 2 0 3 recv 23 xv 11 0 137 -418304 9512448 -15614696 21558639588 14981840 2 0 3 recv 23 xv 11 0 465 -77696 9434752 -15728921 21558753813 15116876 2 0 2 recv 23 xv 11 0 446 -219648 9215104 -16170362 21559195254 15533616 2 0 2 recv 23 xv 11 0 2 -107136 9107968 -16572275 21559597167 15961548 2 0 3 recv 23 xv 11 0 453 -311296 8796672 -16928290 21559953182 16289039 2 0 3 recv 23 xv 11 0 130 -711424 8085248 -28452779 21571477671 27827273 2 0 3 recv 23 xv 11 0 40 -416512 7668736 -28707936 21571732828 28054184 2 0 1 recv 23 xv 11 0 192 -1066112 6602624 -28977681 21572002573 28364131 2 0 3 recv 23 xv 11 0 38 -449024 6153600 -30058812 21573083704 29440517 2 0 3 recv 23 xv 11 0 50 -400128 5753472 -30160189 21573185081 29539528 2 0 3 recv 23 xv 11 0 49 -318720 5434752 -30309729 21573334621 29639769 2 0 3 recv 23 xv 11 0 136 -199808 5234944 -30514052 21573538944 29849946 2 0 3 recv 23 xv 11 0 200 -368000 4866944 -31055012 21574079904 30410502 2 0 3 recv 23 xv 11 0 192 -345984 4520960 -38056623 21581081515 37427004 2 0 3 recv 23 xv 11 0 67 -312448 4208512 -38085547 21581110439 37454057 2 0 3 recv 23 xv 11 0 65 -292096 3916416 -38835374 21581860266 38189338 2 0 1 recv 23 xv 11 0 44 -1056768 2859648 -40237431 21583262323 39586043 2 0 1 recv 23 xv 11 0 202 -360832 2498816 -40553084 21583577976 39891964 2 0 1 recv 23 xv 11 0 205 -547968 1950848 -40777829 21583802721 40161544 2 0 1 recv 23 xv 11 0 177 -379648 1571200 -40836656 21583861548 40164686 2 0 1 recv 23 xv 11 0 57 -332160 1239040 -40873449 21583898341 40206496 2 0 1 recv 23 xv 11 0 137 -212608 1026432 -41885432 21584910324 0 2 0 2 send 22 tend_part 7 1 38 2328 1028760 -41896300 21584921192 0 2 0 3 send 22 tend_part 7 1 38 2328 1031088 -41942824 21584967716 57392 2 0 2 send 22 tend_part 7 0 38 -2328 1028760 -41950298 21584975190 53998 2 0 3 send 22 tend_part 7 0 38 -2328 1026432 -43257053 21586281945 42607454 2 0 1 recv 23 xv 11 0 194 -143872 882560 -43529999 21586554891 42887448 2 0 1 recv 23 xv 11 0 45 -328576 553984 -43591111 21586616003 42950385 2 0 1 recv 23 xv 11 0 118 -553984 0 -48599962 21591624854 0 2 0 2 send 22 tend_part 7 1 138 2712 2712 -48619658 21591644550 0 2 0 1 send 22 tend_part 7 1 138 2712 5424 -48642246 21591667138 0 2 0 3 send 22 tend_part 7 1 138 2712 8136 -48695836 21591720728 53590 2 0 3 send 22 tend_part 7 0 138 -2712 5424 -48704250 21591729142 104288 2 0 2 send 22 tend_part 7 0 138 -2712 2712 -48924980 21591949872 305322 2 0 1 send 22 tend_part 7 0 138 -2712 0 -48963599 21591988491 0 2 0 1 send 22 tend_part 7 1 192 6936 6936 -48968850 21591993742 5251 2 0 1 send 22 tend_part 7 0 192 -6936 0 -48990086 21592014978 0 2 0 3 send 22 tend_part 7 1 192 6936 6936 -48995836 21592020728 5750 2 0 3 send 22 tend_part 7 0 192 -6936 0 +608817 21543633709 0 2 0 2 recv 26 xv 11 1 445 142720 142720 +610727 21543635619 0 2 0 3 recv 26 xv 11 1 453 311296 454016 +612045 21543636937 0 2 0 2 recv 26 xv 11 1 446 219648 673664 +613550 21543638442 0 2 0 3 recv 26 xv 11 1 38 449024 1122688 +615327 21543640219 0 2 0 3 recv 26 xv 11 1 62 282624 1405312 +616285 21543641177 0 2 0 1 recv 26 xv 11 1 177 379648 1784960 +618295 21543643187 0 2 0 3 recv 26 xv 11 1 50 400128 2185088 +619247 21543644139 0 2 0 2 recv 26 xv 11 1 160 94208 2279296 +620661 21543645553 0 2 0 3 recv 26 xv 11 1 49 318720 2598016 +623155 21543648047 0 2 0 2 recv 26 xv 11 1 158 131072 2729088 +625506 21543650398 0 2 0 3 recv 26 xv 11 1 40 416512 3145600 +627165 21543652057 0 2 0 2 recv 26 xv 11 1 156 111360 3256960 +629619 21543654511 0 2 0 3 recv 26 xv 11 1 67 312448 3569408 +631490 21543656382 0 2 0 3 recv 26 xv 11 1 65 292096 3861504 +632856 21543657748 0 2 0 3 recv 26 xv 11 1 465 77696 3939200 +635292 21543660184 0 2 0 2 recv 26 xv 11 1 3 583168 4522368 +636746 21543661638 0 2 0 2 recv 26 xv 11 1 2 107136 4629504 +639251 21543664143 0 2 0 3 recv 26 xv 11 1 130 711424 5340928 +640726 21543665618 0 2 0 1 recv 26 xv 11 1 118 553984 5894912 +642551 21543667443 0 2 0 1 recv 26 xv 11 1 45 328576 6223488 +644510 21543669402 0 2 0 3 recv 26 xv 11 1 192 345984 6569472 +646036 21543670928 0 2 0 1 recv 26 xv 11 1 44 1056768 7626240 +647586 21543672478 0 2 0 3 recv 26 xv 11 1 230 255232 7881472 +649599 21543674491 0 2 0 1 recv 26 xv 11 1 194 143872 8025344 +651388 21543676280 0 2 0 1 recv 26 xv 11 1 202 360832 8386176 +653752 21543678644 0 2 0 1 recv 26 xv 11 1 192 1066112 9452288 +655523 21543680415 0 2 0 2 recv 26 xv 11 1 249 28544 9480832 +657245 21543682137 0 2 0 2 recv 26 xv 11 1 245 83840 9564672 +659152 21543684044 0 2 0 2 recv 26 xv 11 1 244 78720 9643392 +661120 21543686012 0 2 0 1 recv 26 xv 11 1 205 547968 10191360 +664106 21543688998 0 2 0 3 recv 26 xv 11 1 200 368000 10559360 +666953 21543691845 0 2 0 1 recv 26 xv 11 1 137 212608 10771968 +668385 21543693277 0 2 0 3 recv 26 xv 11 1 137 418304 11190272 +669960 21543694852 0 2 0 3 recv 26 xv 11 1 136 199808 11390080 +671970 21543696862 0 2 0 1 recv 26 xv 11 1 57 332160 11722240 +6447305 21549472197 5791782 2 0 2 recv 26 xv 11 0 249 -28544 11693696 +11419335 21554444227 10796180 2 0 2 recv 26 xv 11 0 158 -131072 11562624 +11758915 21554783807 11111329 2 0 3 recv 26 xv 11 0 230 -255232 11307392 +11799763 21554824655 11172598 2 0 2 recv 26 xv 11 0 156 -111360 11196032 +11838133 21554863025 11222806 2 0 3 recv 26 xv 11 0 62 -282624 10913408 +11910866 21554935758 11253621 2 0 2 recv 26 xv 11 0 245 -83840 10829568 +13014189 21556039081 12405372 2 0 2 recv 26 xv 11 0 445 -142720 10686848 +13215819 21556240711 12580527 2 0 2 recv 26 xv 11 0 3 -583168 10103680 +13730903 21556755795 13111656 2 0 2 recv 26 xv 11 0 160 -94208 10009472 +14435788 21557460680 13776636 2 0 2 recv 26 xv 11 0 244 -78720 9930752 +15491583 21558516475 14823198 2 0 3 recv 26 xv 11 0 137 -418304 9512448 +15614696 21558639588 14981840 2 0 3 recv 26 xv 11 0 465 -77696 9434752 +15728921 21558753813 15116876 2 0 2 recv 26 xv 11 0 446 -219648 9215104 +16170362 21559195254 15533616 2 0 2 recv 26 xv 11 0 2 -107136 9107968 +16572275 21559597167 15961548 2 0 3 recv 26 xv 11 0 453 -311296 8796672 +16928290 21559953182 16289039 2 0 3 recv 26 xv 11 0 130 -711424 8085248 +28452779 21571477671 27827273 2 0 3 recv 26 xv 11 0 40 -416512 7668736 +28707936 21571732828 28054184 2 0 1 recv 26 xv 11 0 192 -1066112 6602624 +28977681 21572002573 28364131 2 0 3 recv 26 xv 11 0 38 -449024 6153600 +30058812 21573083704 29440517 2 0 3 recv 26 xv 11 0 50 -400128 5753472 +30160189 21573185081 29539528 2 0 3 recv 26 xv 11 0 49 -318720 5434752 +30309729 21573334621 29639769 2 0 3 recv 26 xv 11 0 136 -199808 5234944 +30514052 21573538944 29849946 2 0 3 recv 26 xv 11 0 200 -368000 4866944 +31055012 21574079904 30410502 2 0 3 recv 26 xv 11 0 192 -345984 4520960 +38056623 21581081515 37427004 2 0 3 recv 26 xv 11 0 67 -312448 4208512 +38085547 21581110439 37454057 2 0 3 recv 26 xv 11 0 65 -292096 3916416 +38835374 21581860266 38189338 2 0 1 recv 26 xv 11 0 44 -1056768 2859648 +40237431 21583262323 39586043 2 0 1 recv 26 xv 11 0 202 -360832 2498816 +40553084 21583577976 39891964 2 0 1 recv 26 xv 11 0 205 -547968 1950848 +40777829 21583802721 40161544 2 0 1 recv 26 xv 11 0 177 -379648 1571200 +40836656 21583861548 40164686 2 0 1 recv 26 xv 11 0 57 -332160 1239040 +40873449 21583898341 40206496 2 0 1 recv 26 xv 11 0 137 -212608 1026432 +41885432 21584910324 0 2 0 2 send 25 tend_part 7 1 38 2328 1028760 +41896300 21584921192 0 2 0 3 send 25 tend_part 7 1 38 2328 1031088 +41942824 21584967716 57392 2 0 2 send 25 tend_part 7 0 38 -2328 1028760 +41950298 21584975190 53998 2 0 3 send 25 tend_part 7 0 38 -2328 1026432 +43257053 21586281945 42607454 2 0 1 recv 26 xv 11 0 194 -143872 882560 +43529999 21586554891 42887448 2 0 1 recv 26 xv 11 0 45 -328576 553984 +43591111 21586616003 42950385 2 0 1 recv 26 xv 11 0 118 -553984 0 +48599962 21591624854 0 2 0 2 send 25 tend_part 7 1 138 2712 2712 +48619658 21591644550 0 2 0 1 send 25 tend_part 7 1 138 2712 5424 +48642246 21591667138 0 2 0 3 send 25 tend_part 7 1 138 2712 8136 +48695836 21591720728 53590 2 0 3 send 25 tend_part 7 0 138 -2712 5424 +48704250 21591729142 104288 2 0 2 send 25 tend_part 7 0 138 -2712 2712 +48924980 21591949872 305322 2 0 1 send 25 tend_part 7 0 138 -2712 0 +48963599 21591988491 0 2 0 1 send 25 tend_part 7 1 192 6936 6936 +48968850 21591993742 5251 2 0 1 send 25 tend_part 7 0 192 -6936 0 +48990086 21592014978 0 2 0 3 send 25 tend_part 7 1 192 6936 6936 +48995836 21592020728 5750 2 0 3 send 25 tend_part 7 0 192 -6936 0 ## ## Number of requests: 42 ## Maximum request size: 1.0167 (MB) @@ -90,56 +90,56 @@ ## Mean of all requests: 0.2668 (MB) ## # stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum -12953800 21558567216 0 2 1 3 recv 23 tend_part 7 1 32 1368 1368 -12964804 21558578220 0 2 1 0 recv 23 tend_part 7 1 138 2712 4080 -12970634 21558584050 0 2 1 0 recv 23 tend_part 7 1 192 6936 11016 -12983064 21558596480 0 2 1 2 recv 23 xv 11 1 321 310784 321800 -12985183 21558598599 0 2 1 3 recv 23 xv 11 1 440 876032 1197832 -12986818 21558600234 0 2 1 3 recv 23 xv 11 1 403 171136 1368968 -12989387 21558602803 0 2 1 2 recv 23 xv 11 1 395 487040 1856008 -12993612 21558607028 0 2 1 2 recv 23 xv 11 1 383 146048 2002056 -12998091 21558611507 0 2 1 2 recv 23 xv 11 1 322 235904 2237960 -18459228 21564072644 0 2 1 0 send 22 xv 11 1 192 1066112 3304072 -23740217 21569353633 0 2 1 0 send 22 xv 11 1 44 1056768 4360840 -25940939 21571554355 0 2 1 0 send 22 xv 11 1 205 547968 4908808 -28186716 21573800132 0 2 1 0 send 22 xv 11 1 177 379648 5288456 -29239247 21574852663 0 2 1 3 send 22 xv 11 1 453 282752 5571208 -30275585 21575889001 17281973 2 1 2 recv 23 xv 11 0 383 -146048 5425160 -30291458 21575904874 0 2 1 0 send 22 xv 11 1 202 360832 5785992 -33012165 21578625581 0 2 1 0 send 22 xv 11 1 57 332160 6118152 -33756463 21579369879 20771280 2 1 3 recv 23 xv 11 0 440 -876032 5242120 -33892572 21579505988 20903185 2 1 2 recv 23 xv 11 0 395 -487040 4755080 -33925103 21579538519 0 2 1 0 send 22 xv 11 1 137 212608 4967688 -34725100 21580338516 0 2 1 0 send 22 xv 11 1 194 143872 5111560 -36049973 21581663389 0 2 1 0 send 22 xv 11 1 45 328576 5440136 -36265488 21581878904 17806260 2 1 0 send 22 xv 11 0 192 -1066112 4374024 -38193925 21583807341 0 2 1 3 send 22 xv 11 1 56 472064 4846088 -38822832 21584436248 15082615 2 1 0 send 22 xv 11 0 44 -1056768 3789320 -40220268 21585833684 12033552 2 1 0 send 22 xv 11 0 177 -379648 3409672 -40540832 21586154248 7528667 2 1 0 send 22 xv 11 0 57 -332160 3077512 -40705244 21586318660 6780141 2 1 0 send 22 xv 11 0 137 -212608 2864904 -40722144 21586335560 10430686 2 1 0 send 22 xv 11 0 202 -360832 2504072 -40735977 21586349393 14795038 2 1 0 send 22 xv 11 0 205 -547968 1956104 -42881032 21588494448 0 2 1 0 send 22 xv 11 1 118 553984 2510088 -43243043 21588856459 5049118 2 1 3 send 22 xv 11 0 56 -472064 2038024 -43254760 21588868176 14015513 2 1 3 send 22 xv 11 0 453 -282752 1755272 -43431687 21589045103 7381714 2 1 0 send 22 xv 11 0 45 -328576 1426696 -43441434 21589054850 8716334 2 1 0 send 22 xv 11 0 194 -143872 1282824 -43447599 21589061015 566567 2 1 0 send 22 xv 11 0 118 -553984 728840 -44195173 21589808589 0 2 1 3 send 22 xv 11 1 59 183552 912392 -44432889 21590046305 0 2 1 3 send 22 xv 11 1 47 46720 959112 -44593620 21590207036 160731 2 1 3 send 22 xv 11 0 47 -46720 912392 -44602907 21590216323 407734 2 1 3 send 22 xv 11 0 59 -183552 728840 -44639981 21590253397 31656917 2 1 2 recv 23 xv 11 0 321 -310784 418056 -44960058 21590573474 31961967 2 1 2 recv 23 xv 11 0 322 -235904 182152 -48461460 21594074876 35474642 2 1 3 recv 23 xv 11 0 403 -171136 11016 -48919281 21594532697 35954477 2 1 0 recv 23 tend_part 7 0 138 -2712 8304 -48984643 21594598059 36030843 2 1 3 recv 23 tend_part 7 0 32 -1368 6936 -49409002 21595022418 36438368 2 1 0 recv 23 tend_part 7 0 192 -6936 0 -56492967 21602106383 0 2 1 2 send 22 tend_part 7 1 359 2520 2520 -56507501 21602120917 14534 2 1 2 send 22 tend_part 7 0 359 -2520 0 -56512908 21602126324 0 2 1 3 send 22 tend_part 7 1 359 2520 2520 -56518112 21602131528 5204 2 1 3 send 22 tend_part 7 0 359 -2520 0 +12953800 21558567216 0 2 1 3 recv 26 tend_part 7 1 32 1368 1368 +12964804 21558578220 0 2 1 0 recv 26 tend_part 7 1 138 2712 4080 +12970634 21558584050 0 2 1 0 recv 26 tend_part 7 1 192 6936 11016 +12983064 21558596480 0 2 1 2 recv 26 xv 11 1 321 310784 321800 +12985183 21558598599 0 2 1 3 recv 26 xv 11 1 440 876032 1197832 +12986818 21558600234 0 2 1 3 recv 26 xv 11 1 403 171136 1368968 +12989387 21558602803 0 2 1 2 recv 26 xv 11 1 395 487040 1856008 +12993612 21558607028 0 2 1 2 recv 26 xv 11 1 383 146048 2002056 +12998091 21558611507 0 2 1 2 recv 26 xv 11 1 322 235904 2237960 +18459228 21564072644 0 2 1 0 send 25 xv 11 1 192 1066112 3304072 +23740217 21569353633 0 2 1 0 send 25 xv 11 1 44 1056768 4360840 +25940939 21571554355 0 2 1 0 send 25 xv 11 1 205 547968 4908808 +28186716 21573800132 0 2 1 0 send 25 xv 11 1 177 379648 5288456 +29239247 21574852663 0 2 1 3 send 25 xv 11 1 453 282752 5571208 +30275585 21575889001 17281973 2 1 2 recv 26 xv 11 0 383 -146048 5425160 +30291458 21575904874 0 2 1 0 send 25 xv 11 1 202 360832 5785992 +33012165 21578625581 0 2 1 0 send 25 xv 11 1 57 332160 6118152 +33756463 21579369879 20771280 2 1 3 recv 26 xv 11 0 440 -876032 5242120 +33892572 21579505988 20903185 2 1 2 recv 26 xv 11 0 395 -487040 4755080 +33925103 21579538519 0 2 1 0 send 25 xv 11 1 137 212608 4967688 +34725100 21580338516 0 2 1 0 send 25 xv 11 1 194 143872 5111560 +36049973 21581663389 0 2 1 0 send 25 xv 11 1 45 328576 5440136 +36265488 21581878904 17806260 2 1 0 send 25 xv 11 0 192 -1066112 4374024 +38193925 21583807341 0 2 1 3 send 25 xv 11 1 56 472064 4846088 +38822832 21584436248 15082615 2 1 0 send 25 xv 11 0 44 -1056768 3789320 +40220268 21585833684 12033552 2 1 0 send 25 xv 11 0 177 -379648 3409672 +40540832 21586154248 7528667 2 1 0 send 25 xv 11 0 57 -332160 3077512 +40705244 21586318660 6780141 2 1 0 send 25 xv 11 0 137 -212608 2864904 +40722144 21586335560 10430686 2 1 0 send 25 xv 11 0 202 -360832 2504072 +40735977 21586349393 14795038 2 1 0 send 25 xv 11 0 205 -547968 1956104 +42881032 21588494448 0 2 1 0 send 25 xv 11 1 118 553984 2510088 +43243043 21588856459 5049118 2 1 3 send 25 xv 11 0 56 -472064 2038024 +43254760 21588868176 14015513 2 1 3 send 25 xv 11 0 453 -282752 1755272 +43431687 21589045103 7381714 2 1 0 send 25 xv 11 0 45 -328576 1426696 +43441434 21589054850 8716334 2 1 0 send 25 xv 11 0 194 -143872 1282824 +43447599 21589061015 566567 2 1 0 send 25 xv 11 0 118 -553984 728840 +44195173 21589808589 0 2 1 3 send 25 xv 11 1 59 183552 912392 +44432889 21590046305 0 2 1 3 send 25 xv 11 1 47 46720 959112 +44593620 21590207036 160731 2 1 3 send 25 xv 11 0 47 -46720 912392 +44602907 21590216323 407734 2 1 3 send 25 xv 11 0 59 -183552 728840 +44639981 21590253397 31656917 2 1 2 recv 26 xv 11 0 321 -310784 418056 +44960058 21590573474 31961967 2 1 2 recv 26 xv 11 0 322 -235904 182152 +48461460 21594074876 35474642 2 1 3 recv 26 xv 11 0 403 -171136 11016 +48919281 21594532697 35954477 2 1 0 recv 26 tend_part 7 0 138 -2712 8304 +48984643 21594598059 36030843 2 1 3 recv 26 tend_part 7 0 32 -1368 6936 +49409002 21595022418 36438368 2 1 0 recv 26 tend_part 7 0 192 -6936 0 +56492967 21602106383 0 2 1 2 send 25 tend_part 7 1 359 2520 2520 +56507501 21602120917 14534 2 1 2 send 25 tend_part 7 0 359 -2520 0 +56512908 21602126324 0 2 1 3 send 25 tend_part 7 1 359 2520 2520 +56518112 21602131528 5204 2 1 3 send 25 tend_part 7 0 359 -2520 0 ## ## Number of requests: 25 ## Maximum request size: 1.0167 (MB) @@ -147,52 +147,52 @@ ## Mean of all requests: 0.3132 (MB) ## # stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum -987251 21546628055 0 2 2 3 recv 23 tend_part 7 1 32 1368 1368 -1025615 21546666419 0 2 2 0 recv 23 tend_part 7 1 38 2328 3696 -1029326 21546670130 0 2 2 0 recv 23 tend_part 7 1 138 2712 6408 -1038964 21546679768 0 2 2 1 recv 23 tend_part 7 1 359 2520 8928 -1304769 21546945573 0 2 2 3 send 22 xv 11 1 444 98816 107744 -1404115 21547044919 0 2 2 3 send 22 xv 11 1 2 107136 214880 -1406430 21547047234 0 2 2 0 send 22 xv 11 1 2 107136 322016 -1892700 21547533504 0 2 2 0 send 22 xv 11 1 446 219648 541664 -3170516 21548811320 0 2 2 1 send 22 xv 11 1 395 487040 1028704 -3551697 21549192501 0 2 2 0 send 22 xv 11 1 3 583168 1611872 -3608675 21549249479 0 2 2 1 send 22 xv 11 1 383 146048 1757920 -3722238 21549363042 0 2 2 0 send 22 xv 11 1 445 142720 1900640 -4474542 21550115346 0 2 2 1 send 22 xv 11 1 322 235904 2136544 -4660091 21550300895 0 2 2 1 send 22 xv 11 1 321 310784 2447328 -4802628 21550443432 0 2 2 0 send 22 xv 11 1 249 28544 2475872 -4947390 21550588194 144762 2 2 0 send 22 xv 11 0 249 -28544 2447328 -7943938 21553584742 6537508 2 2 0 send 22 xv 11 0 2 -107136 2340192 -7961657 21553602461 6068957 2 2 0 send 22 xv 11 0 446 -219648 2120544 -7971824 21553612628 4420127 2 2 0 send 22 xv 11 0 3 -583168 1537376 -8524640 21554165444 0 2 2 0 send 22 xv 11 1 158 131072 1668448 -8974356 21554615160 0 2 2 0 send 22 xv 11 1 156 111360 1779808 -9378345 21555019149 0 2 2 0 send 22 xv 11 1 160 94208 1874016 -9846859 21555487663 0 2 2 0 send 22 xv 11 1 245 83840 1957856 -10082623 21555723427 6360385 2 2 0 send 22 xv 11 0 445 -142720 1815136 -10425140 21556065944 0 2 2 3 send 22 xv 11 1 1 64512 1879648 -11258279 21556899083 1411420 2 2 0 send 22 xv 11 0 245 -83840 1795808 -12594280 21558235084 2169140 2 2 3 send 22 xv 11 0 1 -64512 1731296 -12624810 21558265614 11220695 2 2 3 send 22 xv 11 0 2 -107136 1624160 -12806376 21558447180 3428031 2 2 0 send 22 xv 11 0 160 -94208 1529952 -12864222 21558505026 3889866 2 2 0 send 22 xv 11 0 156 -111360 1418592 -12923342 21558564146 11618573 2 2 3 send 22 xv 11 0 444 -98816 1319776 -13069781 21558710585 4545141 2 2 0 send 22 xv 11 0 158 -131072 1188704 -13360446 21559001250 0 2 2 3 send 22 xv 11 1 30 79232 1267936 -13833681 21559474485 0 2 2 3 send 22 xv 11 1 44 64128 1332064 -13842742 21559483546 9061 2 2 3 send 22 xv 11 0 44 -64128 1267936 -14347042 21559987846 0 2 2 0 send 22 xv 11 1 244 78720 1346656 -14447458 21560088262 100416 2 2 0 send 22 xv 11 0 244 -78720 1267936 -14549561 21560190365 1189115 2 2 3 send 22 xv 11 0 30 -79232 1188704 -29619186 21575259990 26448670 2 2 1 send 22 xv 11 0 395 -487040 701664 -29973620 21575614424 26364945 2 2 1 send 22 xv 11 0 383 -146048 555616 -30232105 21575872909 25757563 2 2 1 send 22 xv 11 0 322 -235904 319712 -30884935 21576525739 26224844 2 2 1 send 22 xv 11 0 321 -310784 8928 -41905408 21587546212 40879793 2 2 0 recv 23 tend_part 7 0 38 -2328 6600 -48456241 21594097045 47468990 2 2 3 recv 23 tend_part 7 0 32 -1368 5232 -48605647 21594246451 47576321 2 2 0 recv 23 tend_part 7 0 138 -2712 2520 -56499592 21602140396 55460628 2 2 1 recv 23 tend_part 7 0 359 -2520 0 +987251 21546628055 0 2 2 3 recv 26 tend_part 7 1 32 1368 1368 +1025615 21546666419 0 2 2 0 recv 26 tend_part 7 1 38 2328 3696 +1029326 21546670130 0 2 2 0 recv 26 tend_part 7 1 138 2712 6408 +1038964 21546679768 0 2 2 1 recv 26 tend_part 7 1 359 2520 8928 +1304769 21546945573 0 2 2 3 send 25 xv 11 1 444 98816 107744 +1404115 21547044919 0 2 2 3 send 25 xv 11 1 2 107136 214880 +1406430 21547047234 0 2 2 0 send 25 xv 11 1 2 107136 322016 +1892700 21547533504 0 2 2 0 send 25 xv 11 1 446 219648 541664 +3170516 21548811320 0 2 2 1 send 25 xv 11 1 395 487040 1028704 +3551697 21549192501 0 2 2 0 send 25 xv 11 1 3 583168 1611872 +3608675 21549249479 0 2 2 1 send 25 xv 11 1 383 146048 1757920 +3722238 21549363042 0 2 2 0 send 25 xv 11 1 445 142720 1900640 +4474542 21550115346 0 2 2 1 send 25 xv 11 1 322 235904 2136544 +4660091 21550300895 0 2 2 1 send 25 xv 11 1 321 310784 2447328 +4802628 21550443432 0 2 2 0 send 25 xv 11 1 249 28544 2475872 +4947390 21550588194 144762 2 2 0 send 25 xv 11 0 249 -28544 2447328 +7943938 21553584742 6537508 2 2 0 send 25 xv 11 0 2 -107136 2340192 +7961657 21553602461 6068957 2 2 0 send 25 xv 11 0 446 -219648 2120544 +7971824 21553612628 4420127 2 2 0 send 25 xv 11 0 3 -583168 1537376 +8524640 21554165444 0 2 2 0 send 25 xv 11 1 158 131072 1668448 +8974356 21554615160 0 2 2 0 send 25 xv 11 1 156 111360 1779808 +9378345 21555019149 0 2 2 0 send 25 xv 11 1 160 94208 1874016 +9846859 21555487663 0 2 2 0 send 25 xv 11 1 245 83840 1957856 +10082623 21555723427 6360385 2 2 0 send 25 xv 11 0 445 -142720 1815136 +10425140 21556065944 0 2 2 3 send 25 xv 11 1 1 64512 1879648 +11258279 21556899083 1411420 2 2 0 send 25 xv 11 0 245 -83840 1795808 +12594280 21558235084 2169140 2 2 3 send 25 xv 11 0 1 -64512 1731296 +12624810 21558265614 11220695 2 2 3 send 25 xv 11 0 2 -107136 1624160 +12806376 21558447180 3428031 2 2 0 send 25 xv 11 0 160 -94208 1529952 +12864222 21558505026 3889866 2 2 0 send 25 xv 11 0 156 -111360 1418592 +12923342 21558564146 11618573 2 2 3 send 25 xv 11 0 444 -98816 1319776 +13069781 21558710585 4545141 2 2 0 send 25 xv 11 0 158 -131072 1188704 +13360446 21559001250 0 2 2 3 send 25 xv 11 1 30 79232 1267936 +13833681 21559474485 0 2 2 3 send 25 xv 11 1 44 64128 1332064 +13842742 21559483546 9061 2 2 3 send 25 xv 11 0 44 -64128 1267936 +14347042 21559987846 0 2 2 0 send 25 xv 11 1 244 78720 1346656 +14447458 21560088262 100416 2 2 0 send 25 xv 11 0 244 -78720 1267936 +14549561 21560190365 1189115 2 2 3 send 25 xv 11 0 30 -79232 1188704 +29619186 21575259990 26448670 2 2 1 send 25 xv 11 0 395 -487040 701664 +29973620 21575614424 26364945 2 2 1 send 25 xv 11 0 383 -146048 555616 +30232105 21575872909 25757563 2 2 1 send 25 xv 11 0 322 -235904 319712 +30884935 21576525739 26224844 2 2 1 send 25 xv 11 0 321 -310784 8928 +41905408 21587546212 40879793 2 2 0 recv 26 tend_part 7 0 38 -2328 6600 +48456241 21594097045 47468990 2 2 3 recv 26 tend_part 7 0 32 -1368 5232 +48605647 21594246451 47576321 2 2 0 recv 26 tend_part 7 0 138 -2712 2520 +56499592 21602140396 55460628 2 2 1 recv 26 tend_part 7 0 359 -2520 0 ## ## Number of requests: 23 ## Maximum request size: 0.5562 (MB) @@ -200,70 +200,70 @@ ## Mean of all requests: 0.1320 (MB) ## # stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum -8154566 21553806876 0 2 3 0 recv 23 tend_part 7 1 38 2328 2328 -8241482 21553893792 0 2 3 0 recv 23 tend_part 7 1 138 2712 5040 -8252462 21553904772 0 2 3 1 recv 23 xv 11 1 59 183552 188592 -8255236 21553907546 0 2 3 2 recv 23 xv 11 1 44 64128 252720 -8265527 21553917837 0 2 3 0 recv 23 tend_part 7 1 192 6936 259656 -8267192 21553919502 0 2 3 1 recv 23 xv 11 1 47 46720 306376 -8281049 21553933359 0 2 3 1 recv 23 tend_part 7 1 359 2520 308896 -8285576 21553937886 0 2 3 1 recv 23 xv 11 1 56 472064 780960 -8290638 21553942948 0 2 3 1 recv 23 xv 11 1 453 282752 1063712 -8312534 21553964844 0 2 3 2 recv 23 xv 11 1 444 98816 1162528 -8314496 21553966806 0 2 3 2 recv 23 xv 11 1 30 79232 1241760 -8318186 21553970496 0 2 3 2 recv 23 xv 11 1 2 107136 1348896 -8321900 21553974210 0 2 3 2 recv 23 xv 11 1 1 64512 1413408 -8551050 21554203360 0 2 3 0 send 22 xv 11 1 465 77696 1491104 -9240343 21554892653 0 2 3 0 send 22 xv 11 1 62 282624 1773728 -9516048 21555168358 0 2 3 0 send 22 xv 11 1 230 255232 2028960 -10222671 21555874981 1910137 2 3 2 recv 23 xv 11 0 444 -98816 1930144 -11396780 21557049090 3074880 2 3 2 recv 23 xv 11 0 1 -64512 1865632 -11919637 21557571947 0 2 3 0 send 22 xv 11 1 130 711424 2577056 -12416538 21558068848 496901 2 3 0 send 22 xv 11 0 130 -711424 1865632 -12449098 21558101408 0 2 3 1 send 22 xv 11 1 403 171136 2036768 -12525243 21558177553 4207057 2 3 2 recv 23 xv 11 0 2 -107136 1929632 -13886887 21559539197 0 2 3 0 send 22 xv 11 1 453 311296 2240928 -13993363 21559645673 5738127 2 3 2 recv 23 xv 11 0 44 -64128 2176800 -14353159 21560005469 6038663 2 3 2 recv 23 xv 11 0 30 -79232 2097568 -14361689 21560013999 0 2 3 0 send 22 xv 11 1 137 418304 2515872 -15260362 21560912672 898673 2 3 0 send 22 xv 11 0 137 -418304 2097568 -16761542 21562413852 0 2 3 0 send 22 xv 11 1 200 368000 2465568 -17561805 21563214115 0 2 3 0 send 22 xv 11 1 136 199808 2665376 -17768823 21563421133 0 2 3 1 send 22 xv 11 1 440 876032 3541408 -18876067 21564528377 0 2 3 0 send 22 xv 11 1 192 345984 3887392 -19483556 21565135866 0 2 3 0 send 22 xv 11 1 40 416512 4303904 -20583795 21566236105 0 2 3 0 send 22 xv 11 1 38 449024 4752928 -21166982 21566819292 0 2 3 0 send 22 xv 11 1 49 318720 5071648 -21367099 21567019409 7480212 2 3 0 send 22 xv 11 0 453 -311296 4760352 -21428196 21567080506 11912148 2 3 0 send 22 xv 11 0 230 -255232 4505120 -21489677 21567141987 12249334 2 3 0 send 22 xv 11 0 62 -282624 4222496 -21545174 21567197484 12994124 2 3 0 send 22 xv 11 0 465 -77696 4144800 -27653028 21573305338 10891486 2 3 0 send 22 xv 11 0 200 -368000 3776800 -27725141 21573377451 10163336 2 3 0 send 22 xv 11 0 136 -199808 3576992 -27785208 21573437518 8909141 2 3 0 send 22 xv 11 0 192 -345984 3231008 -27844800 21573497110 8361244 2 3 0 send 22 xv 11 0 40 -416512 2814496 -27904150 21573556460 7320355 2 3 0 send 22 xv 11 0 38 -449024 2365472 -29544767 21575197077 0 2 3 0 send 22 xv 11 1 50 400128 2765600 -29899938 21575552248 355171 2 3 0 send 22 xv 11 0 50 -400128 2365472 -29964381 21575616691 8797399 2 3 0 send 22 xv 11 0 49 -318720 2046752 -31237607 21576889917 0 2 3 0 send 22 xv 11 1 67 312448 2359200 -32076277 21577728587 23785639 2 3 1 recv 23 xv 11 0 453 -282752 2076448 -32997616 21578649926 20548518 2 3 1 send 22 xv 11 0 403 -171136 1905312 -33045540 21578697850 15276717 2 3 1 send 22 xv 11 0 440 -876032 1029280 -35555962 21581208272 0 2 3 0 send 22 xv 11 1 65 292096 1321376 -37724703 21583377013 6487096 2 3 0 send 22 xv 11 0 67 -312448 1008928 -38038978 21583691288 2483016 2 3 0 send 22 xv 11 0 65 -292096 716832 -39312145 21584964455 31026569 2 3 1 recv 23 xv 11 0 56 -472064 244768 -41913009 21587565319 33758443 2 3 0 recv 23 tend_part 7 0 38 -2328 242440 -44653176 21590305486 36385984 2 3 1 recv 23 xv 11 0 47 -46720 195720 -44672181 21590324491 36419719 2 3 1 recv 23 xv 11 0 59 -183552 12168 -48439936 21594092246 0 2 3 2 send 22 tend_part 7 1 32 1368 13536 -48448809 21594101119 0 2 3 1 send 22 tend_part 7 1 32 1368 14904 -48521474 21594173784 81538 2 3 2 send 22 tend_part 7 0 32 -1368 13536 -48529891 21594182201 81082 2 3 1 send 22 tend_part 7 0 32 -1368 12168 -48659245 21594311555 40417763 2 3 0 recv 23 tend_part 7 0 138 -2712 9456 -48993631 21594645941 40728104 2 3 0 recv 23 tend_part 7 0 192 -6936 2520 -56516622 21602168932 48235573 2 3 1 recv 23 tend_part 7 0 359 -2520 0 +8154566 21553806876 0 2 3 0 recv 26 tend_part 7 1 38 2328 2328 +8241482 21553893792 0 2 3 0 recv 26 tend_part 7 1 138 2712 5040 +8252462 21553904772 0 2 3 1 recv 26 xv 11 1 59 183552 188592 +8255236 21553907546 0 2 3 2 recv 26 xv 11 1 44 64128 252720 +8265527 21553917837 0 2 3 0 recv 26 tend_part 7 1 192 6936 259656 +8267192 21553919502 0 2 3 1 recv 26 xv 11 1 47 46720 306376 +8281049 21553933359 0 2 3 1 recv 26 tend_part 7 1 359 2520 308896 +8285576 21553937886 0 2 3 1 recv 26 xv 11 1 56 472064 780960 +8290638 21553942948 0 2 3 1 recv 26 xv 11 1 453 282752 1063712 +8312534 21553964844 0 2 3 2 recv 26 xv 11 1 444 98816 1162528 +8314496 21553966806 0 2 3 2 recv 26 xv 11 1 30 79232 1241760 +8318186 21553970496 0 2 3 2 recv 26 xv 11 1 2 107136 1348896 +8321900 21553974210 0 2 3 2 recv 26 xv 11 1 1 64512 1413408 +8551050 21554203360 0 2 3 0 send 25 xv 11 1 465 77696 1491104 +9240343 21554892653 0 2 3 0 send 25 xv 11 1 62 282624 1773728 +9516048 21555168358 0 2 3 0 send 25 xv 11 1 230 255232 2028960 +10222671 21555874981 1910137 2 3 2 recv 26 xv 11 0 444 -98816 1930144 +11396780 21557049090 3074880 2 3 2 recv 26 xv 11 0 1 -64512 1865632 +11919637 21557571947 0 2 3 0 send 25 xv 11 1 130 711424 2577056 +12416538 21558068848 496901 2 3 0 send 25 xv 11 0 130 -711424 1865632 +12449098 21558101408 0 2 3 1 send 25 xv 11 1 403 171136 2036768 +12525243 21558177553 4207057 2 3 2 recv 26 xv 11 0 2 -107136 1929632 +13886887 21559539197 0 2 3 0 send 25 xv 11 1 453 311296 2240928 +13993363 21559645673 5738127 2 3 2 recv 26 xv 11 0 44 -64128 2176800 +14353159 21560005469 6038663 2 3 2 recv 26 xv 11 0 30 -79232 2097568 +14361689 21560013999 0 2 3 0 send 25 xv 11 1 137 418304 2515872 +15260362 21560912672 898673 2 3 0 send 25 xv 11 0 137 -418304 2097568 +16761542 21562413852 0 2 3 0 send 25 xv 11 1 200 368000 2465568 +17561805 21563214115 0 2 3 0 send 25 xv 11 1 136 199808 2665376 +17768823 21563421133 0 2 3 1 send 25 xv 11 1 440 876032 3541408 +18876067 21564528377 0 2 3 0 send 25 xv 11 1 192 345984 3887392 +19483556 21565135866 0 2 3 0 send 25 xv 11 1 40 416512 4303904 +20583795 21566236105 0 2 3 0 send 25 xv 11 1 38 449024 4752928 +21166982 21566819292 0 2 3 0 send 25 xv 11 1 49 318720 5071648 +21367099 21567019409 7480212 2 3 0 send 25 xv 11 0 453 -311296 4760352 +21428196 21567080506 11912148 2 3 0 send 25 xv 11 0 230 -255232 4505120 +21489677 21567141987 12249334 2 3 0 send 25 xv 11 0 62 -282624 4222496 +21545174 21567197484 12994124 2 3 0 send 25 xv 11 0 465 -77696 4144800 +27653028 21573305338 10891486 2 3 0 send 25 xv 11 0 200 -368000 3776800 +27725141 21573377451 10163336 2 3 0 send 25 xv 11 0 136 -199808 3576992 +27785208 21573437518 8909141 2 3 0 send 25 xv 11 0 192 -345984 3231008 +27844800 21573497110 8361244 2 3 0 send 25 xv 11 0 40 -416512 2814496 +27904150 21573556460 7320355 2 3 0 send 25 xv 11 0 38 -449024 2365472 +29544767 21575197077 0 2 3 0 send 25 xv 11 1 50 400128 2765600 +29899938 21575552248 355171 2 3 0 send 25 xv 11 0 50 -400128 2365472 +29964381 21575616691 8797399 2 3 0 send 25 xv 11 0 49 -318720 2046752 +31237607 21576889917 0 2 3 0 send 25 xv 11 1 67 312448 2359200 +32076277 21577728587 23785639 2 3 1 recv 26 xv 11 0 453 -282752 2076448 +32997616 21578649926 20548518 2 3 1 send 25 xv 11 0 403 -171136 1905312 +33045540 21578697850 15276717 2 3 1 send 25 xv 11 0 440 -876032 1029280 +35555962 21581208272 0 2 3 0 send 25 xv 11 1 65 292096 1321376 +37724703 21583377013 6487096 2 3 0 send 25 xv 11 0 67 -312448 1008928 +38038978 21583691288 2483016 2 3 0 send 25 xv 11 0 65 -292096 716832 +39312145 21584964455 31026569 2 3 1 recv 26 xv 11 0 56 -472064 244768 +41913009 21587565319 33758443 2 3 0 recv 26 tend_part 7 0 38 -2328 242440 +44653176 21590305486 36385984 2 3 1 recv 26 xv 11 0 47 -46720 195720 +44672181 21590324491 36419719 2 3 1 recv 26 xv 11 0 59 -183552 12168 +48439936 21594092246 0 2 3 2 send 25 tend_part 7 1 32 1368 13536 +48448809 21594101119 0 2 3 1 send 25 tend_part 7 1 32 1368 14904 +48521474 21594173784 81538 2 3 2 send 25 tend_part 7 0 32 -1368 13536 +48529891 21594182201 81082 2 3 1 send 25 tend_part 7 0 32 -1368 12168 +48659245 21594311555 40417763 2 3 0 recv 26 tend_part 7 0 138 -2712 9456 +48993631 21594645941 40728104 2 3 0 recv 26 tend_part 7 0 192 -6936 2520 +56516622 21602168932 48235573 2 3 1 recv 26 tend_part 7 0 359 -2520 0 ## ## Number of requests: 32 ## Maximum request size: 0.8354 (MB)