Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
S
swiftmpistepsim
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
SWIFT
swiftmpistepsim
Commits
63cf8177
Commit
63cf8177
authored
5 years ago
by
Peter W. Draper
Browse files
Options
Downloads
Patches
Plain Diff
Add scripts that change the number of messages, but preserve the data volume
parent
db32f19a
Branches
Branches containing commit
No related tags found
1 merge request
!7
Add a scale factor for messages
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
testdata/doubledata.py
+166
-0
166 additions, 0 deletions
testdata/doubledata.py
testdata/pairdata.py
+130
-0
130 additions, 0 deletions
testdata/pairdata.py
with
296 additions
and
0 deletions
testdata/doubledata.py
0 → 100755
+
166
−
0
View file @
63cf8177
#!/usr/bin/env python
"""
Usage:
doubledata.py [options] input-log doubled-log
Split pairs of send and recv log data entries to increase the number of
messages that we send to represent the data of a step. Preserves the data
volume, but not the tagging.
This file is part of SWIFT.
Copyright (C) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
All Rights Reserved.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import
argparse
import
sys
from
operator
import
itemgetter
# Handle the command line.
parser
=
argparse
.
ArgumentParser
(
description
=
"
Pair MPI logs
"
)
parser
.
add_argument
(
"
input
"
,
help
=
"
Output log from simulator
"
)
parser
.
add_argument
(
"
output
"
,
help
=
"
Doubled log
"
)
parser
.
add_argument
(
"
-v
"
,
"
--verbose
"
,
dest
=
"
verbose
"
,
help
=
"
Verbose output
"
,
default
=
False
,
action
=
"
store_true
"
,
)
args
=
parser
.
parse_args
()
infile
=
args
.
input
outfile
=
args
.
output
# Indices for words in a line.
sticcol
=
0
eticcol
=
1
dticcol
=
2
stepcol
=
3
rankcol
=
4
otherrankcol
=
5
typecol
=
6
itypecol
=
7
subtypecol
=
8
isubtypecol
=
9
activationcol
=
10
tagcol
=
11
sizecol
=
12
sumcol
=
13
# The plan is to pair the sends and receives across all ranks so that
# we can split each pair into two and give everything new tags that also
# match.
# Keyed lines.
keysends
=
{}
keyrecvs
=
{}
# Indexed lines.
sends
=
[]
recvs
=
[]
nsends
=
0
nrecvs
=
0
# Generate keys that are unique between send/recv pairs and gather the
# associated lines initiation lines.
with
open
(
infile
,
"
r
"
)
as
fp
:
for
line
in
fp
:
if
line
[
0
]
==
'
#
'
:
continue
words
=
line
.
split
()
if
words
[
activationcol
]
==
0
:
continue
if
words
[
itypecol
]
==
"
22
"
:
key
=
words
[
otherrankcol
]
+
"
/
"
+
\
words
[
rankcol
]
+
"
/
"
+
\
words
[
isubtypecol
]
+
"
/
"
+
\
words
[
tagcol
]
+
"
/
"
+
\
words
[
sizecol
]
if
not
key
in
keysends
:
keysends
[
key
]
=
[
nsends
]
else
:
keysends
[
key
].
append
(
nsends
)
sends
.
append
(
words
)
nsends
=
nsends
+
1
elif
words
[
itypecol
]
==
"
23
"
:
key
=
words
[
rankcol
]
+
"
/
"
+
\
words
[
otherrankcol
]
+
"
/
"
+
\
words
[
isubtypecol
]
+
"
/
"
+
\
words
[
tagcol
]
+
"
/
"
+
\
words
[
sizecol
]
if
not
key
in
keyrecvs
:
keyrecvs
[
key
]
=
[
nrecvs
]
else
:
keyrecvs
[
key
].
append
(
nrecvs
)
recvs
.
append
(
words
)
nrecvs
=
nrecvs
+
1
print
"
# Read
"
+
str
(
nsends
)
+
"
sends and
"
+
str
(
nrecvs
)
+
"
recvs
"
# Now get the indices of the matches.
msends
=
[
None
]
*
nsends
for
key
in
keysends
:
if
key
in
keyrecvs
:
isend
=
keysends
[
key
][
0
]
irecv
=
keyrecvs
[
key
][
0
]
msends
[
isend
]
=
irecv
# Reorder recvs to same order as sends.
recvs
=
[
recvs
[
i
]
for
i
in
msends
]
# Now we can split and retag.
splits
=
[]
ctag
=
0
for
i
in
range
(
nsends
):
swords
=
sends
[
i
]
rwords
=
recvs
[
i
]
size
=
str
(
float
(
swords
[
sizecol
])
/
2.0
)
swords
[
sizecol
]
=
size
rwords
[
sizecol
]
=
size
ctag
=
ctag
+
1
stag
=
str
(
ctag
)
swords
[
tagcol
]
=
stag
rwords
[
tagcol
]
=
stag
splits
.
append
(
swords
)
splits
.
append
(
rwords
)
ctag
=
ctag
+
1
stag
=
str
(
ctag
)
swords
[
tagcol
]
=
stag
rwords
[
tagcol
]
=
stag
splits
.
append
(
swords
)
splits
.
append
(
rwords
)
# Sort by tic.
splits
=
sorted
(
splits
,
key
=
lambda
x
:
int
(
x
[
sticcol
]))
# And output.
with
open
(
outfile
,
"
w
"
)
as
fp
:
fp
.
write
(
"
# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum
\n
"
)
for
line
in
splits
:
fp
.
write
(
"
"
.
join
(
line
)
+
"
\n
"
)
sys
.
exit
(
0
)
This diff is collapsed.
Click to expand it.
testdata/pairdata.py
0 → 100755
+
130
−
0
View file @
63cf8177
#!/usr/bin/env python
"""
Usage:
pairdata.py [options] output-log paired-log
Combine pairs of send and recv log data entries to reduce the number of
messages that we send to represent the data of a step. Uses a simple strategy
identifying sends and recvs of the same ranks and type/subtypes that are close
in time and combining them. We need to keep the tags and sizes matched across
the ranks so we pick the tag of the first in time and sum the sizes.
This file is part of SWIFT.
Copyright (C) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
All Rights Reserved.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import
argparse
import
sys
from
operator
import
itemgetter
# Handle the command line.
parser
=
argparse
.
ArgumentParser
(
description
=
"
Pair MPI logs
"
)
parser
.
add_argument
(
"
input
"
,
help
=
"
Output log from simulator
"
)
parser
.
add_argument
(
"
output
"
,
help
=
"
Paired log
"
)
parser
.
add_argument
(
"
-v
"
,
"
--verbose
"
,
dest
=
"
verbose
"
,
help
=
"
Verbose output
"
,
default
=
False
,
action
=
"
store_true
"
,
)
args
=
parser
.
parse_args
()
infile
=
args
.
input
outfile
=
args
.
output
# Indices for words in a line.
sticcol
=
0
eticcol
=
1
dticcol
=
2
stepcol
=
3
rankcol
=
4
otherrankcol
=
5
typecol
=
6
itypecol
=
7
subtypecol
=
8
isubtypecol
=
9
activationcol
=
10
tagcol
=
11
sizecol
=
12
sumcol
=
13
# Indexed lines.
lines
=
[]
nlines
=
0
# Gather lines from the input log. We only want activation lines.
with
open
(
infile
,
"
r
"
)
as
fp
:
for
line
in
fp
:
if
line
[
0
]
==
'
#
'
:
continue
words
=
line
.
split
()
if
words
[
activationcol
]
==
"
1
"
:
lines
.
append
(
words
)
nlines
=
nlines
+
1
# Sort by tag as we need a stable join (sends and recvs are not ordered).
lines
=
sorted
(
lines
,
key
=
lambda
x
:
float
(
x
[
tagcol
]))
if
args
.
verbose
:
print
"
# Read
"
,
nlines
,
"
activation logs from
"
,
infile
# Now locate pairs.
nmatches
=
0
nnlines
=
0
with
open
(
outfile
,
"
w
"
)
as
fp
:
for
n
in
range
(
0
,
nlines
):
nwords
=
lines
[
n
]
# If not already used.
if
nwords
[
rankcol
]
==
"
-1
"
:
continue
# Check remaining lines for a match.
for
m
in
range
(
n
+
1
,
nlines
):
mwords
=
lines
[
m
]
if
(
nwords
[
rankcol
]
==
mwords
[
rankcol
]
and
nwords
[
otherrankcol
]
==
mwords
[
otherrankcol
]
and
nwords
[
itypecol
]
==
mwords
[
itypecol
]
and
nwords
[
isubtypecol
]
==
mwords
[
isubtypecol
]):
# Matching send or recv of same type to and from same
# ranks. Use sum of sizes and min of tags and tics.
nwords
[
sizecol
]
=
str
(
int
(
nwords
[
sizecol
])
+
int
(
mwords
[
sizecol
]))
nwords
[
tagcol
]
=
str
(
min
(
int
(
nwords
[
tagcol
]),
int
(
mwords
[
tagcol
])))
nwords
[
sticcol
]
=
str
(
min
(
int
(
nwords
[
sticcol
]),
int
(
mwords
[
sticcol
])))
# Don't use this other entry again. Will be skipped or
# fail to match.
mwords
[
rankcol
]
=
"
-1
"
nmatches
=
nmatches
+
1
break
# And output.
fp
.
write
(
"
"
.
join
(
nwords
)
+
"
\n
"
)
nnlines
=
nnlines
+
1
if
args
.
verbose
:
print
"
# Matched
"
,
nmatches
,
"
wrote
"
,
nnlines
,
"
lines to
"
,
outfile
sys
.
exit
(
0
)
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment