diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..d9d719b44035a97cfc1698727173ea5e2aa02c38
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+__pycache__
+*.log
diff --git a/conf/cosma7.py b/conf/cosma7.py
index 22c53aa2598c03d33f579d35be4c5980bbb04c70..95c8682603732b3ade6aba7d4b2334fa733b620f 100644
--- a/conf/cosma7.py
+++ b/conf/cosma7.py
@@ -24,13 +24,22 @@ site_configuration = {
'max_jobs': 100,
},
{
- 'name': 'cpu_multi_node',
+ 'name': 'cpu_multi_node_aps',
'descr': 'cpu nodes - multi node',
'scheduler': 'slurm',
'launcher': 'mpirun',
'access': ['-p cosma7', '-A ds007'],
'environs': ['intelcosma7'],
'max_jobs': 100,
+ },
+ {
+ 'name': 'cpu_multi_node_perf_report',
+ 'descr': 'cpu nodes - multi node',
+ 'scheduler': 'slurm',
+ 'launcher': 'local',
+ 'access': ['-p cosma7', '-A ds007'],
+ 'environs': ['intelcosma7'],
+ 'max_jobs': 100,
}
]
}
@@ -39,7 +48,7 @@ site_configuration = {
{
'modules': ['intel_comp/2018', 'intel_mpi/2018',
'parmetis/4.0.3', 'parallel_hdf5/1.10.3',
- 'gsl/2.4', 'fftw/3.3.8cosma7',
+ 'gsl/2.4', 'fftw/3.3.8epyc',
'python/3.6.5', 'vtune',
'allinea/ddt/20.2.1'],
'name': 'intelcosma7',
diff --git a/conf/dine.py b/conf/dine.py
index 76eea15182773526d83623faf3de2c0ce6ab6366..66a6db8e5aac115025774d16c30f03922d93e078 100644
--- a/conf/dine.py
+++ b/conf/dine.py
@@ -24,13 +24,22 @@ site_configuration = {
'max_jobs': 100,
},
{
- 'name': 'cpu_multi_node',
+ 'name': 'cpu_multi_node_aps',
'descr': 'cpu nodes - multi node',
'scheduler': 'slurm',
'launcher': 'mpirun',
'access': ['-p bluefield1', '-A durham'],
'environs': ['intel'],
'max_jobs': 100,
+ },
+ {
+ 'name': 'cpu_multi_node_perf_report',
+ 'descr': 'cpu nodes - multi node',
+ 'scheduler': 'slurm',
+ 'launcher': 'local',
+ 'access': ['-p bluefield1', '-A durham'],
+ 'environs': ['intel'],
+ 'max_jobs': 100,
}
]
}
@@ -39,14 +48,15 @@ site_configuration = {
{
'modules': ['intel_comp/2018', 'intel_mpi/2018',
'parmetis/4.0.3', 'parallel_hdf5/1.10.3',
- 'gsl/2.4', 'fftw/3.3.8cosma7',
+ 'gsl/2.4', 'fftw/3.3.8epyc',
'python/3.6.5', 'vtune',
'allinea/ddt/20.2.1'],
'name': 'intel',
'cc': 'icc',
'cxx': 'icpc',
'ftn': '',
- 'target_systems': ['dine']
+ 'target_systems': ['dine'],
+ 'variables': [['MPICC','mpiicc'],['I_MPI_FABRICS','shm:tcp'],['MPS_STAT_LEVEL','2']]
}
],
'logging': [
diff --git a/env.sh b/env.sh
old mode 100644
new mode 100755
diff --git a/run.sh b/run.sh
old mode 100644
new mode 100755
diff --git a/src/common.py b/src/common.py
index 10de243b61b87ba01663292137186f509d12cf53..7b6e568d9773af18cb8c190c8580dbca4f95ef02 100644
--- a/src/common.py
+++ b/src/common.py
@@ -2,17 +2,13 @@ import reframe as rfm
import reframe.utility.sanity as sn
-GIT_REV = '0bb2f777a298531d5c60214314a2b2679df1fc6a'
-def setup(test, num_tasks, num_tasks_per_node=1):
- test.time_limit = '2d'
+def setup(test, launcher, num_tasks, num_tasks_per_node=1):
+ test.time_limit = '1h'
test.num_tasks = num_tasks
- test.num_tasks_per_node = num_tasks_per_node
+ test.num_tasks_per_node = 1
test.valid_prog_environs = ['*']
- test.valid_systems = ['cosma7:cpu_single_node',
- # 'cosma7:cpu_multi_node',
- 'dine:cpu_single_node']
test.build_system = 'Autotools'
test.sourcesdir = None
@@ -20,7 +16,7 @@ def setup(test, num_tasks, num_tasks_per_node=1):
'rm -rf swiftsim',
'git clone git@gitlab:swift/swiftsim.git',
'pushd swiftsim',
- f'git checkout {GIT_REV}',
+ f'git checkout master',
'git clean -x -f -d',
'./autogen.sh'
]
@@ -29,11 +25,26 @@ def setup(test, num_tasks, num_tasks_per_node=1):
test.sanity_patterns = sn.assert_found(r'main: done. Bye.', test.stdout)
ic_dir = None
- if rfm.utility.osext.osuser() == 'dc-turn5' and \
- test.current_system.name == 'cosma7':
- ic_dir = '/cosma7/data/ds007/dc-turn5/swift_initial_conditions'
- elif test.current_system.name == 'cosma7':
- ic_dir = '/cosma5/data/do008/dc-fraw1/swift_initial_conditions'
+ if rfm.utility.osext.osuser() == 'dc-turn5':
+ if test.current_system.name == 'cosma7':
+ ic_dir = '/cosma7/data/ds007/dc-turn5/swift_initial_conditions'
+ if launcher == 'mpirun':
+ test.valid_systems = ['cosma7:cpu_multi_node_aps']
+ elif launcher == 'perf-report':
+ test.valid_systems = ['cosma7:cpu_multi_node_perf_report']
+ elif test.current_system.name == 'dine':
+ ic_dir = '/cosma5/data/durham/dc-turn5/swift_initial_conditions'
+ if launcher == 'mpirun':
+ test.valid_systems = ['dine:cpu_multi_node_aps']
+ elif launcher == 'perf-report':
+ test.valid_systems = ['dine:cpu_multi_node_perf_report']
+ else:
+ raise ValueError(f'Need to handle {test.current_system.name} for {rfm.utility.osext.osuser()}')
+ elif rfm.utility.osext.osuser() == 'dc-fraw1':
+ if test.current_system.name == 'cosma7' or test.current_system.name == 'dine':
+ ic_dir = '/cosma5/data/do008/dc-fraw1/swift_initial_conditions'
+ else:
+ raise ValueError(f'Need to handle {test.current_system.name} for {rfm.utility.osext.osuser()}')
else:
raise ValueError(f'Need to handle {test.current_system.name} for {rfm.utility.osext.osuser()}')
return ic_dir
diff --git a/src/pmill-768.py b/src/pmill-768.py
index 4189a20948c5a5d2b4ed9966720869e5bf92d580..e0f732c42c97974cdcc8949dd324577edb94b9f1 100644
--- a/src/pmill-768.py
+++ b/src/pmill-768.py
@@ -4,24 +4,17 @@ import reframe.core.launchers.mpi
import common
-@rfm.parameterized_test(*([tasks, threads, iteration, executable]
- for tasks in [8]
- for threads in list(reversed([64]))
+@rfm.parameterized_test(*([tasks, threads, iteration, launcher]
+ for tasks in [2]
+ for threads in list(reversed([56]))
for iteration in [1]
- for executable in ['mpirun','perf-report']))
+ for launcher in ['mpirun','perf-report']))
class PMillenniumTest(rfm.RegressionTest):
- def __init__(self, num_tasks, num_threads, iteration, executable):
- ic_dir = common.setup(self, num_tasks, num_tasks_per_node=num_tasks)
-
- if num_threads == 64:
- self.time_limit = '40m'
- else:
- self.time_limit = '2h'
-
-
+ def __init__(self, num_tasks, num_threads, iteration, launcher):
+ ic_dir = common.setup(self, launcher, num_tasks, num_tasks_per_node=num_tasks)
+
test_dir = 'swiftsim/examples/PMillennium/PMillennium-768'
- num_threads = num_threads // num_tasks
- # self.cpus_per_task = num_threads #? may still be needed
+ self.cpus_per_task = 32 # added
print(f'Running with threads : <{num_threads}>')
self.keep_files = [test_dir]
@@ -31,17 +24,18 @@ class PMillenniumTest(rfm.RegressionTest):
'--with-parmetis'
]
self.prerun_cmds = [f'pushd {test_dir}']
- num_iterations = 1
+ num_iterations = 5
+
- self.executable = executable
- if executable == 'mpirun':
- self.executable_opts = ['aps', '--collection-mode=mpi,omp']
- elif executable == 'perf-report':
- self.executable_opts = ['--mpi=intel-mpi']
+ if launcher == 'mpirun':
+ self.executable = 'aps'
+ self.executable_opts = ['--collection-mode=mpi,omp', '../../swift_mpi']
+ elif launcher == 'perf-report':
+ self.executable = 'perf-report'
+ self.executable_opts = ['--mpi=intel-mpi', '../../swift_mpi']
self.time_limit = '2h40m'
self.executable_opts += [
- '../../swift_mpi',
'--cosmology',
'--self-gravity',
'-v', '1',
@@ -52,7 +46,3 @@ class PMillenniumTest(rfm.RegressionTest):
'p-mill-768.yml'
]
- #@rfm.run_before('run')
- #def set_memory_limit(self):
- # self.job.options += [f'--cpus-per-task={self.cpus_per_task}']
-
diff --git a/src/sodshock_3d.py b/src/sodshock_3d.py
new file mode 100644
index 0000000000000000000000000000000000000000..65509aecfebb4789c732c8d5a8508c8d3dbeb4fe
--- /dev/null
+++ b/src/sodshock_3d.py
@@ -0,0 +1,48 @@
+import reframe as rfm
+import reframe.core.launchers.mpi
+
+import common
+
+
+@rfm.parameterized_test(*([tasks, threads, iteration, launcher]
+ for tasks in [2]
+ for threads in list(reversed([56]))
+ for iteration in [1]
+ for launcher in ['mpirun', 'perf-report']))
+class SodShock3dTest(rfm.RegressionTest):
+ def __init__(self, num_tasks, num_threads, iteration, launcher):
+ ic_dir = common.setup(self, launcher, num_tasks, num_tasks_per_node=num_tasks)
+
+ test_dir = 'swiftsim/examples/HydroTests/SodShock_3D'
+ self.cpus_per_task = 32 # added
+ print(f'Running with threads : <{num_threads}>')
+
+ self.keep_files = [test_dir]
+ self.build_system.config_opts = [
+ '--enable-ipo',
+ '--with-tbbmalloc',
+ '--with-parmetis'
+ ]
+ self.prerun_cmds = [f'pushd {test_dir}']
+ num_iterations = 10000
+
+
+ if launcher == 'mpirun':
+ self.executable = 'aps'
+ self.executable_opts = ['--collection-mode=mpi,omp', '../../swift_mpi',
+ '-P', 'Snapshots:time_first:999999999']
+ elif launcher == 'perf-report':
+ self.executable = 'perf-report'
+ self.executable_opts = ['--mpi=intel-mpi', '../../swift_mpi']
+ self.time_limit = '2h40m'
+
+ self.executable_opts += [
+ '--hydro',
+ '-v', '1',
+ f'--threads={num_threads}',
+ '-n', f'{num_iterations}',
+ '-P', 'Restarts:enable:0',
+ '-P', f'InitialConditions:file_name:{ic_dir}/sodshock/glassCube_64.hdf5',
+ 'sodShock.yml'
+ ]
+