diff --git a/src/common.py b/src/common.py new file mode 100644 index 0000000000000000000000000000000000000000..10de243b61b87ba01663292137186f509d12cf53 --- /dev/null +++ b/src/common.py @@ -0,0 +1,40 @@ +import reframe as rfm +import reframe.utility.sanity as sn + + +GIT_REV = '0bb2f777a298531d5c60214314a2b2679df1fc6a' + +def setup(test, num_tasks, num_tasks_per_node=1): + test.time_limit = '2d' + test.num_tasks = num_tasks + test.num_tasks_per_node = num_tasks_per_node + + test.valid_prog_environs = ['*'] + test.valid_systems = ['cosma7:cpu_single_node', + # 'cosma7:cpu_multi_node', + 'dine:cpu_single_node'] + + test.build_system = 'Autotools' + test.sourcesdir = None + test.prebuild_cmds = [ + 'rm -rf swiftsim', + 'git clone git@gitlab:swift/swiftsim.git', + 'pushd swiftsim', + f'git checkout {GIT_REV}', + 'git clean -x -f -d', + './autogen.sh' + ] + test.sourcepath = './' + test.build_system.max_concurrency = 32 + test.sanity_patterns = sn.assert_found(r'main: done. Bye.', test.stdout) + + ic_dir = None + if rfm.utility.osext.osuser() == 'dc-turn5' and \ + test.current_system.name == 'cosma7': + ic_dir = '/cosma7/data/ds007/dc-turn5/swift_initial_conditions' + elif test.current_system.name == 'cosma7': + ic_dir = '/cosma5/data/do008/dc-fraw1/swift_initial_conditions' + else: + raise ValueError(f'Need to handle {test.current_system.name} for {rfm.utility.osext.osuser()}') + return ic_dir + diff --git a/src/pmill-768.py b/src/pmill-768.py index 26e6332e4a6fb7c1062facf983c54697e6a3eb53..4189a20948c5a5d2b4ed9966720869e5bf92d580 100644 --- a/src/pmill-768.py +++ b/src/pmill-768.py @@ -1,48 +1,46 @@ import reframe as rfm -import reframe.utility.sanity as sn +import reframe.core.launchers.mpi +import common -#GIT_REV = 'master' -GIT_REV = '0bb2f777a298531d5c60214314a2b2679df1fc6a' -@rfm.simple_test +@rfm.parameterized_test(*([tasks, threads, iteration, executable] + for tasks in [8] + for threads in list(reversed([64])) + for iteration in [1] + for executable in ['mpirun','perf-report'])) class PMillenniumTest(rfm.RegressionTest): - def __init__(self): - self.valid_systems = ['cosma7:cpu_single_node', - 'cosma7:cpu_multi_node', - 'dine:cpu_single_node'] - self.valid_prog_environs = ['*'] - self.num_tasks = 1 - self.build_system = 'Autotools' - self.sourcesdir = None # Added - self.prebuild_cmds = [ - 'rm -rf swiftsim', - #'git clone https://gitlab.cosma.dur.ac.uk/swift/swiftsim.git', - 'git clone git@gitlab:swift/swiftsim.git', - 'pushd swiftsim', - f'git checkout {GIT_REV}', - 'git clean -x -f -d', - './autogen.sh', - 'cd ./' - ] - self.build_system.max_concurrency = 32 + def __init__(self, num_tasks, num_threads, iteration, executable): + ic_dir = common.setup(self, num_tasks, num_tasks_per_node=num_tasks) + + if num_threads == 64: + self.time_limit = '40m' + else: + self.time_limit = '2h' + + + test_dir = 'swiftsim/examples/PMillennium/PMillennium-768' + num_threads = num_threads // num_tasks + # self.cpus_per_task = num_threads #? may still be needed + print(f'Running with threads : <{num_threads}>') + + self.keep_files = [test_dir] self.build_system.config_opts = [ '--enable-ipo', '--with-tbbmalloc', '--with-parmetis' ] - - test_dir = 'swiftsim/examples/PMillennium/PMillennium-768' - self.keep_files = [test_dir] self.prerun_cmds = [f'pushd {test_dir}'] - self.executable = 'mpirun' - #self.executable = 'aps' #temp copied out num_iterations = 1 - num_threads = 64 - #ic_dir = '/cosma/home/durham/dc-turn5/swiftsim/examples' - ic_dir = '/cosma5/data/do008/dc-fraw1/swift_initial_conditions/pmillenium' - self.executable_opts = [ - 'aps', '--collection-mode=mpi,omp', + + self.executable = executable + if executable == 'mpirun': + self.executable_opts = ['aps', '--collection-mode=mpi,omp'] + elif executable == 'perf-report': + self.executable_opts = ['--mpi=intel-mpi'] + self.time_limit = '2h40m' + + self.executable_opts += [ '../../swift_mpi', '--cosmology', '--self-gravity', @@ -53,3 +51,8 @@ class PMillenniumTest(rfm.RegressionTest): f'-PInitialConditions:file_name:{ic_dir}/pmillenium/PMill-768.hdf5', 'p-mill-768.yml' ] + + #@rfm.run_before('run') + #def set_memory_limit(self): + # self.job.options += [f'--cpus-per-task={self.cpus_per_task}'] +