diff --git a/benchmark-fast/build.sh b/benchmark-fast/build.sh index ffa20bed112dbe2d2b4aa2a7dc746b8f672c7754..afb9c37dd092be0e9d9c05160c66dc0119f9c7bd 100644 --- a/benchmark-fast/build.sh +++ b/benchmark-fast/build.sh @@ -1,5 +1,5 @@ -module load intel_comp/2018 -module load intel_mpi/2018 +module load intel_comp/2018-update2 +module load intel_mpi/2018-update2 module load parmetis/4.0.3 module load parallel_hdf5/1.10.3 module load gsl/2.4 diff --git a/conf/cosma7.py b/conf/cosma7.py new file mode 100644 index 0000000000000000000000000000000000000000..3dcb16ddb807b0899156840fa672221cf8676e15 --- /dev/null +++ b/conf/cosma7.py @@ -0,0 +1,88 @@ +site_configuration = { + 'systems': [ + { + 'name': 'cosma7', + 'descr': 'Cosma7', + 'hostnames': ['login7b.pri.cosma7.alces.network', 'login7c.pri.cosma7.alces.network', 'login7a.pri.cosma7.alces.network'], + 'modules_system': 'tmod4', + 'partitions': [ + { + 'name': 'login', + 'descr': 'login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['intelcosma7'], + 'max_jobs': 100, + }, + { + 'name': 'cpu_single_node', + 'descr': 'cpu nodes - single node', + 'scheduler': 'slurm', + 'launcher': 'local', + 'access': ['-p cosma7', '-A ds007'], + 'environs': ['intelcosma7'], + 'max_jobs': 100, + }, + { + 'name': 'cpu_multi_node', + 'descr': 'cpu nodes - multi node', + 'scheduler': 'slurm', + 'launcher': 'mpirun', + 'access': ['-p cosma7', '-A ds007'], + 'environs': ['intelcosma7'], + 'max_jobs': 100, + } + ] + } + ], + 'environments': [ + { + 'modules': ['intel_comp/2018', 'intel_mpi/2018', + 'parmetis/4.0.3', 'parallel_hdf5/1.10.3', + 'gsl/2.4', 'fftw/3.3.8cosma7', + 'python/3.6.5'], + 'name': 'intelcosma7', + 'cc': 'icc', + 'cxx': 'icpc', + 'ftn': '', + 'target_systems': ['cosma7'] + } + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' + }, + { + 'type': 'file', + 'name': 'reframe.log', + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 + 'append': False + } + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': ( + '%(check_job_completion_time)s|reframe %(version)s|' + '%(check_info)s|jobid=%(check_jobid)s|' + '%(check_perf_var)s=%(check_perf_value)s|' + 'ref=%(check_perf_ref)s ' + '(l=%(check_perf_lower_thres)s, ' + 'u=%(check_perf_upper_thres)s)|' + '%(check_perf_unit)s' + ), + 'append': True + } + ] + } + ], +} diff --git a/conf/dine.py b/conf/dine.py new file mode 100644 index 0000000000000000000000000000000000000000..27e7d04f51eb89936f4d316ec3abf65c46c6c90a --- /dev/null +++ b/conf/dine.py @@ -0,0 +1,88 @@ +site_configuration = { + 'systems': [ + { + 'name': 'dine', + 'descr': 'DINE', + 'hostnames': ['b101.pri.cosma7.alces.network'], + 'modules_system': 'tmod4', + 'partitions': [ + { + 'name': 'login', + 'descr': 'login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['intel'], + 'max_jobs': 100, + }, + { + 'name': 'cpu_single_node', + 'descr': 'cpu nodes - single node', + 'scheduler': 'slurm', + 'launcher': 'local', + 'access': ['-p bluefield1', '-A ds007'], + 'environs': ['intel'], + 'max_jobs': 100, + }, + { + 'name': 'cpu_multi_node', + 'descr': 'cpu nodes - multi node', + 'scheduler': 'slurm', + 'launcher': 'mpirun', + 'access': ['-p bluefield1', '-A ds007'], + 'environs': ['intel'], + 'max_jobs': 100, + } + ] + } + ], + 'environments': [ + { + 'modules': ['intel_comp/2018', 'intel_mpi/2018', + 'parmetis/4.0.3', 'parallel_hdf5/1.10.3', + 'gsl/2.4', 'fftw/3.3.8cosma7', + 'python/3.6.5'], + 'name': 'intel', + 'cc': 'icc', + 'cxx': 'icpc', + 'ftn': '', + 'target_systems': ['dine'] + } + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' + }, + { + 'type': 'file', + 'name': 'reframe.log', + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 + 'append': False + } + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': ( + '%(check_job_completion_time)s|reframe %(version)s|' + '%(check_info)s|jobid=%(check_jobid)s|' + '%(check_perf_var)s=%(check_perf_value)s|' + 'ref=%(check_perf_ref)s ' + '(l=%(check_perf_lower_thres)s, ' + 'u=%(check_perf_upper_thres)s)|' + '%(check_perf_unit)s' + ), + 'append': True + } + ] + } + ], +} diff --git a/env.sh b/env.sh new file mode 100644 index 0000000000000000000000000000000000000000..d73ca1bc4aa5dd2beabecc04b0d0839517475f95 --- /dev/null +++ b/env.sh @@ -0,0 +1,10 @@ +#!/bin/bash +module purge +module load python/3.6.5 +source ~/reframe_env/bin/activate + +export STAGE_DIR=$HOME/reframe_stage +export OUTPUT_DIR=$HOME/reframe_output +export RFM_SAVE_LOG_FILES=1 + +export PYTHONPATH=$PYTHONPATH:`pwd`/src diff --git a/run.sh b/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..889b0503e030b93311d5478ea2ba2b1862402faa --- /dev/null +++ b/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +source env.sh + +~/build/reframe/bin/reframe \ + --max-retries=0 \ + --exec-policy async \ + --stage $STAGE_DIR \ + --purge-env \ + -C conf/dine.py \ + -c src \ + -r \ + -v \ + --performance-report \ + -n \ + 'PMillenniumTest' \ + --output \ + $OUTPUT_DIR \ + --report-file=pmill_768.log + diff --git a/src/pmill-768.py b/src/pmill-768.py new file mode 100644 index 0000000000000000000000000000000000000000..26e6332e4a6fb7c1062facf983c54697e6a3eb53 --- /dev/null +++ b/src/pmill-768.py @@ -0,0 +1,55 @@ +import reframe as rfm +import reframe.utility.sanity as sn + + +#GIT_REV = 'master' +GIT_REV = '0bb2f777a298531d5c60214314a2b2679df1fc6a' + +@rfm.simple_test +class PMillenniumTest(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['cosma7:cpu_single_node', + 'cosma7:cpu_multi_node', + 'dine:cpu_single_node'] + self.valid_prog_environs = ['*'] + self.num_tasks = 1 + self.build_system = 'Autotools' + self.sourcesdir = None # Added + self.prebuild_cmds = [ + 'rm -rf swiftsim', + #'git clone https://gitlab.cosma.dur.ac.uk/swift/swiftsim.git', + 'git clone git@gitlab:swift/swiftsim.git', + 'pushd swiftsim', + f'git checkout {GIT_REV}', + 'git clean -x -f -d', + './autogen.sh', + 'cd ./' + ] + self.build_system.max_concurrency = 32 + self.build_system.config_opts = [ + '--enable-ipo', + '--with-tbbmalloc', + '--with-parmetis' + ] + + test_dir = 'swiftsim/examples/PMillennium/PMillennium-768' + self.keep_files = [test_dir] + self.prerun_cmds = [f'pushd {test_dir}'] + self.executable = 'mpirun' + #self.executable = 'aps' #temp copied out + num_iterations = 1 + num_threads = 64 + #ic_dir = '/cosma/home/durham/dc-turn5/swiftsim/examples' + ic_dir = '/cosma5/data/do008/dc-fraw1/swift_initial_conditions/pmillenium' + self.executable_opts = [ + 'aps', '--collection-mode=mpi,omp', + '../../swift_mpi', + '--cosmology', + '--self-gravity', + '-v', '1', + f'--threads={num_threads}', + '-n', f'{num_iterations}', + '-P', 'Restarts:enable:0', + f'-PInitialConditions:file_name:{ic_dir}/pmillenium/PMill-768.hdf5', + 'p-mill-768.yml' + ]