add scalasca on DINE

conf/cube_scalasca_dine.py

+site_configuration = {
+    'systems': [
+        {
+            'name': 'dine',
+            'descr': 'DINE',
+            'hostnames': ['b101.pri.cosma7.alces.network'],
+            'modules_system': 'tmod4',
+            'partitions': [
+                {
+                    'name': 'login',
+                    'descr': 'login nodes',
+                    'scheduler': 'local',
+                    'launcher': 'local',
+                    'environs': ['intel'],
+                    'max_jobs': 100,
+                },
+                {
+                    'name': 'cpu_single_node',
+                    'descr': 'cpu nodes - single node',
+                    'scheduler': 'slurm',
+                    'launcher': 'local',
+                    'access': ['-p bluefield1', '-A durham'],
+                    'environs': ['intel'],
+                    'max_jobs': 100,
+                }
+            ]
+        }
+    ],
+    'environments': [
+        {
+            'modules': ['intel_comp/2018', 'intel_mpi/2018',
+                        # 'intel_comp/2020-update2', 'intel_mpi/2020-update2',
+                        'parmetis/4.0.3', 'parallel_hdf5/1.10.3',
+                        'gsl/2.4', 'fftw/3.3.8epyc',
+                        'python/3.6.5', # 'scalasca', 
+                        'scorep/6.0', 'cube/4.5'],
+            'name': 'intel',
+            'cc': 'icc',
+            'cxx': 'icpc',
+            'ftn': '',
+            'target_systems': ['dine'],
+            'variables': [['SCOREP_ENABLE_PROFILING','1'],['SCOREP_EXPERIMENT_DIRECTORY','scorep_profiler_output'],['MPICC','mpiicc'],['I_MPI_FABRICS','shm:tcp'],['MPS_STAT_LEVEL','2']]
+        }
+    ],
+    'logging': [
+        {
+            'level': 'debug',
+            'handlers': [
+                {
+                    'type': 'stream',
+                    'name': 'stdout',
+                    'level': 'info',
+                    'format': '%(message)s'
+                },
+                {
+                    'type': 'file',
+                    'name': 'reframe.log',
+                    'level': 'debug',
+                    'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s',   # noqa: E501
+                    'append': False
+                }
+            ],
+            'handlers_perflog': [
+                {
+                    'type': 'filelog',
+                    'prefix': '%(check_system)s/%(check_partition)s',
+                    'level': 'info',
+                    'format': (
+                        '%(check_job_completion_time)s|reframe %(version)s|'
+                        '%(check_info)s|jobid=%(check_jobid)s|'
+                        '%(check_perf_var)s=%(check_perf_value)s|'
+                        'ref=%(check_perf_ref)s '
+                        '(l=%(check_perf_lower_thres)s, '
+                        'u=%(check_perf_upper_thres)s)|'
+                        '%(check_perf_unit)s'
+                    ),
+                    'append': True
+                }
+            ]
+        }
+    ],
+}

conf/dine.py

@@ -50,13 +50,13 @@ site_configuration = {
                         'parmetis/4.0.3', 'parallel_hdf5/1.10.3',
                         'gsl/2.4', 'fftw/3.3.8epyc',
                         'python/3.6.5', 'vtune',
-                        'allinea/ddt/20.2.1'],
+                        'allinea/ddt/20.2.1', 'scorep/6.0', 'cube/4.5'],
             'name': 'intel',
             'cc': 'icc',
             'cxx': 'icpc',
             'ftn': '',
             'target_systems': ['dine'],
-            'variables': [['MPICC','mpiicc'],['I_MPI_FABRICS','shm:tcp'],['MPS_STAT_LEVEL','2']]
+            'variables': [['SCOREP_ENABLE_PROFILING','1'],['SCOREP_EXPERIMENT_DIRECTORY','scorep_profiler_output'],['MPICC','mpiicc'],['I_MPI_FABRICS','shm:tcp'],['MPS_STAT_LEVEL','2']]
         }
     ],
     'logging': [

src/common.py

@@ -38,6 +38,8 @@ def setup(test, launcher, num_tasks, num_tasks_per_node=1):
                 test.valid_systems = ['dine:cpu_multi_node_aps']
             elif launcher == 'perf-report':
                 test.valid_systems = ['dine:cpu_multi_node_perf_report']
+            elif launcher == 'scalasca':
+                test.valid_systems = ['dine:cpu_single_node']
         else:
             raise ValueError(f'Need to handle {test.current_system.name} for {rfm.utility.osext.osuser()}')
     elif rfm.utility.osext.osuser() == 'dc-fraw1':

src/sodshock_3d.py

@@ -6,9 +6,9 @@ import common
 
 @rfm.parameterized_test(*([tasks, threads, iteration, launcher]
     for tasks in [2]
-    for threads in list(reversed([56])) 
+    for threads in list(reversed([32])) 
     for iteration in [1]
-    for launcher in ['mpirun', 'perf-report']))
+    for launcher in ['mpirun', 'perf-report', 'scalasca']))
 class SodShock3dTest(rfm.RegressionTest):
     def __init__(self, num_tasks, num_threads, iteration, launcher):
         ic_dir = common.setup(self, launcher, num_tasks, num_tasks_per_node=num_tasks)
@@ -25,7 +25,6 @@ class SodShock3dTest(rfm.RegressionTest):
         ]
         self.prerun_cmds = [f'pushd {test_dir}']
         num_iterations = 10000
-        
 
         if launcher == 'mpirun':
             self.executable = 'aps' 
@@ -35,6 +34,10 @@ class SodShock3dTest(rfm.RegressionTest):
             self.executable = 'perf-report'
             self.executable_opts = ['--mpi=intel-mpi', '../../swift_mpi']
             self.time_limit = '2h40m'
+        elif launcher == 'scalasca':
+            self.build_system.make_opts = ['CC="scorep --user --thread=pthread icc"']
+            self.executable = '../../swift'
+            self.executable_opts = ['-P', 'Snapshots:time_first:999999999'] 
 
         self.executable_opts += [
             '--hydro',

src/sodshock_scalasca.py

+import reframe as rfm
+import reframe.core.launchers.mpi
+
+import common
+
+
+@rfm.parameterized_test(*([tasks, threads, iteration, launcher]
+    for tasks in [2]
+    for threads in list(reversed([32])) 
+    for iteration in [1]
+    for launcher in ['scalasca']))
+class SodShock3dTest(rfm.RegressionTest):
+    def __init__(self, num_tasks, num_threads, iteration, launcher):
+        ic_dir = common.setup(self, launcher, num_tasks, num_tasks_per_node=num_tasks)
+    
+        test_dir = 'swiftsim/examples/HydroTests/SodShock_3D'
+        self.cpus_per_task = 32 # added
+        print(f'Running with threads : <{num_threads}>')
+
+        self.keep_files = [test_dir]
+        self.build_system.config_opts = [
+            '--disable-ipo',
+            '--with-tbbmalloc',
+            '--with-parmetis'
+        ]
+        self.prerun_cmds = [f'pushd {test_dir}']
+        num_iterations = 10000
+        
+        # self.build_system.options = ['CC=scorep mpicc', 'CFLAGS=-fopenmp']    #Added for scalasca
+        self.build_system.make_opts = ['CC="scorep --user --thread=pthread icc"']
+
+        if launcher == 'scalasca':
+            self.executable = '../../swift'
+
+        self.executable_opts += [
+            '-P', 'Snapshots:time_first:999999999',
+            '--hydro',
+            '-v', '1',
+            f'--threads={num_threads}',
+            '-n', f'{num_iterations}',
+            '-P', 'Restarts:enable:0',
+            '-P', f'InitialConditions:file_name:{ic_dir}/sodshock/glassCube_64.hdf5',
+            'sodShock.yml'
+        ]
+