Commit 22f7b677 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

First version of parallel output using parallel HDF5. The structure of the...

First version of parallel output using parallel HDF5. The structure of the output file is unchanged.        
Each MPI task writes its particles to the same file in parallel. 
Attributes are unchanged. 
Chunking and compression to reduce file size still has to be investigated.



Former-commit-id: 9efb575b93016b2c27146e8702e7094d5a8e4d15
parent 7a885c52
......@@ -828,7 +828,11 @@ int main ( int argc , char *argv[] ) {
/* Read particles and space information from (GADGET) IC */
tic = getticks();
#ifdef WITH_MPI
read_ic_parallel( ICfileName , dim , &parts , &N , &periodic );
#else
read_ic( ICfileName , dim , &parts , &N , &periodic );
#endif
if ( myrank == 0 )
message( "reading particle properties took %.3f ms." , ((double)(getticks() - tic)) / CPU_TPS * 1000 ); fflush(stdout);
......@@ -900,11 +904,13 @@ int main ( int argc , char *argv[] ) {
#endif
/* Write the state of the system as it is before starting time integration. */
if ( myrank == 0 ) {
tic = getticks();
write_output(&e);
message( "writing particle properties took %.3f ms." , ((double)(getticks() - tic)) / CPU_TPS * 1000 ); fflush(stdout);
}
tic = getticks();
#ifdef WITH_MPI
write_output_parallel(&e, myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
#else
write_output(&e);
#endif
message( "writing particle properties took %.3f ms." , ((double)(getticks() - tic)) / CPU_TPS * 1000 ); fflush(stdout);
/* Init the runner history. */
#ifdef HIST
......@@ -942,8 +948,14 @@ int main ( int argc , char *argv[] ) {
/* Take a step. */
engine_step( &e );
if ( myrank == 0 && j % 100 == 0 )
write_output(&e);
if ( j % 100 == 0 )
{
#ifdef WITH_MPI
write_output_parallel(&e, myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
#else
write_output(&e);
#endif
}
/* Dump a line of agregate output. */
if ( myrank == 0 ) {
......@@ -983,10 +995,13 @@ int main ( int argc , char *argv[] ) {
(e.sched.tasks[k].cj==NULL)?0:e.sched.tasks[k].cj->count ); */
/* Write final output. */
if ( myrank == 0 )
write_output( &e );
#ifdef WITH_MPI
write_output_parallel( &e, myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL );
#else
write_output( &e );
#endif
#ifdef WITH_MPI
#ifdef WITH_MPI
if ( MPI_Finalize() != MPI_SUCCESS )
error( "call to MPI_Finalize failed with error %i." , res );
#endif
......
......@@ -38,11 +38,11 @@ endif
# List required headers
include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
engine.h swift.h io.h timers.h debug.h scheduler.h proxy.h
engine.h swift.h io.h timers.h debug.h scheduler.h proxy.h parallel_io.h
# Common source files
AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c \
io.c timers.c debug.c scheduler.c proxy.c
io.c timers.c debug.c scheduler.c proxy.c parallel_io.c
# Sources and flags for regular library
libswiftsim_la_SOURCES = $(AM_SOURCES)
......
......@@ -21,7 +21,7 @@
/* Config parameters. */
#include "../config.h"
#ifdef HAVE_HDF5
#if defined(HAVE_HDF5) && !defined(WITH_MPI)
/* Some standard headers. */
......@@ -32,11 +32,6 @@
#include <hdf5.h>
#include <math.h>
/* MPI headers. */
#ifdef WITH_MPI
#include <mpi.h>
#endif
#include "const.h"
#include "cycle.h"
#include "lock.h"
......@@ -543,7 +538,7 @@ void writeArrayBackEnd(hid_t grp, char* fileName, FILE* xmfFile, char* name, enu
if(temp == NULL)
error("Unable to allocate memory for temporary buffer");
/* Copy temporary buffer to particle data */
/* Copy particle data to temporary buffer */
temp_c = temp;
for(i=0; i<N; ++i)
memcpy(&temp_c[i*copySize], part_c+i*partSize, copySize);
......@@ -626,7 +621,7 @@ void writeArrayBackEnd(hid_t grp, char* fileName, FILE* xmfFile, char* name, enu
* Calls #error() if an error occurs.
*
*/
void write_output (struct engine *e)
void write_output (struct engine *e, int mpi_rank)
{
hid_t h_file=0, h_grp=0, h_grpsph=0;
......@@ -662,7 +657,7 @@ void write_output (struct engine *e)
error( "Error while opening file '%s'." , fileName );
}
/* Open header to read simulation properties */
/* Open header to write simulation properties */
/* message("Writing runtime parameters..."); */
h_grp = H5Gcreate1(h_file, "/RuntimePars", 0);
if(h_grp < 0)
......
......@@ -18,10 +18,11 @@
******************************************************************************/
#if defined(HAVE_HDF5) && !defined(WITH_MPI)
void read_ic ( char* fileName, double dim[3], struct part **parts, int* N, int* periodic);
void write_output ( struct engine* e);
#endif
This diff is collapsed.
/*******************************************************************************
* This file is part of SWIFT.
* Coypright (c) 2012 Matthieu Schaller (matthieu.schaller@durham.ac.uk).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
******************************************************************************/
#if defined(HAVE_HDF5) && defined(WITH_MPI)
void read_ic_parallel ( char* fileName, double dim[3], struct part **parts, int* N, int* periodic);
void write_output_parallel ( struct engine* e, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info);
#endif
......@@ -37,6 +37,7 @@
#include "runner.h"
#include "engine.h"
#include "io.h"
#include "parallel_io.h"
#include "debug.h"
#ifdef LEGACY_GADGET2_SPH
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment