Commit 10e7b958 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Merge branch 'master' of gitlab.cosma.dur.ac.uk:swift/swiftsim

parents aa04e99b c85355b3
...@@ -8,7 +8,6 @@ InternalUnitSystem: ...@@ -8,7 +8,6 @@ InternalUnitSystem:
Scheduler: Scheduler:
max_top_level_cells: 8 max_top_level_cells: 8
tasks_per_cell: 50
# Parameters governing the time integration # Parameters governing the time integration
TimeIntegration: TimeIntegration:
......
...@@ -505,18 +505,20 @@ int main(int argc, char *argv[]) { ...@@ -505,18 +505,20 @@ int main(int argc, char *argv[]) {
read_ic_parallel(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, read_ic_parallel(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas,
&Ngpart, &Nspart, &periodic, &flag_entropy_ICs, with_hydro, &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
(with_external_gravity || with_self_gravity), with_stars, (with_external_gravity || with_self_gravity), with_stars,
myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run); myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, nr_threads,
dry_run);
#else #else
read_ic_serial(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart, read_ic_serial(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart,
&Nspart, &periodic, &flag_entropy_ICs, with_hydro, &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
(with_external_gravity || with_self_gravity), with_stars, (with_external_gravity || with_self_gravity), with_stars,
myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run); myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, nr_threads,
dry_run);
#endif #endif
#else #else
read_ic_single(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart, read_ic_single(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart,
&Nspart, &periodic, &flag_entropy_ICs, with_hydro, &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
(with_external_gravity || with_self_gravity), with_stars, (with_external_gravity || with_self_gravity), with_stars,
dry_run); nr_threads, dry_run);
#endif #endif
if (myrank == 0) { if (myrank == 0) {
clocks_gettime(&toc); clocks_gettime(&toc);
......
...@@ -406,6 +406,8 @@ void io_write_code_description(hid_t h_file) { ...@@ -406,6 +406,8 @@ void io_write_code_description(hid_t h_file) {
H5Gclose(h_grpcode); H5Gclose(h_grpcode);
} }
#endif /* HAVE_HDF5 */
/** /**
* @brief Mapper function to copy #part or #gpart fields into a buffer. * @brief Mapper function to copy #part or #gpart fields into a buffer.
*/ */
...@@ -608,39 +610,16 @@ void io_copy_temp_buffer(void* temp, const struct engine* e, ...@@ -608,39 +610,16 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
} }
} }
#endif /* HAVE_HDF5 */ void io_prepare_dm_gparts_mapper(void* restrict data, int Ndm, void* dummy) {
/* ------------------------------------------------------------------------------------------------
* This part writes the XMF file descriptor enabling a visualisation through
* ParaView
* ------------------------------------------------------------------------------------------------
*/
/**
* @brief Prepares the XMF file for the new entry
*
* Creates a temporary file on the disk in order to copy the right lines.
*
* @todo Use a proper XML library to avoid stupid copies.
*/
/** struct gpart* restrict gparts = (struct gpart*)data;
* @brief Prepare the DM particles (in gparts) read in for the addition of the
* other particle types
*
* This function assumes that the DM particles are all at the start of the
* gparts array
*
* @param gparts The array of #gpart freshly read in.
* @param Ndm The number of DM particles read in.
*/
void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) {
/* Let's give all these gparts a negative id */ /* Let's give all these gparts a negative id */
for (size_t i = 0; i < Ndm; ++i) { for (int i = 0; i < Ndm; ++i) {
/* 0 or negative ids are not allowed */ /* 0 or negative ids are not allowed */
if (gparts[i].id_or_neg_offset <= 0) if (gparts[i].id_or_neg_offset <= 0)
error("0 or negative ID for DM particle %zu: ID=%lld", i, error("0 or negative ID for DM particle %i: ID=%lld", i,
gparts[i].id_or_neg_offset); gparts[i].id_or_neg_offset);
/* Set gpart type */ /* Set gpart type */
...@@ -649,22 +628,43 @@ void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) { ...@@ -649,22 +628,43 @@ void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) {
} }
/** /**
* @brief Copy every #part into the corresponding #gpart and link them. * @brief Prepare the DM particles (in gparts) read in for the addition of the
* other particle types
* *
* This function assumes that the DM particles are all at the start of the * This function assumes that the DM particles are all at the start of the
* gparts array and adds the hydro particles afterwards * gparts array
* *
* @param parts The array of #part freshly read in. * @param tp The current #threadpool.
* @param gparts The array of #gpart freshly read in with all the DM particles * @param gparts The array of #gpart freshly read in.
* at the start
* @param Ngas The number of gas particles read in.
* @param Ndm The number of DM particles read in. * @param Ndm The number of DM particles read in.
*/ */
void io_duplicate_hydro_gparts(struct part* const parts, void io_prepare_dm_gparts(struct threadpool* tp, struct gpart* const gparts,
struct gpart* const gparts, size_t Ngas,
size_t Ndm) { size_t Ndm) {
for (size_t i = 0; i < Ngas; ++i) { threadpool_map(tp, io_prepare_dm_gparts_mapper, gparts, Ndm,
sizeof(struct gpart), 0, NULL);
}
struct duplication_data {
struct part* parts;
struct gpart* gparts;
struct spart* sparts;
int Ndm;
int Ngas;
int Nstars;
};
void io_duplicate_hydro_gparts_mapper(void* restrict data, int Ngas,
void* restrict extra_data) {
struct duplication_data* temp = (struct duplication_data*)extra_data;
const int Ndm = temp->Ndm;
struct part* parts = (struct part*)data;
const ptrdiff_t offset = parts - temp->parts;
struct gpart* gparts = temp->gparts + offset;
for (int i = 0; i < Ngas; ++i) {
/* Duplicate the crucial information */ /* Duplicate the crucial information */
gparts[i + Ndm].x[0] = parts[i].x[0]; gparts[i + Ndm].x[0] = parts[i].x[0];
...@@ -687,22 +687,40 @@ void io_duplicate_hydro_gparts(struct part* const parts, ...@@ -687,22 +687,40 @@ void io_duplicate_hydro_gparts(struct part* const parts,
} }
/** /**
* @brief Copy every #spart into the corresponding #gpart and link them. * @brief Copy every #part into the corresponding #gpart and link them.
* *
* This function assumes that the DM particles and gas particles are all at * This function assumes that the DM particles are all at the start of the
* the start of the gparts array and adds the star particles afterwards * gparts array and adds the hydro particles afterwards
* *
* @param sparts The array of #spart freshly read in. * @param tp The current #threadpool.
* @param gparts The array of #gpart freshly read in with all the DM and gas * @param parts The array of #part freshly read in.
* particles at the start. * @param gparts The array of #gpart freshly read in with all the DM particles
* @param Nstars The number of stars particles read in. * at the start
* @param Ndm The number of DM and gas particles read in. * @param Ngas The number of gas particles read in.
* @param Ndm The number of DM particles read in.
*/ */
void io_duplicate_star_gparts(struct spart* const sparts, void io_duplicate_hydro_gparts(struct threadpool* tp, struct part* const parts,
struct gpart* const gparts, size_t Nstars, struct gpart* const gparts, size_t Ngas,
size_t Ndm) { size_t Ndm) {
struct duplication_data data;
data.parts = parts;
data.gparts = gparts;
data.Ndm = Ndm;
threadpool_map(tp, io_duplicate_hydro_gparts_mapper, parts, Ngas,
sizeof(struct part), 0, &data);
}
void io_duplicate_hydro_sparts_mapper(void* restrict data, int Nstars,
void* restrict extra_data) {
struct duplication_data* temp = (struct duplication_data*)extra_data;
const int Ndm = temp->Ndm;
struct spart* sparts = (struct spart*)data;
const ptrdiff_t offset = sparts - temp->sparts;
struct gpart* gparts = temp->gparts + offset;
for (size_t i = 0; i < Nstars; ++i) { for (int i = 0; i < Nstars; ++i) {
/* Duplicate the crucial information */ /* Duplicate the crucial information */
gparts[i + Ndm].x[0] = sparts[i].x[0]; gparts[i + Ndm].x[0] = sparts[i].x[0];
...@@ -724,6 +742,32 @@ void io_duplicate_star_gparts(struct spart* const sparts, ...@@ -724,6 +742,32 @@ void io_duplicate_star_gparts(struct spart* const sparts,
} }
} }
/**
* @brief Copy every #spart into the corresponding #gpart and link them.
*
* This function assumes that the DM particles and gas particles are all at
* the start of the gparts array and adds the star particles afterwards
*
* @param tp The current #threadpool.
* @param sparts The array of #spart freshly read in.
* @param gparts The array of #gpart freshly read in with all the DM and gas
* particles at the start.
* @param Nstars The number of stars particles read in.
* @param Ndm The number of DM and gas particles read in.
*/
void io_duplicate_star_gparts(struct threadpool* tp, struct spart* const sparts,
struct gpart* const gparts, size_t Nstars,
size_t Ndm) {
struct duplication_data data;
data.gparts = gparts;
data.sparts = sparts;
data.Ndm = Ndm;
threadpool_map(tp, io_duplicate_hydro_sparts_mapper, sparts, Nstars,
sizeof(struct spart), 0, &data);
}
/** /**
* @brief Copy every DM #gpart into the dmparts array. * @brief Copy every DM #gpart into the dmparts array.
* *
......
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
/* Avoid cyclic inclusion problems */ /* Avoid cyclic inclusion problems */
struct io_props; struct io_props;
struct engine; struct engine;
struct threadpool;
#if defined(HAVE_HDF5) #if defined(HAVE_HDF5)
...@@ -86,11 +87,12 @@ void io_copy_temp_buffer(void* temp, const struct engine* e, ...@@ -86,11 +87,12 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
void io_collect_dm_gparts(const struct gpart* const gparts, size_t Ntot, void io_collect_dm_gparts(const struct gpart* const gparts, size_t Ntot,
struct gpart* const dmparts, size_t Ndm); struct gpart* const dmparts, size_t Ndm);
void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm); void io_prepare_dm_gparts(struct threadpool* tp, struct gpart* const gparts,
void io_duplicate_hydro_gparts(struct part* const parts, size_t Ndm);
void io_duplicate_hydro_gparts(struct threadpool* tp, struct part* const parts,
struct gpart* const gparts, size_t Ngas, struct gpart* const gparts, size_t Ngas,
size_t Ndm); size_t Ndm);
void io_duplicate_star_gparts(struct spart* const sparts, void io_duplicate_star_gparts(struct threadpool* tp, struct spart* const sparts,
struct gpart* const gparts, size_t Nstars, struct gpart* const gparts, size_t Nstars,
size_t Ndm); size_t Ndm);
......
...@@ -55,7 +55,7 @@ void darkmatter_read_particles(struct gpart* gparts, struct io_props* list, ...@@ -55,7 +55,7 @@ void darkmatter_read_particles(struct gpart* gparts, struct io_props* list,
UNIT_CONV_SPEED, gparts, v_full); UNIT_CONV_SPEED, gparts, v_full);
list[2] = io_make_input_field("Masses", FLOAT, 1, COMPULSORY, UNIT_CONV_MASS, list[2] = io_make_input_field("Masses", FLOAT, 1, COMPULSORY, UNIT_CONV_MASS,
gparts, mass); gparts, mass);
list[3] = io_make_input_field("ParticleIDs", ULONGLONG, 1, COMPULSORY, list[3] = io_make_input_field("ParticleIDs", LONGLONG, 1, COMPULSORY,
UNIT_CONV_NO_UNITS, gparts, id_or_neg_offset); UNIT_CONV_NO_UNITS, gparts, id_or_neg_offset);
} }
......
...@@ -530,6 +530,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, ...@@ -530,6 +530,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile,
* @param mpi_size The number of MPI ranks * @param mpi_size The number of MPI ranks
* @param comm The MPI communicator * @param comm The MPI communicator
* @param info The MPI information object * @param info The MPI information object
* @param n_threads The number of threads to use for local operations.
* @param dry_run If 1, don't read the particle. Only allocates the arrays. * @param dry_run If 1, don't read the particle. Only allocates the arrays.
* *
*/ */
...@@ -539,7 +540,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units, ...@@ -539,7 +540,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
int dry_run) { int n_threads, int dry_run) {
hid_t h_file = 0, h_grp = 0; hid_t h_file = 0, h_grp = 0;
/* GADGET has only cubic boxes (in cosmological mode) */ /* GADGET has only cubic boxes (in cosmological mode) */
...@@ -759,16 +760,24 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units, ...@@ -759,16 +760,24 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
H5Gclose(h_grp); H5Gclose(h_grp);
} }
if (!dry_run && with_gravity) {
/* Let's initialise a bit of thread parallelism here */
struct threadpool tp;
threadpool_init(&tp, n_threads);
/* Prepare the DM particles */ /* Prepare the DM particles */
if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm); io_prepare_dm_gparts(&tp, *gparts, Ndm);
/* Duplicate the hydro particles into gparts */ /* Duplicate the hydro particles into gparts */
if (!dry_run && with_gravity && with_hydro) if (with_hydro) io_duplicate_hydro_gparts(&tp, *parts, *gparts, *Ngas, Ndm);
io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
/* Duplicate the star particles into gparts */ /* Duplicate the star particles into gparts */
if (!dry_run && with_gravity && with_stars) if (with_stars)
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas); io_duplicate_star_gparts(&tp, *sparts, *gparts, *Nstars, Ndm + *Ngas);
threadpool_clean(&tp);
}
/* message("Done Reading particles..."); */ /* message("Done Reading particles..."); */
......
...@@ -40,7 +40,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units, ...@@ -40,7 +40,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
size_t* Nsparts, int* periodic, int* flag_entropy, size_t* Nsparts, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
int dry_run); int nr_threads, int dry_run);
void write_output_parallel(struct engine* e, const char* baseName, void write_output_parallel(struct engine* e, const char* baseName,
const struct unit_system* internal_units, const struct unit_system* internal_units,
......
...@@ -380,6 +380,7 @@ void writeArray(const struct engine* e, hid_t grp, char* fileName, ...@@ -380,6 +380,7 @@ void writeArray(const struct engine* e, hid_t grp, char* fileName,
* @param mpi_size The number of MPI ranks * @param mpi_size The number of MPI ranks
* @param comm The MPI communicator * @param comm The MPI communicator
* @param info The MPI information object * @param info The MPI information object
* @param n_threads The number of threads to use for local operations.
* @param dry_run If 1, don't read the particle. Only allocates the arrays. * @param dry_run If 1, don't read the particle. Only allocates the arrays.
* *
* Opens the HDF5 file fileName and reads the particles contained * Opens the HDF5 file fileName and reads the particles contained
...@@ -396,7 +397,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units, ...@@ -396,7 +397,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
int dry_run) { int n_threads, int dry_run) {
hid_t h_file = 0, h_grp = 0; hid_t h_file = 0, h_grp = 0;
/* GADGET has only cubic boxes (in cosmological mode) */ /* GADGET has only cubic boxes (in cosmological mode) */
...@@ -647,16 +648,25 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units, ...@@ -647,16 +648,25 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
MPI_Barrier(comm); MPI_Barrier(comm);
} }
/* Duplicate the parts for gravity */
if (!dry_run && with_gravity) {
/* Let's initialise a bit of thread parallelism here */
struct threadpool tp;
threadpool_init(&tp, n_threads);
/* Prepare the DM particles */ /* Prepare the DM particles */
if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm); io_prepare_dm_gparts(&tp, *gparts, Ndm);
/* Duplicate the hydro particles into gparts */ /* Duplicate the hydro particles into gparts */
if (!dry_run && with_gravity && with_hydro) if (with_hydro) io_duplicate_hydro_gparts(&tp, *parts, *gparts, *Ngas, Ndm);
io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
/* Duplicate the star particles into gparts */ /* Duplicate the star particles into gparts */
if (!dry_run && with_gravity && with_stars) if (with_stars)
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas); io_duplicate_star_gparts(&tp, *sparts, *gparts, *Nstars, Ndm + *Ngas);
threadpool_clean(&tp);
}
/* message("Done Reading particles..."); */ /* message("Done Reading particles..."); */
......
...@@ -40,7 +40,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units, ...@@ -40,7 +40,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
int dry_run); int nr_threads, int dry_run);
void write_output_serial(struct engine* e, const char* baseName, void write_output_serial(struct engine* e, const char* baseName,
const struct unit_system* internal_units, const struct unit_system* internal_units,
......
...@@ -310,7 +310,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units, ...@@ -310,7 +310,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
struct spart** sparts, size_t* Ngas, size_t* Ngparts, struct spart** sparts, size_t* Ngas, size_t* Ngparts,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int dry_run) { int n_threads, int dry_run) {
hid_t h_file = 0, h_grp = 0; hid_t h_file = 0, h_grp = 0;
/* GADGET has only cubic boxes (in cosmological mode) */ /* GADGET has only cubic boxes (in cosmological mode) */
...@@ -515,16 +515,25 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units, ...@@ -515,16 +515,25 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
H5Gclose(h_grp); H5Gclose(h_grp);
} }
/* Duplicate the parts for gravity */
if (!dry_run && with_gravity) {
/* Let's initialise a bit of thread parallelism here */
struct threadpool tp;
threadpool_init(&tp, n_threads);
/* Prepare the DM particles */ /* Prepare the DM particles */
if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm); io_prepare_dm_gparts(&tp, *gparts, Ndm);
/* Duplicate the hydro particles into gparts */ /* Duplicate the hydro particles into gparts */
if (!dry_run && with_gravity && with_hydro) if (with_hydro) io_duplicate_hydro_gparts(&tp, *parts, *gparts, *Ngas, Ndm);
io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
/* Duplicate the star particles into gparts */ /* Duplicate the star particles into gparts */
if (!dry_run && with_gravity && with_stars) if (with_stars)
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas); io_duplicate_star_gparts(&tp, *sparts, *gparts, *Nstars, Ndm + *Ngas);
threadpool_clean(&tp);
}
/* message("Done Reading particles..."); */ /* message("Done Reading particles..."); */
......
...@@ -34,7 +34,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units, ...@@ -34,7 +34,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
struct spart** sparts, size_t* Ngas, size_t* Ndm, struct spart** sparts, size_t* Ngas, size_t* Ndm,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int dry_run); int nr_threads, int dry_run);
void write_output_single(struct engine* e, const char* baseName, void write_output_single(struct engine* e, const char* baseName,
const struct unit_system* internal_units, const struct unit_system* internal_units,
......
...@@ -45,7 +45,7 @@ int main() { ...@@ -45,7 +45,7 @@ int main() {
/* Read data */ /* Read data */
read_ic_single("input.hdf5", &us, dim, &parts, &gparts, &sparts, &Ngas, read_ic_single("input.hdf5", &us, dim, &parts, &gparts, &sparts, &Ngas,
&Ngpart, &Nspart, &periodic, &flag_entropy_ICs, 1, 1, 0, 0); &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, 1, 1, 0, 1, 0);
/* Check global properties read are correct */ /* Check global properties read are correct */
assert(dim[0] == boxSize); assert(dim[0] == boxSize);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment