Skip to content
Snippets Groups Projects
Commit af2dc556 authored by loikki's avatar loikki
Browse files

Format + fix error in return value

parent e1822f91
No related branches found
No related tags found
1 merge request!53Cleanup index
......@@ -63,7 +63,7 @@ for f in args.files:
filename = f[:-5]
else:
raise Exception("It seems that you are not providing a logfile (.dump)")
with csds.Reader(filename, verbose=0, number_index=10,
with csds.Reader(filename, verbose=2, number_index=10,
restart_init=False, use_cache=True) as reader:
# Check the time limits
......
......@@ -65,18 +65,17 @@ IndexFile::IndexFile(const std::string filename, int verbose) {
MapFile(filename, 0, verbose);
/* Read times */
pmIndex->ReadData(csds_index_time_offset, csds_index_time_size,
&mTime);
pmIndex->ReadData(csds_index_time_offset, csds_index_time_size, &mTime);
pmIndex->ReadData(csds_index_integer_time_offset,
csds_index_integer_time_size, &mIntegerTime);
csds_index_integer_time_size, &mIntegerTime);
/* Read the number of particles. */
pmIndex->ReadData(csds_index_npart_offset, csds_index_npart_size,
mNParts.data());
mNParts.data());
/* Read whether the file is sorted. */
pmIndex->ReadData(csds_index_is_sorted_offset, csds_index_is_sorted_size,
&mIsSorted);
&mIsSorted);
/* Compute the position of the history of new particles. */
size_t count = 0;
......@@ -116,8 +115,8 @@ void IndexFile::WriteSorted() {
const char is_sorted = 1;
/* Write the value */
pmIndex->WriteData(csds_index_is_sorted_offset,
csds_index_is_sorted_size, &is_sorted);
pmIndex->WriteData(csds_index_is_sorted_offset, csds_index_is_sorted_size,
&is_sorted);
mIsSorted = true;
}
......@@ -138,8 +137,7 @@ struct index_data *IndexFile::GetData(enum part_type type) const {
}
const size_t type_offset = count * sizeof(struct index_data);
const char *ret =
pmIndex->GetMap() + csds_index_data_offset + type_offset;
const char *ret = pmIndex->GetMap() + csds_index_data_offset + type_offset;
return (struct index_data *)ret;
}
......@@ -153,7 +151,7 @@ struct index_data *IndexFile::GetData(enum part_type type) const {
struct index_data *IndexFile::GetCreatedHistory(enum part_type type) const {
/* Get the position after the previous array. */
char *ret = (char *) GetData(csds_type_count);
char *ret = (char *)GetData(csds_type_count);
/* Get the offset of particles of this type by skipping entries of lower
* types. */
......@@ -178,7 +176,7 @@ struct index_data *IndexFile::GetCreatedHistory(enum part_type type) const {
struct index_data *IndexFile::GetRemovedHistory(enum part_type type) const {
/* Get the position after the previous array. */
char *ret = (char *) GetCreatedHistory(csds_type_count);
char *ret = (char *)GetCreatedHistory(csds_type_count);
/* Get the offset of particles of this type by skipping entries of lower
* types. */
......@@ -218,8 +216,7 @@ bool IndexFile::ContainsTimeArray() {
* @param sorted Does the file needs to be sorted?
* @param verbose the verbose level
*/
void IndexFile::MapFile(const std::string filename,
int sorted, int verbose) {
void IndexFile::MapFile(const std::string filename, int sorted, int verbose) {
/* Un-map previous file */
if (pmIndex != NULL) {
csds_error("Trying to remap.");
......@@ -232,8 +229,8 @@ void IndexFile::MapFile(const std::string filename,
}
/* Map the index file */
pmIndex = new MappedFile(filename,
/* read_only */ 0,
/* track_mmap */ 0);
/* read_only */ 0,
/* track_mmap */ 0);
/* Sort the file */
for (int i = 0; i < csds_type_count; i++) {
if (mNParts[i] != 0) {
......@@ -256,7 +253,7 @@ void IndexFile::MapFile(const std::string filename,
/* Map the index file */
pmIndex = new MappedFile(filename, /* read_only */ 1,
/* track_mmap */ 0);
/* track_mmap */ 0);
}
/**
......@@ -280,8 +277,10 @@ IndexFile::~IndexFile() {
*
* @return Did we find the particle?
*/
bool IndexFile::GetOffset(int64_t id, enum part_type type, uint64_t &offset) const {
bool IndexFile::GetOffset(int64_t id, enum part_type type,
uint64_t &offset) const {
#ifdef CSDS_DEBUG_CHECKS
/* Ensure that the file is sorted according to the ids. */
if (!mIsSorted) {
csds_error("The index file should be already sorted.");
......@@ -290,19 +289,20 @@ bool IndexFile::GetOffset(int64_t id, enum part_type type, uint64_t &offset) con
if (mNParts[type] == 0) {
csds_error("Trying to find a particle in an empty index.");
}
#endif
/* Get the data structure. */
struct index_data *data = GetData(type);
/* Search for the value (binary search) */
struct index_data value = {.id = id, .offset = 0};
struct index_data *it = std::lower_bound(data, data + mNParts[type],
value, CompareIndexDataId);
struct index_data *it =
std::lower_bound(data, data + mNParts[type], value, CompareIndexDataId);
/* Check if we found the particle */
if (it->id == id) {
offset = it->offset;
return false;
return true;
}
/* Try to find it in the history */
......@@ -311,10 +311,10 @@ bool IndexFile::GetOffset(int64_t id, enum part_type type, uint64_t &offset) con
for (uint64_t i = 0; i < mNPartsCreated[type]; i++) {
if (data[i].id == id) {
offset = data[i].offset;
return false;
return true;
}
}
/* The particle was not found */
return true;
return false;
}
......@@ -41,8 +41,7 @@ class IndexFile {
~IndexFile();
bool ContainsTimeArray();
void MapFile(const std::string filename,
int sorted, int verbose);
void MapFile(const std::string filename, int sorted, int verbose);
bool GetOffset(int64_t id, enum part_type type, uint64_t &offset) const;
// TODO convert the 3 functions into particle accessor (e.g. return const&)
......@@ -50,9 +49,11 @@ class IndexFile {
struct index_data *GetCreatedHistory(enum part_type type) const;
struct index_data *GetRemovedHistory(enum part_type type) const;
double GetTime() const {return mTime;}
integertime_t GetIntegerTime() const {return mIntegerTime;}
uint64_t GetNumberParticles(enum part_type type) const {return mNParts[type];}
double GetTime() const { return mTime; }
integertime_t GetIntegerTime() const { return mIntegerTime; }
uint64_t GetNumberParticles(enum part_type type) const {
return mNParts[type];
}
uint64_t GetNumberCreatedParticles(enum part_type type) const {
return mNPartsCreated[type];
}
......@@ -70,11 +71,10 @@ class IndexFile {
return d1.id < d2.id;
}
protected:
protected:
void SortFile();
void WriteSorted();
/* Time of the index file */
double mTime;
......
......@@ -201,13 +201,13 @@ void Reader::SetTime(double time) {
/* Read the file */
this->mIndex.index_prev = new IndexFile(filename_prev, this->mVerbose);
this->mIndex.index_prev->MapFile(
filename_prev, /* sorted */ 1, this->mVerbose);
this->mIndex.index_prev->MapFile(filename_prev, /* sorted */ 1,
this->mVerbose);
if ((int)index_ind != this->mIndex.n_files - 1) {
this->mIndex.index_next = new IndexFile(filename_next, this->mVerbose);
this->mIndex.index_next->MapFile(
filename_next, /* sorted */ 1, this->mVerbose);
this->mIndex.index_next->MapFile(filename_next, /* sorted */ 1,
this->mVerbose);
}
/* Get the offset of the time chunk */
......@@ -243,7 +243,7 @@ size_t Reader::CountNumberNewParticles(enum part_type part_type) {
/* Get the history of created particles. */
struct index_data *data =
this->mIndex.index_next->GetCreatedHistory(part_type);
this->mIndex.index_next->GetCreatedHistory(part_type);
/* Do we have any new particle? */
if (this->mIndex.index_next->GetNumberCreatedParticles(part_type) == 0 ||
......@@ -253,7 +253,8 @@ size_t Reader::CountNumberNewParticles(enum part_type part_type) {
/* Get the last created particle */
// TODO move inside index file
uint64_t nparts = this->mIndex.index_next->GetNumberCreatedParticles(part_type);
uint64_t nparts =
this->mIndex.index_next->GetNumberCreatedParticles(part_type);
struct index_data value = {.id = 0, .offset = threshold};
struct index_data *it = std::lower_bound(data, data + nparts, value,
IndexFile::CompareIndexDataOffset);
......@@ -297,7 +298,7 @@ size_t Reader::CountNumberRemovedParticles(enum part_type part_type) {
/* Get the history of created particles. */
struct index_data *data =
this->mIndex.index_next->GetRemovedHistory(part_type);
this->mIndex.index_next->GetRemovedHistory(part_type);
/* Do we have any new particle? */
if (this->mIndex.index_next->GetNumberRemovedParticles(part_type) == 0 ||
......@@ -306,7 +307,8 @@ size_t Reader::CountNumberRemovedParticles(enum part_type part_type) {
}
/* Get the last created particle */
const uint64_t nparts = this->mIndex.index_next->GetNumberCreatedParticles(part_type);
const uint64_t nparts =
this->mIndex.index_next->GetNumberCreatedParticles(part_type);
struct index_data value = {.id = 0, .offset = threshold};
struct index_data *it = std::lower_bound(data, data + nparts, value,
IndexFile::CompareIndexDataOffset);
......@@ -343,7 +345,8 @@ void Reader::GetNumberParticles(
std::array<uint64_t, csds_type_count> &n_parts,
const std::array<bool, csds_type_count> &read_types) {
const bool final_index = this->mIndex.index_prev->GetTime() == this->GetTimeEnd();
const bool final_index =
this->mIndex.index_prev->GetTime() == this->GetTimeEnd();
for (size_t i = 0; i < n_parts.size(); i++) {
/* Should we skip this type of particles? */
......@@ -360,19 +363,22 @@ void Reader::GetNumberParticles(
/* Here we simply need to count the number of particles removed between
the two last time records */
struct index_data *removed =
this->mIndex.index_prev->GetRemovedHistory((part_type)i);
size_t size_max = this->mIndex.index_prev->GetNumberRemovedParticles((part_type)i);
this->mIndex.index_prev->GetRemovedHistory((part_type)i);
size_t size_max =
this->mIndex.index_prev->GetNumberRemovedParticles((part_type)i);
struct index_data value = {.id = 0, .offset = this->mTime.time_offset};
struct index_data *it = std::upper_bound(
removed, removed + size_max, value, IndexFile::CompareIndexDataOffset);
struct index_data *it =
std::upper_bound(removed, removed + size_max, value,
IndexFile::CompareIndexDataOffset);
n_parts[i] = size_max - (it - removed);
/* In between the last two records, we also store the particles active
in the last step, therefore we need to count the particles created */
struct index_data *created =
this->mIndex.index_prev->GetCreatedHistory((part_type)i);
size_max = this->mIndex.index_prev->GetNumberCreatedParticles((part_type)i);
this->mIndex.index_prev->GetCreatedHistory((part_type)i);
size_max =
this->mIndex.index_prev->GetNumberCreatedParticles((part_type)i);
it = std::upper_bound(created, created + size_max, value,
IndexFile::CompareIndexDataOffset);
......@@ -382,8 +388,9 @@ void Reader::GetNumberParticles(
else {
/* Count the number of particles present in the last index file. */
const uint64_t count_prev =
final_index ? this->mIndex.index_prev->GetNumberRemovedParticles((part_type)i)
: this->mIndex.index_prev->GetNumberParticles((part_type)i);
final_index
? this->mIndex.index_prev->GetNumberRemovedParticles((part_type)i)
: this->mIndex.index_prev->GetNumberParticles((part_type)i);
/* Count the number of particles that have been created since last index.
*/
const uint64_t count_new =
......@@ -609,9 +616,7 @@ void Reader::ReadAllParticlesSingleType(
struct index_data *data = this->mIndex.index_prev->GetData(type);
struct index_data *data_created =
have_next_index
? this->mIndex.index_next->GetCreatedHistory(type)
: NULL;
have_next_index ? this->mIndex.index_next->GetCreatedHistory(type) : NULL;
/* Current index to read */
size_t current_index = 0;
......@@ -758,8 +763,8 @@ void Reader::ReadParticlesFromIdsSingleType(
/* Get the offset */
size_t offset = 0;
bool found = this->mIndex.index_prev->GetOffset(
(int64_t)ids[i], type, offset);
bool found =
this->mIndex.index_prev->GetOffset((int64_t)ids[i], type, offset);
/* Deal with the particles not found */
if (!found) {
......@@ -912,7 +917,6 @@ int Reader::UpdateSingleParticle(size_t index, enum part_type type, double time,
csds_error("Found particles that do not correspond.");
}
#endif
if (flag == csds_flag_change_type || flag == csds_flag_mpi_exit ||
flag == csds_flag_delete) {
return 1;
......@@ -1019,6 +1023,7 @@ void Reader::UpdateParticlesSingleType(
/* Update the particles */
const int64_t initial_size_cache = this->mCache[type].Size();
for (int64_t part = 0; part < initial_size_cache; part++) {
/* Get the current particle index */
size_t ind = this->mCache[type].GetCurrentIndexAndIncrement();
......@@ -1046,8 +1051,9 @@ void Reader::UpdateParticlesSingleType(
if (have_next_index &&
current_field.GetField() == field_enum_particles_ids) {
struct index_data *next_removed =
this->mIndex.index_next->GetRemovedHistory(type);
for (size_t i = 0; i < this->mIndex.index_next->GetNumberRemovedParticles(type);
this->mIndex.index_next->GetRemovedHistory(type);
for (size_t i = 0;
i < this->mIndex.index_next->GetNumberRemovedParticles(type);
i++) {
/* Stop as soon as we reach the correct time */
if (next_removed[i].offset > offset_time) break;
......@@ -1058,10 +1064,11 @@ void Reader::UpdateParticlesSingleType(
/* Ensures that it is not simply a transitioning particle */
struct index_data *next_created =
this->mIndex.index_next->GetCreatedHistory(type);
this->mIndex.index_next->GetCreatedHistory(type);
bool found = false;
for (size_t j = 0;
j < this->mIndex.index_next->GetNumberCreatedParticles(type); j++) {
j < this->mIndex.index_next->GetNumberCreatedParticles(type);
j++) {
/* Stop as soon as we reach the correct time */
if (next_created[j].offset > offset_time) break;
......@@ -1088,7 +1095,7 @@ void Reader::UpdateParticlesSingleType(
/* Check if we need to update the cache with the previous index file */
size_t offset_index =
this->mLog->times.GetIndexFromTime(this->mIndex.index_prev->GetTime());
this->mLog->times.GetIndexFromTime(this->mIndex.index_prev->GetTime());
offset_index = this->mLog->times[offset_index].offset;
const bool do_previous =
this->mCache[type].GetLastTimeOffset() < offset_index;
......@@ -1097,22 +1104,23 @@ void Reader::UpdateParticlesSingleType(
struct index_data *prev_created =
this->mIndex.index_prev->GetCreatedHistory(type);
struct index_data *next_created =
have_next_index
? this->mIndex.index_next->GetCreatedHistory(type)
: NULL;
have_next_index ? this->mIndex.index_next->GetCreatedHistory(type) : NULL;
/* Go to the last particle created */
struct index_data *index_data = do_previous ? prev_created : next_created;
const size_t size_next =
have_next_index ? this->mIndex.index_next->GetNumberCreatedParticles(type) : 0;
have_next_index ? this->mIndex.index_next->GetNumberCreatedParticles(type)
: 0;
size_t size_max =
do_previous ? this->mIndex.index_prev->GetNumberCreatedParticles(type) : size_next;
do_previous ? this->mIndex.index_prev->GetNumberCreatedParticles(type)
: size_next;
/* Get the last created particle */
struct index_data value = {.id = 0,
.offset = this->mCache[type].GetLastTimeOffset()};
struct index_data *it = std::upper_bound(
index_data, index_data + size_max, value, IndexFile::CompareIndexDataOffset);
struct index_data *it =
std::upper_bound(index_data, index_data + size_max, value,
IndexFile::CompareIndexDataOffset);
size_t center = it - index_data;
/* Create a temporary buffer */
......
......@@ -698,7 +698,8 @@ void Reader::GenerateIndexFiles(int number_index, int current_index) {
/* Loop over all the particle types */
for (int i = 0; i < csds_type_count; i++) {
/* Allocate the array for the current state */
current_state[i].reserve(hashmap_overallocation * index.GetNumberParticles((part_type)i));
current_state[i].reserve(hashmap_overallocation *
index.GetNumberParticles((part_type)i));
/* Copy the index file into the arrays. */
for (size_t p = 0; p < index.GetNumberParticles((part_type)i); p++) {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment