Skip to content
Snippets Groups Projects

Cleanup index

Merged Cleanup index
cleanup_index into master
Merged Loic Hausammann requested to merge cleanup_index into master
Compare
and
9 files
+ 296
− 282
Compare changes
  • Side-by-side
  • Inline
Files
9
src/index_file.cpp
+ 43
− 199
@@ -21,203 +21,99 @@
#include "index_file.hpp"
/* Include the standard headers */
#include <algorithm>
#include <errno.h>
#include <fcntl.h>
/* Include local headers */
#include "mapped_file.hpp"
/* Define the place and size of each element in the header. */
/* The time in double precision. */
#define csds_index_time_offset 0
#define csds_index_time_size sizeof(double)
/* The time on the integer timeline. */
#define csds_index_integer_time_offset \
csds_index_time_offset + csds_index_time_size
#define csds_index_integer_time_size sizeof(integertime_t)
/* The number of particle (for each type). */
#define csds_index_npart_offset \
csds_index_integer_time_offset + csds_index_integer_time_size
#define csds_index_npart_size sizeof(uint64_t) * csds_type_count
/* The flag used for checking if the file is sorted or not. */
#define csds_index_is_sorted_offset \
csds_index_npart_offset + csds_index_npart_size
#define csds_index_is_sorted_size sizeof(char)
/* The array containing the offset and ids. Rounding to a multiplier of 8 in
* order to align the memory */
#define csds_index_data_offset \
((csds_index_is_sorted_offset + csds_index_is_sorted_size + 7) & ~7)
/**
* @brief Read the index file header.
*
* @param index The #csds_index.
* @param filename The filename of the index file.
* @param verbose The verbose level
*/
IndexFile::IndexFile(const std::string filename, int verbose) {
this->index = NULL;
pmIndex = NULL;
/* Open the file */
if (verbose > 1) {
message("Reading " << filename);
}
csds_index_map_file(this, filename, 0, verbose);
MapFile(filename, 0, verbose);
/* Read times */
this->index->ReadData(csds_index_time_offset, csds_index_time_size,
&this->time);
this->index->ReadData(csds_index_integer_time_offset,
csds_index_integer_time_size, &this->integer_time);
pmIndex->ReadData(csds_index_time_offset, csds_index_time_size, &mTime);
pmIndex->ReadData(csds_index_integer_time_offset,
csds_index_integer_time_size, &mIntegerTime);
/* Read the number of particles. */
this->index->ReadData(csds_index_npart_offset, csds_index_npart_size,
this->nparts);
pmIndex->ReadData(csds_index_npart_offset, csds_index_npart_size,
mNParts.data());
/* Read whether the file is sorted. */
this->index->ReadData(csds_index_is_sorted_offset, csds_index_is_sorted_size,
&this->is_sorted);
pmIndex->ReadData(csds_index_is_sorted_offset, csds_index_is_sorted_size,
&mIsSorted);
/* Compute the position of the history of new particles. */
size_t count = 0;
for (int i = 0; i < csds_type_count; i++) {
count += this->nparts[i];
count += mNParts[i];
}
count *= sizeof(struct index_data);
/* Read the number of new particles. */
this->index->ReadData(csds_index_data_offset + count, csds_index_npart_size,
this->nparts_created);
pmIndex->ReadData(csds_index_data_offset + count, csds_index_npart_size,
mNPartsCreated.data());
/* Compute the position of the history of particles removed. */
size_t count_new = 0;
for (int i = 0; i < csds_type_count; i++) {
count_new += this->nparts_created[i];
count_new += mNPartsCreated[i];
}
count_new *= sizeof(struct index_data);
/* Read the number of particles removed. */
this->index->ReadData(
pmIndex->ReadData(
csds_index_data_offset + count + count_new + csds_index_npart_size,
csds_index_npart_size, this->nparts_removed);
csds_index_npart_size, mNPartsRemoved.data());
/* Close the file */
delete this->index;
this->index = NULL;
delete pmIndex;
pmIndex = NULL;
}
/**
* @brief Write that the file is sorted.
*
* WARNING The file must be mapped.
*
* @param index The #csds_index.
*/
void csds_index_write_sorted(IndexFile *index) {
if (index->index == NULL) {
csds_error("File is not mapped");
}
void IndexFile::WriteSorted() {
/* Set the value */
const char is_sorted = 1;
/* Write the value */
index->index->WriteData(csds_index_is_sorted_offset,
csds_index_is_sorted_size, &is_sorted);
}
/**
* @brief Get the #index_data of a particle type.
*
* The #index_data contains the ids and offset of the particle.
* This function should be used when looking for the last known particles.
*
* @param index The #csds_index.
* @param type The particle type.
*/
struct index_data *csds_index_get_data(IndexFile *index, int type) {
/* Get the offset of particles of this type by skipping entries of lower
* types. */
size_t count = 0;
for (int i = 0; i < type; i++) {
count += index->nparts[i];
}
const size_t type_offset = count * sizeof(struct index_data);
pmIndex->WriteData(csds_index_is_sorted_offset, csds_index_is_sorted_size,
&is_sorted);
const char *ret =
index->index->GetMap() + csds_index_data_offset + type_offset;
return (struct index_data *)ret;
}
/**
* @brief Get the #index_data for the particles created.
*
* The #index_data contains the ids and offset of the particle.
* This function should be used when looking for the particles created.
*
* @param index The #csds_index.
* @param type The particle type.
*/
struct index_data *csds_index_get_created_history(IndexFile *index, int type) {
/* Get the position after the previous array. */
char *ret = (char *)csds_index_get_data(index, csds_type_count);
/* Get the offset of particles of this type by skipping entries of lower
* types. */
size_t count = 0;
for (int i = 0; i < type; i++) {
count += index->nparts_created[i];
}
const size_t type_offset = count * sizeof(struct index_data);
ret += type_offset + csds_index_npart_size;
return (struct index_data *)ret;
}
/**
* @brief Get the #index_data for the particles removed.
*
* The #index_data contains the ids and offset of the particle.
* This function should be used when looking for the particles removed.
*
* @param index The #csds_index.
* @param type The particle type.
*/
struct index_data *csds_index_get_removed_history(IndexFile *index, int type) {
/* Get the position after the previous array. */
char *ret = (char *)csds_index_get_created_history(index, csds_type_count);
/* Get the offset of particles of this type by skipping entries of lower
* types. */
size_t count = 0;
for (int i = 0; i < type; i++) {
count += index->nparts_removed[i];
}
const size_t type_offset = count * sizeof(struct index_data);
ret += type_offset + csds_index_npart_size;
return (struct index_data *)ret;
mIsSorted = true;
}
/**
* @brief Check if a time array is written at the end.
*
* @param index The #csds_index.
*/
int csds_index_contains_time_array(IndexFile *index) {
bool IndexFile::ContainsTimeArray() {
/* Only the first index file should have a time array */
if (index->integer_time != 0) {
if (mIntegerTime != 0) {
csds_error("Only the first index file can have a time array.");
}
/* Get the file size */
const size_t file_size = index->index->GetFileSize();
const size_t file_size = pmIndex->GetFileSize();
/* Get the position after the previous array. */
char *ret = (char *)csds_index_get_removed_history(index, csds_type_count);
const size_t before_time_array = ret - index->index->GetMap();
char *ret = (char *)GetRemovedHistory(csds_type_count);
const size_t before_time_array = ret - pmIndex->GetMap();
return file_size != before_time_array;
}
@@ -225,41 +121,39 @@ int csds_index_contains_time_array(IndexFile *index) {
/**
* @brief Map the file and if required sort it.
*
* @param index The #csds_index.
* @param filename The index filename.
* @param sorted Does the file needs to be sorted?
* @param verbose the verbose level
*/
void csds_index_map_file(IndexFile *index, const std::string filename,
int sorted, int verbose) {
void IndexFile::MapFile(const std::string filename, int sorted, int verbose) {
/* Un-map previous file */
if (index->index != NULL) {
if (pmIndex != NULL) {
csds_error("Trying to remap.");
}
/* Check if need to sort the file */
if (sorted && !index->is_sorted) {
if (sorted && !mIsSorted) {
if (verbose > 1) {
message("Sorting the index file.");
}
/* Map the index file */
index->index = new MappedFile(filename,
/* read_only */ 0,
/* track_mmap */ 0);
pmIndex = new MappedFile(filename,
/* read_only */ 0,
/* track_mmap */ 0);
/* Sort the file */
for (int i = 0; i < csds_type_count; i++) {
if (index->nparts[i] != 0) {
struct index_data *data = csds_index_get_data(index, i);
std::sort(data, data + index->nparts[i], csds_compare_index_data_id);
if (mNParts[i] != 0) {
struct index_data *data = GetData((part_type)i);
std::sort(data, data + mNParts[i], CompareIndexDataId);
}
}
/* Write that the file is sorted */
csds_index_write_sorted(index);
WriteSorted();
/* Free the index file before opening it again in read only */
delete index->index;
index->index = NULL;
delete pmIndex;
pmIndex = NULL;
if (verbose > 1) {
message("Sorting done.");
@@ -267,8 +161,8 @@ void csds_index_map_file(IndexFile *index, const std::string filename,
}
/* Map the index file */
index->index = new MappedFile(filename, /* read_only */ 1,
/* track_mmap */ 0);
pmIndex = new MappedFile(filename, /* read_only */ 1,
/* track_mmap */ 0);
}
/**
@@ -277,58 +171,8 @@ void csds_index_map_file(IndexFile *index, const std::string filename,
* @param index The #csds_index.
*/
IndexFile::~IndexFile() {
if (this->index != NULL) {
delete this->index;
this->index = NULL;
}
}
/**
* @brief Give the offset of a particle given its id.
*
* @param index The #csds_index.
* @param id The ID of the particle.
* @param type The type of particles.
* @param offset (output) The offset of the particle.
*
* @return Did we find the particle?
*/
int csds_index_get_offset(IndexFile *index, int64_t id, int type,
uint64_t *offset) {
/* Ensure that the file is sorted according to the ids. */
if (!index->is_sorted) {
csds_error("The index file should be already sorted.");
if (pmIndex != NULL) {
delete pmIndex;
pmIndex = NULL;
}
/* Ensure that we have some particles */
if (index->nparts[type] == 0) {
csds_error("Trying to find a particle in an empty index.");
}
/* Get the data structure. */
struct index_data *data = csds_index_get_data(index, type);
/* Search for the value (binary search) */
struct index_data value = {.id = id, .offset = 0};
struct index_data *it = std::lower_bound(data, data + index->nparts[type],
value, csds_compare_index_data_id);
/* Check if we found the particle */
if (it->id == id) {
*offset = it->offset;
return 1;
}
/* Try to find it in the history */
// TODO Improve this with a sort?
data = csds_index_get_created_history(index, type);
for (uint64_t i = 0; i < index->nparts_created[type]; i++) {
if (data[i].id == id) {
*offset = data[i].offset;
return 1;
}
}
/* The particle was not found */
return 0;
}