Skip to content

Parallel reading of serial basis files #316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dylan-copeland merged 6 commits into from
Sep 16, 2025
Merged

Parallel reading of serial basis files #316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0bc6d4c
Enabling BasisReader to load a serial basis on each rank independentl…
dylan-copeland Mar 18, 2025
d7e2a50
Merge branch 'master' of github.com:LLNL/libROM into serial-reader
dylan-copeland Aug 31, 2025
df46dd3
Adding distributed flag to database read.
dylan-copeland Sep 3, 2025
b900a1b
Adding unit test to read a small serial file in parallel.
dylan-copeland Sep 4, 2025
f162ca5
Fixed new unit test.
dylan-copeland Sep 4, 2025
47678e0
Verify that BasisReader comm is NULL or WORLD.
dylan-copeland Sep 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 23 additions & 17 deletions lib/linalg/BasisReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,22 @@ namespace CAROM {
BasisReader::BasisReader(
const std::string& base_file_name,
Database::formats db_format,
const int dim) :
const int dim,
MPI_Comm comm) :
d_dim(dim),
full_file_name(""),
base_file_name_(base_file_name),
d_format(db_format)
{
CAROM_ASSERT(!base_file_name.empty());
CAROM_VERIFY(!base_file_name.empty());
CAROM_VERIFY(comm == MPI_COMM_NULL || comm == MPI_COMM_WORLD);
d_distributed = comm != MPI_COMM_NULL;
Copy link
Collaborator

@dreamer2368 dreamer2368 Sep 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This concerns me a little bit, since libROM currently does not support custom MPI communicator other than MPI_COMM_WORLD.

Should we add a line of CAROM_VERIFY for checking if the communicator is MPI_COMM_WORLD?


int mpi_init;
MPI_Initialized(&mpi_init);
int rank;
if (mpi_init) {
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (mpi_init && d_distributed) {
MPI_Comm_rank(comm, &rank);
}
else {
rank = 0;
Expand All @@ -55,15 +58,15 @@ BasisReader::BasisReader(
We allow 0 local dimension. (global dimension still needs to be positive)
*/
std::vector<int> tmp;
d_global_dim = get_global_offsets(d_dim, tmp, MPI_COMM_WORLD);
d_global_dim = get_global_offsets(d_dim, tmp, comm);
CAROM_VERIFY(d_dim >= 0);
CAROM_VERIFY(d_global_dim > 0);
d_database = new HDFDatabaseMPIO();
}
else
CAROM_ERROR("BasisWriter only supports HDF5/HDF5_MPIO data format!\n");

d_database->open(full_file_name, "r", MPI_COMM_WORLD);
d_database->open(full_file_name, "r", comm);
}

BasisReader::~BasisReader()
Expand All @@ -78,12 +81,12 @@ BasisReader::getSpatialBasis()
int num_rows = getDim("basis");
int num_cols = getNumSamples("basis");

Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, true);
Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, d_distributed);

d_database->getDoubleArray("spatial_basis",
&spatial_basis_vectors->item(0, 0),
num_rows*num_cols,
true);
d_distributed);
return std::unique_ptr<Matrix>(spatial_basis_vectors);
}

Expand All @@ -107,15 +110,16 @@ BasisReader::getSpatialBasis(
CAROM_VERIFY(start_col <= end_col && end_col <= num_cols);
int num_cols_to_read = end_col - start_col + 1;

Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read, true);
Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read,
d_distributed);
sprintf(tmp, "spatial_basis");
d_database->getDoubleArray(tmp,
&spatial_basis_vectors->item(0, 0),
num_rows*num_cols_to_read,
start_col - 1,
num_cols_to_read,
num_cols,
true);
d_distributed);
return std::unique_ptr<Matrix>(spatial_basis_vectors);
}

Expand Down Expand Up @@ -156,7 +160,7 @@ BasisReader::getTemporalBasis()
sprintf(tmp, "temporal_basis");
d_database->getDoubleArray(tmp,
&temporal_basis_vectors->item(0, 0),
num_rows*num_cols);
num_rows*num_cols, d_distributed);
return std::unique_ptr<Matrix>(temporal_basis_vectors);
}

Expand Down Expand Up @@ -187,7 +191,8 @@ BasisReader::getTemporalBasis(
num_rows*num_cols_to_read,
start_col - 1,
num_cols_to_read,
num_cols);
num_cols,
d_distributed);
return std::unique_ptr<Matrix>(temporal_basis_vectors);
}

Expand Down Expand Up @@ -229,7 +234,8 @@ BasisReader::getSingularValues()
sprintf(tmp, "singular_value");
d_database->getDoubleArray(tmp,
&singular_values->item(0),
size);
size,
d_distributed);
return std::unique_ptr<Vector>(singular_values);
}

Expand Down Expand Up @@ -324,12 +330,12 @@ BasisReader::getSnapshotMatrix()
int num_cols = getNumSamples("snapshot");

char tmp[100];
Matrix* snapshots = new Matrix(num_rows, num_cols, true);
Matrix* snapshots = new Matrix(num_rows, num_cols, d_distributed);
sprintf(tmp, "snapshot_matrix");
d_database->getDoubleArray(tmp,
&snapshots->item(0, 0),
num_rows*num_cols,
true);
d_distributed);
return std::unique_ptr<Matrix>(snapshots);
}

Expand All @@ -353,15 +359,15 @@ BasisReader::getSnapshotMatrix(
int num_cols_to_read = end_col - start_col + 1;

char tmp[100];
Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, true);
Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, d_distributed);
sprintf(tmp, "snapshot_matrix");
d_database->getDoubleArray(tmp,
&snapshots->item(0, 0),
num_rows*num_cols_to_read,
start_col - 1,
num_cols_to_read,
num_cols,
true);
d_distributed);
return std::unique_ptr<Matrix>(snapshots);
}
}
5 changes: 4 additions & 1 deletion lib/linalg/BasisReader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ class BasisReader {
BasisReader(
const std::string& base_file_name,
Database::formats db_format = Database::formats::HDF5,
const int dim = -1);
const int dim = -1,
MPI_Comm comm = MPI_COMM_WORLD);

/**
* @brief Destructor.
Expand Down Expand Up @@ -314,6 +315,8 @@ class BasisReader {
* If negative, use the dimension from the rank-specific local file.
*/
int d_global_dim;

bool d_distributed;
};

}
Expand Down
2 changes: 2 additions & 0 deletions unit_tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ endforeach()
# Copy testing inputs and scripts to build/tests/
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/baselines/basis_conversion/
DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/baselines/basis_conversion/")
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/basis_data/
DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/basis_data/")
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/s_opt_data/
DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/s_opt_data/")

Expand Down
Binary file added unit_tests/basis_data/basis.000000
View file
Open in desktop
Binary file not shown.
29 changes: 29 additions & 0 deletions unit_tests/test_HDFDatabase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -543,6 +543,35 @@ TEST(HDF5, Test_selective_parallel_reading)

#endif

TEST(HDF5, Test_serial_file_parallel_reading)
{
// Read the matrix on all ranks.
CAROM::BasisReader reader("./basis_data/basis.000000",
CAROM::Database::formats::HDF5, -1,
MPI_COMM_NULL);
std::unique_ptr<CAROM::Matrix> B = reader.getSpatialBasis();

EXPECT_EQ(B->numRows(), 4);
EXPECT_EQ(B->numColumns(), 2);
EXPECT_FALSE(B->distributed());

double error = 0.0;
for (int i=0; i<B->numRows(); ++i)
for (int j=0; j<B->numColumns(); ++j)
{
const double B_ij = (i == 1 && j == 0) || (i == 2 && j == 1) ? -1.0 : 0.0;
error = std::max(error, std::abs(B_ij - (*B)(i, j)));
}

std::unique_ptr<CAROM::Vector> sv = reader.getSingularValues();
EXPECT_EQ(sv->dim(), 2);

for (int i=0; i<sv->dim(); ++i)
error = std::max(error, std::abs((*sv)(i) - 1.0));

EXPECT_NEAR(error, 0.0, threshold);
}

TEST(BasisGeneratorIO, HDFDatabase)
{
// Get the rank of this process, and the number of processors.
Expand Down