Skip to content

Commit

Permalink
Merge pull request #2 from hpc-io/develop
Browse files Browse the repository at this point in the history 
Update linux.yml
  • Loading branch information
@houjun
houjun committed Aug 16, 2023
2 parents 990c456 + b06d31c commit 9dcddc6
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 103 deletions.
50 changes: 20 additions & 30 deletions .github/workflows/linux.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,20 +42,21 @@ jobs:
sudo apt-get install libtool
sudo apt install cmake
# hdf5
git clone --depth 1 https://github.com/HDFGroup/hdf5.git
git clone https://github.com/HDFGroup/hdf5.git
# libfabric 1.12.1
wget https://github.com/ofiwg/libfabric/archive/refs/tags/v1.12.1.tar.gz
tar xf v1.12.1.tar.gz
# mercury
git clone https://github.com/mercury-hpc/mercury.git
git clone https://github.com/mercury-hpc/mercury.git --recursive
# mpi
sudo apt-get install libopenmpi-dev
# zlib
# sudo apt-get install zlib1g-dev
# python3
sudo apt-get install python3
# pdc
git clone --depth 1 https://github.com/hpc-io/pdc.git
git clone https://github.com/hpc-io/pdc.git
- name: Installation
run: |
Expand All @@ -66,17 +67,17 @@ jobs:
export VOL_DIR=$mydir/vol-pdc
export MERCURY_DIR=$mydir/mercury/install
export LIBFABRIC_DIR=$mydir/libfabric-1.12.1/install
export PDC_DIR=$mydir/pdc/src/install
export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
export PDC_DIR=$mydir/pdc/install
export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$PDC_DIR/lib:$VOL_DIR/install/lib:$LD_LIBRARY_PATH"
export PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH"
# Compile HDF5
mkdir $HDF5_DIR
cd hdf5
git checkout hdf5-1_14_1-2
export HDF5_LIBTOOL=/usr/bin/libtoolize
./autogen.sh
./configure --prefix=$HDF5_DIR --enable-parallel --enable-threadsafe --enable-unsupported \
--disable-tests --disable-hl --disable-fortran
./configure CC=mpicc --prefix=$HDF5_DIR --enable-parallel --disable-tests --disable-hl --disable-fortran
make -j2 && make install
# Compile libfabric
Expand All @@ -91,45 +92,34 @@ jobs:
# Compile Mercury
mkdir $MERCURY_DIR
cd $MERCURY_DIR
git submodule update --init
cmake ../ -DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF
git checkout v2.1.0
cmake ../ -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DMERCURY_USE_CHECKSUMS=OFF
make -j2 && sudo make install
ctest
# Compile PDC
mkdir $PDC_DIR
cd $PDC_DIR
cmake ../ -DBUILD_MPI_TESTING=OFF -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc
git checkout develop
cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc
make -j2 && sudo make install
# Compile VOL-PDC
cd $VOL_DIR
mkdir build
cd build
cmake ../ -DHDF5_INCLUDE_DIR=$HDF5_INCLUDE_DIR -DHDF5_LIBRARY=$HDF5_LIBRARY -DBUILD_SHARED_LIBS=ON -DHDF5_DIR=$HDF5_DIR
cmake ../ -DCMAKE_INSTALL_PREFIX=$VOL_DIR/install -DHDF5_INCLUDE_DIR=$HDF5_INCLUDE_DIR -DHDF5_LIBRARY=$HDF5_LIBRARY -DBUILD_SHARED_LIBS=ON -DHDF5_DIR=$HDF5_DIR -DPDC_DIR=$PDC_DIR/share/cmake/pdc -DBUILD_EXAMPLES=ON
make && sudo make install
- name: Compile and Test VOL-PDC
run: |
mydir="$PWD"
# Set Environmental Variables
export VOL_DIR=$mydir/vol-pdc
export H5_DIR=$mydir/hdf5
export HDF5_DIR=$mydir/hdf5/install
export MERCURY_DIR=$mydir/mercury/install
export LIBFABRIC_DIR=$mydir/libfabric-1.12.1/install
export PDC_DIR=$mydir/pdc/src/install
export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
export HDF5_PLUGIN_PATH="$VOL_DIR/build/bin"
export HDF5_PLUGIN_PATH=$VOL_DIR/install/lib
export HDF5_VOL_CONNECTOR="pdc under_vol=0;under_info={}"
# Compile example
cd $VOL_DIR/examples
cmake .
make
export LD_PRELOAD=$VOL_DIR/install/lib/libhdf5_vol_pdc.so
# Run test
mpirun -N 1 -n 1 -c 1 $PDC_DIR/bin/pdc_server.exe &
mpirun -N 1 -n 1 -c 1 ./h5pdc_vpicio test
mpirun -N 1 -n 1 -c 1 ./bin/h5pdc_vpicio test
mpirun -N 1 -n 1 -c 1 $PDC_DIR/bin/close_server
- name: Setup tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
4 changes: 2 additions & 2 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,8 +300,8 @@ install(

install(
FILES
${CMAKE_CURRENT_SOURCE_DIR}/examples/h5pdc_vpicio
${CMAKE_CURRENT_SOURCE_DIR}/examples/h5pdc_vpicio_open
${HDF5_VOL_PDC_BINARY_DIR}/bin/h5pdc_vpicio
${HDF5_VOL_PDC_BINARY_DIR}/bin/h5pdc_vpicio_open
DESTINATION
${CMAKE_INSTALL_PREFIX}/bin
)
Expand Down
132 changes: 64 additions & 68 deletions examples/h5pdc_vpicio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ uniform_random_number()
int
main(int argc, char *argv[])
{
hid_t file_id, group_id, fapl_id, lcpl, gcpl;
hid_t file_id, group_id, fapl_id, dxpl_id;
hid_t dset_id1, dset_id2, dset_id3, dset_id4, dset_id5, dset_id6, dset_id7, dset_id8;
hid_t r_dset_id1, r_dset_id2, r_dset_id3, r_dset_id4, r_dset_id5, r_dset_id6, r_dset_id7, r_dset_id8;
hid_t r_dset_id1, r_dset_id2;
hid_t filespace, memspace;
hid_t attr1;
hid_t aid1;
Expand All @@ -34,24 +34,18 @@ main(int argc, char *argv[])
my_rank = 0;
num_procs = 1;
// Variables and dimensions
// long numparticles = 8388608; // 8 meg particles per process
long numparticles = 4;
long numparticles = 8388608; // 8 meg particles per process
/* long numparticles = 4; */
long long total_particles, offset;

float *x, *y, *z;
float *xr;
float *px, *py, *pz;
int * id1, *id2;
int x_dim = 64;
int y_dim = 64;
int z_dim = 64;
int i, j;

float matrix[ADIM1][ADIM2]; /* Attribute data */
for (i = 0; i < ADIM1; i++) { /* Values of the array attribute */
for (j = 0; j < ADIM2; j++)
matrix[i][j] = -1.;
}
int i;
double stime, etime;

MPI_Init(&argc, &argv);
MPI_Comm_dup(MPI_COMM_WORLD, &comm);
Expand All @@ -66,16 +60,12 @@ main(int argc, char *argv[])
if (my_rank == 0)
printf("Number of paritcles: %ld \n", numparticles);

x = (float *)malloc(numparticles * sizeof(float));
y = (float *)malloc(numparticles * sizeof(float));
z = (float *)malloc(numparticles * sizeof(float));

xr = (float *)malloc(numparticles * sizeof(float));

px = (float *)malloc(numparticles * sizeof(float));
py = (float *)malloc(numparticles * sizeof(float));
pz = (float *)malloc(numparticles * sizeof(float));

x = (float *)malloc(numparticles * sizeof(float));
y = (float *)malloc(numparticles * sizeof(float));
z = (float *)malloc(numparticles * sizeof(float));
px = (float *)malloc(numparticles * sizeof(float));
py = (float *)malloc(numparticles * sizeof(float));
pz = (float *)malloc(numparticles * sizeof(float));
id1 = (int *)malloc(numparticles * sizeof(int));
id2 = (int *)malloc(numparticles * sizeof(int));

Expand All @@ -95,24 +85,18 @@ main(int argc, char *argv[])
printf("H5Pcreate() error\n");
H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);

/* Initialize VOL */
// pdc_vol_id = H5VLregister_connector_by_name("pdc", H5P_DEFAULT); // not used if choosing to use
// environmental variable
memspace = H5Screate_simple(1, (hsize_t *)&numparticles, NULL);
filespace = H5Screate_simple(1, (hsize_t *)&total_particles, NULL);

MPI_Barrier(MPI_COMM_WORLD);
stime = MPI_Wtime();

/* Create file */
if ((file_id = H5Fcreate(argv[1], H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0)
printf("H5Fcreate() error\n");

memspace = H5Screate_simple(1, (hsize_t *)&numparticles, NULL);
filespace = H5Screate_simple(1, (hsize_t *)&total_particles, NULL);

if ((lcpl = H5Pcreate(H5P_LINK_CREATE)) < 0)
printf("lcpl H5Pcreate() error\n");
if ((gcpl = H5Pcreate(H5P_GROUP_CREATE)) < 0)
printf("gcpl H5Pcreate() error\n");

/* Create group */
if ((group_id = H5Gcreate2(file_id, "group1", lcpl, gcpl, H5P_DEFAULT)) < 0)
if ((group_id = H5Gcreate2(file_id, "group1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
printf("H5Gcreate() error\n");

/* Create dataset */
Expand All @@ -125,64 +109,79 @@ main(int argc, char *argv[])
dset_id7 = H5Dcreate(file_id, "id1", H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
dset_id8 = H5Dcreate(file_id, "id2", H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);

fapl_id = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(fapl_id, H5FD_MPIO_COLLECTIVE);
MPI_Barrier(MPI_COMM_WORLD);
etime = MPI_Wtime();
if (my_rank == 0)
printf("File/Group/Dataset create took %.2f seconds\n", etime - stime);

dxpl_id = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, (hsize_t *)&offset, NULL, (hsize_t *)&numparticles, NULL);

MPI_Barrier(comm);
ierr = H5Dwrite(dset_id1, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, x);
stime = MPI_Wtime();

ierr = H5Dwrite(dset_id1, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, x);
if (ierr < 0)
printf("write x failed\n");
ierr = H5Dwrite(dset_id2, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, y);
ierr = H5Dwrite(dset_id2, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, y);
if (ierr < 0)
printf("write y failed\n");
ierr = H5Dwrite(dset_id3, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, z);
ierr = H5Dwrite(dset_id3, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, z);
if (ierr < 0)
printf("write z failed\n");
ierr = H5Dwrite(dset_id4, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, px);
ierr = H5Dwrite(dset_id4, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, px);
if (ierr < 0)
printf("write px failed\n");
ierr = H5Dwrite(dset_id5, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, py);
ierr = H5Dwrite(dset_id5, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, py);
if (ierr < 0)
printf("write py failed\n");
ierr = H5Dwrite(dset_id6, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, pz);
ierr = H5Dwrite(dset_id6, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, pz);
if (ierr < 0)
printf("write pz failed\n");
ierr = H5Dwrite(dset_id7, H5T_NATIVE_INT, memspace, filespace, fapl_id, id1);
ierr = H5Dwrite(dset_id7, H5T_NATIVE_INT, memspace, filespace, dxpl_id, id1);
if (ierr < 0)
printf("write id1 failed\n");
ierr = H5Dwrite(dset_id8, H5T_NATIVE_INT, memspace, filespace, fapl_id, id2);
ierr = H5Dwrite(dset_id8, H5T_NATIVE_INT, memspace, filespace, dxpl_id, id2);
if (ierr < 0)
printf("write id2 failed\n");

r_dset_id1 = H5Dopen2(file_id, "x", H5P_DEFAULT);
// r_dset_id2 = H5Dopen2(file_id, "y", H5P_DEFAULT);
// r_dset_id3 = H5Dopen2(file_id, "z", H5P_DEFAULT);
// r_dset_id4 = H5Dopen2(file_id, "px", H5P_DEFAULT);
// r_dset_id5 = H5Dopen2(file_id, "py", H5P_DEFAULT);
// r_dset_id6 = H5Dopen2(file_id, "pz", H5P_DEFAULT);
// r_dset_id7 = H5Dopen2(file_id, "id1", H5P_DEFAULT);
// r_dset_id8 = H5Dopen2(file_id, "id2", H5P_DEFAULT);
MPI_Barrier(MPI_COMM_WORLD);
etime = MPI_Wtime();
if (my_rank == 0)
printf("Dataset write took %.2f seconds\n", etime - stime);

r_dset_id1 = H5Dopen2(file_id, "id1", H5P_DEFAULT);
r_dset_id2 = H5Dopen2(file_id, "id2", H5P_DEFAULT);

ierr = H5Dread(dset_id1, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, xr);
ierr = H5Dread(r_dset_id1, H5T_NATIVE_INT, memspace, filespace, dxpl_id, id1);
if (ierr < 0)
printf("read dset1 failed\n");

fprintf(stderr, "\nxr vals:\n");
ierr = H5Dread(r_dset_id2, H5T_NATIVE_INT, memspace, filespace, dxpl_id, id2);
if (ierr < 0)
printf("read r_dset2 failed\n");

if (H5Dclose(r_dset_id1) < 0)
printf("H5Dclose dataset1 error\n");

if (H5Dclose(r_dset_id2) < 0)
printf("H5Dclose dataset2 error\n");

for (loop = 0; loop < numparticles; loop++) {
fprintf(stderr, "%f ", xr[loop]);
if (id1[loop] != loop) {
fprintf(stderr, "Error id1: %d / %d\n", id1[loop], loop);
break;
}
}
fprintf(stderr, "\nx vals:\n");

for (loop = 0; loop < numparticles; loop++) {
fprintf(stderr, "%f ", x[loop]);
if (id2[loop] != loop * 2) {
fprintf(stderr, "Error id2: %d / %d\n", id2[loop], loop * 2);
break;
}
}

ierr = H5Dread(r_dset_id1, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, xr);
if (ierr < 0)
printf("read r_dset1 failed\n");
if (H5Dclose(r_dset_id1) < 0)
printf("H5Dclose dataset8 error\n");

/* create attribute */
aid1 = H5Screate(H5S_SCALAR);
attr1 = H5Acreate2(dset_id1, "Integer attribute", H5T_NATIVE_INT, aid1, H5P_DEFAULT, H5P_DEFAULT);
Expand Down Expand Up @@ -234,9 +233,7 @@ main(int argc, char *argv[])
printf("H5Fclose error\n");
if (H5Pclose(fapl_id) < 0)
printf("H5Pclose error\n");
if (H5Pclose(lcpl) < 0)
printf("H5Pclose error\n");
if (H5Pclose(gcpl) < 0)
if (H5Pclose(dxpl_id) < 0)
printf("H5Pclose error\n");

if (my_rank == 0) {
Expand All @@ -252,7 +249,6 @@ main(int argc, char *argv[])
free(id1);
free(id2);

free(xr);
(void)MPI_Finalize();
MPI_Finalize();
return 0;
}
12 changes: 9 additions & 3 deletions src/H5VLpdc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
#define H5VL_PDC_SEQ_LIST_LEN 128
/* (Uncomment to enable) */
//#define ENABLE_LOGGING
#define USE_REGION_TRANSFER
#define USE_REGION_TRANSFER 1

/* Remove warnings when connector does not use callback arguments */
#if defined(__cplusplus)
Expand Down Expand Up @@ -1312,7 +1312,7 @@ H5VL_pdc_dataset_write(size_t count, void *_dset[], hid_t mem_type_id[], hid_t m
#endif

H5VL_pdc_obj_t *dset;
uint64_t * offset;
uint64_t * offset = NULL;
size_t type_size;
int ndim;
pdcid_t region_x, region_xx;
Expand Down Expand Up @@ -1349,15 +1349,19 @@ H5VL_pdc_dataset_write(size_t count, void *_dset[], hid_t mem_type_id[], hid_t m
}
else
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "data dimension not supported");

/* printf("Rank %d: mem offset %lu\n", dset->my_rank, offset[0]); */
/* printf("Rank %d: mem count %lu\n", dset->my_rank, dims[0]); */
region_x = PDCregion_create(ndim, offset, dims);
dset->reg_id_from = region_x;

type_size = H5Tget_size(mem_type_id[u]);
H5VL__pdc_sel_to_recx_iov(file_space_id[u], type_size, offset);

/* printf("Rank %d: file offset %lu\n", dset->my_rank, offset[0]); */
/* printf("Rank %d: file count %lu\n", dset->my_rank, dims[0]); */
region_xx = PDCregion_create(ndim, offset, dims);
dset->reg_id_to = region_xx;
free(offset);

#ifdef USE_REGION_TRANSFER
transfer_request =
Expand Down Expand Up @@ -1397,6 +1401,8 @@ H5VL_pdc_dataset_write(size_t count, void *_dset[], hid_t mem_type_id[], hid_t m
HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't unmap object");
}
#endif
if (offset)
free(offset);
}

done:
Expand Down

0 comments on commit 9dcddc6

Please sign in to comment.