diff --git a/configs/make.lassen b/configs/make.lassen new file mode 100644 index 00000000..1039820b --- /dev/null +++ b/configs/make.lassen @@ -0,0 +1,9 @@ +# # Modules loaded +# 1) StdEnv (S) 2) cuda/11.7.0 3) fftw/3.3.9 4) xl/2022.08.19 5) spectrum-mpi/rolling-release 6) lapack/3.8.0-gcc-4.9.3 +# +etree=no +SW4ROOT=/usr/apps/wpp +FC=mpifort +CXX=mpicxx +#EXTRA_LINK_FLAGS = -L/usr/tce/packages/xl/xl-2019.02.07/xlf/16.1.1/lib -L/usr/tcetmp/packages/lapack/lapack-3.8.0-gcc-4.9.3/lib -llapack -lblas -lxlfmath -lxlf90 -lgfortran +EXTRA_LINK_FLAGS = -L/usr/tce/packages/xl/xl-2022.08.19/xlf/16.1.1/lib -L$(LAPACK_DIR) -llapack -lblas -lxlfmath -lxlf90 -lgfortran diff --git a/configs/make.quartz b/configs/make.quartz index ead07fb3..c4fe6c3b 100644 --- a/configs/make.quartz +++ b/configs/make.quartz @@ -1,23 +1,29 @@ +# module load intel-classic/2021.6.0 mvapich2/2.3.6 fftw/3.3.10 mkl/2022.1.0 hdf5-parallel/1.14.0 etree = no - proj = yes + proj = no fftw = yes hdf5 = yes - SW4ROOT = /usr/apps/wpp -# assumes module load intel +# Requires module load intel (e.g., intel-classic/2021.6.0) # FC = mpiifort FC = mpif90 CXX = mpicxx - MKL_PATH = /usr/tce/packages/mkl/mkl-2019.0/lib - HDF5ROOT = ${HDF5} - FFTWHOME = /usr/tce/packages/fftw/fftw-3.3.8-mvapich2-2.3-intel-19.0.3 +# Location of user built modules (e.g., proj4, which has not been rebuilt with the above toolchain) + SW4ROOT = /usr/apps/wpp +# MKLROOT environmental variable is set upon module load mkl/2022.1.0 +# MKLROOT = /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 + MKL_PATH = ${MKLROOT}/lib/intel64 +#OLD MKL_PATH = /usr/tce/packages/mkl/mkl-2019.0/lib +# HDF5 environmental variable is set upon module load hdf5-parallel/1.14.0 +# HDF5 = /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 + HDF5ROOT = ${HDF5} +# FFTWHOME info is from module display fftw/3.3.10, no root environmental variable is set :( + FFTWHOME = /usr/tce/packages/fftw/fftw-3.3.10-mvapich2-2.3.6-intel-classic-2021.6.0 +#OLD FFTWHOME = /usr/tce/packages/fftw/fftw-3.3.8-mvapich2-2.3-intel-19.0.3 -EXTRA_LINK_FLAGS = -Wl,-rpath=$(FFTWHOME)/lib -Wl,-rpath=$(SW4ROOT)/lib -Wl,-rpath=${MKL_PATH} -L${MKL_PATH} -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl -lifcore +EXTRA_LINK_FLAGS = -Wl,-rpath=$(FFTWHOME)/lib -Wl,-rpath=$(SW4ROOT)/lib -Wl,-rpath=$(MKL_PATH) -L$(MKL_PATH) -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl -lifcore # EXTRA_CXX_FLAGS = -O # EXTRA_FORT_FLAGS = -O -# must load appropriate modules for hdf5-parallel library support -# ( module add hdf5-parallel ) -# debugdir := debug_quartz optdir := optimize_quartz diff --git a/configs/make.quartz.fftw b/configs/make.quartz.fftw deleted file mode 100644 index caa31e96..00000000 --- a/configs/make.quartz.fftw +++ /dev/null @@ -1,23 +0,0 @@ - etree = no - proj = yes - SW4ROOT = /usr/apps/wpp -# assumes module load intel -# FC = mpiifort - FC = mpif90 - CXX = mpicxx - MKL_PATH = /usr/tce/packages/mkl/mkl-2019.0/lib - HDF5ROOT = ${HDF5} - - fftw = yes - FFTWHOME = /usr/tce/packages/fftw/fftw-3.3.8-mvapich2-2.3-intel-19.0.3 - -EXTRA_LINK_FLAGS = -Wl,-rpath=$(FFTWHOME)/lib -Wl,-rpath=$(SW4ROOT)/lib -Wl,-rpath=${MKL_PATH} -L${MKL_PATH} -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl -lifcore - -# EXTRA_CXX_FLAGS = -O -# EXTRA_FORT_FLAGS = -O -# must load appropriate modules for hdf5-parallel library support -# ( module add hdf5-parallel ) -# - - debugdir := debug_quartz - optdir := optimize_quartz diff --git a/pytest-sw4mopt/test_sw4mopt.py b/pytest-sw4mopt/test_sw4mopt.py index ccd810e2..e2928226 100755 --- a/pytest-sw4mopt/test_sw4mopt.py +++ b/pytest-sw4mopt/test_sw4mopt.py @@ -208,6 +208,9 @@ def guess_mpi_cmd(mpi_tasks, omp_threads, cpu_allocation, verbose): elif 'ray' in node_name: if mpi_tasks<=0: mpi_tasks = 16 mpirun_cmd="mpirun -gpu -np " + str(mpi_tasks)+" mpibind" + elif 'lassen' in node_name: + if mpi_tasks<=0: mpi_tasks = 16 + mpirun_cmd="lrun -T40 " elif 'sierra' in node_name: if mpi_tasks<=0: mpi_tasks = 16 mpirun_cmd="lrun -T16 -p" + str(mpi_tasks) diff --git a/pytest/test_sw4.py b/pytest/test_sw4.py index 8f7eaa3a..9bdb3a2b 100755 --- a/pytest/test_sw4.py +++ b/pytest/test_sw4.py @@ -160,7 +160,7 @@ def guess_mpi_cmd(mpi_tasks, omp_threads, cpu_allocation, verbose): elif 'lassen' in node_name: os.environ["PSM2_DEVICES"] = "" if mpi_tasks<=0: mpi_tasks = 4 - mpirun_cmd="lrun -T4 -M -gpu" + mpirun_cmd="lrun -T40 " # add more machine names here elif 'Linux' in sys_name: if omp_threads<=0: omp_threads=1; diff --git a/src/AllDims.C b/src/AllDims.C index 8252943a..a937fc60 100644 --- a/src/AllDims.C +++ b/src/AllDims.C @@ -104,15 +104,15 @@ AllDims::AllDims( int nprocs, int ibg, int ieg, int jbg, int jeg, ptrdiff_t fftw_alloc_local = fftw_mpi_local_size_3d( nig, njg, nkg, m_communicator, &ni, &ib ); m_fftw_alloc_local = static_cast(fftw_alloc_local); #else - int ni, ib; + ptrdiff_t ni, ib; #endif - std::vector niloc(m_nproci), ibloc(m_nproci); + std::vector niloc(m_nproci), ibloc(m_nproci); niloc[m_myid1d] = ni; ibloc[m_myid1d] = ib; - MPI_Allgather( &ni, 1, MPI_INT, &niloc[0], 1, MPI_INT, m_communicator ); - MPI_Allgather( &ib, 1, MPI_INT, &ibloc[0], 1, MPI_INT, m_communicator ); + MPI_Allgather( &ni, 1, MPI_AINT, &niloc[0], 1, MPI_AINT, m_communicator ); + MPI_Allgather( &ib, 1, MPI_AINT, &ibloc[0], 1, MPI_AINT, m_communicator ); m_ib.resize(m_nproci); m_ie.resize(m_nproci); diff --git a/src/parseInputFile.C b/src/parseInputFile.C index 438ccccd..21d2a31b 100644 --- a/src/parseInputFile.C +++ b/src/parseInputFile.C @@ -5775,16 +5775,17 @@ void EW::processSource(char* buffer, vector > & a_GlobalUniqueSo { tDep = iDiscrete6moments; fname = basename + ".xx"; - npar = 6*(npts+1); } else { tDep = iDiscrete3forces; fname = basename + ".x"; - npar = 3*(npts+1); } bool byteswap; readSACheader( fname.c_str(), dt, t0, latsac, lonsac, cmpazsac, cmpincsac, utcsac, npts, byteswap ); + npar = 3*(npts+1); + if( isMomentType ) npar = 6*(npts+1); + if( !useB ) t0 = 0;