Build quartz with new modules

configs/make.lassen

@@ -0,0 +1,9 @@
+# # Modules loaded
+# 1) StdEnv (S)   2) cuda/11.7.0   3) fftw/3.3.9   4) xl/2022.08.19   5) spectrum-mpi/rolling-release   6) lapack/3.8.0-gcc-4.9.3
+#
+etree=no
+SW4ROOT=/usr/apps/wpp
+FC=mpifort
+CXX=mpicxx
+#EXTRA_LINK_FLAGS =  -L/usr/tce/packages/xl/xl-2019.02.07/xlf/16.1.1/lib -L/usr/tcetmp/packages/lapack/lapack-3.8.0-gcc-4.9.3/lib -llapack -lblas -lxlfmath -lxlf90 -lgfortran
+EXTRA_LINK_FLAGS =  -L/usr/tce/packages/xl/xl-2022.08.19/xlf/16.1.1/lib -L$(LAPACK_DIR)  -llapack -lblas -lxlfmath -lxlf90 -lgfortran

configs/make.quartz

@@ -1,23 +1,29 @@
+# module load intel-classic/2021.6.0  mvapich2/2.3.6 fftw/3.3.10 mkl/2022.1.0 hdf5-parallel/1.14.0
   etree = no
-  proj = yes
+  proj = no
   fftw = yes
   hdf5 = yes
-  SW4ROOT = /usr/apps/wpp
-# assumes module load intel
+# Requires module load intel (e.g., intel-classic/2021.6.0)
 # FC = mpiifort
   FC = mpif90
   CXX = mpicxx
-  MKL_PATH = /usr/tce/packages/mkl/mkl-2019.0/lib
-  HDF5ROOT   = ${HDF5}
-  FFTWHOME = /usr/tce/packages/fftw/fftw-3.3.8-mvapich2-2.3-intel-19.0.3
+# Location of user built modules (e.g., proj4, which has not been rebuilt with the above toolchain)
+  SW4ROOT = /usr/apps/wpp
+# MKLROOT environmental variable is set upon module load mkl/2022.1.0
+#  MKLROOT = /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0
+  MKL_PATH = ${MKLROOT}/lib/intel64
+#OLD  MKL_PATH = /usr/tce/packages/mkl/mkl-2019.0/lib
+# HDF5 environmental variable is set upon module load hdf5-parallel/1.14.0
+#  HDF5 = /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0  
+  HDF5ROOT = ${HDF5}
+# FFTWHOME info is from module display fftw/3.3.10, no root environmental variable is set :(
+  FFTWHOME = /usr/tce/packages/fftw/fftw-3.3.10-mvapich2-2.3.6-intel-classic-2021.6.0
+#OLD  FFTWHOME = /usr/tce/packages/fftw/fftw-3.3.8-mvapich2-2.3-intel-19.0.3
 
-EXTRA_LINK_FLAGS = -Wl,-rpath=$(FFTWHOME)/lib -Wl,-rpath=$(SW4ROOT)/lib -Wl,-rpath=${MKL_PATH} -L${MKL_PATH} -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl -lifcore
+EXTRA_LINK_FLAGS = -Wl,-rpath=$(FFTWHOME)/lib -Wl,-rpath=$(SW4ROOT)/lib -Wl,-rpath=$(MKL_PATH) -L$(MKL_PATH) -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl -lifcore
 
 #  EXTRA_CXX_FLAGS = -O
 #  EXTRA_FORT_FLAGS = -O
-# must load appropriate modules for hdf5-parallel library support
-# ( module add hdf5-parallel )
-#
 
   debugdir := debug_quartz
   optdir := optimize_quartz

pytest-sw4mopt/test_sw4mopt.py

@@ -208,6 +208,9 @@ def guess_mpi_cmd(mpi_tasks, omp_threads, cpu_allocation, verbose):
     elif 'ray' in node_name:
         if mpi_tasks<=0: mpi_tasks = 16
         mpirun_cmd="mpirun -gpu -np " + str(mpi_tasks)+" mpibind"
+    elif 'lassen' in node_name:
+        if mpi_tasks<=0: mpi_tasks = 16
+        mpirun_cmd="lrun -T40 " 
     elif 'sierra' in node_name:
         if mpi_tasks<=0: mpi_tasks = 16
         mpirun_cmd="lrun -T16 -p" + str(mpi_tasks)

pytest/test_sw4.py

@@ -160,7 +160,7 @@ def guess_mpi_cmd(mpi_tasks, omp_threads, cpu_allocation, verbose):
     elif 'lassen' in node_name:
         os.environ["PSM2_DEVICES"] = ""
         if mpi_tasks<=0: mpi_tasks = 4
-        mpirun_cmd="lrun -T4 -M -gpu"
+        mpirun_cmd="lrun -T40 "
     # add more machine names here
     elif 'Linux' in sys_name:
         if omp_threads<=0: omp_threads=1;

src/AllDims.C

@@ -104,15 +104,15 @@ AllDims::AllDims( int nprocs, int ibg, int ieg, int jbg, int jeg,
    ptrdiff_t fftw_alloc_local = fftw_mpi_local_size_3d( nig, njg, nkg, m_communicator, &ni, &ib );
    m_fftw_alloc_local = static_cast<size_t>(fftw_alloc_local);
 #else
-   int ni, ib;
+   ptrdiff_t ni, ib;
 #endif
 
-   std::vector<int> niloc(m_nproci), ibloc(m_nproci);
+   std::vector<ptrdiff_t> niloc(m_nproci), ibloc(m_nproci);
    niloc[m_myid1d] = ni;
    ibloc[m_myid1d] = ib;
 
-   MPI_Allgather( &ni, 1, MPI_INT, &niloc[0], 1, MPI_INT, m_communicator );
-   MPI_Allgather( &ib, 1, MPI_INT, &ibloc[0], 1, MPI_INT, m_communicator );
+   MPI_Allgather( &ni, 1, MPI_AINT, &niloc[0], 1, MPI_AINT, m_communicator );
+   MPI_Allgather( &ib, 1, MPI_AINT, &ibloc[0], 1, MPI_AINT, m_communicator );
 
    m_ib.resize(m_nproci);
    m_ie.resize(m_nproci);

src/parseInputFile.C

@@ -5775,16 +5775,17 @@ void EW::processSource(char* buffer, vector<vector<Source*> > & a_GlobalUniqueSo
      {
 	tDep = iDiscrete6moments;
 	fname = basename + ".xx";
-	npar  = 6*(npts+1);
      }
      else
      {
 	tDep = iDiscrete3forces;
 	fname = basename + ".x";
-	npar  = 3*(npts+1);
      }
      bool byteswap;
      readSACheader( fname.c_str(), dt, t0, latsac, lonsac, cmpazsac, cmpincsac, utcsac, npts, byteswap );
+     npar = 3*(npts+1);
+     if( isMomentType ) npar  = 6*(npts+1);
+
      if( !useB )
 	t0 = 0;