#!/bin/bash

-
# This script is a generic build script for Trilinos using the SEMS provided modules at Sandia
# It assumes you have the SEMS modules files as well as the Kokkos subcategory enabled
# Assuming basic SEMS Module usage has been set up you can add the Kokkos modules through
# source $SEMS_MODULE_ROOT/utils/sems-modules-init.sh kokkos
# The script also assumes BLAS and LAPACK are available in the system paths.
# A 'modules' file will be created which can be used to reload the same modules as used during configuration. 

# Choose options below for configurations. 

rm -f CMakeCache.txt;
rm -rf CMakeFiles
echo ""
echo "WARNING: In order to get modules correctly loaded source this files, don't just execute it."
echo ""
EXTRA_ARGS=$@

# Trilinos Path:
TRILINOS_PATH=${HOME}/Trilinos

# Top Level Configuration Options
TESTS=ON
EXAMPLES=ON

CUDA=OFF
OPENMP=ON
PTHREAD=OFF
SERIAL=OFF

COMPLEX=ON


#----------------------------------------------------
#TPL Paths-------------------------------------------
#----------------------------------------------------

#Generic parts of the paths, change this only to switch for example 
#compiler while using same version numbers for tpl libraries
#This is meant for Sandia SEMS module system and similar setups.
TPL_PATH=/projects/install/rhel6-x86_64/sems/tpl

#Options: gcc/4.7.2 gcc/4.8.4 gcc/4.9.2 gcc/5.1.0 intel/14.0.4 intel/15.0.2 clang/3.5.2 clang/3.6.1
COMPILER_SUFFIX=gcc/4.7.2
#Options: cuda/6.5.14 cuda/7.0.28 cuda/7.5.18
CUDA_SUFFIX=cuda/7.5.18
#Options: openmpi/1.6.5 openmpi/1.8.7 openmpi/1.10.1
MPI_SUFFIX=openmpi/1.8.7


#Using Boost from sems
BOOST_SUFFIX=boost/1.58.0

#SuperLU is not correctly installed in the sems modules so use your own
SUPERLU_SUFFIX=superlu/4.3

#Using HDF5 from sems
HDF5_SUFFIX=hdf5/1.8.12

#Using NetCDF from sems
NETCDF_SUFFIX=netcdf/4.3.2

#Using Scotch from sems
SCOTCH_SUFFIX=scotch/6.0.3/nopthread_32bit_parallel

#Zlib doesn't exist in the sems modules yet
ZLIB_SUFFIX=zlib/1.2.8

#--------------------------------------------------
#--------------------------------------------------

module purge 
module load sems-env
module load kokkos-env
module load sems-cmake/3.5.2
module load sems-${COMPILER_SUFFIX}
if [ "${CUDA}" == 'ON' ]; then
  module load kokkos-${CUDA_SUFFIX}
  module load kokkos-${MPI_SUFFIX}/cuda
  export OMPI_CXX=${TRILINOS_PATH}/packages/kokkos/config/nvcc_wrapper
  export CUDA_LAUNCH_BLOCKING=1
  export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
else
  unset OMPI_CXX
  module load sems-${MPI_SUFFIX}
fi
module load sems-${BOOST_SUFFIX}/base
module load sems-${HDF5_SUFFIX}/parallel
module load sems-${NETCDF_SUFFIX}/parallel
module load sems-${ZLIB_SUFFIX}/base
module load sems-${SCOTCH_SUFFIX}

echo "module purge" > modules
echo "module load sems-env" >> modules
echo "module load kokkos-env" >> modules
echo "module load sems-cmake/3.5.2" >> modules
echo "module load sems-${COMPILER_SUFFIX}" >> modules
if [ "${CUDA}" == 'ON' ]; then
echo "  module load kokkos-${CUDA_SUFFIX}" >> modules
echo "  module load kokkos-${MPI_SUFFIX}/cuda" >> modules
echo "  export OMPI_CXX=${TRILINOS_PATH}/packages/kokkos/config/nvcc_wrapper" >> modules
echo "  export CUDA_LAUNCH_BLOCKING=1" >> modules
echo "  export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1" >> modules
else
echo "  unset OMPI_CXX" >> modules
echo "  module load sems-${MPI_SUFFIX}" >> modules
fi
echo "module load sems-${BOOST_SUFFIX}/base" >> modules
echo "module load sems-${HDF5_SUFFIX}/parallel" >> modules
echo "module load sems-${NETCDF_SUFFIX}/parallel" >> modules
echo "module load sems-${ZLIB_SUFFIX}/base" >> modules
echo "module load sems-${SCOTCH_SUFFIX}" >> modules


# CMAKE Command structured
#
# Basic Options for backends (Enable OpenMP etc.)
# \
# Build control (flags, debug, ETI)
# \
# Generic CMake options 
# \
# TPL Setup
# \
# Packages (on/off, tests, examples)

cmake \
-D Trilinos_ENABLE_OpenMP=${OPENMP} \
-D Kokkos_ENABLE_Pthread=${PTHREAD} \
-D TPL_ENABLE_CUDA=${CUDA} \
  -D Kokkos_ENABLE_Cuda_UVM:BOOL=ON \
-D Teuchos_ENABLE_COMPLEX=${COMPLEX} \
\
-D CMAKE_CXX_FLAGS="-ansi -pedantic -Wno-unknown-pragmas -Wno-narrowing -Wno-pragmas -Wno-unused-but-set-variable -Wno-delete-non-virtual-dtor -Wno-inline -Wshadow" \
-D CMAKE_C_FLAGS="-ansi -pedantic -Wno-unknown-pragmas -Wno-narrowing -Wno-pragmas -Wno-unused-but-set-variable -Wno-inline -Wshadow" \
-D Trilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON \
-D Trilinos_ENABLE_DEBUG:BOOL=OFF \
\
-D CMAKE_INSTALL_PREFIX:PATH="$PWD/../install/" \
-D Trilinos_ENABLE_INSTALL_CMAKE_CONFIG_FILES:BOOL=ON \
-D CMAKE_BUILD_TYPE:STRING=RELEASE \
-D CMAKE_VERBOSE_MAKEFILE:BOOL=OFF \
-D Trilinos_ENABLE_CHECKED_STL:BOOL=OFF \
-D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \
-D Trilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=OFF \
-D BUILD_SHARED_LIBS:BOOL=OFF \
-D DART_TESTING_TIMEOUT:STRING=600 \
-D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING="" \
-D Trilinos_ENABLE_CXX11=ON \
  -D Trilinos_CXX11_FLAGS:STRING="--std=c++11" \
\
-D TPL_ENABLE_MPI=ON \
  -D MPI_EXEC_POST_NUMPROCS_FLAGS:STRING="-bind-to;socket;-map-by;socket" \
-D TPL_ENABLE_BinUtils=OFF \
-D TPL_ENABLE_SuperLU=OFF \
  -D TPL_SuperLU_LIBRARIES:STRING="${SEMS_SUPERLU_ROOT}/lib/libsuperlu.a" \
  -D TPL_SuperLU_INCLUDE_DIRS:STRING="${SEMS_SUPERLU_ROOT}/include" \
-D TPL_ENABLE_BLAS=ON \
-D TPL_ENABLE_LAPACK=ON \
-D TPL_ENABLE_Boost=ON \
   -D Boost_INCLUDE_DIRS:PATH="${SEMS_BOOST_ROOT}/include" \
   -D Boost_LIBRARY_DIRS:PATH="${SEMS_BOOST_ROOT}/lib" \
-D TPL_ENABLE_BoostLib=ON \
   -D BoostLib_INCLUDE_DIRS:PATH="${SEMS_BOOST_ROOT}/include" \
   -D BoostLib_LIBRARY_DIRS:PATH="${SEMS_BOOST_ROOT}/lib" \
-D TPL_ENABLE_Netcdf=ON \
  -D Netcdf_INCLUDE_DIRS:PATH="${SEMS_NETCDF_ROOT}/include" \
  -D Netcdf_LIBRARY_DIRS:PATH="${SEMS_NETCDF_ROOT}/lib" \
  -D TPL_Netcdf_PARALLEL:BOOL=ON \
-D TPL_ENABLE_HDF5=ON \
  -D HDF5_INCLUDE_DIRS:PATH="${SEMS_HDF5_ROOT}/include" \
  -D TPL_HDF5_LIBRARIES:PATH="${SEMS_HDF5_ROOT}/lib/libhdf5_hl.a;${SEMS_HDF5_ROOT}/lib/libhdf5.a;${SEMS_ZLIB_ROOT}/lib/libz.a" \
-D TPL_ENABLE_Zlib=ON \
  -D Zlib_INCLUDE_DIRS:PATH="${SEMS_ZLIB_ROOT}/include" \
  -D TPL_Zlib_LIBRARIES:PATH="${SEMS_ZLIB_ROOT}/lib/libz.a" \
-D TPL_ENABLE_Scotch=ON \
  -D Scotch_INCLUDE_DIRS:PATH="${SEMS_SCOTCH_ROOT}/include" \
  -D Scotch_LIBRARY_DIRS:PATH="${SEMS_SCOTCH_ROOT}/lib" \
-D TPL_ENABLE_DLlib=ON \
\
\
-D Trilinos_ENABLE_Amesos=ON \
-D Trilinos_ENABLE_Amesos2=ON \
  -D Amesos2_ENABLE_TESTS:BOOL=${TESTS} \
  -D Amesos2_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_Anasazi=ON \
  -D Anasazi_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_AztecOO=ON \
-D Trilinos_ENABLE_Belos=ON \
  -D Belos_ENABLE_TESTS:BOOL=${TESTS} \
  -D Belos_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_Epetra=ON \
-D Trilinos_ENABLE_EpetraExt=ON \
-D Trilinos_ENABLE_Ifpack=ON \
  -D Ifpack_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_Ifpack2=ON \
  -D Ifpack2_ENABLE_TESTS:BOOL=${TESTS} \
  -D Ifpack2_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_Intrepid=ON \
  -D Intrepid_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
  -D Intrepid_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_Intrepid2=ON \
  -D Intrepid2_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
  -D Intrepid2_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_Isorropia=ON \
  -D Isorropia_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_Kokkos=ON \
  -D Kokkos_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_ML=ON \
  -D ML_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_MueLu=ON \
  -D MueLu_INST_DOUBLE_INT_LONGINT:BOOL=ON \
  -D MueLu_ENABLE_TESTS:BOOL=${TESTS} \
  -D MueLu_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_NOX=ON \
  -D NOX_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_Panzer=ON \
  -D Panzer_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_ROL=ON \
  -D ROL_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_Sacado=ON \
  -D Sacado_ENABLE_UNINIT:BOOL=ON \
  -D Sacado_ENABLE_TESTS:BOOL=${TESTS} \
  -D Sacado_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_ShyLU=ON \
  -D ShyLU_ENABLE_TESTS:BOOL=${TESTS} \
  -D ShyLU_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_ShyLUTacho=ON \
  -D ShyLUTacho_ENABLE_TESTS:BOOL=${TESTS} \
  -D ShyLUTacho_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_Stokhos=ON \
  -D Stokhos_ENABLE_TESTS:BOOL=${TESTS} \
  -D Stokhos_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_Stratimikos=ON \
  -D Stratimikos_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_Thyra=ON \
  -D Thyra_ENABLE_TESTS:BOOL=${TESTS} \
-D Trilinos_ENABLE_Tpetra=ON \
  -D Tpetra_INST_SERIAL:BOOL=${SERIAL} \
  -D Tpetra_INST_OPENMP:BOOL=${OPENMP} \
  -D Tpetra_INST_PTHREAD:BOOL=${PTHREAD} \
  -D Tpetra_INST_CUDA:BOOL=${CUDA} \
  -D Tpetra_INST_COMPLEX_DOUBLE:BOOL=${COMPLEX} \
  -D Tpetra_ENABLE_TESTS:BOOL=${TESTS} \
  -D Tpetra_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_TrilinosCouplings=ON \
  -D TrilinosCouplings_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_Triutils=ON \
-D Trilinos_ENABLE_Xpetra=ON \
  -D Xpetra_ENABLE_TESTS:BOOL=${TESTS} \
  -D Xpetra_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
-D Trilinos_ENABLE_Zoltan=ON \
-D Trilinos_ENABLE_Zoltan2=ON \
  -D Zoltan2_ENABLE_TESTS:BOOL=${TESTS} \
  -D Zoltan2_ENABLE_EXAMPLES:BOOL=${EXAMPLES} \
\
  -D Trilinos_ENABLE_STKMesh:BOOL=ON \
  -D Trilinos_ENABLE_STKIO:BOOL=ON \
  -D Trilinos_ENABLE_STKTransfer:BOOL=ON \
  -D Trilinos_ENABLE_STKSearch:BOOL=ON \
  -D Trilinos_ENABLE_STKUtil:BOOL=ON \
  -D Trilinos_ENABLE_STKTopology:BOOL=ON \
  -D Trilinos_ENABLE_STKClassic:BOOL=OFF \
  -D Trilinos_ENABLE_SEACASExodus:BOOL=ON \
  -D Trilinos_ENABLE_SEACASEpu:BOOL=ON \
  -D Trilinos_ENABLE_SEACASExodiff:BOOL=ON \
  -D Trilinos_ENABLE_SEACASNemspread:BOOL=ON \
  -D Trilinos_ENABLE_SEACASNemslice:BOOL=ON \
\
\
$EXTRA_ARGS \
${TRILINOS_PATH}

