goolfc

#!/bin/bash
# goolfc toolchain steps:
# 0. openblas with gcc
# 1. cuda with gcc
# 2. gcccuda toolchain
# 3. openmpi with gcccuda
# 4. gompic toolchain
# 5. scalapack with gompic
# 6. fftw with gompic
# 7. goolfc toolchain
#
# Ruoshi Sun
# 2020-04-29

GCC_VER=9.2.0 #8.3.0 #6.5.0
CUDA_VER=11.0.228 #10.2.89 #10.1.168
OPENMPI_VER=3.1.6 #3.1.4
OPENBLAS_VER=0.3.7
SCALAPACK_VER=2.1.0
FFTW_VER=3.3.8

# toolchain version
GCCCUDA_VER=${GCC_VER}_${CUDA_VER}
GOMPIC_VER=${GCC_VER}_${OPENMPI_VER}
GOOLFC_VER=${GCC_VER}_${OPENMPI_VER}_${CUDA_VER}

# easyconfig filename
EB_OPENBLAS="openblas-${OPENBLAS_VER}-gcc-${GCC_VER}.eb"
EB_CUDA="cuda-${CUDA_VER}-gcc-${GCC_VER}.eb"
EB_GCCCUDA="gcccuda-${GCCCUDA_VER}.eb"
EB_OPENMPI="openmpi-${OPENMPI_VER}-gcccuda-${GCCCUDA_VER}.eb"
EB_GOMPIC="gompic-${GOMPIC_VER}.eb"
EB_SCALAPACK="scalapack-${SCALAPACK_VER}-gompic-${GOMPIC_VER}.eb"
EB_FFTW="fftw-${FFTW_VER}-gompic-${GOMPIC_VER}.eb"
EB_GOOLFC="goolfc-${GOOLFC_VER}.eb"

DATE=$(date +%Y-%m-%d)
EB_HEADER="# $DATE generated by goolfc script (Ruoshi Sun)"

# temporary file for appsync commands
APPSYNC_FILE="$(mktemp)"
APPSYNC=(
    "compiler gcc/$GCC_VER openblas/$OPENBLAS_VER"
    "compiler gcc/$GCC_VER cuda/$CUDA_VER "
    "toolchains gcccuda/$GCCCUDA_VER"
    "compiler gcc-cuda/${GCCCUDA_VER/_/-} openmpi/$OPENMPI_VER"
    "toolchains gompic/$GOMPIC_VER"
    "mpi gcc-cuda/${GCCCUDA_VER/_/-} openmpi/$OPENMPI_VER scalapack/$SCALAPACK_VER"
    "mpi gcc-cuda/${GCCCUDA_VER/_/-} openmpi/$OPENMPI_VER fftw/$FFTW_VER"
    "toolchains goolfc/$GOOLFC_VER"
)

# menu and prompt
MESSAGE=(
    "openblas/$OPENBLAS_VER with gcc/$GCC_VER"
    "cuda/$CUDA_VER with gcc/$GCC_VER"
    "gcccuda/$GCCCUDA_VER toolchain"
    "openmpi/$OPENMPI_VER with gcccuda/$GCCCUDA_VER"
    "gompic/$GOMPIC_VER toolchain"
    "scalapack/$SCALAPACK_VER with gompic/$GOMPIC_VER"
    "fftw/$FFTW_VER with gompic/$GOMPIC_VER"
    "goolfc/$GOOLFC_VER toolchain"
)

echo "   <G>cc"
echo "   <O>penmpi"
echo "   <O>penblas"
echo "sca<L>apack"
echo "   <F>ftw"
echo "   <C>uda      automatic installation (Ruoshi Sun)"
echo

for i in "${!MESSAGE[@]}"; do 
    echo "$i. ${MESSAGE[$i]}"
done
echo

read -p "Select initial step [0-7]: " step
if ! [[ "$step" =~ ^[0-7]$ ]]; then
    echo "Invalid option."
    exit 1
fi

#----------------------------------------- 
# begin installation

module purge
module load easybuild

# 0. openblas with gcc
if [ $step -eq 0 ]; then
    cat > $EB_OPENBLAS <<EOF
$EB_HEADER
easyblock = 'ConfigureMake'

name = 'openblas'
version = '$OPENBLAS_VER'

homepage = 'http://xianyi.github.com/OpenBLAS/'
description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version."

toolchain = {'name': 'gcc', 'version': '$GCC_VER'}

source_urls = [
    'http://www.netlib.org/lapack/timing/',
    'https://github.com/xianyi/OpenBLAS/archive/',
]
sources = ['v%(version)s.tar.gz']

patches = [
    ('large.tgz', '.'),
    ('timing.tgz', '.'),
]

buildopts = 'TARGET=SANDYBRIDGE DYNAMIC_ARCH=1 BINARY=64 USE_THREAD=0 CC="\$CC" FC="\$F77" NO_AFFINITY=1'
installopts = 'USE_THREAD=0 PREFIX=%(installdir)s'

skipsteps = ['configure']

sanity_check_paths = {
    'files': ['include/cblas.h', 'include/f77blas.h', 'include/lapacke_config.h', 'include/lapacke.h', 'include/lapacke_mangling.h', 'include/lapacke_utils.h', 'include/openblas_config.h', 'lib/libopenblas.a', 'lib/libopenblas.so'],
    'dirs': [],
}

moduleclass = 'numlib'
EOF

    eb -f $EB_OPENBLAS
fi

# 1. cuda with gcc
if [ $step -le 1 ]; then
    cat > $EB_CUDA <<EOF
$EB_HEADER
easyblock='EB_CUDA'

name = 'cuda'
version = '$CUDA_VER'
version_url = '11.0.3' # in URLs only

homepage = 'https://developer.nvidia.com/cuda-toolkit'
description = """CUDA (formerly Compute Unified Device Architecture) is a parallel
 computing platform and programming model created by NVIDIA and implemented by the
 graphics processing units (GPUs) that they produce. CUDA gives developers access
 to the virtual instruction set and memory of the parallel computational elements in CUDA GPUs."""

toolchain = {'name': 'gcc', 'version': '$GCC_VER'}

source_urls = ['https://developer.download.nvidia.com/compute/%%(name)s/%s/local_installers/' % version_url]
sources = ['%%(name)s_%s_450.51.06_linux.run' % version_url]

moduleclass = 'system'
EOF

    eb -f $EB_CUDA
fi

# 2. gcccuda toolchain
if [ $step -le 2 ]; then
    cat > $EB_GCCCUDA <<EOF
$EB_HEADER
easyblock = 'Toolchain'

name = 'gcccuda'
gccver = '$GCC_VER'
cudaver = '$CUDA_VER'
version = '%s_%s' % (gccver, cudaver)

homepage = '(none)'
description = "GNU Compiler Collection (GCC) based compiler toolchain, along with CUDA toolkit."

toolchain = {'name': 'dummy', 'version': ''}

# compiler toolchain dependencies
comp = ('gcc', gccver)

dependencies = [
    comp,
    ('cuda', cudaver, '', comp),
]

allow_prepend_abs_path = True
modextrapaths = {'MODULEPATH': '/apps/modulefiles/standard/compiler/gcc-cuda/%s-%s' % (gccver, cudaver)}

moduleclass = 'toolchain'
EOF

    eb -f $EB_GCCCUDA
fi

# 3. openmpi with gcccuda
if [ $step -le 3 ]; then
    cat > $EB_OPENMPI <<EOF
$EB_HEADER
easyblock = 'ConfigureMake'

name = 'openmpi'
version = '$OPENMPI_VER'

homepage = 'https://www.open-mpi.org/'
description = """The Open MPI Project is an open source MPI-3 implementation."""

toolchain = {'name': 'gcccuda', 'version': '$GCCCUDA_VER'}

source_urls = ['https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
sources = [SOURCELOWER_TAR_GZ]

# needed for --with-verbs
osdependencies = ['libibverbs-devel', 'zlib']

configopts = '--enable-shared --with-verbs --with-cuda=\${CUDA_HOME} '
configopts += '--enable-mpirun-prefix-by-default '  # suppress failure modes in relation to mpirun path

# to enable SLURM integration (site-specific)
configopts += '--with-slurm --with-pmi=/opt/slurm/current --with-pmi-libdir=/opt/slurm/current/lib'

libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
sanity_check_paths = {
    'files': ["bin/%s" % binfile for binfile in ["ompi_info", "opal_wrapper", "orterun"]] +
             ["lib/lib%s.%s" % (libfile, SHLIB_EXT) for libfile in libs] +
             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config", "mpif", "mpi", "mpi_portable_platform"]],
    'dirs': [],
}

modextravars = {
    'SLURM_MPI_TYPE': 'pmi2',
    'SLURM_CPU_BIND_TYPE': 'cores',
}

moduleclass = 'mpi'
EOF

    eb -f $EB_OPENMPI
fi

# 4. gompic toolchain
if [ $step -le 4 ]; then
    cat > $EB_GOMPIC <<EOF
$EB_HEADER
easyblock = 'Toolchain'

name = 'gompic'

gccver  = '$GCC_VER'
mpiver = '$OPENMPI_VER'
cudaver = '$CUDA_VER'

version = '%s_%s' % (gccver, mpiver)

homepage = '(none)'
description = """GNU Compiler Collection (GCC) based compiler toolchain along with CUDA toolkit,
 including OpenMPI for MPI support with CUDA features enabled."""

toolchain = {'name': 'dummy', 'version': ''}

comp = ('gcc', gccver)
gcccuda_tc = ('gcccuda', '%s_%s' % (gccver, cudaver))

dependencies = [
    comp,
    ('openmpi', mpiver, '', gcccuda_tc),
    ('cuda',    cudaver, '', comp),
]

moduleclass = 'toolchain'
EOF
    eb -f $EB_GOMPIC
fi

# 5. scalapack with gompic
if [ $step -le 5 ]; then
    cat > $EB_SCALAPACK <<EOF
$EB_HEADER
easyblock = 'EB_ScaLAPACK'

name = 'scalapack'
version = '$SCALAPACK_VER'

homepage = 'http://www.netlib.org/scalapack/'
description = """The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines
 redesigned for distributed memory MIMD parallel computers."""

gccver = '$GCC_VER'
mpiver = '$OPENMPI_VER'

toolchain = {'name': 'gompic', 'version': '%s_%s' % (gccver, mpiver)}
toolchainopts = {'pic': True, 'usempi': True}

source_urls = [homepage]
sources = [SOURCELOWER_TGZ]

blaslib = 'openblas'
blasver = '$OPENBLAS_VER'

dependencies = [
	(blaslib, blasver, '', ('gcc', gccver)),
]

# parallel build tends to fail, so disabling it
parallel = 1

moduleclass = 'numlib'
EOF

    eb -f $EB_SCALAPACK
fi

# 6. fftw with gompic
if [ $step -le 6 ]; then
    cat > $EB_FFTW <<EOF
$EB_HEADER
easyblock = 'ConfigureMake'

name = 'fftw'
version = '$FFTW_VER'

homepage = 'http://www.fftw.org'
description = """FFTW is a C subroutine library for computing the discrete Fourier transform (DFT)
 in one or more dimensions, of arbitrary input size, and of both real and complex data."""

toolchain = {'name': 'gompic', 'version': '$GOMPIC_VER'}
toolchainopts = {'optarch': True, 'usempi': True, 'pic': True}

#sources = [SOURCELOWER_TAR_GZ]
sources = ['%(name)s-%(version)s.tar.gz']
source_urls = [homepage]

common_configopts = "--enable-threads --enable-openmp --enable-shared --with-pic"

configopts = [
    common_configopts + " --enable-single --enable-avx --enable-mpi",
    common_configopts + " --enable-long-double --enable-mpi",
    common_configopts + " --enable-quad-precision",
    common_configopts + " --enable-avx --enable-mpi",  # default as last
]

moduleclass = 'numlib'
EOF

    eb -f $EB_FFTW
fi

# 7. goolfc
if [ $step -le 7 ]; then
    cat > $EB_GOOLFC <<EOF
$EB_HEADER
easyblock = 'Toolchain'

name = 'goolfc'

gccver  = '$GCC_VER'
mpiver = '$OPENMPI_VER'
cudaver = '$CUDA_VER'

version = '%s_%s_%s' % (gccver,mpiver,cudaver)

homepage = '(none)'
description = """GNU Compiler Collection (GCC) based compiler toolchain along with CUDA toolkit,
 including OpenMPI for MPI support with CUDA features enabled, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK with CUDA features enabled."""

toolchain = {'name': 'dummy', 'version': 'dummy'}

comp       = ('gcc',     gccver)
gcccuda_tc = ('gcccuda', '%s_%s' % (gccver, cudaver))
gompic_tc  = ('gompic',  '%s_%s' % (gccver, mpiver))

# compiler toolchain dependencies
dependencies = [
    comp,
    ('openmpi',   mpiver,  '', gcccuda_tc),
    ('openblas',  '$OPENBLAS_VER', '', comp),
    ('scalapack', '$SCALAPACK_VER', '', gompic_tc),
    ('fftw',      '$FFTW_VER', '', gompic_tc),
    ('cuda',      cudaver, '', comp),
]

allow_prepend_abs_path = True
modextrapaths = {'MODULEPATH': '/apps/modulefiles/standard/mpi/gcc-cuda/%s-%s/openmpi/%s' % (gccver, cudaver, mpiver)}

moduleclass = 'toolchain'
EOF

    eb -f $EB_GOOLFC
fi

#----------------------------------------- 
# write appsync commands

YYYYMM_SRC=`ls -l /apps | awk -F'/' '{print $NF}'`
YYYYMM_DEST=${YYYYMM_SRC/_build/}

cat > $APPSYNC_FILE <<EOF
for i in $YYYYMM_DEST; do
EOF

for i in "${!APPSYNC[@]}"; do 
    if [ $i -ge $step ]; then
        echo "    appsync $YYYYMM_DEST \$i ${APPSYNC[$i]}" >>$APPSYNC_FILE
    fi
done

echo "done" >>$APPSYNC_FILE
chmod +x $APPSYNC_FILE

echo
echo "appsync commands in $APPSYNC_FILE"
cat $APPSYNC_FILE