Phoenix Runner

.github/workflows/ice.yml

@@ -1,7 +1,6 @@
 name: 'Test on ICE'
 
 on:
- push:
  pull_request:
  workflow_dispatch:
 
@@ -17,32 +16,21 @@ jobs:
  uses: actions/checkout@v4
 
  # use mkl option
- - name: Build
+ - name: Build Packages
  run: |
- ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-c/4.9.2-mva2-hdf5-1.14 netcdf-cxx/4.2-mva2-hdf5-1.14 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva
- bash rbc.sh install-with-mkl
+ ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva cmake
+ ./rbc.sh install
 
  # will stop on any errors
- - name: Compile Common
+ - name: Compile Cases
  run: |
- ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-c/4.9.2-mva2-hdf5-1.14 netcdf-cxx/4.2-mva2-hdf5-1.14 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva
+ ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva cmake
  set -e -x
- cd common
- make .depend
+ export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+ export PETSC_ARCH=arch-linux-c-opt
+ mkdir build
+ cd build
+ cmake ..
  make
- echo "/common compiled successfully!"
+ echo "/common and all cases in /examples compiled successfully!"
 
- - name: Compile Example Cases
- run: |
- ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-c/4.9.2-mva2-hdf5-1.14 netcdf-cxx/4.2-mva2-hdf5-1.14 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva
- set -e -x
- base_dir="examples"
-
- cases=$(find "$base_dir" -maxdepth 1 -type d ! -path "$base_dir")
-
- for case in $cases; do
- make -C "$case" .depend
- make -C "$case"
- done
-
- echo "All cases in /examples compiled successfully!"

.github/workflows/phoenix.yml

@@ -0,0 +1,49 @@
+name: 'Test on Phoenix'
+
+on:
+ pull_request:
+ workflow_dispatch:
+
+jobs:
+ self:
+ name: Phoenix Runner
+ runs-on:
+ labels: gt
+ if: github.repository == 'Comp-Physics/RBC3D'
+ continue-on-error: true
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Build Packages
+ run: |
+ ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv
+ ./rbc.sh install-phoenix
+
+ - name: Make Case with Makefiles
+ run: |
+ ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
+ set -e -x
+ export FFTWROOT=`pwd`/packages/fftw-3.3.10/build
+ export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+ export PETSC_ARCH=arch-linux-c-opt
+ cd common
+ make .depend
+ make
+ cd ../examples/case
+ make .depend
+ make
+ echo "/common and case compiled successfully!"
+
+ - name: Compile Cases with CMake
+ run: |
+ ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
+ set -e -x
+ export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+ export PETSC_ARCH=arch-linux-c-opt
+ mkdir build
+ cd build
+ cmake ..
+ make
+ echo "/common and all cases in /examples compiled successfully!"
+

.gitignore

@@ -12,5 +12,6 @@ tube
 libcommon.a
 .vscode
 field
-packages
+packages*
 traction
+build

CMakeLists.txt

@@ -0,0 +1,167 @@
+# To generate build files for common and all example cases:
+# mkdir build
+# cd build
+# cmake ..
+# make
+# Regenerate case files after changes:
+# cmake .
+# make case
+# Show each build command line as it is launched
+# cmake -DCMAKE_VERBOSE_MAKEFILE=ON ..
+# See compile commands in build directory
+# cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
+
+include(CMakePrintHelpers)
+cmake_minimum_required(VERSION 3.20)
+
+project(RBC3D LANGUAGES Fortran C CXX)
+
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+set(USE_MKL OFF)
+
+if (DEFINED ENV{MKLROOT})
+ set(USE_MKL ON)
+ message("Using MKL lapack because MKLROOT=$ENV{MKLROOT}")
+ # use mkl implementation of lapack
+ set(BLA_VENDOR Intel10_64lp)
+endif()
+
+if (NOT DEFINED ENV{PETSC_DIR})
+ message(FATAL_ERROR "PETSc environment variables are not set. Please export PETSC_DIR.")
+endif()
+
+if (NOT DEFINED ENV{PETSC_ARCH})
+ # PETSC_ARCH is not part of PETSc module
+ message(WARNING "PETSC_ARCH is not set. Please export PETSC_ARCH if using manually installed PETSc package.")
+ set(PETSC $ENV{PETSC_DIR})
+else ()
+ # full path if PETSC_DIR and PETSC_ARCH env variables are set
+ set(PETSC $ENV{PETSC_DIR}/$ENV{PETSC_ARCH})
+endif()
+
+set(ENV{PKG_CONFIG_PATH} ${PETSC}/lib/pkgconfig)
+
+cmake_print_variables(PETSC)
+
+# use the same fortran compiler petsc was configured with
+execute_process (COMMAND pkg-config PETSc --variable=fcompiler COMMAND tr -d '\n' OUTPUT_VARIABLE FORTRAN_COMPILER)
+cmake_print_variables(FORTRAN_COMPILER)
+
+if (FORTRAN_COMPILER)
+ set(CMAKE_Fortran_COMPILER ${FORTRAN_COMPILER})
+ # same compile options as Makefile.in
+ add_compile_options(-fallow-argument-mismatch -freal-4-real-8 -O3)
+ add_link_options(-ldl -lstdc++ -Wl,--copy-dt-needed-entries)
+else ()
+ message(FATAL_ERROR "PETSc Fortran compiler not found. Please ensure PETSc was configured with a Fortran compiler.")
+endif (FORTRAN_COMPILER)
+
+find_package(PkgConfig REQUIRED)
+
+if (PKG_CONFIG_FOUND)
+ pkg_check_modules(PETSC PETSc)
+else()
+ message(FATAL_ERROR "PkgConfig was not found.")
+endif()
+
+if (NOT PETSC_FOUND)
+ message(FATAL_ERROR "PETSc was not found through PkgConfig.")
+else()
+ if (NOT DEFINED ENV{PETSC_ARCH})
+ # not having PETSC_ARCH set makes include dirs null for some reason
+ set(PETSC_INCLUDE_DIRS "$ENV{PETSC_DIR}/include")
+ endif()
+endif()
+
+if (LAPACK_FOUND)
+ message(STATUS "LAPACK libraries: ${LAPACK_LIBRARIES}")
+elseif(NOT USE_MKL)
+ # LAPACK_LIBRARIES doesn't need to be set if MKL is module loaded
+ set(LAPACK_LIBRARIES "-L${CMAKE_CURRENT_SOURCE_DIR}/packages/lapack-3.11 -llapack -lrefblas")
+ cmake_print_variables(LAPACK_LIBRARIES)
+endif(LAPACK_FOUND)
+
+# look for FFTWROOT or FFTW_ROOT
+if (DEFINED ENV{FFTWROOT})
+ set(FFTW_INCLUDE "$ENV{FFTWROOT}/include")
+ set(FFTW_LIB "-L$ENV{FFTWROOT}/lib -lfftw3")
+elseif(DEFINED ENV{FFTWROOT})
+ set(FFTW_INCLUDE "$ENV{FFTW_ROOT}/include")
+ set(FFTW_LIB "-L$ENV{FFTW_ROOT}/lib -lfftw3")
+else ()
+ message(WARNING "FFTWROOT and FFTW_ROOT environment variables from module load fftw not found. Assuming you installed FFTW in packages.")
+ set(FFTW_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/fftw-3.3.10/build/include")
+ set(FFTW_LIB "-L${CMAKE_CURRENT_SOURCE_DIR}/packages/fftw-3.3.10/build/lib -lfftw3")
+endif()
+
+# look for NETCDF_FORTRANROOT or NETCDF_FORTRAN_ROOT 
+if (DEFINED ENV{NETCDF_FORTRANROOT})
+ set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRANROOT}/include")
+ set(NETCDF_LIB "-L$ENV{NETCDF_FORTRANROOT}/lib -lnetcdff")
+elseif (DEFINED ENV{NETCDF_FORTRAN_ROOT})
+ set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRAN_ROOT}/include")
+ set(NETCDF_LIB "-L$ENV{NETCDF_FORTRAN_ROOT}/lib -lnetcdff")
+else ()
+ message(FATAL_ERROR "NETCDF_FORTRANROOT or NETCDF_FORTRAN_ROOT environment variables not found. Please module load netcdf-fortran.")
+endif()
+
+# spherepack library
+if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/packages/spherepack3.2/lib")
+ set(SPHPK_LIB "-L${CMAKE_CURRENT_SOURCE_DIR}/packages/spherepack3.2/lib -lspherepack")
+else()
+ message(FATAL_ERROR "Spherepack is not installed in ${CMAKE_CURRENT_SOURCE_DIR}/packages/spherepack3.2")
+endif()
+
+# build common
+add_subdirectory(common)
+install(TARGETS common LIBRARY DESTINATION "bin")
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}/common")
+
+set(examples_dir "examples")
+
+# glob for the directories in /examples
+file(GLOB examples_list RELATIVE ${CMAKE_SOURCE_DIR}/${examples_dir} ${CMAKE_SOURCE_DIR}/${examples_dir}/*)
+set(dirlist "")
+foreach (example ${examples_list})
+ if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/${examples_dir}/${example})
+ list(APPEND dirlist ${example})
+ endif()
+endforeach()
+
+cmake_print_variables(LAPACK_LIBRARIES)
+
+foreach (casedir ${dirlist})
+
+ # make binary dir inside /build for case executables
+ set(case_binary_dir "${CMAKE_CURRENT_BINARY_DIR}/${casedir}")
+ file(MAKE_DIRECTORY ${case_binary_dir})
+ # copy input directory to case binary dir and create D directory
+ make_directory("${case_binary_dir}/D")
+ file(GLOB inputs "${CMAKE_CURRENT_SOURCE_DIR}/${examples_dir}/${casedir}/Input/*")
+ file(COPY ${inputs} DESTINATION "${case_binary_dir}/Input")
+
+ # find all F90 files in the examples/case directory
+ file(GLOB case_F90s "${CMAKE_CURRENT_SOURCE_DIR}/${examples_dir}/${casedir}/*.F90")
+
+ # create a custom target with the name of the current case directory
+ add_custom_target(${casedir})
+
+ # loop through each F90 file
+ foreach(F90_filepath ${case_F90s})
+ # get the file name without extension
+ get_filename_component(file_target ${F90_filepath} NAME_WE)
+ add_executable(${file_target} ${F90_filepath})
+
+ # add each target to its own case build directory
+ set_target_properties(${file_target} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${case_binary_dir})
+
+ # set include directories and link libraries
+ # petsc might not be necessary for all files (i.e. initcond) but it doesn't hurt to add it
+ target_include_directories(${file_target} PRIVATE ${PETSC_INCLUDE_DIRS})
+ target_link_libraries(${file_target} PRIVATE ${LAPACK_LIBRARIES} common ${PETSC_LINK_LIBRARIES})
+
+ # add dependency to the custom target
+ add_dependencies(${casedir} ${file_target})
+ endforeach()
+endforeach()

Makefile.in

@@ -18,9 +18,14 @@ vpath $(WORK_DIR)/common
 PETSC_INCLUDE = $(PETSC_DIR)/include
 PETSC_ARCH_INCLUDE = $(PETSC_ARCH)/include
 NETCDF_INCLUDE = $(NETCDF_FORTRANROOT)/include
+# If you installed FFTW in packages through install/install-phoenix.sh
+# run: export FFTWROOT=`pwd`/packages/fftw-3.3.10/build from RBC3D root directory in terminal
+# or make a variable in this file
+FFTW_INCLUDE = $(FFTWROOT)/include
 
 COMMON_INCLUDE = -I$(WORK_DIR)/common
-INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(PETSC_ARCH_INCLUDE) -I$(NETCDF_INCLUDE)
+# you can remove -I$(FFTW_INCLUDE) if you module loaded fftw
+INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(PETSC_ARCH_INCLUDE) -I$(NETCDF_INCLUDE) -I$(FFTW_INCLUDE)
 
 # Libraries
 COMMON_LIB = $(WORK_DIR)/common/libcommon.a

README.md

@@ -28,30 +28,43 @@ This codebase solves the boundary integral form of the Stokes equations via an a
 To install on PACE Phoenix, you need to salloc a node to make sure srun is available and then run this in the RBC3D root directory: 
 
 ```shell
-ml gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-c netcdf-cxx netcdf-fortran fftw
-bash rbc.sh install-with-mkl
+ml gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake
+./rbc.sh install-phoenix
+```
+
+Note that if the `gcc`, `mvapich2`, `mkl`, and `fftw` modules work on your Phoenix account, you should use this installer script for a faster build.
+```shell
+ml gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake
+./rbc.sh install
 ```
 
 Or if you're on COC-ICE, you just need to load different modules to run the installer script.
 
 ```shell
-ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-c/4.9.2-mva2-hdf5-1.14 netcdf-cxx/4.2-mva2-hdf5-1.14 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva2
-bash rbc.sh install-with-mkl
+ml gcc/12.3.0 mvapich2/2.3.7-1 intel-oneapi-mkl/2023.1.0 python/3.10.10 netcdf-fortran/4.6.0-mva2-hdf5-1.14 fftw/3.3.10-mva2 cmake
+./rbc.sh install
+```
+
+Before you can run cmake, you must set these environment variables. You can place them in your `~/.bashrc`. If you didn't place `RBC3D` in your `$HOME` directory, then replace it with where you placed `RBC3D`.
+```shell
+export PETSC_DIR=$HOME/RBC3D/packages/petsc-3.19.6
+export PETSC_ARCH=arch-linux-c-opt
 ```
 
 Then to execute and run a case, you can:
 ```shell
-cd common
-make .depend
-make
-cd ..examples/case
-make .depend
-make
+mkdir build
+cd build
+cmake ..
+make case # or just `make` to make common and all the cases
+cd case
 srun -n 1 ./initcond
 srun ./tube
 ```
 
-On other supercomputing clusters, it should be easy to replace the module loads with the modules available on your system and change the directories in `Makefile.in` to point to those modules. If one of these isn't available, you can follow the manual build instructions [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
+This will generate output files in `build/case/D`. To keep output files in `examples/case/D`, you can `cd examples/case` and `srun ../../build/case/initcond` and same for tube.
+
+On other supercomputing clusters, it should be easy to replace the module loads with the modules available on your system. If one of these isn't available, you can follow the manual build instructions [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
 
 ### Papers that use RBC3D
 

common/CMakeLists.txt

@@ -0,0 +1,6 @@
+file(GLOB common_F90s "*.F90")
+add_library(common STATIC "${common_F90s}")
+
+target_include_directories(common PRIVATE ${PETSC_INCLUDE_DIRS} ${NETCDF_INCLUDE} ${FFTW_INCLUDE})
+target_link_libraries(common ${LAPACK_LIBRARIES} ${PETSC_LINK_LIBRARIES} ${SPHPK_LIB} ${FFTW_LIB} ${NETCDF_LIB})
+

common/ModIntOnWalls.F90

@@ -243,7 +243,7 @@ subroutine PrepareSingIntOnWall(wall)
  end do ! i
 
  ! nnz = nnz*2 ! a very conservative estimate
- nnz = nnz*12 ! a very conservative estimate
+ nnz = nnz*12 ! SLM edit for PETSc to not throw malloc warnings
  call MatCreateSeqAIJ(PETSC_COMM_SELF, nrow, nrow, 0, nnz, wall%lhs, ierr)
 
  ! Assemble the matrix