Merge pull request #460 from etmc/quda_work

work on lib_wrapper / QUDA interface and CVC integration
etmc · Mar 13, 2024 · d601a31 · d601a31
2 parents 7d4d17c + 7343ebb
commit d601a31
Show file tree

Hide file tree

Showing 427 changed files with 56,396 additions and 28,004 deletions.
diff --git a/.github/patches/DDalphaAMG.patch b/.github/patches/DDalphaAMG.patch
@@ -0,0 +1,18 @@
+diff --git a/Makefile b/Makefile
+index ae45ecc..7299fc7 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,10 +1,10 @@
+ # --- COMPILER ----------------------------------------
+-CC = mpiicc 
++CC = mpicc 
+
+ # --- CFLAGS -----------------------------------------
+-CFLAGS_gnu = -std=gnu99 -Wall -pedantic -O3 -ffast-math -msse4.2 -fopenmp 
++CFLAGS_gnu = -std=gnu99 -Wall -pedantic -O3 -ffast-math -mavx2 -mfma -mtune=haswell -march=haswell -fopenmp 
+ CFLAGS_intel = -std=gnu99 -Wall -pedantic -O3  -xHOST -qopenmp 
+-CFLAGS = $(CFLAGS_intel)
++CFLAGS = ${CFLAGS_gnu}
+
+ # --- DO NOT CHANGE -----------------------------------
+ CPP = cpp
diff --git a/.github/workflows/basic-build.yaml b/.github/workflows/basic-build.yaml
@@ -0,0 +1,148 @@
+name: basic-build
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      # handling of environment variable: https://stackoverflow.com/a/57969570
+      - name: lscpu
+        if: ${{ always() }}
+        run: |
+          lscpu; \
+          echo "nb_cores=$(lscpu | grep \^CPU\(s\)\: | awk '{print $2}')" >> $GITHUB_ENV
+
+      - name: gcc_version
+        if: ${{ always() }}
+        run: gcc --version
+
+      - name: echo_nb_cores
+        if: ${{ always() }}
+        run: echo "Number of cores ${nb_cores}"
+
+      - name: get_packages
+        run: sudo apt-get update && sudo apt install openmpi-bin openmpi-common libopenmpi3 libopenmpi-dev numdiff
+
+      - name: mpicc_version
+        if: ${{ always() }}
+        run: mpicc --version
+
+      - name: get_lime
+        uses: actions/checkout@v3
+        with:
+          repository: usqcd-software/c-lime
+          path: lime
+
+      - name: autogen_lime
+        working-directory: ${{github.workspace}}/lime
+        run: ./autogen.sh && mkdir build
+
+      - name: build_lime
+        working-directory: ${{github.workspace}}/lime/build
+        run: |
+          CC=gcc \
+            CFLAGS="-march=haswell -mtune=haswell -O2" \
+            ../configure --prefix=$(pwd)/install_dir
+          make -j
+          make install
+
+      - name: Archive lime config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lime_config_output
+          path: ${{github.workspace}}/lime/build/config.log 
+
+      - name: get_lemon
+        uses: actions/checkout@v3
+        with:
+          repository: etmc/lemon
+          path: lemon
+
+      - name: autogen_lemon
+        working-directory: ${{github.workspace}}/lemon
+        run: |
+          autoreconf -i -f
+          mkdir build
+
+      - name: build_lemon
+        working-directory: ${{github.workspace}}/lemon/build
+        run: |
+          CC=mpicc \
+            CFLAGS="-march=haswell -mtune=haswell -O2" \
+            ../configure --prefix=$(pwd)/install_dir
+          make -j
+          make install
+      
+      - name: Archive lemon config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lemon_config_output
+          path: ${{github.workspace}}/lemon/build/config.log 
+
+      - name: get_tmlqcd
+        uses: actions/checkout@v3
+        with:
+          path: main
+
+      - name: create_builddir
+        shell: bash
+        run: mkdir ${{github.workspace}}/main/build
+
+      - name: autogen_tmlqcd
+        working-directory: ${{github.workspace}}/main
+        run: autoconf
+
+      - name: configure_and_build
+        shell: bash
+        working-directory: ${{github.workspace}}/main/build
+        run: |
+          CC=mpicc CXX=mpicxx \
+            LDFLAGS="-fopenmp" \
+            CFLAGS="-O2 -mtune=haswell -march=haswell -mavx2 -mfma -DOMPI_SKIP_MPICXX -fopenmp" \
+            CXXFLAGS="-O2 -mtune=haswell -march=haswell -mavx2 -mfma -DOMPI_SKIP_MPICXX -fopenmp" \
+            ../configure \
+            --enable-mpi \
+            --with-mpidimension=4 \
+            --enable-omp \
+            --disable-sse2 \
+            --disable-sse3 \
+            --with-limedir=${{github.workspace}}/lime/build/install_dir \
+            --with-lemondir=${{github.workspace}}/lemon/build/install_dir \
+            --with-lapack="-lblas -llapack" || cat config.log
+          make -j
+
+      - name: Archive tmLQCD config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmlqcd_config_output
+          path: ${{github.workspace}}/main/build/config.log 
+
+      - name: nf2_rgmixedcg_hmc_tmcloverdetratio
+        working-directory: ${{github.workspace}}/main/build
+        run: |
+          mpirun -np 2 ./hmc_tm \
+            -f ../sample-input/sample-hmc-rgmixedcg-tmcloverdetratio.input
+      
+      - name: Archive nf2_rgmixedcg_hmc_tmcloverdetratio output
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmlqcd_test_output
+          path: ${{github.workspace}}/main/build/output.data
+
+      - name: compare_nf2_rgmixedcg_hmc_tmcloverdetratio
+        working-directory: ${{github.workspace}}/main/build
+        run: |
+          refpath=${{github.workspace}}/main/sample-output/hmc-rgmixedcg-tmcloverdetratio
+          numdiff -r 1e-4 -X 1:10 -X 1:5-8 -X 2:10 -X 2:5-8 output.data ${refpath}/output.data
+          for i in $(seq 0 2 18); do \
+            f=onlinemeas.$(printf %06d $i); \
+            numdiff -r 1e-5 ${f} ${refpath}/${f}; \
+          done
+
+
diff --git a/.github/workflows/ddalphaamg-build.yaml b/.github/workflows/ddalphaamg-build.yaml
@@ -0,0 +1,161 @@
+name: ddalphaamg-build
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: get_tmlqcd
+        uses: actions/checkout@v3
+        with:
+          path: main
+
+
+      # handling of environment variable: https://stackoverflow.com/a/57969570
+      - name: lscpu
+        run: |
+          lscpu; \
+          echo "nb_cores=$(lscpu | grep \^CPU\(s\)\: | awk '{print $2}')" >> $GITHUB_ENV
+
+      - name: gcc_version
+        if: ${{ always() }}
+        run: gcc --version
+
+      - name: echo_nb_cores
+        if: ${{ always() }}
+        run: echo "Number of cores ${nb_cores}"
+
+      - name: get_packages
+        run: sudo apt update && sudo apt install openmpi-bin openmpi-common libopenmpi3 libopenmpi-dev numdiff
+
+      - name: mpicc_version
+        if: ${{ always() }}
+        run: mpicc --version
+
+      - name: get_lime
+        uses: actions/checkout@v3
+        with:
+          repository: usqcd-software/c-lime
+          path: lime
+
+      - name: autogen_lime
+        working-directory: ${{github.workspace}}/lime
+        run: ./autogen.sh
+
+      - name: create_lime_builddir
+        run: mkdir ${{github.workspace}}/lime/build
+
+      - name: build_lime
+        working-directory: ${{github.workspace}}/lime/build
+        run: |
+          CC=gcc \
+            CFLAGS="-march=haswell -mtune=haswell -O2" \
+            ../configure --prefix=$(pwd)/install_dir
+          make -j
+          make install
+
+      - name: Archive lime config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lime_config_output
+          path: ${{github.workspace}}/lime/build/config.log 
+
+      - name: get_lemon
+        uses: actions/checkout@v3
+        with:
+          repository: etmc/lemon
+          path: lemon
+
+      - name: create_lemon_builddir
+        run: mkdir ${{github.workspace}}/lemon/build
+
+      - name: autogen_lemon
+        working-directory: ${{github.workspace}}/lemon
+        run: autoreconf -i -f
+
+      - name: build_lemon
+        working-directory: ${{github.workspace}}/lemon/build
+        run: |
+          CC=mpicc \
+            CFLAGS="-march=haswell -mtune=haswell -O2" \
+            ../configure \
+            --prefix=$(pwd)/install_dir
+          make -j
+          make install
+
+      - name: Archive lemon config.log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: lemon_config_output
+          path: ${{github.workspace}}/lemon/build/config.log 
+
+      - name: get_ddalphaamg
+        uses: actions/checkout@v3
+        with:
+          repository: kostrzewa/ddalphaamg #sbacchio/ddalphaamg
+          ref: fix_main_build
+          path: ddalphaamg
+
+      - name: build_ddalphaamg
+        working-directory: ${{github.workspace}}/ddalphaamg
+        run: |
+          patch Makefile ${{ github.workspace }}/main/.github/patches/DDalphaAMG.patch
+          LIMEDIR=${{github.workspace}}/lime/build/install_dir \
+            make -j
+
+      - name: create_builddir
+        shell: bash
+        run: mkdir ${{github.workspace}}/main/build
+
+      - name: autogen_tmlqcd
+        working-directory: ${{github.workspace}}/main
+        run: autoconf
+
+      - name: configure_and_build
+        shell: bash
+        working-directory: ${{github.workspace}}/main/build
+        run: |
+          CC=mpicc CXX=mpicxx \
+            LDFLAGS="-fopenmp" \
+            CFLAGS="-O2 -mtune=haswell -march=haswell -mavx2 -mfma -DOMPI_SKIP_MPICXX -fopenmp" \
+            CXXFLAGS="-O2 -mtune=haswell -march=haswell -mavx2 -mfma -DOMPI_SKIP_MPICXX -fopenmp" \
+            ../configure \
+            --enable-mpi \
+            --with-mpidimension=4 \
+            --enable-omp \
+            --disable-sse2 \
+            --disable-sse3 \
+            --with-limedir=${{github.workspace}}/lime/build/install_dir \
+            --with-lemondir=${{github.workspace}}/lemon/build/install_dir \
+            --with-DDalphaAMG=${{github.workspace}}/ddalphaamg \
+            --with-lapack="-lblas -llapack" || cat config.log
+          make -j
+
+      - name: nf2_ddalphaamg_hmc_tmcloverdetratio
+        working-directory: ${{github.workspace}}/main/build
+        run: |
+          mpirun -np 2 ./hmc_tm \
+            -f ../sample-input/sample-hmc-ddalphaamg-tmcloverdetratio.input
+
+      - name: Archive nf2_ddalphaamg_hmc_tmcloverdetratio output
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: tmlqcd_test_output
+          path: ${{github.workspace}}/main/build/output.data
+
+      - name: compare_nf2_ddalphaamg_hmc_tmcloverdetratio
+        working-directory: ${{github.workspace}}/main/build
+        run: |
+          refpath=${{github.workspace}}/main/sample-output/hmc-ddalphaamg-tmcloverdetratio
+          numdiff -r 1e-4 -X 1:10 -X 1:5-8 -X 2:10 -X 2:5-8 output.data ${refpath}/output.data
+          for i in $(seq 0 2 18); do \
+            f=onlinemeas.$(printf %06d $i); \
+            numdiff -r 1e-5 ${f} ${refpath}/${f}; \
+          done
+
+