x86_64: Add portable_atomic_vmovdqa_atomic cfg

taiki-e · Dec 25, 2022 · c52f729 · c52f729
1 parent 7ba52ed
commit c52f729
Show file tree

Hide file tree

Showing 5 changed files with 94 additions and 6 deletions.
diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt
@@ -63,3 +63,4 @@ uwrite
 uwriteln
 virt
 vmlinux
+vmovdqa
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -186,6 +186,31 @@ jobs:
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b ${{ env.RANDOMIZE_LAYOUT }}
         if: (matrix.target == '' || startsWith(matrix.target, 'x86_64')) && !startsWith(matrix.os, 'macos') && startsWith(matrix.rust, 'nightly')
 
+      # +cmpxchg16b,+avx with portable_atomic_vmovdqa_atomic cfg
+      - run: cargo test -vv --workspace --exclude bench --all-features $TARGET $BUILD_STD $DOCTEST_XCOMPILE
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+        if: matrix.target == '' || startsWith(matrix.target, 'x86_64')
+      - run: cargo test -vv --workspace --exclude bench --all-features --release $TARGET $BUILD_STD $DOCTEST_XCOMPILE
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+        if: matrix.target == '' || startsWith(matrix.target, 'x86_64')
+      # LTO + doctests is very slow on some platforms
+      - run: cargo test -vv --workspace --exclude bench --all-features --release $TARGET $BUILD_STD $DOCTEST_XCOMPILE --tests
+        env:
+          CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
+          CARGO_PROFILE_RELEASE_LTO: fat
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic ${{ env.RANDOMIZE_LAYOUT }}
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic ${{ env.RANDOMIZE_LAYOUT }}
+        if: matrix.target == '' || startsWith(matrix.target, 'x86_64')
+      - run: cargo careful test -vv --workspace --exclude bench --all-features $TARGET $DOCTEST_XCOMPILE
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic
+        if: (matrix.target == '' || startsWith(matrix.target, 'x86_64')) && startsWith(matrix.rust, 'nightly')
+
       # +lse
       - run: cargo test -vv --workspace --exclude bench --all-features $TARGET $BUILD_STD $DOCTEST_XCOMPILE
         env:
@@ -425,6 +450,7 @@ jobs:
       - name: Install Rust
         run: rustup toolchain add nightly --no-self-update --component rust-src && rustup default nightly
       - uses: taiki-e/install-action@valgrind
+
       # doctests on Valgrind are very slow
       - run: cargo test -vv --workspace --exclude bench --all-features --tests
         env:
@@ -440,6 +466,7 @@ jobs:
           CARGO_PROFILE_RELEASE_LTO: fat
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg valgrind
           RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg valgrind
+
       # +cmpxchg16b
       - run: cargo test -vv --workspace --exclude bench --all-features --tests
         env:
@@ -456,6 +483,22 @@ jobs:
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b --cfg valgrind
           RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b --cfg valgrind
 
+      # Sandy Bridge (the first Intel chip that introduced AVX) with portable_atomic_vmovdqa_atomic cfg
+      - run: cargo test -vv --workspace --exclude bench --all-features --tests
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+      - run: cargo test -vv --workspace --exclude bench --all-features --release --tests
+        env:
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+      - run: cargo test -vv --workspace --exclude bench --all-features --release --tests
+        env:
+          CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 1
+          CARGO_PROFILE_RELEASE_LTO: fat
+          RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+          RUSTFLAGS: ${{ env.RUSTFLAGS }} -C target-feature=+cmpxchg16b,+avx --cfg portable_atomic_vmovdqa_atomic --cfg valgrind
+
   codegen:
     runs-on: ubuntu-latest
     permissions:

diff --git a/src/imp/atomic128/cpuid.rs b/src/imp/atomic128/cpuid.rs
@@ -4,8 +4,9 @@
     any(
         portable_atomic_no_outline_atomics,
         not(target_feature = "sse"),
+        portable_atomic_vmovdqa_atomic,
         miri,
-        portable_atomic_sanitize_thread
+        portable_atomic_sanitize_thread,
     ),
     allow(dead_code)
 )]

diff --git a/src/imp/atomic128/x86_64.rs b/src/imp/atomic128/x86_64.rs
@@ -4,7 +4,8 @@
 // - x86 and amd64 instruction reference https://www.felixcloutier.com/x86
 //
 // Generated asm:
-// - x86_64 (+cmpxchg16b) https://godbolt.org/z/vbz7bG156
+// - x86_64 (+cmpxchg16b) https://godbolt.org/z/Kc5s5xaMY
+// - x86_64 (+cmpxchg16b,+avx,vmovdqa_atomic) https://godbolt.org/z/4WYG9fasM
 
 include!("macros.rs");
 
@@ -291,19 +292,38 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
         // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
         // Miri and Sanitizer do not support inline assembly.
         #[cfg(any(
-            portable_atomic_no_outline_atomics,
+            all(
+                portable_atomic_no_outline_atomics,
+                not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))
+            ),
             not(target_feature = "sse"),
             miri,
             portable_atomic_sanitize_thread
         ))]
         // SAFETY: the caller must uphold the safety contract for `atomic_load`.
         () => unsafe { _atomic_load_cmpxchg16b(src, order) },
         #[cfg(not(any(
-            portable_atomic_no_outline_atomics,
+            all(
+                portable_atomic_no_outline_atomics,
+                not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))
+            ),
             not(target_feature = "sse"),
             miri,
             portable_atomic_sanitize_thread
         )))]
+        #[cfg(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))]
+        // SAFETY: the caller must uphold the safety contract for `atomic_load`.
+        () => unsafe { _atomic_load_vmovdqa(src, order) },
+        #[cfg(not(any(
+            all(
+                portable_atomic_no_outline_atomics,
+                not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))
+            ),
+            not(target_feature = "sse"),
+            miri,
+            portable_atomic_sanitize_thread
+        )))]
+        #[cfg(not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx")))]
         // SAFETY: the caller must uphold the safety contract for `atomic_load`.
         () => unsafe {
             ifunc!(unsafe fn(src: *mut u128, order: Ordering) -> u128;
@@ -336,19 +356,38 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
         // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
         // Miri and Sanitizer do not support inline assembly.
         #[cfg(any(
-            portable_atomic_no_outline_atomics,
+            all(
+                portable_atomic_no_outline_atomics,
+                not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))
+            ),
             not(target_feature = "sse"),
             miri,
             portable_atomic_sanitize_thread
         ))]
         // SAFETY: the caller must uphold the safety contract for `atomic_store`.
         () => unsafe { _atomic_store_cmpxchg16b(dst, val, order) },
         #[cfg(not(any(
-            portable_atomic_no_outline_atomics,
+            all(
+                portable_atomic_no_outline_atomics,
+                not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))
+            ),
+            not(target_feature = "sse"),
+            miri,
+            portable_atomic_sanitize_thread
+        )))]
+        #[cfg(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))]
+        // SAFETY: the caller must uphold the safety contract for `atomic_store`.
+        () => unsafe { _atomic_store_vmovdqa(dst, val, order) },
+        #[cfg(not(any(
+            all(
+                portable_atomic_no_outline_atomics,
+                not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx"))
+            ),
             not(target_feature = "sse"),
             miri,
             portable_atomic_sanitize_thread
         )))]
+        #[cfg(not(all(portable_atomic_vmovdqa_atomic, target_feature = "avx")))]
         // SAFETY: the caller must uphold the safety contract for `atomic_store`.
         () => unsafe {
             ifunc!(unsafe fn(dst: *mut u128, val: u128, order: Ordering);

diff --git a/tools/build.sh b/tools/build.sh
@@ -93,6 +93,7 @@ known_cfgs=(
     portable_atomic_s_mode
     portable_atomic_disable_fiq
     portable_atomic_no_outline_atomics
+    portable_atomic_vmovdqa_atomic
 )
 
 x() {
@@ -296,6 +297,9 @@ build() {
                         x_cargo "${args[@]}" --target-dir target/cmpxchg16b "$@"
                     ;;
             esac
+            # +cmpxchg16b,+avx with portable_atomic_vmovdqa_atomic cfg
+            RUSTFLAGS="${target_rustflags} -C target-feature=+cmpxchg16b,+sse,+avx --cfg portable_atomic_vmovdqa_atomic" \
+                x_cargo "${args[@]}" --target-dir target/vmovdqa_atomic "$@"
             ;;
         aarch64* | arm64*)
             # macOS is skipped because it is +lse,+lse2 by default