diff --git a/crates/core_arch/src/aarch64/mod.rs b/crates/core_arch/src/aarch64/mod.rs index ebd7a31781..594c6d18c6 100644 --- a/crates/core_arch/src/aarch64/mod.rs +++ b/crates/core_arch/src/aarch64/mod.rs @@ -6,6 +6,10 @@ //! [arm_ref]: http://infocenter.arm.com/help/topic/com.arm.doc.ihi0073a/IHI0073A_arm_neon_intrinsics_ref.pdf //! [arm_dat]: https://developer.arm.com/technologies/neon/intrinsics +mod mte; +#[unstable(feature = "stdarch_aarch64_mte", issue = "129010")] +pub use self::mte::*; + // NEON intrinsics are currently broken on big-endian, so don't expose them. (#1484) #[cfg(target_endian = "little")] mod neon; diff --git a/crates/core_arch/src/aarch64/mte.rs b/crates/core_arch/src/aarch64/mte.rs new file mode 100644 index 0000000000..014a9feafb --- /dev/null +++ b/crates/core_arch/src/aarch64/mte.rs @@ -0,0 +1,171 @@ +//! AArch64 Memory tagging intrinsics +//! +//! [ACLE documentation](https://arm-software.github.io/acle/main/acle.html#markdown-toc-mte-intrinsics) + +extern "unadjusted" { + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.irg" + )] + fn irg_(ptr: *const (), exclude: i64) -> *const (); + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.gmi" + )] + fn gmi_(ptr: *const (), exclude: i64) -> i64; + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.ldg" + )] + fn ldg_(ptr: *const (), tag_ptr: *const ()) -> *const (); + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.stg" + )] + fn stg_(tagged_ptr: *const (), addr_to_tag: *const ()); + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.addg" + )] + fn addg_(ptr: *const (), value: i64) -> *const (); + #[cfg_attr( + any(target_arch = "aarch64", target_arch = "arm64ec"), + link_name = "llvm.aarch64.subp" + )] + fn subp_(ptr_a: *const (), ptr_b: *const ()) -> i64; +} + +/// Return a pointer containing a randomly generated logical address tag. +/// +/// `src`: A pointer containing an address. +/// `mask`: A mask where each of the lower 16 bits specifies logical +/// tags which must be excluded from consideration. Zero excludes no +/// tags. +/// +/// The returned pointer contains a copy of the `src` address, but with a +/// randomly generated logical tag, excluding any specified by `mask`. +/// +/// SAFETY: The pointer provided by this intrinsic will be invalid until the memory +/// has been appropriately tagged with `__arm_mte_set_tag`. If using that intrinsic +/// on the provided pointer is itself invalid, then it will be permanently invalid +/// and Undefined Behavior to dereference it. +#[inline] +#[target_feature(enable = "mte")] +#[unstable(feature = "stdarch_aarch64_mte", issue = "129010")] +pub unsafe fn __arm_mte_create_random_tag(src: *const T, mask: u64) -> *const T { + irg_(src as *const (), mask as i64) as *const T +} + +/// Return a pointer with the logical address tag offset by a value. +/// +/// `src`: A pointer containing an address and a logical tag. +/// `OFFSET`: A compile-time constant value in the range [0, 15]. +/// +/// Adds offset to the logical address tag in `src`, wrapping if the result is +/// outside of the valid 16 tags. +/// +/// SAFETY: See `__arm_mte_create_random_tag`. +#[inline] +#[target_feature(enable = "mte")] +#[unstable(feature = "stdarch_aarch64_mte", issue = "129010")] +pub unsafe fn __arm_mte_increment_tag(src: *const T) -> *const T { + addg_(src as *const (), OFFSET) as *const T +} + +/// Add a logical tag to the set of excluded logical tags. +/// +/// `src`: A pointer containing an address and a logical tag. +/// `excluded`: A mask where the lower 16 bits each specify currently-excluded +/// logical tags. +/// +/// Adds the logical tag stored in `src` to the set in `excluded`, and returns +/// the result. +#[inline] +#[target_feature(enable = "mte")] +#[unstable(feature = "stdarch_aarch64_mte", issue = "129010")] +pub unsafe fn __arm_mte_exclude_tag(src: *const T, excluded: u64) -> u64 { + gmi_(src as *const (), excluded as i64) as u64 +} + +/// Store an allocation tag for the 16-byte granule of memory. +/// +/// `tag_address`: A pointer containing an address and a logical tag, which +/// must be 16-byte aligned. +/// +/// SAFETY: `tag_address` must be 16-byte aligned. The tag will apply to the +/// entire 16-byte memory granule. +#[inline] +#[target_feature(enable = "mte")] +#[unstable(feature = "stdarch_aarch64_mte", issue = "129010")] +pub unsafe fn __arm_mte_set_tag(tag_address: *const T) { + stg_(tag_address as *const (), tag_address as *const ()); +} + +/// Load an allocation tag from memory, returning a new pointer with the +/// corresponding logical tag. +/// +/// `address`: A pointer containing an address from which allocation tag memory +/// is read. This does not need to be 16-byte aligned. +#[inline] +#[target_feature(enable = "mte")] +#[unstable(feature = "stdarch_aarch64_mte", issue = "129010")] +pub unsafe fn __arm_mte_get_tag(address: *const T) -> *const T { + ldg_(address as *const (), address as *const ()) as *const T +} + +/// Calculate the difference between the address parts of two pointers, ignoring +/// the tags, and sign-extending the result. +#[inline] +#[target_feature(enable = "mte")] +#[unstable(feature = "stdarch_aarch64_mte", issue = "129010")] +pub unsafe fn __arm_mte_ptrdiff(a: *const T, b: *const U) -> i64 { + subp_(a as *const (), b as *const ()) +} + +#[cfg(test)] +mod test { + use super::*; + use stdarch_test::assert_instr; + + #[cfg_attr(test, assert_instr(irg))] + #[allow(dead_code)] + #[target_feature(enable = "mte")] + unsafe fn test_arm_mte_create_random_tag(src: *const (), mask: u64) -> *const () { + __arm_mte_create_random_tag(src, mask) + } + + #[cfg_attr(test, assert_instr(addg))] + #[allow(dead_code)] + #[target_feature(enable = "mte")] + unsafe fn test_arm_mte_increment_tag(src: *const ()) -> *const () { + __arm_mte_increment_tag::<1, _>(src) + } + + #[cfg_attr(test, assert_instr(gmi))] + #[allow(dead_code)] + #[target_feature(enable = "mte")] + unsafe fn test_arm_mte_exclude_tag(src: *const (), excluded: u64) -> u64 { + __arm_mte_exclude_tag(src, excluded) + } + + #[cfg_attr(test, assert_instr(stg))] + #[allow(dead_code)] + #[target_feature(enable = "mte")] + unsafe fn test_arm_mte_set_tag(src: *const ()) { + __arm_mte_set_tag(src) + } + + #[cfg_attr(test, assert_instr(ldg))] + #[allow(dead_code)] + #[target_feature(enable = "mte")] + unsafe fn test_arm_mte_get_tag(src: *const ()) -> *const () { + __arm_mte_get_tag(src) + } + + #[cfg_attr(test, assert_instr(subp))] + #[allow(dead_code)] + #[target_feature(enable = "mte")] + unsafe fn test_arm_mte_ptrdiff(a: *const (), b: *const ()) -> i64 { + __arm_mte_ptrdiff(a, b) + } +} diff --git a/crates/stdarch-verify/src/lib.rs b/crates/stdarch-verify/src/lib.rs index 3a5588bbfe..1afb556aab 100644 --- a/crates/stdarch-verify/src/lib.rs +++ b/crates/stdarch-verify/src/lib.rs @@ -97,11 +97,11 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { arguments.push(to_type(ty)); } for generic in f.sig.generics.params.iter() { - let ty = match *generic { - syn::GenericParam::Const(ref c) => &c.ty, + match *generic { + syn::GenericParam::Const(ref c) => const_arguments.push(to_type(&c.ty)), + syn::GenericParam::Type(ref _t) => (), _ => panic!("invalid generic argument on {name}"), }; - const_arguments.push(to_type(ty)); } let ret = match f.sig.output { syn::ReturnType::Default => quote! { None }, @@ -345,6 +345,10 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "v4f32" => quote! { &v4f32 }, "v2f64" => quote! { &v2f64 }, + // Generic types + "T" => quote! { &GENERICT }, + "U" => quote! { &GENERICU }, + s => panic!("unsupported type: \"{s}\""), }, syn::Type::Ptr(syn::TypePtr { diff --git a/crates/stdarch-verify/tests/arm.rs b/crates/stdarch-verify/tests/arm.rs index 1b18dbdaad..ac1d5793d0 100644 --- a/crates/stdarch-verify/tests/arm.rs +++ b/crates/stdarch-verify/tests/arm.rs @@ -28,6 +28,8 @@ static U32: Type = Type::PrimUnsigned(32); static U64: Type = Type::PrimUnsigned(64); static U8: Type = Type::PrimUnsigned(8); static NEVER: Type = Type::Never; +static GENERICT: Type = Type::GenericParam("T"); +static GENERICU: Type = Type::GenericParam("U"); static F16X4: Type = Type::F(16, 4, 1); static F16X4X2: Type = Type::F(16, 4, 2); @@ -157,6 +159,7 @@ enum Type { PrimPoly(u8), MutPtr(&'static Type), ConstPtr(&'static Type), + GenericParam(&'static str), I(u8, u8, u8), U(u8, u8, u8), P(u8, u8, u8), @@ -456,6 +459,7 @@ fn verify_all_signatures() { && !rust.file.ends_with("v7.rs\"") && !rust.file.ends_with("v8.rs\"") && !rust.file.ends_with("tme.rs\"") + && !rust.file.ends_with("mte.rs\"") && !rust.file.ends_with("ex.rs\"") && !skip_intrinsic_verify.contains(&rust.name) {