CFI: Recursively transform predicates in ty::Dynamic rather than using identity (Redux)

compiler/rustc_codegen_llvm/src/builder.rs

@@ -22,10 +22,7 @@ use rustc_middle::ty::layout::{
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
 use rustc_span::Span;
-use rustc_symbol_mangling::typeid::{
-    kcfi_typeid_for_fnabi, kcfi_typeid_for_instance, typeid_for_fnabi, typeid_for_instance,
-    TypeIdOptions,
-};
+use rustc_symbol_mangling::typeid;
 use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange};
 use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
 use smallvec::SmallVec;
@@ -1632,18 +1629,18 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
                 return;
             }
 
-            let mut options = TypeIdOptions::empty();
+            let mut options = typeid::Options::empty();
             if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
-                options.insert(TypeIdOptions::GENERALIZE_POINTERS);
+                options.insert(typeid::Options::GENERALIZE_POINTERS);
             }
             if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
-                options.insert(TypeIdOptions::NORMALIZE_INTEGERS);
+                options.insert(typeid::Options::NORMALIZE_INTEGERS);
             }
 
             let typeid = if let Some(instance) = instance {
-                typeid_for_instance(self.tcx, instance, options)
+                typeid::from_instance(self.tcx, instance, options)
             } else {
-                typeid_for_fnabi(self.tcx, fn_abi, options)
+                typeid::from_fnabi(self.tcx, fn_abi, options)
             };
             let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
 
@@ -1680,18 +1677,18 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
                 return None;
             }
 
-            let mut options = TypeIdOptions::empty();
+            let mut options = typeid::Options::empty();
             if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
-                options.insert(TypeIdOptions::GENERALIZE_POINTERS);
+                options.insert(typeid::Options::GENERALIZE_POINTERS);
             }
             if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
-                options.insert(TypeIdOptions::NORMALIZE_INTEGERS);
+                options.insert(typeid::Options::NORMALIZE_INTEGERS);
             }
 
             let kcfi_typeid = if let Some(instance) = instance {
-                kcfi_typeid_for_instance(self.tcx, instance, options)
+                typeid::from_instance_kcfi(self.tcx, instance, options)
             } else {
-                kcfi_typeid_for_fnabi(self.tcx, fn_abi, options)
+                typeid::from_fnabi_kcfi(self.tcx, fn_abi, options)
             };
 
             Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))

compiler/rustc_codegen_llvm/src/declare.rs

@@ -22,10 +22,7 @@ use itertools::Itertools;
 use rustc_codegen_ssa::traits::TypeMembershipMethods;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_middle::ty::{Instance, Ty};
-use rustc_symbol_mangling::typeid::{
-    kcfi_typeid_for_fnabi, kcfi_typeid_for_instance, typeid_for_fnabi, typeid_for_instance,
-    TypeIdOptions,
-};
+use rustc_symbol_mangling::typeid;
 use smallvec::SmallVec;
 
 /// Declare a function.
@@ -145,27 +142,27 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             if let Some(instance) = instance {
                 let mut typeids = FxIndexSet::default();
                 for options in [
-                    TypeIdOptions::GENERALIZE_POINTERS,
-                    TypeIdOptions::NORMALIZE_INTEGERS,
-                    TypeIdOptions::ERASE_SELF_TYPE,
+                    typeid::Options::GENERALIZE_POINTERS,
+                    typeid::Options::NORMALIZE_INTEGERS,
+                    typeid::Options::ERASE_SELF_TYPE,
                 ]
                 .into_iter()
                 .powerset()
-                .map(TypeIdOptions::from_iter)
+                .map(typeid::Options::from_iter)
                 {
-                    let typeid = typeid_for_instance(self.tcx, instance, options);
+                    let typeid = typeid::from_instance(self.tcx, instance, options);
                     if typeids.insert(typeid.clone()) {
                         self.add_type_metadata(llfn, typeid);
                     }
                 }
             } else {
                 for options in
-                    [TypeIdOptions::GENERALIZE_POINTERS, TypeIdOptions::NORMALIZE_INTEGERS]
+                    [typeid::Options::GENERALIZE_POINTERS, typeid::Options::NORMALIZE_INTEGERS]
                         .into_iter()
                         .powerset()
-                        .map(TypeIdOptions::from_iter)
+                        .map(typeid::Options::from_iter)
                 {
-                    let typeid = typeid_for_fnabi(self.tcx, fn_abi, options);
+                    let typeid = typeid::from_fnabi(self.tcx, fn_abi, options);
                     self.add_type_metadata(llfn, typeid);
                 }
             }
@@ -175,19 +172,19 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             // LLVM KCFI does not support multiple !kcfi_type attachments
             // Default to erasing the self type. If we need the concrete type, there will be a
             // hint in the instance.
-            let mut options = TypeIdOptions::ERASE_SELF_TYPE;
+            let mut options = typeid::Options::ERASE_SELF_TYPE;
             if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
-                options.insert(TypeIdOptions::GENERALIZE_POINTERS);
+                options.insert(typeid::Options::GENERALIZE_POINTERS);
             }
             if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
-                options.insert(TypeIdOptions::NORMALIZE_INTEGERS);
+                options.insert(typeid::Options::NORMALIZE_INTEGERS);
             }
 
             if let Some(instance) = instance {
-                let kcfi_typeid = kcfi_typeid_for_instance(self.tcx, instance, options);
+                let kcfi_typeid = typeid::from_instance_kcfi(self.tcx, instance, options);
                 self.set_kcfi_type_metadata(llfn, kcfi_typeid);
             } else {
-                let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options);
+                let kcfi_typeid = typeid::from_fnabi_kcfi(self.tcx, fn_abi, options);
                 self.set_kcfi_type_metadata(llfn, kcfi_typeid);
             }
         }

compiler/rustc_symbol_mangling/src/typeid.rs

@@ -4,15 +4,15 @@
 /// For more information about LLVM CFI and cross-language LLVM CFI support for the Rust compiler,
 /// see design document in the tracking issue #89653.
 use bitflags::bitflags;
-use rustc_middle::ty::{Instance, InstanceDef, ReifyReason, Ty, TyCtxt};
+use rustc_middle::ty::{Instance, InstanceDef, List, ReifyReason, Ty, TyCtxt};
 use rustc_target::abi::call::FnAbi;
 use std::hash::Hasher;
 use twox_hash::XxHash64;
 
 bitflags! {
     /// Options for typeid_for_fnabi.
     #[derive(Clone, Copy, Debug)]
-    pub struct TypeIdOptions: u32 {
+    pub struct Options: u32 {
         /// Generalizes pointers for compatibility with Clang
         /// `-fsanitize-cfi-icall-generalize-pointers` option for cross-language LLVM CFI and KCFI
         /// support.
@@ -25,58 +25,66 @@ bitflags! {
         /// CFI and  KCFI support.
         const NORMALIZE_INTEGERS = 4;
         /// Generalize the instance by erasing the concrete `Self` type where possible.
-        /// Only has an effect on `{kcfi_,}typeid_for_instance`.
+        /// Only has an effect on `from_instance{_kcfi,}`.
         const ERASE_SELF_TYPE = 8;
     }
 }
 
-mod typeid_itanium_cxx_abi;
+mod instance;
+mod itanium_cxx_abi;
+mod ty;
 
 /// Returns a type metadata identifier for the specified FnAbi.
-pub fn typeid_for_fnabi<'tcx>(
+pub fn from_fnabi<'tcx>(
     tcx: TyCtxt<'tcx>,
     fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
-    options: TypeIdOptions,
+    options: Options,
 ) -> String {
-    typeid_itanium_cxx_abi::typeid_for_fnabi(tcx, fn_abi, options)
+    itanium_cxx_abi::typeid_for_fnabi(tcx, fn_abi, options)
 }
 
 /// Returns a type metadata identifier for the specified Instance.
-pub fn typeid_for_instance<'tcx>(
+pub fn from_instance<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
-    options: TypeIdOptions,
+    options: Options,
 ) -> String {
-    typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options)
+    let instance = instance::transform(tcx, instance, options);
+    let fn_abi = tcx
+        .fn_abi_of_instance(tcx.param_env(instance.def_id()).and((instance, List::empty())))
+        .unwrap_or_else(|error| {
+            bug!("typeid_for_instance: couldn't get fn_abi of instance {instance:?}: {error:?}")
+        });
+    itanium_cxx_abi::typeid_for_fnabi(tcx, fn_abi, options)
 }
 
 /// Returns a KCFI type metadata identifier for the specified FnAbi.
-pub fn kcfi_typeid_for_fnabi<'tcx>(
+pub fn from_fnabi_kcfi<'tcx>(
     tcx: TyCtxt<'tcx>,
     fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
-    options: TypeIdOptions,
+    options: Options,
 ) -> u32 {
     // A KCFI type metadata identifier is a 32-bit constant produced by taking the lower half of the
     // xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.)
     let mut hash: XxHash64 = Default::default();
-    hash.write(typeid_itanium_cxx_abi::typeid_for_fnabi(tcx, fn_abi, options).as_bytes());
+    hash.write(itanium_cxx_abi::typeid_for_fnabi(tcx, fn_abi, options).as_bytes());
     hash.finish() as u32
 }
 
 /// Returns a KCFI type metadata identifier for the specified Instance.
-pub fn kcfi_typeid_for_instance<'tcx>(
+pub fn from_instance_kcfi<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
-    mut options: TypeIdOptions,
+    mut options: Options,
 ) -> u32 {
     // If we receive a `ReifyShim` intended to produce a function pointer, we need to remain
     // concrete - abstraction is for vtables.
     if matches!(instance.def, InstanceDef::ReifyShim(_, Some(ReifyReason::FnPtr))) {
-        options.remove(TypeIdOptions::ERASE_SELF_TYPE);
+        options.remove(Options::ERASE_SELF_TYPE);
     }
     // A KCFI type metadata identifier is a 32-bit constant produced by taking the lower half of the
     // xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.)
     let mut hash: XxHash64 = Default::default();
-    hash.write(typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options).as_bytes());
+    hash.write(from_instance(tcx, instance, options).as_bytes());
     hash.finish() as u32
 }