Move alignment checks to codegen

rust-lang · Nov 1, 2023 · 487ffa6 · 487ffa6
1 parent bbcc169
commit 487ffa6
Show file tree

Hide file tree

Showing 35 changed files with 202 additions and 171 deletions.
diff --git a/compiler/rustc_borrowck/src/invalidation.rs b/compiler/rustc_borrowck/src/invalidation.rs
@@ -205,7 +205,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Unreachable
             | TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }
-            | TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => {
+            | TerminatorKind::FalseUnwind { real_target: _, unwind: _ }
+            | TerminatorKind::UbCheck { .. } => {
                 // no data used, thus irrelevant to borrowck
             }
         }

diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
@@ -785,7 +785,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
             | TerminatorKind::Return
             | TerminatorKind::CoroutineDrop
             | TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }
-            | TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => {
+            | TerminatorKind::FalseUnwind { real_target: _, unwind: _ }
+            | TerminatorKind::UbCheck { .. } => {
                 // no data used, thus irrelevant to borrowck
             }
         }
@@ -835,7 +836,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
             | TerminatorKind::Goto { .. }
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::Unreachable
-            | TerminatorKind::InlineAsm { .. } => {}
+            | TerminatorKind::InlineAsm { .. }
+            | TerminatorKind::UbCheck { .. } => {}
         }
     }
 }

diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -1356,7 +1356,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             | TerminatorKind::Drop { .. }
             | TerminatorKind::FalseEdge { .. }
             | TerminatorKind::FalseUnwind { .. }
-            | TerminatorKind::InlineAsm { .. } => {
+            | TerminatorKind::InlineAsm { .. }
+            | TerminatorKind::UbCheck { .. } => {
                 // no checks needed for these
             }
 
@@ -1688,6 +1689,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 }
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
+            TerminatorKind::UbCheck { target, .. } => {
+                self.assert_iscleanup(body, block_data, target, is_cleanup)
+            }
         }
     }
 

diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -488,6 +488,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 let target_block = fx.get_block(*target);
                 fx.bcx.ins().jump(target_block, &[]);
             }
+            TerminatorKind::UbCheck { .. } => {}
         };
     }
 }

diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -508,7 +508,8 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                     | TerminatorKind::Return
                     | TerminatorKind::Unreachable
                     | TerminatorKind::Drop { .. }
-                    | TerminatorKind::Assert { .. } => {}
+                    | TerminatorKind::Assert { .. }
+                    | TerminatorKind::UbCheck { .. } => {}
                     TerminatorKind::Yield { .. }
                     | TerminatorKind::CoroutineDrop
                     | TerminatorKind::FalseEdge { .. }

diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -277,7 +277,8 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
                 | TerminatorKind::SwitchInt { .. }
                 | TerminatorKind::Yield { .. }
                 | TerminatorKind::FalseEdge { .. }
-                | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ }
+                | TerminatorKind::FalseUnwind { .. }
+                | TerminatorKind::UbCheck { .. } => { /* nothing to do */ }
                 TerminatorKind::Call { unwind, .. }
                 | TerminatorKind::InlineAsm { unwind, .. }
                 | TerminatorKind::Assert { unwind, .. }

diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -12,7 +12,7 @@ use crate::MemFlags;
 use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
-use rustc_middle::mir::{self, AssertKind, SwitchTargets, UnwindTerminateReason};
+use rustc_middle::mir::{self, AssertKind, SwitchTargets, UbCheckKind, UnwindTerminateReason};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty};
@@ -737,6 +737,79 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
     }
 
+    fn codegen_alignment_check(
+        &mut self,
+        helper: &TerminatorCodegenHelper<'tcx>,
+        bx: &mut Bx,
+        pointer: &mir::Operand<'tcx>,
+        source_info: mir::SourceInfo,
+        target: mir::BasicBlock,
+    ) -> MergingSucc {
+        let span = source_info.span;
+        let pointer = self.codegen_operand(bx, pointer);
+        let pointee_ty = pointer.layout.ty.builtin_deref(true).unwrap();
+        let pointee_layout = bx.layout_of(pointee_ty.ty);
+
+        let mk_usize = |v: u64| {
+            let layout = bx.layout_of(bx.tcx().types.usize);
+            let rustc_target::abi::Abi::Scalar(abi) = layout.abi else { unreachable!() };
+            let v = rustc_middle::mir::interpret::Scalar::from_target_usize(v, &bx.tcx());
+            bx.scalar_to_backend(v, abi, bx.cx().type_isize())
+        };
+
+        let align = pointee_layout.align.abi.bytes();
+        let mask = mk_usize(align - 1);
+        let zero = mk_usize(0);
+        let required = mk_usize(align);
+
+        let ptr_imm = match pointer.val {
+            crate::mir::OperandValue::Immediate(imm) => imm,
+            crate::mir::OperandValue::Pair(ptr, _) => ptr,
+            _ => {
+                unreachable!("{pointer:?}");
+            }
+        };
+        let int_imm = bx.ptrtoint(ptr_imm, bx.cx().type_isize());
+
+        let masked = bx.and(int_imm, mask);
+
+        let is_zero = bx.icmp(
+            crate::base::bin_op_to_icmp_predicate(mir::BinOp::Eq.to_hir_binop(), false),
+            masked,
+            zero,
+        );
+
+        let lltarget = helper.llbb_with_cleanup(self, target);
+        let panic_block = bx.append_sibling_block("panic");
+
+        bx.cond_br(is_zero, lltarget, panic_block);
+
+        bx.switch_to_block(panic_block);
+        self.set_debug_loc(bx, source_info);
+
+        let location = self.get_caller_location(bx, source_info).immediate();
+
+        let found = int_imm;
+
+        let (lang_item, args) =
+            (LangItem::PanicMisalignedPointerDereference, vec![required, found, location]);
+
+        let (fn_abi, llfn) = common::build_langcall(bx, Some(span), lang_item);
+        let merging_succ = helper.do_call(
+            self,
+            bx,
+            fn_abi,
+            llfn,
+            &args,
+            None,
+            mir::UnwindAction::Unreachable,
+            &[],
+            false,
+        );
+        assert_eq!(merging_succ, MergingSucc::False);
+        MergingSucc::False
+    }
+
     fn codegen_call_terminator(
         &mut self,
         helper: TerminatorCodegenHelper<'tcx>,
@@ -1292,6 +1365,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.instance,
                 mergeable_succ(),
             ),
+
+            mir::TerminatorKind::UbCheck {
+                target,
+                kind: UbCheckKind::PointerAlignment { ref pointer },
+            } => self.codegen_alignment_check(&helper, bx, pointer, terminator.source_info, target),
         }
     }
 

diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
@@ -85,7 +85,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 self.pop_stack_frame(/* unwinding */ false)?
             }
 
-            Goto { target } => self.go_to_block(target),
+            Goto { target } | UbCheck { target, .. } => self.go_to_block(target),
 
             SwitchInt { ref discr, ref targets } => {
                 let discr = self.read_immediate(&self.eval_operand(discr, None)?)?;

diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -1058,7 +1058,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             | TerminatorKind::UnwindResume
             | TerminatorKind::Return
             | TerminatorKind::SwitchInt { .. }
-            | TerminatorKind::Unreachable => {}
+            | TerminatorKind::Unreachable
+            | TerminatorKind::UbCheck { .. } => {}
         }
     }
 }

diff --git a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
@@ -118,7 +118,8 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> {
             | mir::TerminatorKind::Return
             | mir::TerminatorKind::SwitchInt { .. }
             | mir::TerminatorKind::Unreachable
-            | mir::TerminatorKind::Yield { .. } => {}
+            | mir::TerminatorKind::Yield { .. }
+            | mir::TerminatorKind::UbCheck { .. } => {}
         }
     }
 }
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -393,7 +393,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
 
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
         match &terminator.kind {
-            TerminatorKind::Goto { target } => {
+            TerminatorKind::Goto { target } | TerminatorKind::UbCheck { target, .. } => {
                 self.check_edge(location, *target, EdgeKind::Normal);
             }
             TerminatorKind::SwitchInt { targets, discr: _ } => {
@@ -1299,7 +1299,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             | TerminatorKind::UnwindResume
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Return
-            | TerminatorKind::Unreachable => {}
+            | TerminatorKind::Unreachable
+            | TerminatorKind::UbCheck { .. } => {}
         }
 
         self.super_terminator(terminator, location);

diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
@@ -858,6 +858,9 @@ impl<'tcx> TerminatorKind<'tcx> {
                 }
                 write!(fmt, ", options({options:?}))")
             }
+            UbCheck { target: _, kind: UbCheckKind::PointerAlignment { pointer } } => {
+                write!(fmt, "pointer_alignment_check({pointer:?})")
+            }
         }
     }
 
@@ -866,7 +869,7 @@ impl<'tcx> TerminatorKind<'tcx> {
         use self::TerminatorKind::*;
         match *self {
             Return | UnwindResume | UnwindTerminate(_) | Unreachable | CoroutineDrop => vec![],
-            Goto { .. } => vec!["".into()],
+            Goto { .. } | UbCheck { .. } => vec!["".into()],
             SwitchInt { ref targets, .. } => targets
                 .values
                 .iter()

diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
@@ -588,7 +588,9 @@ impl CallSource {
 #[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
 pub enum TerminatorKind<'tcx> {
     /// Block has one successor; we continue execution there.
-    Goto { target: BasicBlock },
+    Goto {
+        target: BasicBlock,
+    },
 
     /// Switches based on the computed value.
     ///
@@ -655,7 +657,12 @@ pub enum TerminatorKind<'tcx> {
     /// The `replace` flag indicates whether this terminator was created as part of an assignment.
     /// This should only be used for diagnostic purposes, and does not have any operational
     /// meaning.
-    Drop { place: Place<'tcx>, target: BasicBlock, unwind: UnwindAction, replace: bool },
+    Drop {
+        place: Place<'tcx>,
+        target: BasicBlock,
+        unwind: UnwindAction,
+        replace: bool,
+    },
 
     /// Roughly speaking, evaluates the `func` operand and the arguments, and starts execution of
     /// the referred to function. The operand types must match the argument types of the function.
@@ -798,6 +805,17 @@ pub enum TerminatorKind<'tcx> {
         /// if and only if InlineAsmOptions::MAY_UNWIND is set.
         unwind: UnwindAction,
     },
+
+    UbCheck {
+        target: BasicBlock,
+        kind: UbCheckKind<'tcx>,
+    },
+}
+
+#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
+#[derive(Debug)]
+pub enum UbCheckKind<'tcx> {
+    PointerAlignment { pointer: Operand<'tcx> },
 }
 
 impl TerminatorKind<'_> {
@@ -819,6 +837,7 @@ impl TerminatorKind<'_> {
             TerminatorKind::FalseEdge { .. } => "FalseEdge",
             TerminatorKind::FalseUnwind { .. } => "FalseUnwind",
             TerminatorKind::InlineAsm { .. } => "InlineAsm",
+            TerminatorKind::UbCheck { .. } => "UbCheck",
         }
     }
 }

diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
@@ -349,9 +349,8 @@ impl<'tcx> TerminatorKind<'tcx> {
             | Assert { target: t, unwind: _, .. }
             | FalseUnwind { real_target: t, unwind: _ }
             | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(t), .. }
-            | InlineAsm { destination: Some(t), unwind: _, .. } => {
-                Some(t).into_iter().chain((&[]).into_iter().copied())
-            }
+            | InlineAsm { destination: Some(t), unwind: _, .. }
+            | UbCheck { target: t, .. } => Some(t).into_iter().chain((&[]).into_iter().copied()),
             UnwindResume
             | UnwindTerminate(_)
             | CoroutineDrop
@@ -391,9 +390,8 @@ impl<'tcx> TerminatorKind<'tcx> {
             | Assert { target: ref mut t, unwind: _, .. }
             | FalseUnwind { real_target: ref mut t, unwind: _ }
             | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
-            | InlineAsm { destination: Some(ref mut t), unwind: _, .. } => {
-                Some(t).into_iter().chain(&mut [])
-            }
+            | InlineAsm { destination: Some(ref mut t), unwind: _, .. }
+            | UbCheck { target: ref mut t, .. } => Some(t).into_iter().chain(&mut []),
             UnwindResume
             | UnwindTerminate(_)
             | CoroutineDrop
@@ -418,7 +416,8 @@ impl<'tcx> TerminatorKind<'tcx> {
             | TerminatorKind::CoroutineDrop
             | TerminatorKind::Yield { .. }
             | TerminatorKind::SwitchInt { .. }
-            | TerminatorKind::FalseEdge { .. } => None,
+            | TerminatorKind::FalseEdge { .. }
+            | TerminatorKind::UbCheck { .. } => None,
             TerminatorKind::Call { ref unwind, .. }
             | TerminatorKind::Assert { ref unwind, .. }
             | TerminatorKind::Drop { ref unwind, .. }
@@ -437,7 +436,8 @@ impl<'tcx> TerminatorKind<'tcx> {
             | TerminatorKind::CoroutineDrop
             | TerminatorKind::Yield { .. }
             | TerminatorKind::SwitchInt { .. }
-            | TerminatorKind::FalseEdge { .. } => None,
+            | TerminatorKind::FalseEdge { .. }
+            | TerminatorKind::UbCheck { .. } => None,
             TerminatorKind::Call { ref mut unwind, .. }
             | TerminatorKind::Assert { ref mut unwind, .. }
             | TerminatorKind::Drop { ref mut unwind, .. }
@@ -520,7 +520,7 @@ impl<'tcx> TerminatorKind<'tcx> {
                 TerminatorEdges::None
             }
 
-            Goto { target } => TerminatorEdges::Single(target),
+            Goto { target } | UbCheck { target, .. } => TerminatorEdges::Single(target),
 
             Assert { target, unwind, expected: _, msg: _, cond: _ }
             | Drop { target, unwind, place: _, replace: _ }

diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
@@ -599,6 +599,12 @@ macro_rules! make_mir_visitor {
                             }
                         }
                     }
+                    TerminatorKind::UbCheck {
+                        kind: UbCheckKind::PointerAlignment { pointer },
+                        target: _,
+                    } => {
+                        self.visit_operand(pointer, location)
+                    }
                 }
             }
 

diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
@@ -1512,7 +1512,8 @@ impl<'tcx> DropTreeBuilder<'tcx> for Unwind {
             | TerminatorKind::Unreachable
             | TerminatorKind::Yield { .. }
             | TerminatorKind::CoroutineDrop
-            | TerminatorKind::FalseEdge { .. } => {
+            | TerminatorKind::FalseEdge { .. }
+            | TerminatorKind::UbCheck { .. } => {
                 span_bug!(term.source_info.span, "cannot unwind from {:?}", term.kind)
             }
         }

diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs
@@ -214,7 +214,8 @@ impl<'mir, 'tcx, C: TerminatorClassifier<'tcx>> TriColorVisitor<BasicBlocks<'tcx
             | TerminatorKind::FalseEdge { .. }
             | TerminatorKind::FalseUnwind { .. }
             | TerminatorKind::Goto { .. }
-            | TerminatorKind::SwitchInt { .. } => ControlFlow::Continue(()),
+            | TerminatorKind::SwitchInt { .. }
+            | TerminatorKind::UbCheck { .. } => ControlFlow::Continue(()),
         }
     }
 

diff --git a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
@@ -143,7 +143,8 @@ where
             | TerminatorKind::Return
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::Unreachable
-            | TerminatorKind::Yield { .. } => {}
+            | TerminatorKind::Yield { .. }
+            | TerminatorKind::UbCheck { .. } => {}
         }
     }
 }