Auto merge of #82936 - oli-obk:valtree, r=RalfJung,lcnr,matthewjasper

Implement (but don't use) valtree and refactor in preparation of use

This PR does not cause any functional change. It refactors various things that are needed to make valtrees possible. This refactoring got big enough that I decided I'd want it reviewed as a PR instead of trying to make one huge PR with all the changes.

cc `@rust-lang/wg-const-eval` on the following commits:

* 2027184 implement valtree
* eeecea9 fallible Scalar -> ScalarInt
* 042f663 ScalarInt convenience methods

cc `@eddyb` on ef04a6d

cc `@rust-lang/wg-mir-opt` for cf1700c (`mir::Constant` can now represent either a `ConstValue` or a `ty::Const`, and it is totally possible to have two different representations for the same value)

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -8,7 +8,7 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{
     read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Pointer, Scalar,
 };
-use rustc_middle::ty::{Const, ConstKind};
+use rustc_middle::ty::ConstKind;
 
 use cranelift_codegen::ir::GlobalValueData;
 use cranelift_module::*;
@@ -39,7 +39,10 @@ impl ConstantCx {
 pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
     let mut all_constants_ok = true;
     for constant in &fx.mir.required_consts {
-        let const_ = fx.monomorphize(constant.literal);
+        let const_ = match fx.monomorphize(constant.literal) {
+            ConstantKind::Ty(ct) => ct,
+            ConstantKind::Val(..) => continue,
+        };
         match const_.val {
             ConstKind::Value(_) => {}
             ConstKind::Unevaluated(def, ref substs, promoted) => {
@@ -113,19 +116,17 @@ pub(crate) fn codegen_constant<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     constant: &Constant<'tcx>,
 ) -> CValue<'tcx> {
-    let const_ = fx.monomorphize(constant.literal);
+    let const_ = match fx.monomorphize(constant.literal) {
+        ConstantKind::Ty(ct) => ct,
+        ConstantKind::Val(val, ty) => return codegen_const_value(fx, val, ty),
+    };
     let const_val = match const_.val {
         ConstKind::Value(const_val) => const_val,
         ConstKind::Unevaluated(def, ref substs, promoted) if fx.tcx.is_static(def.did) => {
             assert!(substs.is_empty());
             assert!(promoted.is_none());
 
-            return codegen_static_ref(
-                fx,
-                def.did,
-                fx.layout_of(fx.monomorphize(&constant.literal.ty)),
-            )
-            .to_cvalue(fx);
+            return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty)).to_cvalue(fx);
         }
         ConstKind::Unevaluated(def, ref substs, promoted) => {
             match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None) {
@@ -422,11 +423,14 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
 pub(crate) fn mir_operand_get_const_val<'tcx>(
     fx: &FunctionCx<'_, '_, 'tcx>,
     operand: &Operand<'tcx>,
-) -> Option<&'tcx Const<'tcx>> {
+) -> Option<ConstValue<'tcx>> {
     match operand {
         Operand::Copy(_) | Operand::Move(_) => None,
-        Operand::Constant(const_) => {
-            Some(fx.monomorphize(const_.literal).eval(fx.tcx, ParamEnv::reveal_all()))
-        }
+        Operand::Constant(const_) => match const_.literal {
+            ConstantKind::Ty(const_) => {
+                fx.monomorphize(const_).eval(fx.tcx, ParamEnv::reveal_all()).val.try_to_value()
+            }
+            ConstantKind::Val(val, _) => Some(val),
+        },
     }
 }

compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs

@@ -53,7 +53,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         };
         llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) {
             let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
-            let flt_cc = match kind_const.val.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
+            let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
                 0 => FloatCC::Equal,
                 1 => FloatCC::LessThan,
                 2 => FloatCC::LessThanOrEqual,
@@ -84,7 +84,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         llvm.x86.sse2.psrli.d, (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
-                let res_lane = match imm8.val.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+                let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
                 };
@@ -94,7 +94,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
         llvm.x86.sse2.pslli.d, (c a, o imm8) {
             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
-                let res_lane = match imm8.val.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
+                let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
                     imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
                     _ => fx.bcx.ins().iconst(types::I32, 0),
                 };

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -85,8 +85,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 use rustc_middle::mir::interpret::*;
                 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
 
-                let idx_bytes = match idx_const.val {
-                    ty::ConstKind::Value(ConstValue::ByRef { alloc, offset }) => {
+                let idx_bytes = match idx_const {
+                    ConstValue::ByRef { alloc, offset } => {
                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
                         let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
                         alloc.get_bytes(fx, ptr, size).unwrap()
@@ -130,7 +130,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 );
             };
 
-            let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
             let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
@@ -159,7 +159,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 return;
             };
 
-            let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
             let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -231,7 +231,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
     fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) {
         let check = match terminator.kind {
             mir::TerminatorKind::Call { func: mir::Operand::Constant(ref c), ref args, .. } => {
-                match *c.literal.ty.kind() {
+                match *c.ty().kind() {
                     ty::FnDef(did, _) => Some((did, args)),
                     _ => None,
                 }

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -635,12 +635,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
                             if let mir::Operand::Constant(constant) = arg {
                                 let c = self.eval_mir_constant(constant);
-                                let (llval, ty) = self.simd_shuffle_indices(
-                                    &bx,
-                                    constant.span,
-                                    constant.literal.ty,
-                                    c,
-                                );
+                                let (llval, ty) =
+                                    self.simd_shuffle_indices(&bx, constant.span, constant.ty(), c);
                                 return OperandRef {
                                     val: Immediate(llval),
                                     layout: bx.layout_of(ty),
@@ -830,7 +826,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         let const_value = self
                             .eval_mir_constant(constant)
                             .unwrap_or_else(|_| span_bug!(span, "asm const cannot be resolved"));
-                        let ty = constant.literal.ty;
+                        let ty = constant.ty();
                         let size = bx.layout_of(ty).size;
                         let scalar = match const_value {
                             ConstValue::Scalar(s) => s,
@@ -864,7 +860,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
                 mir::InlineAsmOperand::SymFn { ref value } => {
                     let literal = self.monomorphize(value.literal);
-                    if let ty::FnDef(def_id, substs) = *literal.ty.kind() {
+                    if let ty::FnDef(def_id, substs) = *literal.ty().kind() {
                         let instance = ty::Instance::resolve_for_fn_ptr(
                             bx.tcx(),
                             ty::ParamEnv::reveal_all(),

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -16,15 +16,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         constant: &mir::Constant<'tcx>,
     ) -> Result<OperandRef<'tcx, Bx::Value>, ErrorHandled> {
         let val = self.eval_mir_constant(constant)?;
-        let ty = self.monomorphize(constant.literal.ty);
+        let ty = self.monomorphize(constant.ty());
         Ok(OperandRef::from_const(bx, val, ty))
     }
 
     pub fn eval_mir_constant(
         &self,
         constant: &mir::Constant<'tcx>,
     ) -> Result<ConstValue<'tcx>, ErrorHandled> {
-        match self.monomorphize(constant.literal).val {
+        let ct = self.monomorphize(constant.literal);
+        let ct = match ct {
+            mir::ConstantKind::Ty(ct) => ct,
+            mir::ConstantKind::Val(val, _) => return Ok(val),
+        };
+        match ct.val {
             ty::ConstKind::Unevaluated(def, substs, promoted) => self
                 .cx
                 .tcx()

compiler/rustc_codegen_ssa/src/mir/debuginfo.rs

@@ -372,7 +372,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         (var_ty, var_kind)
                     }
                     mir::VarDebugInfoContents::Const(c) => {
-                        let ty = self.monomorphize(c.literal.ty);
+                        let ty = self.monomorphize(c.ty());
                         (ty, VariableKind::LocalVariable)
                     }
                 };

compiler/rustc_middle/src/mir/interpret/value.rs

@@ -1,4 +1,4 @@
-use std::convert::TryFrom;
+use std::convert::{TryFrom, TryInto};
 use std::fmt;
 
 use rustc_apfloat::{
@@ -8,12 +8,12 @@ use rustc_apfloat::{
 use rustc_macros::HashStable;
 use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout};
 
-use crate::ty::{ParamEnv, ScalarInt, Ty, TyCtxt};
+use crate::ty::{Lift, ParamEnv, ScalarInt, Ty, TyCtxt};
 
 use super::{AllocId, Allocation, InterpResult, Pointer, PointerArithmetic};
 
 /// Represents the result of const evaluation via the `eval_to_allocation` query.
-#[derive(Clone, HashStable, TyEncodable, TyDecodable, Debug)]
+#[derive(Copy, Clone, HashStable, TyEncodable, TyDecodable, Debug, Hash, Eq, PartialEq)]
 pub struct ConstAlloc<'tcx> {
     // the value lives here, at offset 0, and that allocation definitely is a `AllocKind::Memory`
     // (so you can use `AllocMap::unwrap_memory`).
@@ -47,6 +47,27 @@ pub enum ConstValue<'tcx> {
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(ConstValue<'_>, 32);
 
+impl From<Scalar> for ConstValue<'tcx> {
+    fn from(s: Scalar) -> Self {
+        Self::Scalar(s)
+    }
+}
+
+impl<'a, 'tcx> Lift<'tcx> for ConstValue<'a> {
+    type Lifted = ConstValue<'tcx>;
+    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ConstValue<'tcx>> {
+        Some(match self {
+            ConstValue::Scalar(s) => ConstValue::Scalar(s),
+            ConstValue::Slice { data, start, end } => {
+                ConstValue::Slice { data: tcx.lift(data)?, start, end }
+            }
+            ConstValue::ByRef { alloc, offset } => {
+                ConstValue::ByRef { alloc: tcx.lift(alloc)?, offset }
+            }
+        })
+    }
+}
+
 impl<'tcx> ConstValue<'tcx> {
     #[inline]
     pub fn try_to_scalar(&self) -> Option<Scalar> {
@@ -56,20 +77,20 @@ impl<'tcx> ConstValue<'tcx> {
         }
     }
 
+    pub fn try_to_scalar_int(&self) -> Option<ScalarInt> {
+        Some(self.try_to_scalar()?.assert_int())
+    }
+
     pub fn try_to_bits(&self, size: Size) -> Option<u128> {
-        self.try_to_scalar()?.to_bits(size).ok()
+        self.try_to_scalar_int()?.to_bits(size).ok()
     }
 
     pub fn try_to_bool(&self) -> Option<bool> {
-        match self.try_to_bits(Size::from_bytes(1))? {
-            0 => Some(false),
-            1 => Some(true),
-            _ => None,
-        }
+        self.try_to_scalar_int()?.try_into().ok()
     }
 
     pub fn try_to_machine_usize(&self, tcx: TyCtxt<'tcx>) -> Option<u64> {
-        Some(self.try_to_bits(tcx.data_layout.pointer_size)? as u64)
+        self.try_to_scalar_int()?.try_to_machine_usize(tcx).ok()
     }
 
     pub fn try_to_bits_for_ty(
@@ -503,6 +524,13 @@ impl<Tag> From<Pointer<Tag>> for Scalar<Tag> {
     }
 }
 
+impl<Tag> From<ScalarInt> for Scalar<Tag> {
+    #[inline(always)]
+    fn from(ptr: ScalarInt) -> Self {
+        Scalar::Int(ptr)
+    }
+}
+
 #[derive(Clone, Copy, Eq, PartialEq, TyEncodable, TyDecodable, HashStable, Hash)]
 pub enum ScalarMaybeUninit<Tag = ()> {
     Scalar(Scalar<Tag>),