coverage. Adapt to llvm optimization for conditions limits

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1719,9 +1719,9 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
-        bitmap_bytes: &'ll Value,
+        bitmap_bits: &'ll Value,
     ) {
-        debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes);
+        debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bits);
 
         assert!(llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later");
 
@@ -1730,7 +1730,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()],
             self.cx.type_void(),
         );
-        let args = &[fn_name, hash, bitmap_bytes];
+        let args = &[fn_name, hash, bitmap_bits];
         let args = self.check_call("call", llty, llfn, args);
 
         unsafe {
@@ -1750,29 +1750,22 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
-        bitmap_bytes: &'ll Value,
         bitmap_index: &'ll Value,
         mcdc_temp: &'ll Value,
     ) {
         debug!(
-            "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})",
-            fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp
+            "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?})",
+            fn_name, hash, bitmap_index, mcdc_temp
         );
         assert!(llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later");
 
         let llfn =
             unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) };
         let llty = self.cx.type_func(
-            &[
-                self.cx.type_ptr(),
-                self.cx.type_i64(),
-                self.cx.type_i32(),
-                self.cx.type_i32(),
-                self.cx.type_ptr(),
-            ],
+            &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_ptr()],
             self.cx.type_void(),
         );
-        let args = &[fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp];
+        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
         let args = self.check_call("call", llty, llfn, args);
         unsafe {
             let _ = llvm::LLVMRustBuildCall(
@@ -1792,38 +1785,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
-        cond_loc: &'ll Value,
+        cond_index: &'ll Value,
         mcdc_temp: &'ll Value,
-        bool_value: &'ll Value,
     ) {
         debug!(
-            "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})",
-            fn_name, hash, cond_loc, mcdc_temp, bool_value
+            "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?})",
+            fn_name, hash, cond_index, mcdc_temp
         );
         assert!(llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later");
-        let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) };
-        let llty = self.cx.type_func(
-            &[
-                self.cx.type_ptr(),
-                self.cx.type_i64(),
-                self.cx.type_i32(),
-                self.cx.type_ptr(),
-                self.cx.type_i1(),
-            ],
-            self.cx.type_void(),
-        );
-        let args = &[fn_name, hash, cond_loc, mcdc_temp, bool_value];
-        self.check_call("call", llty, llfn, args);
-        unsafe {
-            let _ = llvm::LLVMRustBuildCall(
-                self.llbuilder,
-                llty,
-                llfn,
-                args.as_ptr() as *const &llvm::Value,
-                args.len() as c_uint,
-                [].as_ptr(),
-                0 as c_uint,
-            );
-        }
+        let align = self.tcx.data_layout.i32_align.abi;
+        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
+        let new_tv_index = self.add(current_tv_index, cond_index);
+        self.store(new_tv_index, mcdc_temp, align);
     }
 }

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -100,14 +100,14 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
         };
 
         // If there are no MC/DC bitmaps to set up, return immediately.
-        if function_coverage_info.mcdc_bitmap_bytes == 0 {
+        if function_coverage_info.mcdc_bitmap_bits == 0 {
             return;
         }
 
         let fn_name = self.get_pgo_func_name_var(instance);
         let hash = self.const_u64(function_coverage_info.function_source_hash);
-        let bitmap_bytes = self.const_u32(function_coverage_info.mcdc_bitmap_bytes);
-        self.mcdc_parameters(fn_name, hash, bitmap_bytes);
+        let bitmap_bits = self.const_u32(function_coverage_info.mcdc_bitmap_bits as u32);
+        self.mcdc_parameters(fn_name, hash, bitmap_bits);
 
         // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps.
         let mut cond_bitmaps = vec![];
@@ -187,35 +187,30 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
             CoverageKind::ExpressionUsed { id } => {
                 func_coverage.mark_expression_id_seen(id);
             }
-            CoverageKind::CondBitmapUpdate { id, value, decision_depth } => {
+            CoverageKind::CondBitmapUpdate { index, decision_depth } => {
                 drop(coverage_map);
-                assert_ne!(
-                    id.as_u32(),
-                    0,
-                    "ConditionId of evaluated conditions should never be zero"
-                );
                 let cond_bitmap = coverage_context
                     .try_get_mcdc_condition_bitmap(&instance, decision_depth)
                     .expect("mcdc cond bitmap should have been allocated for updating");
-                let cond_loc = bx.const_i32(id.as_u32() as i32 - 1);
-                let bool_value = bx.const_bool(value);
+                let cond_index = bx.const_i32(index as i32);
                 let fn_name = bx.get_pgo_func_name_var(instance);
                 let hash = bx.const_u64(function_coverage_info.function_source_hash);
-                bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value);
+                bx.mcdc_condbitmap_update(fn_name, hash, cond_index, cond_bitmap);
             }
             CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => {
                 drop(coverage_map);
                 let cond_bitmap = coverage_context
                                     .try_get_mcdc_condition_bitmap(&instance, decision_depth)
                                     .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap");
-                let bitmap_bytes = function_coverage_info.mcdc_bitmap_bytes;
-                assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range");
+                assert!(
+                    bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits,
+                    "bitmap index of the decision out of range"
+                );
 
                 let fn_name = bx.get_pgo_func_name_var(instance);
                 let hash = bx.const_u64(function_coverage_info.function_source_hash);
-                let bitmap_bytes = bx.const_u32(bitmap_bytes);
                 let bitmap_index = bx.const_u32(bitmap_idx);
-                bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_bytes, bitmap_index, cond_bitmap);
+                bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap);
             }
         }
     }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1635,7 +1635,6 @@ extern "C" {
     pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;
     pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value;
     pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value;
-    pub fn LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(M: &Module) -> &Value;
 
     pub fn LLVMRustBuildCall<'a>(
         B: &Builder<'a>,

compiler/rustc_middle/src/mir/coverage.rs

@@ -136,8 +136,8 @@ pub enum CoverageKind {
 
     /// Marks the point in MIR control flow represented by a evaluated condition.
     ///
-    /// This is eventually lowered to `llvm.instrprof.mcdc.condbitmap.update` in LLVM IR.
-    CondBitmapUpdate { id: ConditionId, value: bool, decision_depth: u16 },
+    /// This is eventually lowered to instruments updating mcdc temp variables.
+    CondBitmapUpdate { index: u32, decision_depth: u16 },
 
     /// Marks the point in MIR control flow represented by a evaluated decision.
     ///
@@ -153,14 +153,8 @@ impl Debug for CoverageKind {
             BlockMarker { id } => write!(fmt, "BlockMarker({:?})", id.index()),
             CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()),
             ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()),
-            CondBitmapUpdate { id, value, decision_depth } => {
-                write!(
-                    fmt,
-                    "CondBitmapUpdate({:?}, {:?}, depth={:?})",
-                    id.index(),
-                    value,
-                    decision_depth
-                )
+            CondBitmapUpdate { index, decision_depth } => {
+                write!(fmt, "CondBitmapUpdate(index={:?}, depth={:?})", index, decision_depth)
             }
             TestVectorBitmapUpdate { bitmap_idx, decision_depth } => {
                 write!(fmt, "TestVectorUpdate({:?}, depth={:?})", bitmap_idx, decision_depth)
@@ -274,7 +268,7 @@ pub struct Mapping {
 pub struct FunctionCoverageInfo {
     pub function_source_hash: u64,
     pub num_counters: usize,
-    pub mcdc_bitmap_bytes: u32,
+    pub mcdc_bitmap_bits: usize,
     pub expressions: IndexVec<ExpressionId, Expression>,
     pub mappings: Vec<Mapping>,
     /// The depth of the deepest decision is used to know how many
@@ -317,6 +311,14 @@ pub struct MCDCBranchSpan {
     pub span: Span,
     pub condition_info: ConditionInfo,
     pub markers: MCDCBranchMarkers,
+    pub false_index: usize,
+    pub true_index: usize,
+}
+
+impl MCDCBranchSpan {
+    pub fn new(span: Span, condition_info: ConditionInfo, markers: MCDCBranchMarkers) -> Self {
+        Self { span, condition_info, markers, false_index: usize::MAX, true_index: usize::MAX }
+    }
 }
 
 #[derive(Clone, Debug)]
@@ -340,4 +342,5 @@ pub struct MCDCDecisionSpan {
     pub span: Span,
     pub end_markers: Vec<BlockMarkerId>,
     pub decision_depth: u16,
+    pub num_test_vectors: usize,
 }

compiler/rustc_middle/src/mir/pretty.rs

@@ -503,18 +503,22 @@ fn write_coverage_branch_info(
         )?;
     }
 
-    for (coverage::MCDCDecisionSpan { span, end_markers, decision_depth }, conditions) in mcdc_spans
+    for (
+        coverage::MCDCDecisionSpan { span, end_markers, decision_depth, num_test_vectors },
+        conditions,
+    ) in mcdc_spans
     {
         let num_conditions = conditions.len();
         writeln!(
             w,
-            "{INDENT}coverage mcdc decision {{ num_conditions: {num_conditions:?}, end: {end_markers:?}, depth: {decision_depth:?} }} => {span:?}"
+            "{INDENT}coverage mcdc decision {{ num_conditions: {num_conditions:?}, end: {end_markers:?}, depth: {decision_depth:?}, num_test_vectors: {num_test_vectors} }} => {span:?}"
         )?;
-        for coverage::MCDCBranchSpan { span, condition_info, markers } in conditions {
+        for coverage::MCDCBranchSpan { span, condition_info, markers, false_index, true_index } in
+            conditions
+        {
             writeln!(
                 w,
-                "{INDENT}coverage mcdc branch {{ condition_id: {:?}, markers: {markers:?} }} => {span:?}",
-                condition_info.condition_id
+                "{INDENT}coverage mcdc branch {{ condition_info: {condition_info:?}, markers: {markers:?}, indices: {{ false: {false_index}, true: {true_index} }} }} => {span:?}",
             )?;
         }
     }

compiler/rustc_mir_build/src/build/coverageinfo/mcdc.rs

@@ -1,6 +1,7 @@
 use std::collections::VecDeque;
 
 use rustc_data_structures::fx::FxIndexMap;
+use rustc_index::IndexVec;
 use rustc_middle::bug;
 use rustc_middle::mir::coverage::{
     BlockMarkerId, ConditionId, ConditionInfo, DecisionId, MCDCBranchMarkers, MCDCBranchSpan,
@@ -14,10 +15,9 @@ use rustc_span::Span;
 use crate::build::Builder;
 use crate::errors::{MCDCExceedsConditionLimit, MCDCExceedsDecisionDepth};
 
-/// The MCDC bitmap scales exponentially (2^n) based on the number of conditions seen,
-/// So llvm sets a maximum value prevents the bitmap footprint from growing too large without the user's knowledge.
-/// This limit may be relaxed if the [upstream change](https://github.com/llvm/llvm-project/pull/82448) is merged.
-const MAX_CONDITIONS_IN_DECISION: usize = 6;
+/// LLVM uses `i16` to represent condition id. Hence `i16::MAX` is the hard limit for number of
+/// conditions in a decision.
+const MAX_CONDITIONS_IN_DECISION: usize = i16::MAX as usize;
 
 /// MCDC allocates an i32 variable on stack for each depth. Ignore decisions nested too much to prevent it
 /// consuming excessive memory.
@@ -41,6 +41,7 @@ impl BooleanDecisionCtx {
                 span: Span::default(),
                 end_markers: vec![],
                 decision_depth: 0,
+                num_test_vectors: 0,
             },
             decision_stack: VecDeque::new(),
             conditions: vec![],
@@ -158,11 +159,11 @@ impl BooleanDecisionCtx {
             self.decision_info.end_markers.push(false_marker);
         }
 
-        self.conditions.push(MCDCBranchSpan {
+        self.conditions.push(MCDCBranchSpan::new(
             span,
             condition_info,
-            markers: MCDCBranchMarkers::Boolean(true_marker, false_marker),
-        });
+            MCDCBranchMarkers::Boolean(true_marker, false_marker),
+        ));
     }
 
     fn is_finished(&self) -> bool {
@@ -257,9 +258,86 @@ struct MCDCTargetInfo {
 }
 
 impl MCDCTargetInfo {
+    fn new(decision: MCDCDecisionSpan, conditions: Vec<MCDCBranchSpan>) -> Self {
+        let mut this = Self { decision, conditions, nested_decisions_id: vec![] };
+        this.calc_test_vectors_index();
+        this
+    }
+
     fn set_depth(&mut self, depth: u16) {
         self.decision.decision_depth = depth;
     }
+
+    // LLVM checks the executed test vector by accumulate indices of tested branches.
+    // We calculate number of all possible test vectors of the decision and assign indices
+    // for each branch here.
+    // See https://discourse.llvm.org/t/rfc-coverage-new-algorithm-and-file-format-for-mc-dc/76798/ for
+    // more details of the algorithm.
+    // The process of this function is mostly like `TVIdxBuilder` at
+    // https://github.com/llvm/llvm-project/blob/d594d9f7f4dc6eb748b3261917db689fdc348b96/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp#L226
+    fn calc_test_vectors_index(&mut self) {
+        let Self { decision, conditions, .. } = self;
+        let mut indegree_stats = IndexVec::<ConditionId, usize>::from_elem_n(0, conditions.len());
+        // `num_paths` is `width` described at the llvm RFC, which indicates how many paths reaching the condition.
+        let mut num_paths_stats = IndexVec::<ConditionId, usize>::from_elem_n(0, conditions.len());
+        let mut next_conditions = conditions
+            .iter_mut()
+            .map(|branch| {
+                let ConditionInfo { condition_id, true_next_id, false_next_id } =
+                    branch.condition_info;
+                [true_next_id, false_next_id]
+                    .into_iter()
+                    .filter_map(std::convert::identity)
+                    .for_each(|next_id| indegree_stats[next_id] += 1);
+                (condition_id, branch)
+            })
+            .collect::<FxIndexMap<_, _>>();
+
+        let mut queue =
+            VecDeque::from_iter(next_conditions.swap_remove(&ConditionId::START).into_iter());
+        num_paths_stats[ConditionId::START] = 1;
+        let mut decision_end_nodes = Vec::new();
+        while let Some(branch) = queue.pop_front() {
+            let MCDCBranchSpan {
+                span: _,
+                condition_info: ConditionInfo { condition_id, true_next_id, false_next_id },
+                markers: _,
+                false_index,
+                true_index,
+            } = branch;
+            let this_paths_count = num_paths_stats[*condition_id];
+            for (next, index) in [(false_next_id, false_index), (true_next_id, true_index)] {
+                if let Some(next_id) = next {
+                    let next_paths_count = &mut num_paths_stats[*next_id];
+                    *index = *next_paths_count;
+                    *next_paths_count = next_paths_count.saturating_add(this_paths_count);
+                    let next_indegree = &mut indegree_stats[*next_id];
+                    *next_indegree -= 1;
+                    if *next_indegree == 0 {
+                        queue.push_back(next_conditions.swap_remove(next_id).expect(
+                            "conditions with non-zero indegree before must be in next_conditions",
+                        ));
+                    }
+                } else {
+                    decision_end_nodes.push((this_paths_count, *condition_id, index));
+                }
+            }
+        }
+        assert!(next_conditions.is_empty(), "the decision tree has untouched nodes");
+        let mut cur_idx = 0;
+        // LLVM hopes the end nodes is sorted in ascending order by `num_paths`.
+        decision_end_nodes.sort_by_key(|(num_paths, _, _)| usize::MAX - *num_paths);
+        for (num_paths, condition_id, index) in decision_end_nodes {
+            assert_eq!(
+                num_paths, num_paths_stats[condition_id],
+                "end nodes should not be updated since they were visited"
+            );
+            assert_eq!(*index, usize::MAX, "end nodes should not be assigned index before");
+            *index = cur_idx;
+            cur_idx += num_paths;
+        }
+        decision.num_test_vectors = cur_idx;
+    }
 }
 
 #[derive(Default)]
@@ -323,7 +401,7 @@ impl MCDCInfoBuilder {
             }
             // Ignore decisions with only one condition given that mcdc for them is completely equivalent to branch coverage.
             2..=MAX_CONDITIONS_IN_DECISION => {
-                let info = MCDCTargetInfo { decision, conditions, nested_decisions_id: vec![] };
+                let info = MCDCTargetInfo::new(decision, conditions);
                 Some(self.mcdc_targets.entry(id).or_insert(info))
             }
             _ => {