Skip to content

Commit

Permalink
Extract common logic for converting vector masks to i1 vectors and ma…
Browse files Browse the repository at this point in the history 
…ke it use the sign bit instead of lowest bit consistently
  • Loading branch information
@jhorstmann
jhorstmann committed Feb 28, 2023
1 parent b8b2b8d commit 9dd9391
Showing 1 changed file with 50 additions and 57 deletions.
107 changes: 50 additions & 57 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -854,6 +854,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
};
}

fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
bx: &mut Builder<'a, 'll, 'tcx>,
i_xn: &'ll Value,
in_elem_bitwidth: u64,
in_len: u64,
) -> &'ll Value {
// Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
let shift_indices = vec![shift_idx; in_len as _];
let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
// Truncate vector to an <i1 x N>
bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
}

let tcx = bx.tcx();
let sig =
tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx));
Expand Down Expand Up @@ -1061,14 +1075,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
m_len == v_len,
InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
);
match m_elem_ty.kind() {
ty::Int(_) => {}
let in_elem_bitwidth = match m_elem_ty.kind() {
ty::Int(i) => i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()),
_ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }),
}
// truncate the mask to a vector of i1s
let i1 = bx.type_i1();
let i1xn = bx.type_vector(i1, m_len as u64);
let m_i1s = bx.trunc(args[0].immediate(), i1xn);
};
let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
}

Expand Down Expand Up @@ -1103,15 +1114,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
}),
};

// Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
let shift_indices =
vec![
bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
in_len as _
];
let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
// Truncate vector to an <i1 x N>
let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len));
let i1xn = vector_mask_to_bitmask(bx, i_xn, in_elem_bitwidth, in_len);
// Bitcast <i1 x N> to iN:
let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));

Expand Down Expand Up @@ -1377,31 +1380,25 @@ fn generic_simd_intrinsic<'ll, 'tcx>(

// The element type of the third argument must be a signed integer type of any width:
let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
match element_ty2.kind() {
ty::Int(_) => (),
let mask_elem_bitwidth = match element_ty2.kind() {
ty::Int(i) => i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()),
_ => {
require!(
false,
InvalidMonomorphization::ThirdArgElementType {
span,
name,
expected_element: element_ty2,
third_arg: arg_tys[2]
}
);
return_error!(InvalidMonomorphization::ThirdArgElementType {
span,
name,
expected_element: element_ty2,
third_arg: arg_tys[2]
})
}
}
};

// Alignment of T, must be a constant integer value:
let alignment_ty = bx.type_i32();
let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);

// Truncate the mask vector to a vector of i1s:
let (mask, mask_ty) = {
let i1 = bx.type_i1();
let i1xn = bx.type_vector(i1, in_len);
(bx.trunc(args[2].immediate(), i1xn), i1xn)
};
let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
let mask_ty = bx.type_vector(bx.type_i1(), in_len);

// Type of the vector of pointers:
let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count);
Expand Down Expand Up @@ -1518,31 +1515,25 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
assert_eq!(underlying_ty, non_ptr(element_ty0));

// The element type of the third argument must be a signed integer type of any width:
match element_ty2.kind() {
ty::Int(_) => (),
let mask_elem_bitwidth = match element_ty2.kind() {
ty::Int(i) => i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()),
_ => {
require!(
false,
InvalidMonomorphization::ThirdArgElementType {
span,
name,
expected_element: element_ty2,
third_arg: arg_tys[2]
}
);
return_error!(InvalidMonomorphization::ThirdArgElementType {
span,
name,
expected_element: element_ty2,
third_arg: arg_tys[2]
});
}
}
};

// Alignment of T, must be a constant integer value:
let alignment_ty = bx.type_i32();
let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);

// Truncate the mask vector to a vector of i1s:
let (mask, mask_ty) = {
let i1 = bx.type_i1();
let i1xn = bx.type_vector(i1, in_len);
(bx.trunc(args[2].immediate(), i1xn), i1xn)
};
let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
let mask_ty = bx.type_vector(bx.type_i1(), in_len);

let ret_t = bx.type_void();

Expand Down Expand Up @@ -1682,8 +1673,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
);
args[0].immediate()
} else {
match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {}
let bitwidth = match in_elem.kind() {
ty::Int(i) => {
i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
}
ty::Uint(i) => {
i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
}
_ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
span,
name,
Expand All @@ -1692,12 +1688,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
in_elem,
ret_ty
}),
}
};

// boolean reductions operate on vectors of i1s:
let i1 = bx.type_i1();
let i1xn = bx.type_vector(i1, in_len as u64);
bx.trunc(args[0].immediate(), i1xn)
vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
};
return match in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
Expand Down

0 comments on commit 9dd9391

Please sign in to comment.