Skip to content

Commit 3a15d31

Browse files
committed
atomicrmw on pointers: move integer-pointer cast hacks into backend
1 parent ace4ac0 commit 3a15d31

File tree

8 files changed

+132
-101
lines changed

8 files changed

+132
-101
lines changed

compiler/rustc_codegen_ssa/messages.ftl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ codegen_ssa_invalid_monomorphization_basic_float_type = invalid monomorphization
9797
9898
codegen_ssa_invalid_monomorphization_basic_integer_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer type, found `{$ty}`
9999
100+
codegen_ssa_invalid_monomorphization_basic_integer_or_ptr_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer or pointer type, found `{$ty}`
101+
100102
codegen_ssa_invalid_monomorphization_cannot_return = invalid monomorphization of `{$name}` intrinsic: cannot return `{$ret_ty}`, expected `u{$expected_int_bits}` or `[u8; {$expected_bytes}]`
101103
102104
codegen_ssa_invalid_monomorphization_cast_wide_pointer = invalid monomorphization of `{$name}` intrinsic: cannot cast wide pointer `{$ty}`

compiler/rustc_codegen_ssa/src/errors.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -764,6 +764,14 @@ pub enum InvalidMonomorphization<'tcx> {
764764
ty: Ty<'tcx>,
765765
},
766766

767+
#[diag(codegen_ssa_invalid_monomorphization_basic_integer_or_ptr_type, code = E0511)]
768+
BasicIntegerOrPtrType {
769+
#[primary_span]
770+
span: Span,
771+
name: Symbol,
772+
ty: Ty<'tcx>,
773+
},
774+
767775
#[diag(codegen_ssa_invalid_monomorphization_basic_float_type, code = E0511)]
768776
BasicFloatType {
769777
#[primary_span]

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

Lines changed: 48 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
9292
let invalid_monomorphization_int_type = |ty| {
9393
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
9494
};
95+
let invalid_monomorphization_int_or_ptr_type = |ty| {
96+
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerOrPtrType {
97+
span,
98+
name,
99+
ty,
100+
});
101+
};
95102

96103
let parse_atomic_ordering = |ord: ty::Value<'tcx>| {
97104
let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
@@ -351,7 +358,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
351358
sym::atomic_load => {
352359
let ty = fn_args.type_at(0);
353360
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
354-
invalid_monomorphization_int_type(ty);
361+
invalid_monomorphization_int_or_ptr_type(ty);
355362
return Ok(());
356363
}
357364
let ordering = fn_args.const_at(1).to_value();
@@ -367,7 +374,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
367374
sym::atomic_store => {
368375
let ty = fn_args.type_at(0);
369376
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
370-
invalid_monomorphization_int_type(ty);
377+
invalid_monomorphization_int_or_ptr_type(ty);
371378
return Ok(());
372379
}
373380
let ordering = fn_args.const_at(1).to_value();
@@ -377,10 +384,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
377384
bx.atomic_store(val, ptr, parse_atomic_ordering(ordering), size);
378385
return Ok(());
379386
}
387+
// These are all AtomicRMW ops
380388
sym::atomic_cxchg | sym::atomic_cxchgweak => {
381389
let ty = fn_args.type_at(0);
382390
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
383-
invalid_monomorphization_int_type(ty);
391+
invalid_monomorphization_int_or_ptr_type(ty);
384392
return Ok(());
385393
}
386394
let succ_ordering = fn_args.const_at(1).to_value();
@@ -407,7 +415,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
407415

408416
return Ok(());
409417
}
410-
// These are all AtomicRMW ops
411418
sym::atomic_max | sym::atomic_min => {
412419
let atom_op = if name == sym::atomic_max {
413420
AtomicRmwBinOp::AtomicMax
@@ -444,15 +451,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
444451
return Ok(());
445452
}
446453
}
447-
sym::atomic_xchg
448-
| sym::atomic_xadd
454+
sym::atomic_xchg => {
455+
let ty = fn_args.type_at(0);
456+
let ordering = fn_args.const_at(fn_args.len() - 1).to_value();
457+
let ptr = args[0].immediate();
458+
let val = args[1].immediate();
459+
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
460+
let atomic_op = AtomicRmwBinOp::AtomicXchg;
461+
bx.atomic_rmw(atomic_op, ptr, val, parse_atomic_ordering(ordering))
462+
} else {
463+
invalid_monomorphization_int_or_ptr_type(ty);
464+
return Ok(());
465+
}
466+
}
467+
sym::atomic_xadd
449468
| sym::atomic_xsub
450469
| sym::atomic_and
451470
| sym::atomic_nand
452471
| sym::atomic_or
453472
| sym::atomic_xor => {
454473
let atom_op = match name {
455-
sym::atomic_xchg => AtomicRmwBinOp::AtomicXchg,
456474
sym::atomic_xadd => AtomicRmwBinOp::AtomicAdd,
457475
sym::atomic_xsub => AtomicRmwBinOp::AtomicSub,
458476
sym::atomic_and => AtomicRmwBinOp::AtomicAnd,
@@ -462,16 +480,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
462480
_ => unreachable!(),
463481
};
464482

465-
let ty = fn_args.type_at(0);
466-
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
467-
let ordering = fn_args.const_at(1).to_value();
468-
let ptr = args[0].immediate();
469-
let val = args[1].immediate();
470-
bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
471-
} else {
472-
invalid_monomorphization_int_type(ty);
473-
return Ok(());
474-
}
483+
// The type of the in-memory data.
484+
let ty_mem = fn_args.type_at(0);
485+
// The type of the 2nd operand, given by-value.
486+
let ty_op = fn_args.type_at(1);
487+
488+
let ordering = fn_args.const_at(2).to_value();
489+
let ptr = args[0].immediate(); // of type "pointer to `ty_mem`"
490+
let val = args[1].immediate(); // of type `ty_op`
491+
// We require either both arguments to have the same integer type, or the first to
492+
// be a pointer and the second to be `usize`.
493+
let val_adjusted =
494+
if int_type_width_signed(ty_mem, bx.tcx()).is_some() && ty_op == ty_mem {
495+
val
496+
} else if ty_mem.is_raw_ptr() && ty_op == bx.tcx().types.usize {
497+
// FIXME: LLVM does not support an atomic "add integer to pointer" operation, so
498+
// we need to instead add two pointers and hope that works out. See
499+
// <https://github.com/llvm/llvm-project/issues/120837>.
500+
bx.inttoptr(val, bx.backend_type(bx.layout_of(ty_mem)))
501+
} else {
502+
invalid_monomorphization_int_or_ptr_type(ty_mem);
503+
return Ok(());
504+
};
505+
bx.atomic_rmw(atom_op, ptr, val_adjusted, parse_atomic_ordering(ordering))
475506
}
476507
sym::atomic_fence => {
477508
let ordering = fn_args.const_at(0).to_value();

compiler/rustc_hir_analysis/src/check/intrinsic.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -652,16 +652,16 @@ pub(crate) fn check_intrinsic_type(
652652
sym::atomic_store => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
653653

654654
sym::atomic_xchg
655-
| sym::atomic_xadd
656-
| sym::atomic_xsub
657-
| sym::atomic_and
658-
| sym::atomic_nand
659-
| sym::atomic_or
660-
| sym::atomic_xor
661655
| sym::atomic_max
662656
| sym::atomic_min
663657
| sym::atomic_umax
664658
| sym::atomic_umin => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
659+
sym::atomic_xadd
660+
| sym::atomic_xsub
661+
| sym::atomic_and
662+
| sym::atomic_nand
663+
| sym::atomic_or
664+
| sym::atomic_xor => (2, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(1)], param(0)),
665665
sym::atomic_fence | sym::atomic_singlethreadfence => (0, 1, Vec::new(), tcx.types.unit),
666666

667667
other => {

library/core/src/intrinsics/mod.rs

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -150,69 +150,63 @@ pub unsafe fn atomic_xchg<T: Copy, const ORD: AtomicOrdering>(dst: *mut T, src:
150150

151151
/// Adds to the current value, returning the previous value.
152152
/// `T` must be an integer or pointer type.
153-
/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new
154-
/// value stored at `*dst` will have the provenance of the old value stored there.
153+
/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type.
155154
///
156155
/// The stabilized version of this intrinsic is available on the
157156
/// [`atomic`] types via the `fetch_add` method. For example, [`AtomicIsize::fetch_add`].
158157
#[rustc_intrinsic]
159158
#[rustc_nounwind]
160-
pub unsafe fn atomic_xadd<T: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
159+
pub unsafe fn atomic_xadd<T: Copy, U: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: U) -> T;
161160

162161
/// Subtract from the current value, returning the previous value.
163162
/// `T` must be an integer or pointer type.
164-
/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new
165-
/// value stored at `*dst` will have the provenance of the old value stored there.
163+
/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type.
166164
///
167165
/// The stabilized version of this intrinsic is available on the
168166
/// [`atomic`] types via the `fetch_sub` method. For example, [`AtomicIsize::fetch_sub`].
169167
#[rustc_intrinsic]
170168
#[rustc_nounwind]
171-
pub unsafe fn atomic_xsub<T: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
169+
pub unsafe fn atomic_xsub<T: Copy, U: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: U) -> T;
172170

173171
/// Bitwise and with the current value, returning the previous value.
174172
/// `T` must be an integer or pointer type.
175-
/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new
176-
/// value stored at `*dst` will have the provenance of the old value stored there.
173+
/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type.
177174
///
178175
/// The stabilized version of this intrinsic is available on the
179176
/// [`atomic`] types via the `fetch_and` method. For example, [`AtomicBool::fetch_and`].
180177
#[rustc_intrinsic]
181178
#[rustc_nounwind]
182-
pub unsafe fn atomic_and<T: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
179+
pub unsafe fn atomic_and<T: Copy, U: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: U) -> T;
183180

184181
/// Bitwise nand with the current value, returning the previous value.
185182
/// `T` must be an integer or pointer type.
186-
/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new
187-
/// value stored at `*dst` will have the provenance of the old value stored there.
183+
/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type.
188184
///
189185
/// The stabilized version of this intrinsic is available on the
190186
/// [`AtomicBool`] type via the `fetch_nand` method. For example, [`AtomicBool::fetch_nand`].
191187
#[rustc_intrinsic]
192188
#[rustc_nounwind]
193-
pub unsafe fn atomic_nand<T: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
189+
pub unsafe fn atomic_nand<T: Copy, U: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: U) -> T;
194190

195191
/// Bitwise or with the current value, returning the previous value.
196192
/// `T` must be an integer or pointer type.
197-
/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new
198-
/// value stored at `*dst` will have the provenance of the old value stored there.
193+
/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type.
199194
///
200195
/// The stabilized version of this intrinsic is available on the
201196
/// [`atomic`] types via the `fetch_or` method. For example, [`AtomicBool::fetch_or`].
202197
#[rustc_intrinsic]
203198
#[rustc_nounwind]
204-
pub unsafe fn atomic_or<T: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
199+
pub unsafe fn atomic_or<T: Copy, U: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: U) -> T;
205200

206201
/// Bitwise xor with the current value, returning the previous value.
207202
/// `T` must be an integer or pointer type.
208-
/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new
209-
/// value stored at `*dst` will have the provenance of the old value stored there.
203+
/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type.
210204
///
211205
/// The stabilized version of this intrinsic is available on the
212206
/// [`atomic`] types via the `fetch_xor` method. For example, [`AtomicBool::fetch_xor`].
213207
#[rustc_intrinsic]
214208
#[rustc_nounwind]
215-
pub unsafe fn atomic_xor<T: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
209+
pub unsafe fn atomic_xor<T: Copy, U: Copy, const ORD: AtomicOrdering>(dst: *mut T, src: U) -> T;
216210

217211
/// Maximum with the current value using a signed comparison.
218212
/// `T` must be a signed integer type.

0 commit comments

Comments
 (0)