@@ -92,6 +92,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
92
92
let invalid_monomorphization_int_type = |ty| {
93
93
bx. tcx ( ) . dcx ( ) . emit_err ( InvalidMonomorphization :: BasicIntegerType { span, name, ty } ) ;
94
94
} ;
95
+ let invalid_monomorphization_int_or_ptr_type = |ty| {
96
+ bx. tcx ( ) . dcx ( ) . emit_err ( InvalidMonomorphization :: BasicIntegerOrPtrType {
97
+ span,
98
+ name,
99
+ ty,
100
+ } ) ;
101
+ } ;
95
102
96
103
let parse_atomic_ordering = |ord : ty:: Value < ' tcx > | {
97
104
let discr = ord. valtree . unwrap_branch ( ) [ 0 ] . unwrap_leaf ( ) ;
@@ -351,7 +358,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
351
358
sym:: atomic_load => {
352
359
let ty = fn_args. type_at ( 0 ) ;
353
360
if !( int_type_width_signed ( ty, bx. tcx ( ) ) . is_some ( ) || ty. is_raw_ptr ( ) ) {
354
- invalid_monomorphization_int_type ( ty) ;
361
+ invalid_monomorphization_int_or_ptr_type ( ty) ;
355
362
return Ok ( ( ) ) ;
356
363
}
357
364
let ordering = fn_args. const_at ( 1 ) . to_value ( ) ;
@@ -367,7 +374,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
367
374
sym:: atomic_store => {
368
375
let ty = fn_args. type_at ( 0 ) ;
369
376
if !( int_type_width_signed ( ty, bx. tcx ( ) ) . is_some ( ) || ty. is_raw_ptr ( ) ) {
370
- invalid_monomorphization_int_type ( ty) ;
377
+ invalid_monomorphization_int_or_ptr_type ( ty) ;
371
378
return Ok ( ( ) ) ;
372
379
}
373
380
let ordering = fn_args. const_at ( 1 ) . to_value ( ) ;
@@ -377,10 +384,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
377
384
bx. atomic_store ( val, ptr, parse_atomic_ordering ( ordering) , size) ;
378
385
return Ok ( ( ) ) ;
379
386
}
387
+ // These are all AtomicRMW ops
380
388
sym:: atomic_cxchg | sym:: atomic_cxchgweak => {
381
389
let ty = fn_args. type_at ( 0 ) ;
382
390
if !( int_type_width_signed ( ty, bx. tcx ( ) ) . is_some ( ) || ty. is_raw_ptr ( ) ) {
383
- invalid_monomorphization_int_type ( ty) ;
391
+ invalid_monomorphization_int_or_ptr_type ( ty) ;
384
392
return Ok ( ( ) ) ;
385
393
}
386
394
let succ_ordering = fn_args. const_at ( 1 ) . to_value ( ) ;
@@ -407,7 +415,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
407
415
408
416
return Ok ( ( ) ) ;
409
417
}
410
- // These are all AtomicRMW ops
411
418
sym:: atomic_max | sym:: atomic_min => {
412
419
let atom_op = if name == sym:: atomic_max {
413
420
AtomicRmwBinOp :: AtomicMax
@@ -444,15 +451,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
444
451
return Ok ( ( ) ) ;
445
452
}
446
453
}
447
- sym:: atomic_xchg
448
- | sym:: atomic_xadd
454
+ sym:: atomic_xchg => {
455
+ let ty = fn_args. type_at ( 0 ) ;
456
+ let ordering = fn_args. const_at ( fn_args. len ( ) - 1 ) . to_value ( ) ;
457
+ let ptr = args[ 0 ] . immediate ( ) ;
458
+ let val = args[ 1 ] . immediate ( ) ;
459
+ if int_type_width_signed ( ty, bx. tcx ( ) ) . is_some ( ) || ty. is_raw_ptr ( ) {
460
+ let atomic_op = AtomicRmwBinOp :: AtomicXchg ;
461
+ bx. atomic_rmw ( atomic_op, ptr, val, parse_atomic_ordering ( ordering) )
462
+ } else {
463
+ invalid_monomorphization_int_or_ptr_type ( ty) ;
464
+ return Ok ( ( ) ) ;
465
+ }
466
+ }
467
+ sym:: atomic_xadd
449
468
| sym:: atomic_xsub
450
469
| sym:: atomic_and
451
470
| sym:: atomic_nand
452
471
| sym:: atomic_or
453
472
| sym:: atomic_xor => {
454
473
let atom_op = match name {
455
- sym:: atomic_xchg => AtomicRmwBinOp :: AtomicXchg ,
456
474
sym:: atomic_xadd => AtomicRmwBinOp :: AtomicAdd ,
457
475
sym:: atomic_xsub => AtomicRmwBinOp :: AtomicSub ,
458
476
sym:: atomic_and => AtomicRmwBinOp :: AtomicAnd ,
@@ -462,16 +480,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
462
480
_ => unreachable ! ( ) ,
463
481
} ;
464
482
465
- let ty = fn_args. type_at ( 0 ) ;
466
- if int_type_width_signed ( ty, bx. tcx ( ) ) . is_some ( ) || ty. is_raw_ptr ( ) {
467
- let ordering = fn_args. const_at ( 1 ) . to_value ( ) ;
468
- let ptr = args[ 0 ] . immediate ( ) ;
469
- let val = args[ 1 ] . immediate ( ) ;
470
- bx. atomic_rmw ( atom_op, ptr, val, parse_atomic_ordering ( ordering) )
471
- } else {
472
- invalid_monomorphization_int_type ( ty) ;
473
- return Ok ( ( ) ) ;
474
- }
483
+ // The type of the in-memory data.
484
+ let ty_mem = fn_args. type_at ( 0 ) ;
485
+ // The type of the 2nd operand, given by-value.
486
+ let ty_op = fn_args. type_at ( 1 ) ;
487
+
488
+ let ordering = fn_args. const_at ( 2 ) . to_value ( ) ;
489
+ let ptr = args[ 0 ] . immediate ( ) ; // of type "pointer to `ty_mem`"
490
+ let val = args[ 1 ] . immediate ( ) ; // of type `ty_op`
491
+ // We require either both arguments to have the same integer type, or the first to
492
+ // be a pointer and the second to be `usize`.
493
+ let val_adjusted =
494
+ if int_type_width_signed ( ty_mem, bx. tcx ( ) ) . is_some ( ) && ty_op == ty_mem {
495
+ val
496
+ } else if ty_mem. is_raw_ptr ( ) && ty_op == bx. tcx ( ) . types . usize {
497
+ // FIXME: LLVM does not support an atomic "add integer to pointer" operation, so
498
+ // we need to instead add two pointers and hope that works out. See
499
+ // <https://github.com/llvm/llvm-project/issues/120837>.
500
+ bx. inttoptr ( val, bx. backend_type ( bx. layout_of ( ty_mem) ) )
501
+ } else {
502
+ invalid_monomorphization_int_or_ptr_type ( ty_mem) ;
503
+ return Ok ( ( ) ) ;
504
+ } ;
505
+ bx. atomic_rmw ( atom_op, ptr, val_adjusted, parse_atomic_ordering ( ordering) )
475
506
}
476
507
sym:: atomic_fence => {
477
508
let ordering = fn_args. const_at ( 0 ) . to_value ( ) ;
0 commit comments