Skip to content

Commit e2df757

Browse files
author
Pavel Samolysov
committed
[SYCL][CUDA] Add __spirv_Atomic operations for generic address space
1 parent 9c4dc1e commit e2df757

File tree

8 files changed

+169
-94
lines changed

8 files changed

+169
-94
lines changed

‎libclc/generic/include/spirv/atomic/atomic_load.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,15 @@
77
//===----------------------------------------------------------------------===//
88

99
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
10-
#define DECL(TYPE, TYPE_MANGLED, AS, AS_MANGLED) \
11-
_CLC_DECL TYPE \
12-
_Z18__spirv_AtomicLoadPU3##AS_MANGLED##K##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE( \
10+
#define DECL(TYPE, TYPE_MANGLED, AS_PREFIX, AS, AS_MANGLED) \
11+
_CLC_DECL TYPE \
12+
_Z18__spirv_AtomicLoadP##AS_PREFIX##AS_MANGLED##K##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE( \
1313
volatile AS const TYPE *, enum Scope, enum MemorySemanticsMask);
1414

15-
#define DECL_AS(TYPE, TYPE_MANGLED) \
16-
DECL(TYPE, TYPE_MANGLED, global, AS1) \
17-
DECL(TYPE, TYPE_MANGLED, local, AS3)
15+
#define DECL_AS(TYPE, TYPE_MANGLED) \
16+
DECL(TYPE, TYPE_MANGLED, U3, global, AS1) \
17+
DECL(TYPE, TYPE_MANGLED, U3, local, AS3) \
18+
DECL(TYPE, TYPE_MANGLED, , , )
1819

1920
DECL_AS(int, i)
2021
DECL_AS(unsigned int, j)

‎libclc/generic/include/spirv/atomic/atomic_store.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,15 @@
77
//===----------------------------------------------------------------------===//
88

99
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
10-
#define DECL(TYPE, TYPE_MANGLED, AS, AS_MANGLED) \
11-
_CLC_DECL void \
12-
_Z19__spirv_AtomicStorePU3##AS_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE##TYPE_MANGLED( \
10+
#define DECL(TYPE, TYPE_MANGLED, AS_PREFIX, AS, AS_MANGLED) \
11+
_CLC_DECL void \
12+
_Z19__spirv_AtomicStoreP##AS_PREFIX##AS_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE##TYPE_MANGLED( \
1313
volatile AS TYPE *, enum Scope, enum MemorySemanticsMask, TYPE);
1414

15-
#define DECL_AS(TYPE, TYPE_MANGLED) \
16-
DECL(TYPE, TYPE_MANGLED, global, AS1) \
17-
DECL(TYPE, TYPE_MANGLED, local, AS3)
15+
#define DECL_AS(TYPE, TYPE_MANGLED) \
16+
DECL(TYPE, TYPE_MANGLED, U3, global, AS1) \
17+
DECL(TYPE, TYPE_MANGLED, U3, local, AS3) \
18+
DECL(TYPE, TYPE_MANGLED, , , )
1819

1920
DECL_AS(int, i)
2021
DECL_AS(unsigned int, j)

‎libclc/generic/libspirv/atomic/atomic_cmpxchg.cl

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,13 @@ _Z29__spirv_AtomicCompareExchangePU3AS1iN5__spv5Scope4FlagENS1_19MemorySemantics
2424
return __sync_val_compare_and_swap(p, cmp, val);
2525
}
2626

27+
_CLC_DEF int
28+
_Z29__spirv_AtomicCompareExchangePiN5__spv5Scope4FlagENS0_19MemorySemanticsMask4FlagES4_ii(
29+
volatile int *p, enum Scope scope, enum MemorySemanticsMask eq,
30+
enum MemorySemanticsMask neq, int val, int cmp) {
31+
return __sync_val_compare_and_swap(p, cmp, val);
32+
}
33+
2734
_CLC_DEF uint
2835
_Z29__spirv_AtomicCompareExchangePU3AS3jN5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagES5_jj(
2936
volatile local uint *p, enum Scope scope, enum MemorySemanticsMask eq,
@@ -38,6 +45,13 @@ _Z29__spirv_AtomicCompareExchangePU3AS1jN5__spv5Scope4FlagENS1_19MemorySemantics
3845
return __sync_val_compare_and_swap(p, cmp, val);
3946
}
4047

48+
_CLC_DEF uint
49+
_Z29__spirv_AtomicCompareExchangePjN5__spv5Scope4FlagENS0_19MemorySemanticsMask4FlagES4_jj(
50+
volatile uint *p, enum Scope scope, enum MemorySemanticsMask eq,
51+
enum MemorySemanticsMask neq, uint val, uint cmp) {
52+
return __sync_val_compare_and_swap(p, cmp, val);
53+
}
54+
4155
#ifdef cl_khr_int64_base_atomics
4256
_CLC_DEF long
4357
_Z29__spirv_AtomicCompareExchangePU3AS3lN5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagES5_ll(
@@ -53,6 +67,13 @@ _Z29__spirv_AtomicCompareExchangePU3AS1lN5__spv5Scope4FlagENS1_19MemorySemantics
5367
return __sync_val_compare_and_swap_8(p, cmp, val);
5468
}
5569

70+
_CLC_DEF long
71+
_Z29__spirv_AtomicCompareExchangePlN5__spv5Scope4FlagENS0_19MemorySemanticsMask4FlagES4_ll(
72+
volatile long *p, enum Scope scope, enum MemorySemanticsMask eq,
73+
enum MemorySemanticsMask neq, long val, long cmp) {
74+
return __sync_val_compare_and_swap_8(p, cmp, val);
75+
}
76+
5677
_CLC_DEF ulong
5778
_Z29__spirv_AtomicCompareExchangePU3AS3mN5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagES5_mm(
5879
volatile local ulong *p, enum Scope scope, enum MemorySemanticsMask eq,
@@ -66,4 +87,11 @@ _Z29__spirv_AtomicCompareExchangePU3AS1mN5__spv5Scope4FlagENS1_19MemorySemantics
6687
enum MemorySemanticsMask neq, ulong val, ulong cmp) {
6788
return __sync_val_compare_and_swap_8(p, cmp, val);
6889
}
90+
91+
_CLC_DEF ulong
92+
_Z29__spirv_AtomicCompareExchangePmN5__spv5Scope4FlagENS0_19MemorySemanticsMask4FlagES4_mm(
93+
volatile ulong *p, enum Scope scope, enum MemorySemanticsMask eq,
94+
enum MemorySemanticsMask neq, ulong val, ulong cmp) {
95+
return __sync_val_compare_and_swap_8(p, cmp, val);
96+
}
6997
#endif

‎libclc/generic/libspirv/atomic/atomic_load.cl

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,29 +10,35 @@
1010

1111
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
1212

13-
#define FDECL(TYPE, PREFIX, AS, BYTE_SIZE, MEM_ORDER) \
14-
TYPE __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_##MEM_ORDER(volatile AS const TYPE *);
15-
16-
#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, PREFIX, BYTE_SIZE) \
17-
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, unordered) \
18-
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, acquire) \
19-
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, seq_cst) \
20-
_CLC_DEF TYPE \
21-
_Z18__spirv_AtomicLoadPU3##AS_MANGLED##K##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE( \
22-
volatile AS const TYPE *p, enum Scope scope, \
23-
enum MemorySemanticsMask semantics) { \
24-
if (semantics & Acquire) { \
25-
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_acquire(p); \
26-
} \
27-
if (semantics & SequentiallyConsistent) { \
28-
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_seq_cst(p); \
29-
} \
30-
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_unordered(p); \
13+
#define FDECL(TYPE, PREFIX, AS, AS_, BYTE_SIZE, MEM_ORDER) \
14+
TYPE __clc__atomic_##PREFIX##load_##AS_##BYTE_SIZE##_##MEM_ORDER(volatile AS const TYPE *);
15+
16+
#define IMPL(TYPE, TYPE_MANGLED, AS_PREFIX, AS, AS_, AS_MANGLED, PREFIX, \
17+
BYTE_SIZE) \
18+
FDECL(TYPE, PREFIX, AS, AS_, BYTE_SIZE, unordered) \
19+
FDECL(TYPE, PREFIX, , , BYTE_SIZE, unordered) \
20+
FDECL(TYPE, PREFIX, AS, AS_, BYTE_SIZE, acquire) \
21+
FDECL(TYPE, PREFIX, , , BYTE_SIZE, acquire) \
22+
FDECL(TYPE, PREFIX, AS, AS_, BYTE_SIZE, seq_cst) \
23+
FDECL(TYPE, PREFIX, , , BYTE_SIZE, seq_cst) \
24+
_CLC_DEF TYPE \
25+
_Z18__spirv_AtomicLoadP##AS_PREFIX##AS_MANGLED##K##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE( \
26+
volatile AS const TYPE *p, enum Scope scope, \
27+
enum MemorySemanticsMask semantics) { \
28+
if (semantics & Acquire) { \
29+
return __clc__atomic_##PREFIX##load_##AS_##BYTE_SIZE##_acquire( \
30+
p); \
31+
} \
32+
if (semantics & SequentiallyConsistent) { \
33+
return __clc__atomic_##PREFIX##load_##AS_##BYTE_SIZE##_seq_cst(p); \
34+
} \
35+
return __clc__atomic_##PREFIX##load_##AS_##BYTE_SIZE##_unordered(p); \
3136
}
3237

33-
#define IMPL_AS(TYPE, TYPE_MANGLED, PREFIX, BYTE_SIZE) \
34-
IMPL(TYPE, TYPE_MANGLED, global, AS1, PREFIX, BYTE_SIZE) \
35-
IMPL(TYPE, TYPE_MANGLED, local, AS3, PREFIX, BYTE_SIZE)
38+
#define IMPL_AS(TYPE, TYPE_MANGLED, PREFIX, BYTE_SIZE) \
39+
IMPL(TYPE, TYPE_MANGLED, U3, global, global_, AS1, PREFIX, BYTE_SIZE) \
40+
IMPL(TYPE, TYPE_MANGLED, U3, local, local_, AS3, PREFIX, BYTE_SIZE) \
41+
IMPL(TYPE, TYPE_MANGLED, , , , , PREFIX, BYTE_SIZE)
3642

3743
IMPL_AS(int, i, , 4)
3844
IMPL_AS(unsigned int, j, u, 4)

‎libclc/generic/libspirv/atomic/atomic_max.cl

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,28 +10,34 @@
1010

1111
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
1212

13-
#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, NAME, PREFIX, SUFFIX) \
13+
#define IMPL(TYPE, TYPE_MANGLED, AS_PREFIX, AS, AS_MANGLED, NAME, PREFIX, SUFFIX) \
1414
_CLC_DEF TYPE \
15-
_Z18##NAME##PU3##AS_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE##TYPE_MANGLED( \
15+
_Z18##NAME##P##AS_PREFIX##AS_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE##TYPE_MANGLED( \
1616
volatile AS TYPE *p, enum Scope scope, \
1717
enum MemorySemanticsMask semantics, TYPE val) { \
1818
return PREFIX##__sync_fetch_and_##SUFFIX(p, val); \
1919
}
2020

21-
IMPL(int, i, global, AS1, __spirv_AtomicSMax, , max)
22-
IMPL(unsigned int, j, global, AS1, __spirv_AtomicUMax, , umax)
23-
IMPL(int, i, local, AS3, __spirv_AtomicSMax, , max)
24-
IMPL(unsigned int, j, local, AS3, __spirv_AtomicUMax, , umax)
21+
IMPL(int, i, U3, global, AS1, __spirv_AtomicSMax, , max)
22+
IMPL(unsigned int, j, U3, global, AS1, __spirv_AtomicUMax, , umax)
23+
IMPL(int, i, U3, local, AS3, __spirv_AtomicSMax, , max)
24+
IMPL(unsigned int, j, U3, local, AS3, __spirv_AtomicUMax, , umax)
25+
IMPL(int, i, , , , __spirv_AtomicSMax, , max)
26+
IMPL(unsigned int, j, , , , __spirv_AtomicUMax, , umax)
2527

2628
#ifdef cl_khr_int64_extended_atomics
2729
unsigned long __clc__sync_fetch_and_max_local_8(volatile local long *, long);
2830
unsigned long __clc__sync_fetch_and_max_global_8(volatile global long *, long);
31+
unsigned long __clc__sync_fetch_and_max_8(volatile long *, long);
2932
unsigned long __clc__sync_fetch_and_umax_local_8(volatile local unsigned long *, unsigned long);
3033
unsigned long __clc__sync_fetch_and_umax_global_8(volatile global unsigned long *, unsigned long);
34+
unsigned long __clc__sync_fetch_and_umax_8(volatile unsigned long *, unsigned long);
3135

32-
IMPL(long, l, global, AS1, __spirv_AtomicSMax, __clc, max_global_8)
33-
IMPL(unsigned long, m, global, AS1, __spirv_AtomicUMax, __clc, umax_global_8)
34-
IMPL(long, l, local, AS3, __spirv_AtomicSMax, __clc, max_local_8)
35-
IMPL(unsigned long, m, local, AS3, __spirv_AtomicUMax, __clc, umax_local_8)
36+
IMPL(long, l, U3, global, AS1, __spirv_AtomicSMax, __clc, max_global_8)
37+
IMPL(unsigned long, m, U3, global, AS1, __spirv_AtomicUMax, __clc, umax_global_8)
38+
IMPL(long, l, U3, local, AS3, __spirv_AtomicSMax, __clc, max_local_8)
39+
IMPL(unsigned long, m, U3, local, AS3, __spirv_AtomicUMax, __clc, umax_local_8)
40+
IMPL(long, l, , , , __spirv_AtomicSMax, __clc, max_8)
41+
IMPL(unsigned long, m, , , , __spirv_AtomicUMax, __clc, umax_8)
3642
#endif
3743
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_min.cl

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,28 +10,35 @@
1010

1111
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
1212

13-
#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, NAME, PREFIX, SUFFIX) \
14-
_CLC_DEF TYPE \
15-
_Z18##NAME##PU3##AS_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE##TYPE_MANGLED( \
16-
volatile AS TYPE *p, enum Scope scope, \
17-
enum MemorySemanticsMask semantics, TYPE val) { \
18-
return PREFIX##__sync_fetch_and_##SUFFIX(p, val); \
13+
#define IMPL(TYPE, TYPE_MANGLED, AS_PREFIX, AS, AS_MANGLED, NAME, PREFIX, \
14+
SUFFIX) \
15+
_CLC_DEF TYPE \
16+
_Z18##NAME##P##AS_PREFIX##AS_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE##TYPE_MANGLED( \
17+
volatile AS TYPE *p, enum Scope scope, \
18+
enum MemorySemanticsMask semantics, TYPE val) { \
19+
return PREFIX##__sync_fetch_and_##SUFFIX(p, val); \
1920
}
2021

21-
IMPL(int, i, global, AS1, __spirv_AtomicSMin, , min)
22-
IMPL(unsigned int, j, global, AS1, __spirv_AtomicUMin, , umin)
23-
IMPL(int, i, local, AS3, __spirv_AtomicSMin, , min)
24-
IMPL(unsigned int, j, local, AS3, __spirv_AtomicUMin, , umin)
22+
IMPL(int, i, U3, global, AS1, __spirv_AtomicSMin, , min)
23+
IMPL(unsigned int, j, U3, global, AS1, __spirv_AtomicUMin, , umin)
24+
IMPL(int, i, U3, local, AS3, __spirv_AtomicSMin, , min)
25+
IMPL(unsigned int, j, U3, local, AS3, __spirv_AtomicUMin, , umin)
26+
IMPL(int, i, , , , __spirv_AtomicSMin, , min)
27+
IMPL(unsigned int, j, , , , __spirv_AtomicUMin, , umin)
2528

2629
#ifdef cl_khr_int64_extended_atomics
2730
unsigned long __clc__sync_fetch_and_min_local_8(volatile local long *, long);
2831
unsigned long __clc__sync_fetch_and_min_global_8(volatile global long *, long);
32+
unsigned long __clc__sync_fetch_and_min_8(volatile long *, long);
2933
unsigned long __clc__sync_fetch_and_umin_local_8(volatile local unsigned long *, unsigned long);
3034
unsigned long __clc__sync_fetch_and_umin_global_8(volatile global unsigned long *, unsigned long);
35+
unsigned long __clc__sync_fetch_and_umin_8(volatile unsigned long *, unsigned long);
3136

32-
IMPL(long, l, global, AS1, __spirv_AtomicSMin, __clc, min_global_8)
33-
IMPL(unsigned long, m, global, AS1, __spirv_AtomicUMin, __clc, umin_global_8)
34-
IMPL(long, l, local, AS3, __spirv_AtomicSMin, __clc, min_local_8)
35-
IMPL(unsigned long, m, local, AS3, __spirv_AtomicUMin, __clc, umin_local_8)
37+
IMPL(long, l, U3, global, AS1, __spirv_AtomicSMin, __clc, min_global_8)
38+
IMPL(unsigned long, m, U3, global, AS1, __spirv_AtomicUMin, __clc, umin_global_8)
39+
IMPL(long, l, U3, local, AS3, __spirv_AtomicSMin, __clc, min_local_8)
40+
IMPL(unsigned long, m, U3, local, AS3, __spirv_AtomicUMin, __clc, umin_local_8)
41+
IMPL(long, l, , , , __spirv_AtomicSMin, __clc, min_8)
42+
IMPL(unsigned long, m, , , , __spirv_AtomicUMin, __clc, umin_8)
3643
#endif
3744
#undef IMPL

0 commit comments

Comments
 (0)