1#ifndef HALIDE_RUNTIME_RUNTIME_ATOMICS_H
2#define HALIDE_RUNTIME_RUNTIME_ATOMICS_H
16namespace Synchronization {
23struct remove_volatile {
typedef T type; };
25struct remove_volatile<volatile T> {
typedef T type; };
29 return __sync_and_and_fetch(addr, val);
33ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
34 return __sync_fetch_and_add(addr, val);
37template<typename T, typename TV = typename remove_volatile<T>::type>
38ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
39 return __sync_fetch_and_add(addr, val);
42template<typename T, typename TV = typename remove_volatile<T>::type>
43ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
44 return __sync_fetch_and_sub(addr, val);
47template<typename T, typename TV = typename remove_volatile<T>::type>
48ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
49 return __sync_fetch_and_or(addr, val);
53ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
54 return __sync_add_and_fetch(addr, val);
58ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
59 return __sync_sub_and_fetch(addr, val);
62template<typename T, typename TV = typename remove_volatile<T>::type>
63ALWAYS_INLINE bool cas_strong_sequentially_consistent_helper(T *addr, TV *expected, TV *desired) {
64 TV oldval = *expected;
65 TV gotval = __sync_val_compare_and_swap(addr, oldval, *desired);
67 return oldval == gotval;
71 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
74template<typename T, typename TV = typename remove_volatile<T>::type>
75ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
76 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
80 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
84ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
85 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
89 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
93 return cas_strong_sequentially_consistent_helper(addr, expected, desired);
98 return __sync_fetch_and_and(addr, val);
101template<typename T, typename TV = typename remove_volatile<T>::type>
102ALWAYS_INLINE T atomic_fetch_and_sequentially_consistent(T *addr, TV val) {
103 return __sync_fetch_and_and(addr, val);
113 __sync_synchronize();
120 return __sync_lock_test_and_set(addr, val);
124 return __sync_or_and_fetch(addr, val);
134 __sync_synchronize();
137template<typename T, typename TV = typename remove_volatile<T>::type>
138ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
140 __sync_synchronize();
144 __sync_synchronize();
147ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
148 __sync_synchronize();
154 return __atomic_and_fetch(addr, val, __ATOMIC_RELEASE);
158ALWAYS_INLINE T atomic_fetch_add_acquire_release(T *addr, T val) {
159 return __atomic_fetch_add(addr, val, __ATOMIC_ACQ_REL);
162template<typename T, typename TV = typename remove_volatile<T>::type>
163ALWAYS_INLINE T atomic_fetch_add_sequentially_consistent(T *addr, TV val) {
164 return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
167template<typename T, typename TV = typename remove_volatile<T>::type>
168ALWAYS_INLINE T atomic_fetch_sub_sequentially_consistent(T *addr, TV val) {
169 return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
172template<typename T, typename TV = typename remove_volatile<T>::type>
173ALWAYS_INLINE T atomic_fetch_or_sequentially_consistent(T *addr, TV val) {
174 return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
178ALWAYS_INLINE T atomic_add_fetch_sequentially_consistent(T *addr, T val) {
179 return __atomic_add_fetch(addr, val, __ATOMIC_SEQ_CST);
183ALWAYS_INLINE T atomic_sub_fetch_sequentially_consistent(T *addr, T val) {
184 return __atomic_sub_fetch(addr, val, __ATOMIC_SEQ_CST);
188 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
191template<typename T, typename TV = typename remove_volatile<T>::type>
192ALWAYS_INLINE bool atomic_cas_strong_sequentially_consistent(T *addr, TV *expected, TV *desired) {
193 return __atomic_compare_exchange(addr, expected, desired,
false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
197ALWAYS_INLINE bool atomic_cas_weak_relacq_relaxed(T *addr, T *expected, T *desired) {
198 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
202 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
206 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
210 return __atomic_compare_exchange(addr, expected, desired,
true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
215 return __atomic_fetch_and(addr, val, __ATOMIC_RELEASE);
218template<typename T, typename TV = typename remove_volatile<T>::type>
220 return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
225 __atomic_load(addr, val, __ATOMIC_RELAXED);
230 __atomic_load(addr, val, __ATOMIC_ACQUIRE);
231 __sync_synchronize();
238 __atomic_exchange(addr, &val, &result, __ATOMIC_ACQUIRE);
243 return __atomic_or_fetch(addr, val, __ATOMIC_RELAXED);
247 __atomic_store(addr, val, __ATOMIC_RELAXED);
252 __atomic_store(addr, val, __ATOMIC_RELEASE);
255template<typename T, typename TV = typename remove_volatile<T>::type>
256ALWAYS_INLINE void atomic_store_sequentially_consistent(T *addr, TV *val) {
257 __atomic_store(addr, val, __ATOMIC_SEQ_CST);
261 __atomic_thread_fence(__ATOMIC_ACQUIRE);
264ALWAYS_INLINE void atomic_thread_fence_sequentially_consistent() {
265 __atomic_thread_fence(__ATOMIC_SEQ_CST);
This file declares the routines used by Halide internally in its runtime.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
__UINTPTR_TYPE__ uintptr_t