diff options
Diffstat (limited to 'platform/linux-generic/include/odp_atomic_internal.h')
| -rw-r--r-- | platform/linux-generic/include/odp_atomic_internal.h | 518 |
1 files changed, 11 insertions, 507 deletions
diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index dca2175ce..3e6c45a27 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2014, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @@ -20,6 +18,7 @@ #include <odp/api/align.h> #include <odp/api/hints.h> #include <odp/api/atomic.h> +#include <odp_types_internal.h> #include <stdbool.h> #ifdef __cplusplus @@ -27,14 +26,6 @@ extern "C" { #endif /** - * Pointer atomic type - */ -typedef struct { - void *v; /**< Actual storage for the atomic variable */ -} _odp_atomic_ptr_t -ODP_ALIGNED(sizeof(void *)); /* Enforce alignement! */ - -/** * Atomic flag (boolean) type * @Note this is not the same as a plain boolean type. * _odp_atomic_flag_t is guaranteed to be able to operate on atomically. @@ -59,469 +50,9 @@ typedef enum { _ODP_MEMMODEL_RLS = __ATOMIC_RELEASE, /** Acquire&release memory ordering, synchronize with acquire loads and release * stores in another (one other) thread */ - _ODP_MEMMODEL_ACQ_RLS = __ATOMIC_ACQ_REL, -/** Sequential consistent memory ordering, synchronize with acquire loads and - * release stores in all threads */ - _ODP_MEMMODEL_SC = __ATOMIC_SEQ_CST -} _odp_memmodel_t; - -/***************************************************************************** - * Operations on 32-bit atomics - * _odp_atomic_u32_load_mm - return current value - * _odp_atomic_u32_store_mm - no return value - * _odp_atomic_u32_xchg_mm - return old value - * _odp_atomic_u32_cmp_xchg_strong_mm - return bool - * _odp_atomic_u32_fetch_add_mm - return old value - * _odp_atomic_u32_add_mm - no return value - * _odp_atomic_u32_fetch_sub_mm - return old value - * _odp_atomic_u32_sub_mm - no return value - *****************************************************************************/ - -/** - * Atomic load of 32-bit atomic variable - * - * @param atom Pointer to a 32-bit atomic variable - * @param mmodel Memory ordering associated with the load operation - * - * @return Value of the variable - */ -static inline uint32_t _odp_atomic_u32_load_mm(const odp_atomic_u32_t *atom, - _odp_memmodel_t mmodel) -{ - return __atomic_load_n(&atom->v, mmodel); -} - -/** - * Atomic store to 32-bit atomic variable - * - * @param[out] atom Pointer to a 32-bit atomic variable - * @param val Value to store in the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_u32_store_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - __atomic_store_n(&atom->v, val, mmodel); -} - -/** - * Atomic exchange (swap) of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val New value to store in the atomic variable - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of the variable - */ -static inline uint32_t _odp_atomic_u32_xchg_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) - -{ - return __atomic_exchange_n(&atom->v, val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of 32-bit atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successul - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u32_cmp_xchg_strong_mm( - odp_atomic_u32_t *atom, - uint32_t *exp, - uint32_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -} - -/** - * Atomic fetch and add of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - * - * @return Value of the atomic variable before the addition - */ -static inline uint32_t _odp_atomic_u32_fetch_add_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic add of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - */ -static inline void _odp_atomic_u32_add_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) - -{ - (void)__atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic fetch and subtract of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - * - * @return Value of the atomic variable before the subtraction - */ -static inline uint32_t _odp_atomic_u32_fetch_sub_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_sub(&atom->v, val, mmodel); -} - -/** - * Atomic subtract of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - */ -static inline void _odp_atomic_u32_sub_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) - -{ - (void)__atomic_fetch_sub(&atom->v, val, mmodel); -} - -/***************************************************************************** - * Operations on 64-bit atomics - * _odp_atomic_u64_load_mm - return current value - * _odp_atomic_u64_store_mm - no return value - * _odp_atomic_u64_xchg_mm - return old value - * _odp_atomic_u64_cmp_xchg_strong_mm - return bool - * _odp_atomic_u64_fetch_add_mm - return old value - * _odp_atomic_u64_add_mm - no return value - * _odp_atomic_u64_fetch_sub_mm - return old value - * _odp_atomic_u64_sub_mm - no return value - *****************************************************************************/ - -/* Check if the compiler support lock-less atomic operations on 64-bit types */ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 -/** - * @internal - * Helper macro for lock-based atomic operations on 64-bit integers - * @param[in,out] atom Pointer to the 64-bit atomic variable - * @param expr Expression used update the variable. - * @param mm Memory order to use. - * @return The old value of the variable. - */ -#define ATOMIC_OP_MM(atom, expr, mm) \ -({ \ - uint64_t old_val; \ - /* Loop while lock is already taken, stop when lock becomes clear */ \ - while (__atomic_test_and_set(&(atom)->lock, \ - (mm) == _ODP_MEMMODEL_SC ? \ - __ATOMIC_SEQ_CST : __ATOMIC_ACQUIRE)) \ - (void)0; \ - old_val = (atom)->v; \ - (expr); /* Perform whatever update is desired */ \ - __atomic_clear(&(atom)->lock, \ - (mm) == _ODP_MEMMODEL_SC ? \ - __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); \ - old_val; /* Return old value */ \ -}) -#endif - -/** - * Atomic load of 64-bit atomic variable - * - * @param atom Pointer to a 64-bit atomic variable - * @param mmodel Memory order associated with the load operation - * - * @return Value of the variable - */ -static inline uint64_t _odp_atomic_u64_load_mm(odp_atomic_u64_t *atom, - _odp_memmodel_t mmodel) -{ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP_MM(atom, (void)0, mmodel); -#else - return __atomic_load_n(&atom->v, mmodel); -#endif -} - -/** - * Atomic store to 64-bit atomic variable - * - * @param[out] atom Pointer to a 64-bit atomic variable - * @param val Value to write to the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_u64_store_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP_MM(atom, atom->v = val, mmodel); -#else - __atomic_store_n(&atom->v, val, mmodel); -#endif -} - -/** - * Atomic exchange (swap) of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val New value to write to the atomic variable - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of variable - */ -static inline uint64_t _odp_atomic_u64_xchg_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) + _ODP_MEMMODEL_ACQ_RLS = __ATOMIC_ACQ_REL -{ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP_MM(atom, atom->v = val, mmodel); -#else - return __atomic_exchange_n(&atom->v, val, mmodel); -#endif -} - -/** - * Atomic compare and exchange (swap) of 64-bit atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successful - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, - uint64_t *exp, - uint64_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - /* Possibly we are a bit pessimistic with the memory models */ - odp_bool_t ret_succ; - /* Loop while lock is already taken, stop when lock becomes clear */ - while (__atomic_test_and_set(&(atom)->lock, - (success) == _ODP_MEMMODEL_SC ? - __ATOMIC_SEQ_CST : __ATOMIC_ACQUIRE)) - (void)0; - if (atom->v == *exp) { - atom->v = val; - ret_succ = 1; - } else { - *exp = atom->v; - ret_succ = 0; - } - __atomic_clear(&(atom)->lock, - (ret_succ ? success : failure) == _ODP_MEMMODEL_SC ? - __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); - return ret_succ; -#else - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -#endif -} - -/** - * Atomic fetch and add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - * - * @return Value of the atomic variable before the addition - */ -static inline uint64_t _odp_atomic_u64_fetch_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP_MM(atom, atom->v += val, mmodel); -#else - return __atomic_fetch_add(&atom->v, val, mmodel); -#endif -} - -/** - * Atomic add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation. - */ -static inline void _odp_atomic_u64_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP_MM(atom, atom->v += val, mmodel); -#else - (void)__atomic_fetch_add(&atom->v, val, mmodel); -#endif -} - -/** - * Atomic fetch and subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - * - * @return Value of the atomic variable before the subtraction - */ -static inline uint64_t _odp_atomic_u64_fetch_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP_MM(atom, atom->v -= val, mmodel); -#else - return __atomic_fetch_sub(&atom->v, val, mmodel); -#endif -} - -/** - * Atomic subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - */ -static inline void _odp_atomic_u64_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP_MM(atom, atom->v -= val, mmodel); -#else - (void)__atomic_fetch_sub(&atom->v, val, mmodel); -#endif -} - -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 -#undef ATOMIC_OP_MM -#endif - -/***************************************************************************** - * Operations on pointer atomics - * _odp_atomic_ptr_init - no return value - * _odp_atomic_ptr_load - return current value - * _odp_atomic_ptr_store - no return value - * _odp_atomic_ptr_xchg - return old value - *****************************************************************************/ - -/** - * Initialization of pointer atomic variable - * - * @param[out] atom Pointer to a pointer atomic variable - * @param val Value to initialize the variable with - */ -static inline void _odp_atomic_ptr_init(_odp_atomic_ptr_t *atom, void *val) -{ - __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); -} - -/** - * Atomic load of pointer atomic variable - * - * @param atom Pointer to a pointer atomic variable - * @param mmodel Memory order associated with the load operation - * - * @return Value of the variable - */ -static inline void *_odp_atomic_ptr_load(const _odp_atomic_ptr_t *atom, - _odp_memmodel_t mmodel) -{ - return __atomic_load_n(&atom->v, mmodel); -} - -/** - * Atomic store to pointer atomic variable - * - * @param[out] atom Pointer to a pointer atomic variable - * @param val Value to write to the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_ptr_store(_odp_atomic_ptr_t *atom, - void *val, - _odp_memmodel_t mmodel) -{ - __atomic_store_n(&atom->v, val, mmodel); -} - -/** - * Atomic exchange (swap) of pointer atomic variable - * - * @param[in,out] atom Pointer to a pointer atomic variable - * @param val New value to write - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of variable - */ -static inline void *_odp_atomic_ptr_xchg(_odp_atomic_ptr_t *atom, - void *val, - _odp_memmodel_t mmodel) -{ - return __atomic_exchange_n(&atom->v, val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of pointer atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a pointer atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successful - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_ptr_cmp_xchg_strong( - _odp_atomic_ptr_t *atom, - void **exp, - void *val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -} +} _odp_memmodel_t; /***************************************************************************** * Operations on flag atomics @@ -541,7 +72,7 @@ static inline int _odp_atomic_ptr_cmp_xchg_strong( * @param val The initial value of the variable */ static inline void _odp_atomic_flag_init(_odp_atomic_flag_t *flag, - odp_bool_t val) + odp_bool_t val) { __atomic_clear(flag, __ATOMIC_RELAXED); if (val) @@ -605,13 +136,11 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag) #endif #ifdef ODP_ATOMIC_U128 -/** An unsigned 128-bit (16-byte) scalar type */ -typedef __int128 _uint128_t; /** Atomic 128-bit type */ -typedef struct { - _uint128_t v; /**< Actual storage for the atomic variable */ -} _odp_atomic_u128_t ODP_ALIGNED(16); +typedef struct ODP_ALIGNED(16) { + _odp_u128_t v; /**< Actual storage for the atomic variable */ +} _odp_atomic_u128_t; /** * 16-byte atomic exchange operation @@ -622,37 +151,12 @@ typedef struct { * @param mmodel Memory model associated with the exchange operation */ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr, - _uint128_t *val, - _uint128_t *old, + _odp_u128_t *val, + _odp_u128_t *old, _odp_memmodel_t mm) { __atomic_exchange(&ptr->v, val, old, mm); } - -/** - * Atomic compare and exchange (swap) of 16-byte atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param ptr Pointer to a 16-byte atomic variable - * @param exp Pointer to expected value (updated on failure) - * @param val Pointer to new value to write - * @param succ Memory model associated with a successful compare-and-swap - * operation - * @param fail Memory model associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successul - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u128_cmp_xchg_mm(_odp_atomic_u128_t *ptr, - _uint128_t *exp, - _uint128_t *val, - _odp_memmodel_t succ, - _odp_memmodel_t fail) -{ - return __atomic_compare_exchange(&ptr->v, exp, val, - false/*strong*/, succ, fail); -} #endif /** |