diff options
Diffstat (limited to 'platform/linux-generic/arch')
| -rw-r--r-- | platform/linux-generic/arch/aarch64/odp_cpu.h | 166 | ||||
| -rw-r--r-- | platform/linux-generic/arch/aarch64/odp_cpu_idling.h | 39 | ||||
| -rw-r--r-- | platform/linux-generic/arch/aarch64/odp_llsc.h | 170 | ||||
| -rw-r--r-- | platform/linux-generic/arch/aarch64/odp_wait_until.h | 100 | ||||
| -rw-r--r-- | platform/linux-generic/arch/arm/odp_atomic.h | 109 | ||||
| -rw-r--r-- | platform/linux-generic/arch/arm/odp_cpu.h | 56 | ||||
| -rw-r--r-- | platform/linux-generic/arch/arm/odp_cpu_idling.h | 39 | ||||
| -rw-r--r-- | platform/linux-generic/arch/arm/odp_llsc.h | 96 | ||||
| -rw-r--r-- | platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h | 10 | ||||
| -rw-r--r-- | platform/linux-generic/arch/default/odp_cpu.h | 2 | ||||
| -rw-r--r-- | platform/linux-generic/arch/default/odp_cpu_idling.h | 31 | ||||
| -rw-r--r-- | platform/linux-generic/arch/default/odp_wait_until.h | 53 | ||||
| -rw-r--r-- | platform/linux-generic/arch/x86/odp_time_cpu.c | 96 |
13 files changed, 416 insertions, 551 deletions
diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h index 84bc4dffd..ad8b36d87 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu.h @@ -14,6 +14,7 @@ #endif #include <odp_debug_internal.h> +#include <odp_types_internal.h> /* * Use LLD/SCD atomic primitives instead of lock-based code path in llqueue @@ -31,20 +32,6 @@ */ #define CONFIG_DMBSTR -/* - * Use ARM event signalling mechanism - * Event signalling minimises spinning (busy waiting) which decreases - * cache coherency traffic when spinning on shared locations (thus faster and - * more scalable) and enables the CPU to enter a sleep state (lower power - * consumption). - */ -#define CONFIG_WFE - -static inline void _odp_dmb(void) -{ - __asm__ volatile("dmb" : : : "memory"); -} - /* Only ARMv8 supports DMB ISHLD */ /* A load only barrier is much cheaper than full barrier */ #define _odp_release_barrier(ro) \ @@ -55,9 +42,156 @@ do { \ __asm__ volatile("dmb ish" ::: "memory"); \ } while (0) -#include "odp_llsc.h" +static inline uint16_t ll8(uint8_t *var, int mm) +{ + uint16_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +static inline uint32_t ll32(uint32_t *var, int mm) +{ + uint32_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else + __asm__ volatile("stxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +static inline uint64_t ll64(uint64_t *var, int mm) +{ + uint64_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc64(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else + __asm__ volatile("stxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +union i128 { + _odp_u128_t i128; + int64_t i64[2]; +}; + +static inline _odp_u128_t lld(_odp_u128_t *var, int mm) +{ + union i128 old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : ); + return old.i128; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((*(union i128 *)&neu)).i64[0]), + "r" (((*(union i128 *)&neu)).i64[1]), + "r" (var) + : "memory"); + else + __asm__ volatile("stxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((*(union i128 *)&neu)).i64[0]), + "r" (((*(union i128 *)&neu)).i64[1]), + "r" (var) + : ); +#pragma GCC diagnostic pop + return ret; +} + #include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED #define _ODP_UNALIGNED 1 diff --git a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h deleted file mode 100644 index a6cea8c63..000000000 --- a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#ifndef CONFIG_WFE - -#include "../default/odp_cpu_idling.h" - -#else /* CONFIG_WFE */ - -static inline void sevl(void) -{ - __asm__ volatile("sevl" : : : ); -} - -static inline int wfe(void) -{ - __asm__ volatile("wfe" : : : "memory"); - return 1; -} - -#define monitor128(addr, mo) lld((addr), (mo)) -#define monitor64(addr, mo) ll64((addr), (mo)) -#define monitor32(addr, mo) ll32((addr), (mo)) -#define monitor8(addr, mo) ll8((addr), (mo)) -#endif /* CONFIG_WFE */ - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h deleted file mode 100644 index 498785bd4..000000000 --- a/platform/linux-generic/arch/aarch64/odp_llsc.h +++ /dev/null @@ -1,170 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#include <odp_types_internal.h> - -static inline uint16_t ll8(uint8_t *var, int mm) -{ - uint16_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxrb %w0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxrb %w0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -static inline uint32_t ll32(uint32_t *var, int mm) -{ - uint32_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxr %w0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxr %w0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) -{ - uint32_t ret; - - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxr %w0, %w1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxr %w0, %w1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - else - _ODP_ABORT(); - return ret; -} - -static inline uint64_t ll(uint64_t *var, int mm) -{ - uint64_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxr %0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxr %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -#define ll64(a, b) ll((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) -{ - uint32_t ret; - - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxr %w0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxr %w0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - else - _ODP_ABORT(); - return ret; -} - -#define sc64(a, b, c) sc((a), (b), (c)) - -union i128 { - _odp_u128_t i128; - int64_t i64[2]; -}; - -static inline _odp_u128_t lld(_odp_u128_t *var, int mm) -{ - union i128 old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxp %0, %1, [%2]" - : "=&r" (old.i64[0]), "=&r" (old.i64[1]) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxp %0, %1, [%2]" - : "=&r" (old.i64[0]), "=&r" (old.i64[1]) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old.i128; -} - -/* Return 0 on success, 1 on failure */ -static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) -{ - uint32_t ret; - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxp %w0, %1, %2, [%3]" - : "=&r" (ret) - : "r" (((*(union i128 *)&neu)).i64[0]), - "r" (((*(union i128 *)&neu)).i64[1]), - "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxp %w0, %1, %2, [%3]" - : "=&r" (ret) - : "r" (((*(union i128 *)&neu)).i64[0]), - "r" (((*(union i128 *)&neu)).i64[1]), - "r" (var) - : ); - else - _ODP_ABORT(); -#pragma GCC diagnostic pop - return ret; -} - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_wait_until.h b/platform/linux-generic/arch/aarch64/odp_wait_until.h new file mode 100644 index 000000000..eca3f9ce5 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_wait_until.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017 ARM Limited + * Copyright (c) 2017-2018 Linaro Limited + * Copyright (c) 2024 Nokia + */ + +#ifndef ODP_AARCH64_WAIT_UNTIL_H_ +#define ODP_AARCH64_WAIT_UNTIL_H_ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#error This file should not be included directly, please include odp_cpu.h +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/cpu.h> + +#include <odp_cpu.h> + +#include <stdint.h> + +static inline void _odp_sevl(void) +{ + __asm__ volatile("sevl" : : : ); +} + +static inline int _odp_wfe(void) +{ + __asm__ volatile("wfe" : : : "memory"); + return 1; +} + +#define _odp_monitor_u8(addr, mo) ll8((addr), (mo)) +#define _odp_monitor_u32(addr, mo) ll32((addr), (mo)) +#define _odp_monitor_u64(addr, mo) ll64((addr), (mo)) +#define _odp_monitor_u128(addr, mo) lld((addr), (mo)) + +#if ATOM_BITSET_SIZE <= 32 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u32(bs, mo); +} +#elif ATOM_BITSET_SIZE <= 64 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u64(bs, mo); +} +#elif ATOM_BITSET_SIZE <= 128 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u128(bs, mo); +} +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +/** + * The _odp_wait_until_eq_*() functions defined in this header are intended to + * be used only with the scalable scheduler and queue implementations. Even + * though these functions use standard non-atomic parameter types, the + * parameters must only be operated using atomic operations. If new functions + * are added to this file, they should use _odp_wait_until_equal_*() prefix and + * atomic parameter types. + */ + +static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_bitset_monitor(val, __ATOMIC_RELAXED != expected)) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u8(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h deleted file mode 100644 index e400f52d4..000000000 --- a/platform/linux-generic/arch/arm/odp_atomic.h +++ /dev/null @@ -1,109 +0,0 @@ -/* Copyright (c) 2017-2021, ARM Limited. All rights reserved. - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#include <odp_types_internal.h> -#include <limits.h> - -#ifdef CONFIG_DMBSTR - -#define atomic_store_release(loc, val, ro) \ -do { \ - _odp_release_barrier(ro); \ - __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ -} while (0) - -#else - -#define atomic_store_release(loc, val, ro) \ - __atomic_store_n(loc, val, __ATOMIC_RELEASE) - -#endif /* CONFIG_DMBSTR */ - -/** Atomic bit set operations with memory ordering */ -#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ - __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ -typedef unsigned long long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) - -#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ -typedef unsigned long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) - -#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) - -#else -/* Target does not support lock-free atomic operations */ -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) -#endif - -#if ATOM_BITSET_SIZE <= 32 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1UL << bit; -} - -#elif ATOM_BITSET_SIZE <= 64 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1ULL << bit; -} - -#elif ATOM_BITSET_SIZE <= 128 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - if (bit < 64) - return 1ULL << bit; - else - return (_odp_u128_t)(1ULL << (bit - 64)) << 64; -} - -#else -#error Unsupported size of bit sets (ATOM_BITSET_SIZE) -#endif - -static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) -{ - return __atomic_load_n(bs, mo); -} - -static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) -{ - (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); -} - -static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) -{ - (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); -} - -static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) -{ - return __atomic_exchange_n(bs, neu, mo); -} - -static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, - bitset_t neu, bool weak, - int mo_success, int mo_failure) -{ - return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, - mo_failure); -} - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h index 82d47325f..6b2674736 100644 --- a/platform/linux-generic/arch/arm/odp_cpu.h +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -31,26 +31,52 @@ */ #define CONFIG_DMBSTR -/* - * Use ARM event signalling mechanism - * Event signalling minimises spinning (busy waiting) which decreases - * cache coherency traffic when spinning on shared locations (thus faster and - * more scalable) and enables the CPU to enter a sleep state (lower power - * consumption). - */ -/* #define CONFIG_WFE */ +static inline uint64_t lld(uint64_t *var, int mm) +{ + uint64_t old; -static inline void _odp_dmb(void) + __asm__ volatile("ldrexd %0, %H0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("dmb" : : : "memory"); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) { - __asm__ volatile("dmb" : : : "memory"); + uint32_t ret; + + /* Barrier before a releasing store */ + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("dmb" : : : "memory"); + __asm__ volatile("strexd %0, %1, %H1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; } -#define _odp_release_barrier(ro) \ - __atomic_thread_fence(__ATOMIC_RELEASE) +#ifdef CONFIG_DMBSTR + +#define atomic_store_release(loc, val, ro) \ +do { \ + __atomic_thread_fence(__ATOMIC_RELEASE); \ + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ +} while (0) + +#else + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +#endif /* CONFIG_DMBSTR */ -#include "odp_llsc.h" -#include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "../default/odp_atomic.h" +#include "../default/odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED #define _ODP_UNALIGNED 1 diff --git a/platform/linux-generic/arch/arm/odp_cpu_idling.h b/platform/linux-generic/arch/arm/odp_cpu_idling.h deleted file mode 100644 index a6cea8c63..000000000 --- a/platform/linux-generic/arch/arm/odp_cpu_idling.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#ifndef CONFIG_WFE - -#include "../default/odp_cpu_idling.h" - -#else /* CONFIG_WFE */ - -static inline void sevl(void) -{ - __asm__ volatile("sevl" : : : ); -} - -static inline int wfe(void) -{ - __asm__ volatile("wfe" : : : "memory"); - return 1; -} - -#define monitor128(addr, mo) lld((addr), (mo)) -#define monitor64(addr, mo) ll64((addr), (mo)) -#define monitor32(addr, mo) ll32((addr), (mo)) -#define monitor8(addr, mo) ll8((addr), (mo)) -#endif /* CONFIG_WFE */ - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H */ diff --git a/platform/linux-generic/arch/arm/odp_llsc.h b/platform/linux-generic/arch/arm/odp_llsc.h deleted file mode 100644 index 2fea6a0dc..000000000 --- a/platform/linux-generic/arch/arm/odp_llsc.h +++ /dev/null @@ -1,96 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -static inline uint32_t ll8(uint8_t *var, int mm) -{ - uint8_t old; - - __asm__ volatile("ldrexb %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -static inline uint32_t ll(uint32_t *var, int mm) -{ - uint32_t old; - - __asm__ volatile("ldrex %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -#define ll32(a, b) ll((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc(uint32_t *var, uint32_t neu, int mm) -{ - uint32_t ret; - - /* Barrier before a releasing store */ - if (mm == __ATOMIC_RELEASE) - _odp_dmb(); - __asm__ volatile("strex %0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - return ret; -} - -#define sc32(a, b, c) sc((a), (b), (c)) - -static inline uint64_t lld(uint64_t *var, int mm) -{ - uint64_t old; - - __asm__ volatile("ldrexd %0, %H0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -#define ll64(a, b) lld((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) -{ - uint32_t ret; - - /* Barrier before a releasing store */ - if (mm == __ATOMIC_RELEASE) - _odp_dmb(); - __asm__ volatile("strexd %0, %1, %H1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - return ret; -} - -#define sc64(a, b, c) scd((a), (b), (c)) - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H */ diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h index af435e495..c6ed86363 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -197,11 +197,11 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 #define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ __extension__ ({ \ int *_ret_ptr = ret_ptr; \ - odp_u128_t *_old_val = old_val; \ - odp_u128_t _new_val = new_val; \ - if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ - ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ - (_atom)->v = (_new_val); \ + odp_u128_t *_cas_old = old_val; \ + odp_u128_t _cas_new = new_val; \ + if (((_atom)->v.u64[0] == (_cas_old)->u64[0]) && \ + ((_atom)->v.u64[1] == (_cas_old)->u64[1])) { \ + (_atom)->v = (_cas_new); \ *(_ret_ptr) = 1; \ } else { \ *(_ret_ptr) = 0; \ diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h index 821956819..6b10966c6 100644 --- a/platform/linux-generic/arch/default/odp_cpu.h +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -21,6 +21,6 @@ __atomic_store_n(loc, val, __ATOMIC_RELEASE) #include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "odp_wait_until.h" #endif diff --git a/platform/linux-generic/arch/default/odp_cpu_idling.h b/platform/linux-generic/arch/default/odp_cpu_idling.h deleted file mode 100644 index 9d23ad20d..000000000 --- a/platform/linux-generic/arch/default/odp_cpu_idling.h +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_DEFAULT_CPU_IDLING_H_ -#define ODP_DEFAULT_CPU_IDLING_H_ - -/****************************************************************************** - * Idle mgmt - *****************************************************************************/ - -static inline void sevl(void) -{ - /* empty */ -} - -static inline int wfe(void) -{ - return 1; -} - -#define monitor128(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor64(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor32(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor8(addr, mo) __atomic_load_n((addr), (mo)) - -#endif diff --git a/platform/linux-generic/arch/default/odp_wait_until.h b/platform/linux-generic/arch/default/odp_wait_until.h new file mode 100644 index 000000000..c51f4355e --- /dev/null +++ b/platform/linux-generic/arch/default/odp_wait_until.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#ifndef ODP_DEFAULT_WAIT_UNTIL_H_ +#define ODP_DEFAULT_WAIT_UNTIL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/plat/cpu_inlines.h> + +#include <stdint.h> + +/** + * The _odp_wait_until_eq_*() functions defined in this header are intended to + * be used only with the scalable scheduler and queue implementations. Even + * though these functions use standard non-atomic parameter types, the + * parameters must only be operated using atomic operations. If new functions + * are added to this file, they should use _odp_wait_until_equal_*() prefix and + * atomic parameter types. + */ + +static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp_time_cpu.c b/platform/linux-generic/arch/x86/odp_time_cpu.c index aa00ac04e..ab897296d 100644 --- a/platform/linux-generic/arch/x86/odp_time_cpu.c +++ b/platform/linux-generic/arch/x86/odp_time_cpu.c @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2024 Nokia */ #include <odp_posix_extensions.h> @@ -14,42 +13,77 @@ #include <odp_debug_internal.h> #include <time.h> +#include <errno.h> +#include <string.h> -/* Measure TSC frequency. Frequency information registers are defined for x86, - * but those are often not enumerated. */ +static int nwait(uint64_t nsec) +{ + struct timespec ts1, ts2; + uint64_t diff; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) + return 1; + + do { + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) + return 1; + + diff = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS + + ts2.tv_nsec - ts1.tv_nsec; + } while (diff < nsec); + + return 0; +} + +static void sort(uint64_t values[], int num) +{ + for (int n = 0; n < num; n++) { + for (int i = n + 1; i < num; i++) { + if (values[i] < values[n]) { + uint64_t tmp = values[i]; + + values[i] = values[n]; + values[n] = tmp; + } + } + } +} + +static uint64_t median(uint64_t values[], int num) +{ + sort(values, num); + if (num % 2 == 0) + return (values[num / 2 - 1] + values[num / 2]) / 2; + else + return values[num / 2]; +} + +/* Measure TSC frequency. */ uint64_t _odp_time_cpu_global_freq(void) { - struct timespec sleep, ts1, ts2; - uint64_t t1, t2, ts_nsec, cycles, hz; + struct timespec ts1, ts2; + uint64_t t1, t2, ts_nsec, cycles; int i; - uint64_t avg = 0; - int rounds = 3; + const int rounds = 6; /* first round is warmup */ int warm_up = 1; + uint64_t hz[rounds]; for (i = 0; i < rounds; i++) { - sleep.tv_sec = 0; + uint64_t wait_nsec = ODP_TIME_SEC_IN_NS / 50; if (warm_up) - sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 1000; - else - sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 4; + wait_nsec = ODP_TIME_SEC_IN_NS / 1000; - if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { - _ODP_ERR("clock_gettime() failed\n"); - return 0; - } + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) + goto err_out; t1 = _odp_time_cpu_global(); - if (nanosleep(&sleep, NULL) < 0) { - _ODP_ERR("nanosleep() failed\n"); - return 0; - } + if (nwait(wait_nsec)) + goto err_out; - if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { - _ODP_ERR("clock_gettime() failed\n"); - return 0; - } + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) + goto err_out; t2 = _odp_time_cpu_global(); @@ -58,13 +92,15 @@ uint64_t _odp_time_cpu_global_freq(void) cycles = t2 - t1; - hz = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; + hz[i] = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; if (warm_up) warm_up = 0; - else - avg += hz; } - return avg / (rounds - 1); + return median(&hz[1], rounds - 1); + +err_out: + _ODP_ERR("clock_gettime() failed (%s)\n", strerror(errno)); + return 0; } |