diff options
Diffstat (limited to 'platform/linux-generic/arch/aarch64/odp')
9 files changed, 618 insertions, 0 deletions
diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h new file mode 100644 index 000000000..14cca3ca0 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifdef __ARM_FEATURE_ATOMICS +#define _ODP_LOCK_FREE_128BIT_ATOMICS +#endif + +#include <odp/api/abi-default/atomic.h> +#include <odp/api/plat/atomic_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h new file mode 100644 index 000000000..3b0f94efe --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h @@ -0,0 +1,278 @@ +/* Copyright (c) 2021, ARM Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_ATOMIC_INLINES_H_ +#define ODP_API_ABI_ATOMIC_INLINES_H_ + +#include <odp/api/atomic.h> + +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Helper macro for lockless atomic CAS operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param oper CAS operation + * @param old_val Old value + * @param new_val New value to be swapped + * @return 1 for success and 0 for fail + */ +#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ +__extension__ ({ \ + odp_u128_t _val; \ + odp_atomic_u128_t *_atom = atom; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + odp_u128_t *ptr = (odp_u128_t *)(_atom); \ + register uint64_t old0 __asm__ ("x0"); \ + register uint64_t old1 __asm__ ("x1"); \ + register uint64_t new0 __asm__ ("x2"); \ + register uint64_t new1 __asm__ ("x3"); \ + old0 = (uint64_t)(_old_val)->u64[0]; \ + old1 = (uint64_t)(_old_val)->u64[1]; \ + new0 = (uint64_t)(_new_val).u64[0]; \ + new1 = (uint64_t)(_new_val).u64[1]; \ + __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ + : [old0] "+r" (old0), [old1] "+r" (old1) \ + : [new0] "r" (new0), [new1] "r" (new1), \ + [ptr] "r" (ptr) \ + : "memory"); \ + _val.u64[0] = old0; \ + _val.u64[1] = old1; \ + val = _val; \ +}) + +#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + atom->v = new_val; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val, exp; + + exp.u64[0] = 0; + exp.u64[1] = 0; + ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); + return val; +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline void _odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stadd %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + int32_t neg_val = (int32_t)-val; + + __asm__ volatile("stadd %w[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val)); +} + +static inline void _odp_atomic_inc_u32(odp_atomic_u32_t *atom) +{ + _odp_atomic_add_u32(atom, 1); +} + +static inline void _odp_atomic_dec_u32(odp_atomic_u32_t *atom) +{ + _odp_atomic_sub_u32(atom, 1); +} + +static inline void _odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stadd %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + int64_t neg_val = (int64_t)-val; + + __asm__ volatile("stadd %[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val)); +} + +static inline void _odp_atomic_inc_u64(odp_atomic_u64_t *atom) +{ + _odp_atomic_add_u64(atom, 1); +} + +static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) +{ + _odp_atomic_sub_u64(atom, 1); +} + +static inline void _odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stumax %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stumin %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stumax %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stumin %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("staddl %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val) + : "memory"); +} + +static inline void _odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + int32_t neg_val = (int32_t)-val; + + __asm__ volatile("staddl %w[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val) + : "memory"); +} + +static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("staddl %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val) + : "memory"); +} + +static inline void _odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + int64_t neg_val = (int64_t)-val; + + __asm__ volatile("staddl %[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val) + : "memory"); +} + +#else /* !_ODP_LOCK_FREE_128BIT_ATOMICS */ + +/* Use generic implementation */ +#include <odp/api/abi/atomic_generic.h> + +#endif +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h new file mode 100644 index 000000000..825ff19d4 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/autoheader_external.h> + +#ifndef ODP_CACHE_LINE_SIZE + #define ODP_CACHE_LINE_SIZE _ODP_CACHE_LINE_SIZE +#endif + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..a26908e66 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/abi/time_cpu.h> + +#include <stdint.h> + +/* CPU frequency is shifted to decrease integer division error */ +#define _ODP_CPU_FREQ_SHIFT 16 + +typedef struct _odp_cpu_cycles_global_t { + uint64_t res; + uint64_t res_shifted; + uint64_t max; + +} _odp_cpu_cycles_global_t; + +extern _odp_cpu_cycles_global_t _odp_cpu_cycles_glob; + +static inline void _odp_cpu_pause(void) +{ + /* YIELD hints the CPU to switch to another thread if possible + * and executes as a NOP otherwise. + * ISB flushes the pipeline, then restarts. This is guaranteed to + * stall the CPU a number of cycles. + */ + __asm volatile("isb" ::: "memory"); +} + +static inline uint64_t _odp_cpu_cycles(void) +{ + return (_odp_time_cpu_global() * _odp_cpu_cycles_glob.res_shifted) >> _ODP_CPU_FREQ_SHIFT; +} + +static inline uint64_t _odp_cpu_cycles_resolution(void) +{ + return _odp_cpu_cycles_glob.res; +} + +static inline uint64_t _odp_cpu_cycles_max(void) +{ + return _odp_cpu_cycles_glob.max; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/hash_crc32.h b/platform/linux-generic/arch/aarch64/odp/api/abi/hash_crc32.h new file mode 100644 index 000000000..fd7bf91c6 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/hash_crc32.h @@ -0,0 +1,103 @@ +/* Copyright (c) 2021 ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_HASH_CRC32_H_ +#define ODP_API_ABI_HASH_CRC32_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +uint32_t _odp_hash_crc32_generic(const void *data, uint32_t data_len, + uint32_t init_val); +uint32_t _odp_hash_crc32c_generic(const void *data, uint32_t data_len, + uint32_t init_val); + +#ifdef __ARM_FEATURE_CRC32 + +#include <arm_acle.h> + +static inline uint32_t _odp_hash_crc32(const void *data_ptr, uint32_t data_len, + uint32_t init_val) +{ + uint32_t i; + uintptr_t pd = (uintptr_t)data_ptr; + + for (i = 0; i < data_len / 8; i++) { + init_val = __crc32d(init_val, *(const uint64_t *)pd); + pd += 8; + } + + if (data_len & 0x4) { + init_val = __crc32w(init_val, *(const uint32_t *)pd); + pd += 4; + } + + if (data_len & 0x2) { + init_val = __crc32h(init_val, *(const uint16_t *)pd); + pd += 2; + } + + if (data_len & 0x1) + init_val = __crc32b(init_val, *(const uint8_t *)pd); + + return init_val; +} + +static inline uint32_t _odp_hash_crc32c(const void *data, uint32_t data_len, + uint32_t init_val) +{ + uint32_t i; + uintptr_t pd = (uintptr_t)data; + + for (i = 0; i < data_len / 8; i++) { + init_val = __crc32cd(init_val, *(const uint64_t *)pd); + pd += 8; + } + + if (data_len & 0x4) { + init_val = __crc32cw(init_val, *(const uint32_t *)pd); + pd += 4; + } + + if (data_len & 0x2) { + init_val = __crc32ch(init_val, *(const uint16_t *)pd); + pd += 2; + } + + if (data_len & 0x1) + init_val = __crc32cb(init_val, *(const uint8_t *)pd); + + return init_val; +} + +#else /* __ARM_FEATURE_CRC32 */ + +/* + * Fall back to software implementation + */ + +static inline uint32_t _odp_hash_crc32(const void *data, uint32_t data_len, + uint32_t init_val) +{ + return _odp_hash_crc32_generic(data, data_len, init_val); +} + +static inline uint32_t _odp_hash_crc32c(const void *data, uint32_t data_len, + uint32_t init_val) +{ + return _odp_hash_crc32c_generic(data, data_len, init_val); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..3d42e7dd8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("dsb ld" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("dsb st" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..aba2799c7 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_TIME_CPU_H_ +#define ODP_API_ABI_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +static inline uint64_t _odp_time_cpu_global(void) +{ + uint64_t cntvct; + + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); + + return cntvct; +} + +static inline uint64_t _odp_time_cpu_global_strict(void) +{ + uint64_t cntvct; + + __asm__ volatile("isb" ::: "memory"); + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); + + return cntvct; +} + +static inline uint64_t _odp_time_cpu_global_freq(void) +{ + uint64_t cntfrq; + + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(cntfrq) : : ); + + return cntfrq; +} + +static inline int _odp_time_cpu_global_freq_is_const(void) +{ + return 1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/wait_until.h b/platform/linux-generic/arch/aarch64/odp/api/abi/wait_until.h new file mode 100644 index 000000000..73a3d476a --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/wait_until.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 ARM Limited + */ + +#ifndef ODP_API_ABI_WAIT_UNTIL_H_ +#define ODP_API_ABI_WAIT_UNTIL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/autoheader_external.h> + +#ifdef _ODP_WFE_LOCKS + +#include <stdint.h> + +#include <odp/api/atomic.h> + +static inline void +_odp_wait_until_equal_acq_u32(odp_atomic_u32_t *addr, uint32_t expected) +{ + uint32_t value; + uint32_t *var = &addr->v; + + __asm__ volatile("sevl" : : : "memory"); + do { + __asm__ volatile("wfe" : : : "memory"); + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (value) + : "r" (var) + : "memory"); + } while (expected != value); +} + +#else /* !_ODP_WFE_LOCKS*/ + +/* Use generic implementation */ +#include <odp/api/abi/wait_until_generic.h> + +#endif + +#ifdef __cplusplus +} +#endif + +#endif |