diff options
Diffstat (limited to 'platform/linux-generic/arch')
69 files changed, 6681 insertions, 385 deletions
diff --git a/platform/linux-generic/arch/aarch64/cpu_flags.c b/platform/linux-generic/arch/aarch64/cpu_flags.c new file mode 100644 index 000000000..9923e9306 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/cpu_flags.c @@ -0,0 +1,1052 @@ +/* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/hints.h> + +#include <odp_debug_internal.h> +#include <odp_macros_internal.h> + +#include "cpu_flags.h" + +#include <asm/hwcap.h> +#include <string.h> +#include <stdlib.h> +#include <sys/auxv.h> + +typedef struct { + const char *name; + const uint64_t bit_mask; +} hwcap_feat_flag_t; + +/* Linux HWCAP and HWCAP2 flags + * + * See https://docs.kernel.org/arch/arm64/elf_hwcaps.html for meaning of each flag. + */ +static hwcap_feat_flag_t hwcap_flags[] = { + { + /* Floating-point support for single-precision and double-precision types */ + .name = "FEAT_FP", +#ifdef HWCAP_FP + .bit_mask = HWCAP_FP, +#endif + }, + + { + /* Advanced SIMD support for: + * - integer byte, halfword, word and doubleword element operations + * - single-precision and double-precision floating-point arithmetic */ + .name = "ASIMD", +#ifdef HWCAP_ASIMD + .bit_mask = HWCAP_ASIMD, +#endif + }, + + { + /* Generic Timer is configured to generate events at approx. 10KHz */ + .name = "EVTSTRM", +#ifdef HWCAP_EVTSTRM + .bit_mask = HWCAP_EVTSTRM, +#endif + }, + + { + /* Advanced SIMD AES Instructions */ + .name = "FEAT_AES", +#ifdef HWCAP_AES + .bit_mask = HWCAP_AES, +#endif + }, + + { + /* Advanced SIMD PMULL Instructions */ + .name = "FEAT_PMULL", +#ifdef HWCAP_PMULL + .bit_mask = HWCAP_PMULL, +#endif + }, + + { + /* Advanced SIMD SHA1 Instructions */ + .name = "FEAT_SHA1", +#ifdef HWCAP_SHA1 + .bit_mask = HWCAP_SHA1, +#endif + }, + + { + /* Advanced SIMD SHA256 Instructions */ + .name = "FEAT_SHA256", +#ifdef HWCAP_SHA2 + .bit_mask = HWCAP_SHA2, +#endif + }, + + { + /* CRC32 Instructions */ + .name = "FEAT_CRC32", +#ifdef HWCAP_CRC32 + .bit_mask = HWCAP_CRC32, +#endif + }, + + { + /* Large System Extensions */ + .name = "FEAT_LSE", +#ifdef HWCAP_ATOMICS + .bit_mask = HWCAP_ATOMICS, +#endif + }, + + { + /* Half-precision Floating-point Data Processing Instructions */ + .name = "FEAT_FP16", +#ifdef HWCAP_FPHP + .bit_mask = HWCAP_FPHP, +#endif + }, + + { + /* Advanced SIMD support with half-precision floating-point arithmetic */ + .name = "ASIMDHP", +#ifdef HWCAP_ASIMDHP + .bit_mask = HWCAP_ASIMDHP, +#endif + }, + + { + /* Availability of EL0 Access to certain ID Registers */ + .name = "CPUID", +#ifdef HWCAP_CPUID + .bit_mask = HWCAP_CPUID, +#endif + }, + + { + /* Rounding Double Multiply Accumulate Extensions */ + .name = "FEAT_RDM", +#ifdef HWCAP_ASIMDRDM + .bit_mask = HWCAP_ASIMDRDM, +#endif + }, + + { + /* JavaScript FJCVTS Conversion Instructions */ + .name = "FEAT_JSCVT", +#ifdef HWCAP_JSCVT + .bit_mask = HWCAP_JSCVT, +#endif + }, + + { + /* Floating-point FCMLA and FCADD Instructions */ + .name = "FEAT_FCMA", +#ifdef HWCAP_FCMA + .bit_mask = HWCAP_FCMA, +#endif + }, + + { + /* Load-acquire RCpc Instructions */ + .name = "FEAT_LRCPC", +#ifdef HWCAP_LRCPC + .bit_mask = HWCAP_LRCPC, +#endif + }, + + { + /* DC CVAP Instructions */ + .name = "FEAT_DPB", +#ifdef HWCAP_DCPOP + .bit_mask = HWCAP_DCPOP, +#endif + }, + + { + /* Advanced SIMD EOR3, RAX1, XAR, and BCAX Instructions */ + .name = "FEAT_SHA3", +#ifdef HWCAP_SHA3 + .bit_mask = HWCAP_SHA3, +#endif + }, + + { + /* Advanced SIMD SM3 Instructions */ + .name = "FEAT_SM3", +#ifdef HWCAP_SM3 + .bit_mask = HWCAP_SM3, +#endif + }, + + { + /* Advanced SIMD SM4 Instructions */ + .name = "FEAT_SM4", +#ifdef HWCAP_SM4 + .bit_mask = HWCAP_SM4, +#endif + }, + + { + /* Advanced SIMD Int8 Dot Product Instructions */ + .name = "FEAT_DotProd", +#ifdef HWCAP_ASIMDDP + .bit_mask = HWCAP_ASIMDDP, +#endif + }, + + { + /* Advanced SIMD SHA512 Instructions */ + .name = "FEAT_SHA512", +#ifdef HWCAP_SHA512 + .bit_mask = HWCAP_SHA512, +#endif + }, + + { + /* Scalable Vector Extensions */ + .name = "FEAT_SVE", +#ifdef HWCAP_SVE + .bit_mask = HWCAP_SVE, +#endif + }, + + { + /* Half-precision Floating-point FMLAL Instructions */ + .name = "FEAT_FHM", +#ifdef HWCAP_ASIMDFHM + .bit_mask = HWCAP_ASIMDFHM, +#endif + }, + + { + /* Data Independent Timing Instructions */ + .name = "FEAT_DIT", +#ifdef HWCAP_DIT + .bit_mask = HWCAP_DIT, +#endif + }, + + { + /* Large System Extensions Version 2 */ + .name = "FEAT_LSE2", +#ifdef HWCAP_USCAT + .bit_mask = HWCAP_USCAT, +#endif + }, + + { + /* Load-acquire RCpc Instructions Version 2 */ + .name = "FEAT_LRCPC2", +#ifdef HWCAP_ILRCPC + .bit_mask = HWCAP_ILRCPC, +#endif + }, + + { + /* Condition Flag Manipulation Extensions */ + .name = "FEAT_FlagM", +#ifdef HWCAP_FLAGM + .bit_mask = HWCAP_FLAGM, +#endif + }, + + { + /* Speculative Store Bypass Safe Instructions */ + .name = "FEAT_SSBS2", +#ifdef HWCAP_SSBS + .bit_mask = HWCAP_SSBS, +#endif + }, + + { + /* Speculation Barrier Instructions */ + .name = "FEAT_SB", +#ifdef HWCAP_SB + .bit_mask = HWCAP_SB, +#endif + }, + + { + /* Pointer Authentication Extensions */ + .name = "FEAT_PAuth", +#ifdef HWCAP_PACA + .bit_mask = HWCAP_PACA, +#endif + }, + + { + /* Generic Authentication Extensions */ + .name = "PACG", +#ifdef HWCAP_PACG + .bit_mask = HWCAP_PACG, +#endif + } +}; + +static hwcap_feat_flag_t hwcap2_flags[] = { + { + /* DC CVADP instructions */ + .name = "FEAT_DPB2", +#ifdef HWCAP2_DCPODP + .bit_mask = HWCAP2_DCPODP, +#endif + }, + + { + /* Scalable Vector Extensions Version 2 */ + .name = "FEAT_SVE2", +#ifdef HWCAP2_SVE2 + .bit_mask = HWCAP2_SVE2, +#endif + }, + + { + /* SVE AES Instructions */ + .name = "FEAT_SVE_AES", +#ifdef HWCAP2_SVEAES + .bit_mask = HWCAP2_SVEAES, +#endif + }, + + { + /* SVE PMULL Instructions */ + .name = "FEAT_SVE_PMULL128", +#ifdef HWCAP2_SVEPMULL + .bit_mask = HWCAP2_SVEPMULL, +#endif + }, + + { + /* SVE Bit Permute Instructions */ + .name = "FEAT_SVE_BitPerm", +#ifdef HWCAP2_SVEBITPERM + .bit_mask = HWCAP2_SVEBITPERM, +#endif + }, + + { + /* SVE SHA-3 Instructions */ + .name = "FEAT_SVE_SHA3", +#ifdef HWCAP2_SVESHA3 + .bit_mask = HWCAP2_SVESHA3, +#endif + }, + + { + /* SVE SM4 Instructions */ + .name = "FEAT_SVE_SM4", +#ifdef HWCAP2_SVESM4 + .bit_mask = HWCAP2_SVESM4, +#endif + }, + + { + /* Condition Flag Manipulation Extensions Version 2 */ + .name = "FEAT_FlagM2", +#ifdef HWCAP2_FLAGM2 + .bit_mask = HWCAP2_FLAGM2, +#endif + }, + + { + /* FRINT32Z, FRINT32X, FRINT64Z, and FRINT64X instructions */ + .name = "FEAT_FRINTTS", +#ifdef HWCAP2_FRINT + .bit_mask = HWCAP2_FRINT, +#endif + }, + + { + /* SVE Int8 Matrix Multiplication Instructions */ + .name = "SVEI8MM", +#ifdef HWCAP2_SVEI8MM + .bit_mask = HWCAP2_SVEI8MM, +#endif + }, + + { + /* SVE Single-precision Floating-point Matrix Multiply Instructions */ + .name = "FEAT_F32MM", +#ifdef HWCAP2_SVEF32MM + .bit_mask = HWCAP2_SVEF32MM, +#endif + }, + + { + /* SVE Double-precision Floating-point Matrix Multiply Instructions */ + .name = "FEAT_F64MM", +#ifdef HWCAP2_SVEF64MM + .bit_mask = HWCAP2_SVEF64MM, +#endif + }, + + { + /* SVE BFloat16 Instructions */ + .name = "SVEBF16", +#ifdef HWCAP2_SVEBF16 + .bit_mask = HWCAP2_SVEBF16, +#endif + }, + + { + /* Advanced SIMD and Floating-point Int8 Matrix Multiplication Instructions */ + .name = "FEAT_I8MM", +#ifdef HWCAP2_I8MM + .bit_mask = HWCAP2_I8MM, +#endif + }, + + { + /* Advanced SIMD and Floating-point BFloat16 Instructions */ + .name = "FEAT_BF16", +#ifdef HWCAP2_BF16 + .bit_mask = HWCAP2_BF16, +#endif + }, + + { + /* Data Gathering Hint Extensions */ + .name = "FEAT_DGH", +#ifdef HWCAP2_DGH + .bit_mask = HWCAP2_DGH, +#endif + }, + + { + /* Random Number Generation Extensions */ + .name = "FEAT_RNG", +#ifdef HWCAP2_RNG + .bit_mask = HWCAP2_RNG, +#endif + }, + + { + /* Branch Target Identification Extensions */ + .name = "FEAT_BTI", +#ifdef HWCAP2_BTI + .bit_mask = HWCAP2_BTI, +#endif + }, + + { + /* Full Memory Tagging Extensions */ + .name = "FEAT_MTE2", +#ifdef HWCAP2_MTE + .bit_mask = HWCAP2_MTE, +#endif + }, + + { + .name = "ECV", +#ifdef HWCAP2_ECV + .bit_mask = HWCAP2_ECV, +#endif + }, + + { + .name = "AFP", +#ifdef HWCAP2_AFP + .bit_mask = HWCAP2_AFP, +#endif + }, + + { + .name = "RPRES", +#ifdef HWCAP2_RPRES + .bit_mask = HWCAP2_RPRES, +#endif + }, + + { + .name = "MTE3", +#ifdef HWCAP2_MTE3 + .bit_mask = HWCAP2_MTE3, +#endif + }, + + { + .name = "SME", +#ifdef HWCAP2_SME + .bit_mask = HWCAP2_SME, +#endif + }, + + { + .name = "SME_I16I64", +#ifdef HWCAP2_SME_I16I64 + .bit_mask = HWCAP2_SME_I16I64, +#endif + }, + + { + .name = "SME_F64F64", +#ifdef HWCAP2_SME_F64F64 + .bit_mask = HWCAP2_SME_F64F64, +#endif + }, + + { + .name = "SME_I8I32", +#ifdef HWCAP2_SME_I8I32 + .bit_mask = HWCAP2_SME_I8I32, +#endif + }, + + { + .name = "SME_F16F32", +#ifdef HWCAP2_SME_F16F32 + .bit_mask = HWCAP2_SME_F16F32, +#endif + }, + + { + .name = "SME_B16F32", +#ifdef HWCAP2_SME_B16F32 + .bit_mask = HWCAP2_SME_B16F32, +#endif + }, + + { + .name = "SME_F32F32", +#ifdef HWCAP2_SME_F32F32 + .bit_mask = HWCAP2_SME_F32F32, +#endif + }, + + { + .name = "SME_FA64", +#ifdef HWCAP2_SME_FA64 + .bit_mask = HWCAP2_SME_FA64, +#endif + }, + + { + .name = "WFXT", +#ifdef HWCAP2_WFXT + .bit_mask = HWCAP2_WFXT, +#endif + }, + + { + .name = "EBF16", +#ifdef HWCAP2_EBF16 + .bit_mask = HWCAP2_EBF16, +#endif + }, + + { + .name = "SVE_EBF16", +#ifdef HWCAP2_SVE_EBF16 + .bit_mask = HWCAP2_SVE_EBF16, +#endif + }, + + { + .name = "CSSC", +#ifdef HWCAP2_CSSC + .bit_mask = HWCAP2_CSSC, +#endif + }, + + { + .name = "RPRFM", +#ifdef HWCAP2_RPRFM + .bit_mask = HWCAP2_RPRFM, +#endif + }, + + { + .name = "SVE2P1", +#ifdef HWCAP2_SVE2P1 + .bit_mask = HWCAP2_SVE2P1, +#endif + }, + + { + .name = "SME2", +#ifdef HWCAP2_SME2 + .bit_mask = HWCAP2_SME2, +#endif + }, + + { + .name = "SME2P1", +#ifdef HWCAP2_SME2P1 + .bit_mask = HWCAP2_SME2P1, +#endif + }, + + { + .name = "SME_I16I32", +#ifdef HWCAP2_SME_I16I32 + .bit_mask = HWCAP2_SME_I16I32, +#endif + }, + + { + .name = "SME_BI32I32", +#ifdef HWCAP2_SME_BI32I32 + .bit_mask = HWCAP2_SME_BI32I32, +#endif + }, + + { + .name = "SME_B16B16", +#ifdef HWCAP2_SME_B16B16 + .bit_mask = HWCAP2_SME_B16B16, +#endif + }, + + { + .name = "SME_F16F16", +#ifdef HWCAP2_SME_F16F16 + .bit_mask = HWCAP2_SME_F16F16, +#endif + }, + + { + .name = "MOPS", +#ifdef HWCAP2_MOPS + .bit_mask = HWCAP2_MOPS, +#endif + }, +}; + +static void _odp_sys_info_print_acle_flags(void) +{ + const char *ndef = "n/a"; + + /* Avoid compiler warning about unused variable */ + (void)ndef; + + /* See ARM C Language Extensions documentation for details */ + _ODP_PRINT("ARM FEATURES:\n"); + + _ODP_PRINT(" __ARM_ALIGN_MAX_PWR "); +#ifdef __ARM_ALIGN_MAX_PWR + _ODP_PRINT("%i\n", __ARM_ALIGN_MAX_PWR); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_ALIGN_MAX_STACK_PWR "); +#ifdef __ARM_ALIGN_MAX_STACK_PWR + _ODP_PRINT("%i\n", __ARM_ALIGN_MAX_STACK_PWR); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_ARCH "); +#ifdef __ARM_ARCH + _ODP_PRINT("%i\n", __ARM_ARCH); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_ARCH_ISA_A64 "); +#ifdef __ARM_ARCH_ISA_A64 + _ODP_PRINT("%i\n", __ARM_ARCH_ISA_A64); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_BIG_ENDIAN "); +#ifdef __ARM_BIG_ENDIAN + _ODP_PRINT("%i\n", __ARM_BIG_ENDIAN); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_BF16_FORMAT_ALTERNATIVE "); +#ifdef __ARM_BF16_FORMAT_ALTERNATIVE + _ODP_PRINT("%i\n", __ARM_BF16_FORMAT_ALTERNATIVE); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_ATOMICS "); +#ifdef __ARM_FEATURE_ATOMICS + _ODP_PRINT("%i\n", __ARM_FEATURE_ATOMICS); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_BF16 "); +#ifdef __ARM_FEATURE_BF16 + _ODP_PRINT("%i\n", __ARM_FEATURE_BF16); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_BTI_DEFAULT "); +#ifdef __ARM_FEATURE_BTI_DEFAULT + _ODP_PRINT("%i\n", __ARM_FEATURE_BTI_DEFAULT); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_CDE "); +#ifdef __ARM_FEATURE_CDE + _ODP_PRINT("%i\n", __ARM_FEATURE_CDE); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_CDE_COPROC "); +#ifdef __ARM_FEATURE_CDE_COPROC + _ODP_PRINT("0x%X\n", __ARM_FEATURE_CDE_COPROC); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_CLZ "); +#ifdef __ARM_FEATURE_CLZ + _ODP_PRINT("%i\n", __ARM_FEATURE_CLZ); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_COMPLEX "); +#ifdef __ARM_FEATURE_COMPLEX + _ODP_PRINT("%i\n", __ARM_FEATURE_COMPLEX); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_COPROC "); +#ifdef __ARM_FEATURE_COPROC + _ODP_PRINT("0x%X\n", __ARM_FEATURE_COPROC); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_CRC32 "); +#ifdef __ARM_FEATURE_CRC32 + _ODP_PRINT("%i\n", __ARM_FEATURE_CRC32); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_CRYPTO "); +#ifdef __ARM_FEATURE_CRYPTO + _ODP_PRINT("%i\n", __ARM_FEATURE_CRYPTO); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_DIRECTED_ROUNDING "); +#ifdef __ARM_FEATURE_DIRECTED_ROUNDING + _ODP_PRINT("%i\n", __ARM_FEATURE_DIRECTED_ROUNDING); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_DOTPROD "); +#ifdef __ARM_FEATURE_DOTPROD + _ODP_PRINT("%i\n", __ARM_FEATURE_DOTPROD); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_FMA "); +#ifdef __ARM_FEATURE_FMA + _ODP_PRINT("%i\n", __ARM_FEATURE_FMA); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_FP16_FML "); +#ifdef __ARM_FEATURE_FP16_FML + _ODP_PRINT("%i\n", __ARM_FEATURE_FP16_FML); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_FRINT "); +#ifdef __ARM_FEATURE_FRINT + _ODP_PRINT("%i\n", __ARM_FEATURE_FRINT); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_IDIV "); +#ifdef __ARM_FEATURE_IDIV + _ODP_PRINT("%i\n", __ARM_FEATURE_IDIV); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_JCVT "); +#ifdef __ARM_FEATURE_JCVT + _ODP_PRINT("%i\n", __ARM_FEATURE_JCVT); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_MATMUL_INT8 "); +#ifdef __ARM_FEATURE_MATMUL_INT8 + _ODP_PRINT("%i\n", __ARM_FEATURE_MATMUL_INT8); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_MEMORY_TAGGING "); +#ifdef __ARM_FEATURE_MEMORY_TAGGING + _ODP_PRINT("%i\n", __ARM_FEATURE_MEMORY_TAGGING); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_MVE "); +#ifdef __ARM_FEATURE_MVE + _ODP_PRINT("0x%X\n", __ARM_FEATURE_MVE); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_NUMERIC_MAXMIN "); +#ifdef __ARM_FEATURE_NUMERIC_MAXMIN + _ODP_PRINT("%i\n", __ARM_FEATURE_NUMERIC_MAXMIN); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_PAC_DEFAULT "); +#ifdef __ARM_FEATURE_PAC_DEFAULT + _ODP_PRINT("0x%X\n", __ARM_FEATURE_PAC_DEFAULT); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_QRDMX "); +#ifdef __ARM_FEATURE_QRDMX + _ODP_PRINT("%i\n", __ARM_FEATURE_QRDMX); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_RNG "); +#ifdef __ARM_FEATURE_RNG + _ODP_PRINT("%i\n", __ARM_FEATURE_RNG); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_SHA3 "); +#ifdef __ARM_FEATURE_SHA3 + _ODP_PRINT("%i\n", __ARM_FEATURE_SHA3); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_SHA512 "); +#ifdef __ARM_FEATURE_SHA512 + _ODP_PRINT("%i\n", __ARM_FEATURE_SHA512); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_SM3 "); +#ifdef __ARM_FEATURE_SM3 + _ODP_PRINT("%i\n", __ARM_FEATURE_SM3); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_SM4 "); +#ifdef __ARM_FEATURE_SM4 + _ODP_PRINT("%i\n", __ARM_FEATURE_SM4); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_TME "); +#ifdef __ARM_FEATURE_TME + _ODP_PRINT("%i\n", __ARM_FEATURE_TME); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FEATURE_UNALIGNED "); +#ifdef __ARM_FEATURE_UNALIGNED + _ODP_PRINT("%i\n", __ARM_FEATURE_UNALIGNED); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FP "); +#ifdef __ARM_FP + _ODP_PRINT("0x%X\n", __ARM_FP); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FP_FAST "); +#ifdef __ARM_FP_FAST + _ODP_PRINT("%i\n", __ARM_FP_FAST); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FP_FENV_ROUNDING "); +#ifdef __ARM_FP_FENV_ROUNDING + _ODP_PRINT("%i\n", __ARM_FP_FENV_ROUNDING); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FP16_ARGS "); +#ifdef __ARM_FP16_ARGS + _ODP_PRINT("%i\n", __ARM_FP16_ARGS); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FP16_FORMAT_ALTERNATIVE "); +#ifdef __ARM_FP16_FORMAT_ALTERNATIVE + _ODP_PRINT("%i\n", __ARM_FP16_FORMAT_ALTERNATIVE); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_FP16_FORMAT_IEEE "); +#ifdef __ARM_FP16_FORMAT_IEEE + _ODP_PRINT("%i\n", __ARM_FP16_FORMAT_IEEE); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_NEON "); +#ifdef __ARM_NEON + _ODP_PRINT("%i\n", __ARM_NEON); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_NEON_FP "); +#ifdef __ARM_NEON_FP + _ODP_PRINT("0x%X\n", __ARM_NEON_FP); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_PCS_AAPCS64 "); +#ifdef __ARM_PCS_AAPCS64 + _ODP_PRINT("%i\n", __ARM_PCS_AAPCS64); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_ROPI "); +#ifdef __ARM_ROPI + _ODP_PRINT("%i\n", __ARM_ROPI); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_RWPI "); +#ifdef __ARM_RWPI + _ODP_PRINT("%i\n", __ARM_RWPI); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_SIZEOF_MINIMAL_ENUM "); +#ifdef __ARM_SIZEOF_MINIMAL_ENUM + _ODP_PRINT("%i\n", __ARM_SIZEOF_MINIMAL_ENUM); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" __ARM_SIZEOF_WCHAR_T "); +#ifdef __ARM_SIZEOF_WCHAR_T + _ODP_PRINT("%i\n", __ARM_SIZEOF_WCHAR_T); +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT(" ARM ISA version: "); +#if defined(__ARM_ARCH) + if (__ARM_ARCH < 8) { + _ODP_PRINT("v%i\n", __ARM_ARCH); + } else if (__ARM_ARCH == 8) { + /* Actually, this checks for new NEON instructions in + * v8.1, but is currently the only way to distinguish + * v8.0 and >=v8.1. */ + #ifdef __ARM_FEATURE_QRDMX + _ODP_PRINT("v8.1 or higher\n"); + #else + _ODP_PRINT("v8.0\n"); + #endif + } else { + /* ACLE 2018 defines that from v8.1 onwards the value includes + * the minor version number: __ARM_ARCH = X * 100 + Y + * E.g. for Armv8.1 __ARM_ARCH = 801 */ + int major = __ARM_ARCH / 100; + int minor = __ARM_ARCH - (major * 100); + + _ODP_PRINT("v%i.%i\n", major, minor); + } +#else + _ODP_PRINT("%s\n", ndef); +#endif + + _ODP_PRINT("\n"); +} + +static void _odp_sys_info_print_hwcap_flags(void) +{ + uint64_t hwcap, hwcap2; + uint32_t size, size2, i; + + hwcap = getauxval(AT_HWCAP); + hwcap2 = getauxval(AT_HWCAP2); + size = _ODP_ARRAY_SIZE(hwcap_flags); + size2 = _ODP_ARRAY_SIZE(hwcap2_flags); + + _ODP_PRINT("ARM FEATURES SUPPORTED BY HARDWARE:\n"); + + /* Supported HWCAP flags */ + for (i = 0; i < size; i++) + if (hwcap & hwcap_flags[i].bit_mask) + _ODP_PRINT("%s ", hwcap_flags[i].name); + + /* Supported HWCAP2 flags */ + for (i = 0; i < size2; i++) + if (hwcap2 & hwcap2_flags[i].bit_mask) + _ODP_PRINT("%s ", hwcap2_flags[i].name); + + _ODP_PRINT("\n\nARM FEATURES NOT SUPPORTED BY HARDWARE:\n"); + + /* Unsupported HWCAP flags */ + for (i = 0; i < size; i++) + if (hwcap_flags[i].bit_mask && (hwcap & hwcap_flags[i].bit_mask) == 0) + _ODP_PRINT("%s ", hwcap_flags[i].name); + + /* Unsupported HWCAP2 flags */ + for (i = 0; i < size2; i++) + if (hwcap2_flags[i].bit_mask && (hwcap2 & hwcap2_flags[i].bit_mask) == 0) + _ODP_PRINT("%s ", hwcap2_flags[i].name); + + _ODP_PRINT("\n\nARM FEATURES UNKNOWN TO LINUX VERSION:\n"); + /* Unknown HWCAP flags */ + for (i = 0; i < size; i++) + if (hwcap_flags[i].bit_mask == 0) + _ODP_PRINT("%s ", hwcap_flags[i].name); + + /* Unknown HWCAP2 flags */ + for (i = 0; i < size2; i++) + if (hwcap2_flags[i].bit_mask == 0) + _ODP_PRINT("%s ", hwcap2_flags[i].name); + + _ODP_PRINT("\n\n"); +} + +void _odp_cpu_flags_print_all(void) +{ + _odp_sys_info_print_acle_flags(); + _odp_sys_info_print_hwcap_flags(); +} diff --git a/platform/linux-generic/arch/aarch64/cpu_flags.h b/platform/linux-generic/arch/aarch64/cpu_flags.h new file mode 100644 index 000000000..177b1c44f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/cpu_flags.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_FLAGS_H_ +#define ODP_PLAT_CPU_FLAGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void _odp_cpu_flags_print_all(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h new file mode 100644 index 000000000..14cca3ca0 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifdef __ARM_FEATURE_ATOMICS +#define _ODP_LOCK_FREE_128BIT_ATOMICS +#endif + +#include <odp/api/abi-default/atomic.h> +#include <odp/api/plat/atomic_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h new file mode 100644 index 000000000..3b0f94efe --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h @@ -0,0 +1,278 @@ +/* Copyright (c) 2021, ARM Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_ATOMIC_INLINES_H_ +#define ODP_API_ABI_ATOMIC_INLINES_H_ + +#include <odp/api/atomic.h> + +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Helper macro for lockless atomic CAS operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param oper CAS operation + * @param old_val Old value + * @param new_val New value to be swapped + * @return 1 for success and 0 for fail + */ +#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ +__extension__ ({ \ + odp_u128_t _val; \ + odp_atomic_u128_t *_atom = atom; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + odp_u128_t *ptr = (odp_u128_t *)(_atom); \ + register uint64_t old0 __asm__ ("x0"); \ + register uint64_t old1 __asm__ ("x1"); \ + register uint64_t new0 __asm__ ("x2"); \ + register uint64_t new1 __asm__ ("x3"); \ + old0 = (uint64_t)(_old_val)->u64[0]; \ + old1 = (uint64_t)(_old_val)->u64[1]; \ + new0 = (uint64_t)(_new_val).u64[0]; \ + new1 = (uint64_t)(_new_val).u64[1]; \ + __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ + : [old0] "+r" (old0), [old1] "+r" (old1) \ + : [new0] "r" (new0), [new1] "r" (new1), \ + [ptr] "r" (ptr) \ + : "memory"); \ + _val.u64[0] = old0; \ + _val.u64[1] = old1; \ + val = _val; \ +}) + +#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + atom->v = new_val; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val, exp; + + exp.u64[0] = 0; + exp.u64[1] = 0; + ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); + return val; +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + return 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return 0; +} + +static inline void _odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stadd %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + int32_t neg_val = (int32_t)-val; + + __asm__ volatile("stadd %w[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val)); +} + +static inline void _odp_atomic_inc_u32(odp_atomic_u32_t *atom) +{ + _odp_atomic_add_u32(atom, 1); +} + +static inline void _odp_atomic_dec_u32(odp_atomic_u32_t *atom) +{ + _odp_atomic_sub_u32(atom, 1); +} + +static inline void _odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stadd %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + int64_t neg_val = (int64_t)-val; + + __asm__ volatile("stadd %[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val)); +} + +static inline void _odp_atomic_inc_u64(odp_atomic_u64_t *atom) +{ + _odp_atomic_add_u64(atom, 1); +} + +static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) +{ + _odp_atomic_sub_u64(atom, 1); +} + +static inline void _odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stumax %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("stumin %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stumax %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("stumin %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val)); +} + +static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + __asm__ volatile("staddl %w[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val) + : "memory"); +} + +static inline void _odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + int32_t neg_val = (int32_t)-val; + + __asm__ volatile("staddl %w[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val) + : "memory"); +} + +static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __asm__ volatile("staddl %[val], %[atom]" + : [atom] "+Q" (atom->v) + : [val] "r" (val) + : "memory"); +} + +static inline void _odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + int64_t neg_val = (int64_t)-val; + + __asm__ volatile("staddl %[neg_val], %[atom]" + : [atom] "+Q" (atom->v) + : [neg_val] "r" (neg_val) + : "memory"); +} + +#else /* !_ODP_LOCK_FREE_128BIT_ATOMICS */ + +/* Use generic implementation */ +#include <odp/api/abi/atomic_generic.h> + +#endif +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h new file mode 100644 index 000000000..825ff19d4 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/autoheader_external.h> + +#ifndef ODP_CACHE_LINE_SIZE + #define ODP_CACHE_LINE_SIZE _ODP_CACHE_LINE_SIZE +#endif + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..a26908e66 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/cpu_inlines.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/abi/time_cpu.h> + +#include <stdint.h> + +/* CPU frequency is shifted to decrease integer division error */ +#define _ODP_CPU_FREQ_SHIFT 16 + +typedef struct _odp_cpu_cycles_global_t { + uint64_t res; + uint64_t res_shifted; + uint64_t max; + +} _odp_cpu_cycles_global_t; + +extern _odp_cpu_cycles_global_t _odp_cpu_cycles_glob; + +static inline void _odp_cpu_pause(void) +{ + /* YIELD hints the CPU to switch to another thread if possible + * and executes as a NOP otherwise. + * ISB flushes the pipeline, then restarts. This is guaranteed to + * stall the CPU a number of cycles. + */ + __asm volatile("isb" ::: "memory"); +} + +static inline uint64_t _odp_cpu_cycles(void) +{ + return (_odp_time_cpu_global() * _odp_cpu_cycles_glob.res_shifted) >> _ODP_CPU_FREQ_SHIFT; +} + +static inline uint64_t _odp_cpu_cycles_resolution(void) +{ + return _odp_cpu_cycles_glob.res; +} + +static inline uint64_t _odp_cpu_cycles_max(void) +{ + return _odp_cpu_cycles_glob.max; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/hash_crc32.h b/platform/linux-generic/arch/aarch64/odp/api/abi/hash_crc32.h new file mode 100644 index 000000000..fd7bf91c6 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/hash_crc32.h @@ -0,0 +1,103 @@ +/* Copyright (c) 2021 ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_HASH_CRC32_H_ +#define ODP_API_ABI_HASH_CRC32_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +uint32_t _odp_hash_crc32_generic(const void *data, uint32_t data_len, + uint32_t init_val); +uint32_t _odp_hash_crc32c_generic(const void *data, uint32_t data_len, + uint32_t init_val); + +#ifdef __ARM_FEATURE_CRC32 + +#include <arm_acle.h> + +static inline uint32_t _odp_hash_crc32(const void *data_ptr, uint32_t data_len, + uint32_t init_val) +{ + uint32_t i; + uintptr_t pd = (uintptr_t)data_ptr; + + for (i = 0; i < data_len / 8; i++) { + init_val = __crc32d(init_val, *(const uint64_t *)pd); + pd += 8; + } + + if (data_len & 0x4) { + init_val = __crc32w(init_val, *(const uint32_t *)pd); + pd += 4; + } + + if (data_len & 0x2) { + init_val = __crc32h(init_val, *(const uint16_t *)pd); + pd += 2; + } + + if (data_len & 0x1) + init_val = __crc32b(init_val, *(const uint8_t *)pd); + + return init_val; +} + +static inline uint32_t _odp_hash_crc32c(const void *data, uint32_t data_len, + uint32_t init_val) +{ + uint32_t i; + uintptr_t pd = (uintptr_t)data; + + for (i = 0; i < data_len / 8; i++) { + init_val = __crc32cd(init_val, *(const uint64_t *)pd); + pd += 8; + } + + if (data_len & 0x4) { + init_val = __crc32cw(init_val, *(const uint32_t *)pd); + pd += 4; + } + + if (data_len & 0x2) { + init_val = __crc32ch(init_val, *(const uint16_t *)pd); + pd += 2; + } + + if (data_len & 0x1) + init_val = __crc32cb(init_val, *(const uint8_t *)pd); + + return init_val; +} + +#else /* __ARM_FEATURE_CRC32 */ + +/* + * Fall back to software implementation + */ + +static inline uint32_t _odp_hash_crc32(const void *data, uint32_t data_len, + uint32_t init_val) +{ + return _odp_hash_crc32_generic(data, data_len, init_val); +} + +static inline uint32_t _odp_hash_crc32c(const void *data, uint32_t data_len, + uint32_t init_val) +{ + return _odp_hash_crc32c_generic(data, data_len, init_val); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..3d42e7dd8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("dsb sy" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("dsb ld" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("dsb st" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..aba2799c7 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_cpu.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_TIME_CPU_H_ +#define ODP_API_ABI_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +static inline uint64_t _odp_time_cpu_global(void) +{ + uint64_t cntvct; + + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); + + return cntvct; +} + +static inline uint64_t _odp_time_cpu_global_strict(void) +{ + uint64_t cntvct; + + __asm__ volatile("isb" ::: "memory"); + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(cntvct) : : "memory"); + + return cntvct; +} + +static inline uint64_t _odp_time_cpu_global_freq(void) +{ + uint64_t cntfrq; + + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(cntfrq) : : ); + + return cntfrq; +} + +static inline int _odp_time_cpu_global_freq_is_const(void) +{ + return 1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/wait_until.h b/platform/linux-generic/arch/aarch64/odp/api/abi/wait_until.h new file mode 100644 index 000000000..73a3d476a --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/wait_until.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 ARM Limited + */ + +#ifndef ODP_API_ABI_WAIT_UNTIL_H_ +#define ODP_API_ABI_WAIT_UNTIL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/autoheader_external.h> + +#ifdef _ODP_WFE_LOCKS + +#include <stdint.h> + +#include <odp/api/atomic.h> + +static inline void +_odp_wait_until_equal_acq_u32(odp_atomic_u32_t *addr, uint32_t expected) +{ + uint32_t value; + uint32_t *var = &addr->v; + + __asm__ volatile("sevl" : : : "memory"); + do { + __asm__ volatile("wfe" : : : "memory"); + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (value) + : "r" (var) + : "memory"); + } while (expected != value); +} + +#else /* !_ODP_WFE_LOCKS*/ + +/* Use generic implementation */ +#include <odp/api/abi/wait_until_generic.h> + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.c b/platform/linux-generic/arch/aarch64/odp_atomic.c new file mode 100644 index 000000000..c6b809768 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_atomic.c @@ -0,0 +1,56 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/atomic.h> + +int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#else + /* All operations are lock-free */ + if (atomic_op) { + atomic_op->all_bits = ~((uint32_t)0); + atomic_op->op.init = 0; + } + + return 2; +#endif +} + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + + return 2; +#elif defined(__SIZEOF_INT128__) + if (__atomic_is_lock_free(16, NULL)) { + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + return 2; + } +#endif + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +} diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h new file mode 100644 index 000000000..d3b8ea4dc --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -0,0 +1,325 @@ +/* Copyright (c) 2017-2021, ARM Limited + * Copyright (c) 2017-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H +#define PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#error This file should not be included directly, please include odp_cpu.h +#endif + +#include <odp_types_internal.h> +#include <limits.h> + +#ifdef CONFIG_DMBSTR + +#define atomic_store_release(loc, val, ro) \ +do { \ + _odp_release_barrier(ro); \ + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ +} while (0) + +#else + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +#endif /* CONFIG_DMBSTR */ + +#define HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) +#define HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) +#define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */ +static inline bool +__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp, + register _odp_u128_t neu, bool weak, int mo_success, + int mo_failure) +{ + (void)weak; /* Always do strong CAS or we can't perform atomic read */ + /* Ignore memory ordering for failure, memory order for + * success must be stronger or equal. */ + (void)mo_failure; + register _odp_u128_t old; + register _odp_u128_t expected; + int ll_mo = LL_MO(mo_success); + int sc_mo = SC_MO(mo_success); + + expected = *exp; + __asm__ volatile("" ::: "memory"); + do { + /* Atomicity of LLD is not guaranteed */ + old = lld(var, ll_mo); + /* Must write back neu or old to verify atomicity of LLD */ + } while (odp_unlikely(scd(var, old == expected ? neu : old, sc_mo))); + *exp = old; /* Always update, atomically read value */ + return old == expected; +} + +static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var, + _odp_u128_t neu, int mo) +{ + register _odp_u128_t old; + int ll_mo = LL_MO(mo); + int sc_mo = SC_MO(mo); + + do { + /* Atomicity of LLD is not guaranteed */ + old = lld(var, ll_mo); + /* Must successfully write back to verify atomicity of LLD */ + } while (odp_unlikely(scd(var, neu, sc_mo))); + return old; +} + +static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) +{ + register _odp_u128_t old; + int ll_mo = LL_MO(mo); + int sc_mo = SC_MO(mo); + + do { + /* Atomicity of LLD is not guaranteed */ + old = lld(var, ll_mo); + /* Must successfully write back to verify atomicity of LLD */ + } while (odp_unlikely(scd(var, old & mask, sc_mo))); + return old; +} + +static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) +{ + register _odp_u128_t old; + int ll_mo = LL_MO(mo); + int sc_mo = SC_MO(mo); + + do { + /* Atomicity of LLD is not guaranteed */ + old = lld(var, ll_mo); + /* Must successfully write back to verify atomicity of LLD */ + } while (odp_unlikely(scd(var, old | mask, sc_mo))); + return old; +} + +#else + +static inline _odp_u128_t cas_u128(_odp_u128_t *ptr, _odp_u128_t old_val, + _odp_u128_t new_val, int mo) +{ + /* CASP instructions require that the first register number is paired */ + register uint64_t old0 __asm__ ("x0"); + register uint64_t old1 __asm__ ("x1"); + register uint64_t new0 __asm__ ("x2"); + register uint64_t new1 __asm__ ("x3"); + + old0 = (uint64_t)old_val; + old1 = (uint64_t)(old_val >> 64); + new0 = (uint64_t)new_val; + new1 = (uint64_t)(new_val >> 64); + + if (mo == __ATOMIC_RELAXED) { + __asm__ volatile("casp %[old0], %[old1], %[new0], %[new1], [%[ptr]]" + : [old0] "+r" (old0), [old1] "+r" (old1) + : [new0] "r" (new0), [new1] "r" (new1), [ptr] "r" (ptr) + : "memory"); + } else if (mo == __ATOMIC_ACQUIRE) { + __asm__ volatile("caspa %[old0], %[old1], %[new0], %[new1], [%[ptr]]" + : [old0] "+r" (old0), [old1] "+r" (old1) + : [new0] "r" (new0), [new1] "r" (new1), [ptr] "r" (ptr) + : "memory"); + } else if (mo == __ATOMIC_ACQ_REL) { + __asm__ volatile("caspal %[old0], %[old1], %[new0], %[new1], [%[ptr]]" + : [old0] "+r" (old0), [old1] "+r" (old1) + : [new0] "r" (new0), [new1] "r" (new1), [ptr] "r" (ptr) + : "memory"); + } else if (mo == __ATOMIC_RELEASE) { + __asm__ volatile("caspl %[old0], %[old1], %[new0], %[new1], [%[ptr]]" + : [old0] "+r" (old0), [old1] "+r" (old1) + : [new0] "r" (new0), [new1] "r" (new1), [ptr] "r" (ptr) + : "memory"); + } else { + abort(); + } + + return ((_odp_u128_t)old0) | (((_odp_u128_t)old1) << 64); +} + +static inline bool +__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp, + register _odp_u128_t neu, bool weak, int mo_success, + int mo_failure) +{ + (void)weak; + (void)mo_failure; + _odp_u128_t old; + _odp_u128_t expected; + + expected = *exp; + old = cas_u128(var, expected, neu, mo_success); + *exp = old; /* Always update, atomically read value */ + return old == expected; +} + +static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var, + _odp_u128_t neu, int mo) +{ + _odp_u128_t old; + _odp_u128_t expected; + + do { + expected = *var; + old = cas_u128(var, expected, neu, mo); + } while (old != expected); + return old; +} + +static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) +{ + _odp_u128_t old; + _odp_u128_t expected; + + do { + expected = *var; + old = cas_u128(var, expected, expected & mask, mo); + } while (old != expected); + return old; +} + +static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var, + _odp_u128_t mask, int mo) +{ + _odp_u128_t old; + _odp_u128_t expected; + + do { + expected = *var; + old = cas_u128(var, expected, expected | mask, mo); + } while (old != expected); + return old; +} + +#endif /* __ARM_FEATURE_QRDMX */ + +static inline _odp_u128_t __lockfree_load_16(_odp_u128_t *var, int mo) +{ + _odp_u128_t old = *var; /* Possibly torn read */ + + /* Do CAS to ensure atomicity + * Either CAS succeeds (writing back the same value) + * Or CAS fails and returns the old value (atomic read) + */ + (void)__lockfree_compare_exchange_16(var, &old, old, false, mo, mo); + return old; +} + +static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic) +{ + return __lockfree_load_16((_odp_u128_t *)atomic, __ATOMIC_RELAXED); +} + +static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, + _odp_u128_t old_val, + _odp_u128_t new_val) +{ + return __lockfree_compare_exchange_16((_odp_u128_t *)atomic, + (_odp_u128_t *)&old_val, + new_val, + 0, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +} + +static inline int lockfree_check_u128(void) +{ + return 1; +} + +/** Atomic bit set operations with memory ordering */ +#if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16 +typedef _odp_u128_t bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__) + +#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) + +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) +#endif + +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (_odp_u128_t)(1ULL << (bit - 64)) << 64; +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ + return __lockfree_load_16(bs, mo); +} + +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__lockfree_fetch_or_16(bs, bitset_mask(bit), mo); +} + +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__lockfree_fetch_and_16(bs, ~bitset_mask(bit), mo); +} + +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ + return __lockfree_exchange_16(bs, neu, mo); +} + +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ + return __lockfree_compare_exchange_16(bs, old, neu, weak, mo_success, + mo_failure); +} + +#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h new file mode 100644 index 000000000..ad8b36d87 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_cpu.h @@ -0,0 +1,202 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#define PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H + +#if !defined(__aarch64__) +#error Use this file only when compiling for ARMv8 architecture +#endif + +#include <odp_debug_internal.h> +#include <odp_types_internal.h> + +/* + * Use LLD/SCD atomic primitives instead of lock-based code path in llqueue + * LLD/SCD is on ARM the fastest way to enqueue and dequeue elements from a + * linked list queue. + */ +#define CONFIG_LLDSCD + +/* + * Use DMB;STR instead of STRL on ARM + * On early ARMv8 implementations (e.g. Cortex-A57) this is noticeably more + * performant than using store-release. + * This also allows for load-only barriers (DMB ISHLD) which are much cheaper + * than a full barrier + */ +#define CONFIG_DMBSTR + +/* Only ARMv8 supports DMB ISHLD */ +/* A load only barrier is much cheaper than full barrier */ +#define _odp_release_barrier(ro) \ +do { \ + if (ro) \ + __asm__ volatile("dmb ishld" ::: "memory"); \ + else \ + __asm__ volatile("dmb ish" ::: "memory"); \ +} while (0) + +static inline uint16_t ll8(uint8_t *var, int mm) +{ + uint16_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +static inline uint32_t ll32(uint32_t *var, int mm) +{ + uint32_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else + __asm__ volatile("stxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +static inline uint64_t ll64(uint64_t *var, int mm) +{ + uint64_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc64(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else + __asm__ volatile("stxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +union i128 { + _odp_u128_t i128; + int64_t i64[2]; +}; + +static inline _odp_u128_t lld(_odp_u128_t *var, int mm) +{ + union i128 old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : ); + return old.i128; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((*(union i128 *)&neu)).i64[0]), + "r" (((*(union i128 *)&neu)).i64[1]), + "r" (var) + : "memory"); + else + __asm__ volatile("stxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((*(union i128 *)&neu)).i64[0]), + "r" (((*(union i128 *)&neu)).i64[1]), + "r" (var) + : ); +#pragma GCC diagnostic pop + return ret; +} + +#include "odp_atomic.h" +#include "odp_wait_until.h" + +#ifdef __ARM_FEATURE_UNALIGNED +#define _ODP_UNALIGNED 1 +#else +#define _ODP_UNALIGNED 0 +#endif + +#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_cpu_cycles.c b/platform/linux-generic/arch/aarch64/odp_cpu_cycles.c new file mode 100644 index 000000000..fba263ee4 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_cpu_cycles.c @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp/api/cpu.h> + +#include <odp/api/abi/cpu_inlines.h> +#include <odp/api/abi/time_cpu.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <string.h> + +#include <odp/visibility_begin.h> + +_odp_cpu_cycles_global_t _odp_cpu_cycles_glob; + +#include <odp/visibility_end.h> + +int _odp_cpu_cycles_init_global(void) +{ + uint64_t cpu_hz, cpu_time_hz; + + memset(&_odp_cpu_cycles_glob, 0, sizeof(_odp_cpu_cycles_global_t)); + + cpu_time_hz = _odp_time_cpu_global_freq(); + if (cpu_time_hz == 0) { + _ODP_ERR("CPU time counter frequency not available\n"); + return -1; + } + + cpu_hz = odp_cpu_hz_max_id(0); + if (cpu_hz == 0) { + _ODP_ERR("CPU frequency not available\n"); + return -1; + } + + _odp_cpu_cycles_glob.res_shifted = (cpu_hz << _ODP_CPU_FREQ_SHIFT) / cpu_time_hz; + + _odp_cpu_cycles_glob.res = cpu_hz > cpu_time_hz ? + (_odp_cpu_cycles_glob.res_shifted >> _ODP_CPU_FREQ_SHIFT) : 1; + + _odp_cpu_cycles_glob.max = (UINT64_MAX >> _ODP_CPU_FREQ_SHIFT) - + (UINT64_MAX % _odp_cpu_cycles_glob.res); + + return 0; +} diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c new file mode 100644 index 000000000..52936dacf --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -0,0 +1,896 @@ +/* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * Copyright (c) 2022-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> + +#include <odp/api/crypto.h> +#include <odp/api/spinlock.h> +#include <odp/api/sync.h> +#include <odp/api/debug.h> +#include <odp/api/align.h> +#include <odp/api/shared_memory.h> +#include <odp/api/hints.h> +#include <odp/api/random.h> + +#include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/queue_inlines.h> +#include <odp/api/plat/thread_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> +#include <odp_packet_internal.h> + +#include "AArch64cryptolib.h" + +#define MAX_SESSIONS 4000 +/* Length in bytes */ +#define ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH 32 +#define ARM_CRYPTO_MAX_AUTH_KEY_LENGTH 32 +#define ARM_CRYPTO_MAX_IV_LENGTH 16 +#define ARM_CRYPTO_MAX_AAD_LENGTH 16 +#define ARM_CRYPTO_MAX_DATA_LENGTH 65536 +#define ARM_CRYPTO_MAX_DIGEST_LENGTH 16 + +#define AES_GCM_IV_LEN 12 +ODP_STATIC_ASSERT(AES_GCM_IV_LEN <= ARM_CRYPTO_MAX_IV_LENGTH, + "AES_GCM_IV_LEN exceeds ARM_CRYPTO_MAX_IV_LENGTH"); + +/* + * ARM crypto library may read up to 15 bytes past the end of input + * data and AAD and write up to 15 bytes past the end of output data. + */ +#define OOB_WRITE_LEN 16 /* rounded up to 16 bytes for efficiency */ + +/* + * Data buffer size must be a multiple of 16, because the ARM crypto + * library will write full 16 byte blocks even if the last data block + * is not a full block. + */ +ODP_STATIC_ASSERT(ARM_CRYPTO_MAX_DATA_LENGTH % 16 == 0, + "Data buffer size not a multiple of 16"); + +/* + * IV buffer size must be a multiple of 16, because the ARM crypto + * library will read in 16 byte blocks even if the last data block + * is not a full block. + */ +ODP_STATIC_ASSERT(ARM_CRYPTO_MAX_IV_LENGTH % 16 == 0, + "IV buffer size not a multiple of 16"); + +/* + * Cipher algorithm capabilities + * + * Keep sorted: first by key length, then by IV length + */ +static const odp_crypto_cipher_capability_t cipher_capa_null[] = { +{.key_len = 0, .iv_len = 0} }; + +#ifdef __ARM_FEATURE_AES +static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = { +{.key_len = 16, .iv_len = AES_GCM_IV_LEN}, +{.key_len = 24, .iv_len = AES_GCM_IV_LEN}, +{.key_len = 32, .iv_len = AES_GCM_IV_LEN} }; +#endif + +/* + * Authentication algorithm capabilities + * + * Keep sorted: first by digest length, then by key length + */ +static const odp_crypto_auth_capability_t auth_capa_null[] = { +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + +#define AES_GCM_TAG_LEN 16 + +#ifdef __ARM_FEATURE_AES +static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = { +{.digest_len = AES_GCM_TAG_LEN, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } }; +#endif + +/** Forward declaration of session structure */ +typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t; + +/** + * Algorithm handler function prototype + */ +typedef +void (*crypto_func_t)(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session); + +/** + * Per crypto session data structure + */ +struct odp_crypto_generic_session_t { + odp_crypto_generic_session_t *next; + + /* Session creation parameters */ + odp_crypto_session_param_t p; + + struct { + uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH]; + } cipher; + + struct { + uint8_t key[ARM_CRYPTO_MAX_AUTH_KEY_LENGTH]; + } auth; + + crypto_func_t func; + unsigned int idx; + armv8_cipher_constants_t cc; +}; + +typedef struct odp_crypto_global_s odp_crypto_global_t; + +struct odp_crypto_global_s { + odp_spinlock_t lock; + odp_crypto_generic_session_t *free; + odp_crypto_generic_session_t sessions[MAX_SESSIONS]; +}; + +static odp_crypto_global_t *global; + +typedef struct crypto_local_t { + uint8_t buffer[ARM_CRYPTO_MAX_DATA_LENGTH]; +} crypto_local_t; + +static __thread crypto_local_t local; + +static +odp_crypto_generic_session_t *alloc_session(void) +{ + odp_crypto_generic_session_t *session = NULL; + + odp_spinlock_lock(&global->lock); + session = global->free; + if (session) { + global->free = session->next; + session->next = NULL; + } + odp_spinlock_unlock(&global->lock); + + if (!session) + return NULL; + + session->idx = session - global->sessions; + + return session; +} + +static +void free_session(odp_crypto_generic_session_t *session) +{ + odp_spinlock_lock(&global->lock); + session->next = global->free; + global->free = session; + odp_spinlock_unlock(&global->lock); +} + +static inline void set_crypto_op_result(odp_packet_t pkt, + odp_crypto_alg_err_t cipher_err, + odp_crypto_alg_err_t auth_err) +{ + odp_crypto_packet_result_t *op_result; + + op_result = &packet_hdr(pkt)->crypto_op_result; + op_result->cipher_status.alg_err = cipher_err; + op_result->auth_status.alg_err = auth_err; +} + +static inline void set_crypto_op_result_ok(odp_packet_t pkt) +{ + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_NONE, + ODP_CRYPTO_ALG_ERR_NONE); +} + +static void +null_crypto_routine(odp_packet_t pkt ODP_UNUSED, + const odp_crypto_packet_op_param_t *param ODP_UNUSED, + odp_crypto_generic_session_t *session ODP_UNUSED) +{ + set_crypto_op_result_ok(pkt); +} + +static inline void copy_aad(uint8_t *dst, const uint8_t *src, uint32_t len) +{ + _ODP_ASSERT(len == 8 || len == 12); + + /* Use constant length memcpy for better optimization result */ + if (len == 8) + memcpy(dst, src, 8); + else + memcpy(dst, src, 12); +} + +static +void aes_gcm_encrypt(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + armv8_cipher_state_t cs = { + .counter = { + .d = {0, 0} + } + }; + uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; + uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; + uint64_t plaintext_bit_length = param->cipher_range.length * 8; + uint64_t aad_bit_length = session->p.auth_aad_len * 8; + uint32_t in_pos = param->cipher_range.offset; + uint32_t in_len = param->cipher_range.length; + odp_bool_t continuous_data; + uint16_t saved_tail[OOB_WRITE_LEN]; + uint8_t tag[AES_GCM_TAG_LEN]; + int rc; + + /* Fail early if cipher_range is too large */ + if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) { + _ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); + goto err; + } + + /* The crypto lib may read 16 bytes. Copy to a big enough buffer */ + _ODP_ASSERT(param->cipher_iv_ptr != NULL); + memcpy(iv_data, param->cipher_iv_ptr, AES_GCM_IV_LEN); + + cs.constants = &session->cc; + + rc = armv8_aes_gcm_set_counter(iv_data, iv_bit_length, &cs); + if (odp_unlikely(rc)) { + _ODP_DBG("ARM Crypto: Failure while setting nonce\n"); + goto err; + } + + /* Copy AAD in a stack to make sure that the ARM crypto library can + * read it in 16 byte chunks. */ + uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH]; + + copy_aad(aad, param->aad_ptr, session->p.auth_aad_len); + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(odp_packet_is_segmented(pkt)) || + odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) { + /* Packet is segmented or it may not be safe to read and write + * beyond the end of packet data. Copy the cipher range to a + * contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + + data = local.buffer; + continuous_data = false; + } else { + /* Save data that might get overwritten */ + memcpy(saved_tail, data + in_len, OOB_WRITE_LEN); + continuous_data = true; + } + + rc = armv8_enc_aes_gcm_from_state(&cs, + aad, aad_bit_length, + data, plaintext_bit_length, + data, + tag); + if (odp_unlikely(rc)) { + _ODP_DBG("ARM Crypto: AES GCM Encoding failed\n"); + goto err; + } + + if (odp_likely(continuous_data)) { + memcpy(data + in_len, saved_tail, OOB_WRITE_LEN); + memcpy(data - in_pos + param->hash_result_offset, + tag, AES_GCM_TAG_LEN); + } else { + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + odp_packet_copy_from_mem(pkt, param->hash_result_offset, + AES_GCM_TAG_LEN, tag); + } + + set_crypto_op_result_ok(pkt); + return; + +err: + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_DATA_SIZE, + ODP_CRYPTO_ALG_ERR_NONE); +} + +static +void aes_gcm_decrypt(odp_packet_t pkt, + const odp_crypto_packet_op_param_t *param, + odp_crypto_generic_session_t *session) +{ + armv8_cipher_state_t cs = { + .counter = { + .d = {0, 0} + } + }; + uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH]; + uint8_t tag[AES_GCM_TAG_LEN]; + uint64_t iv_bit_length = AES_GCM_IV_LEN * 8; + uint64_t plaintext_bit_length = param->cipher_range.length * 8; + uint64_t aad_bit_length = session->p.auth_aad_len * 8; + uint32_t in_pos = param->cipher_range.offset; + uint32_t in_len = param->cipher_range.length; + odp_bool_t continuous_data; + uint16_t saved_tail[OOB_WRITE_LEN]; + int rc; + + /* Fail early if cipher_range is too large */ + if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) { + _ODP_DBG("ARM Crypto: Packet size too large for requested operation\n"); + goto err; + } + + /* The crypto lib may read 16 bytes. Copy to a big enough buffer */ + _ODP_ASSERT(param->cipher_iv_ptr != NULL); + memcpy(iv_data, param->cipher_iv_ptr, AES_GCM_IV_LEN); + + cs.constants = &session->cc; + + rc = armv8_aes_gcm_set_counter(iv_data, iv_bit_length, &cs); + if (odp_unlikely(rc)) { + _ODP_DBG("ARM Crypto: Failure while setting nonce\n"); + goto err; + } + + /* Copy AAD in a stack to make sure that the ARM crypto library can + * read it in 16 byte chunks. */ + uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH]; + + copy_aad(aad, param->aad_ptr, session->p.auth_aad_len); + + uint32_t seg_len = 0; + uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL); + + if (odp_unlikely(odp_packet_is_segmented(pkt)) || + odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) { + /* Packet is segmented or it may not be safe to read and write + * beyond the end of packet data. Copy the cipher range to a + * contiguous buffer. */ + odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer); + data = local.buffer; + /* Copy tag from the packet to a buffer */ + odp_packet_copy_to_mem(pkt, param->hash_result_offset, + AES_GCM_TAG_LEN, tag); + continuous_data = false; + } else { + /* Save data that might get overwritten */ + memcpy(saved_tail, data + in_len, OOB_WRITE_LEN); + /* Copy tag from the packet to a buffer */ + memcpy(tag, data - in_pos + param->hash_result_offset, AES_GCM_TAG_LEN); + continuous_data = true; + } + + rc = armv8_dec_aes_gcm_from_state(&cs, + aad, aad_bit_length, + data, plaintext_bit_length, + tag, + data); + if (odp_unlikely(rc)) { + _ODP_DBG("ARM Crypto: AES GCM Decoding failed\n"); + goto err; + } + + if (odp_likely(continuous_data)) + memcpy(data + in_len, saved_tail, OOB_WRITE_LEN); + else + odp_packet_copy_from_mem(pkt, in_pos, in_len, data); + + set_crypto_op_result_ok(pkt); + return; + +err: + set_crypto_op_result(pkt, + ODP_CRYPTO_ALG_ERR_NONE, + ODP_CRYPTO_ALG_ERR_ICV_CHECK); +} + +static int process_aes_gcm_param(odp_crypto_generic_session_t *session) +{ + /* Verify Key len is valid */ + if (16 != session->p.cipher_key.length && + 24 != session->p.cipher_key.length && + 32 != session->p.cipher_key.length) + return -1; + + /* Verify IV len is correct */ + if (session->p.cipher_iv_len != AES_GCM_IV_LEN) + return -1; + + if (ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH < session->p.cipher_key.length) + return -1; + + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); + + /* Set function */ + if (ODP_CRYPTO_OP_ENCODE == session->p.op) + session->func = aes_gcm_encrypt; + else + session->func = aes_gcm_decrypt; + + return 0; +} + +int odp_crypto_capability(odp_crypto_capability_t *capa) +{ + if (NULL == capa) + return -1; + + /* Initialize crypto capability structure */ + memset(capa, 0, sizeof(odp_crypto_capability_t)); + + capa->sync_mode = ODP_SUPPORT_PREFERRED; + capa->async_mode = ODP_SUPPORT_YES; + capa->queue_type_plain = 1; + capa->queue_type_sched = 1; + + capa->ciphers.bit.null = 1; + capa->auths.bit.null = 1; + +#ifdef __ARM_FEATURE_AES + capa->ciphers.bit.aes_gcm = 1; + capa->auths.bit.aes_gcm = 1; +#endif + + capa->max_sessions = MAX_SESSIONS; + + return 0; +} + +int odp_crypto_cipher_capability(odp_cipher_alg_t cipher, + odp_crypto_cipher_capability_t dst[], + int num_copy) +{ + const odp_crypto_cipher_capability_t *src; + int num; + int size = sizeof(odp_crypto_cipher_capability_t); + + switch (cipher) { + case ODP_CIPHER_ALG_NULL: + src = cipher_capa_null; + num = sizeof(cipher_capa_null) / size; + break; +#ifdef __ARM_FEATURE_AES + case ODP_CIPHER_ALG_AES_GCM: + src = cipher_capa_aes_gcm; + num = sizeof(cipher_capa_aes_gcm) / size; + break; +#endif + default: + return -1; + } + + if (num < num_copy) + num_copy = num; + + memcpy(dst, src, num_copy * size); + + return num; +} + +int odp_crypto_auth_capability(odp_auth_alg_t auth, + odp_crypto_auth_capability_t dst[], int num_copy) +{ + const odp_crypto_auth_capability_t *src; + int num; + int size = sizeof(odp_crypto_auth_capability_t); + + switch (auth) { + case ODP_AUTH_ALG_NULL: + src = auth_capa_null; + num = sizeof(auth_capa_null) / size; + break; +#ifdef __ARM_FEATURE_AES + case ODP_AUTH_ALG_AES_GCM: + src = auth_capa_aes_gcm; + num = sizeof(auth_capa_aes_gcm) / size; + break; +#endif + default: + return -1; + } + + if (num < num_copy) + num_copy = num; + + memcpy(dst, src, num_copy * size); + + return num; +} + +int +odp_crypto_session_create(const odp_crypto_session_param_t *param, + odp_crypto_session_t *session_out, + odp_crypto_ses_create_err_t *status) +{ + int rc = 0; + odp_crypto_generic_session_t *session; + + if (odp_global_ro.disable.crypto) { + _ODP_ERR("Crypto is disabled\n"); + /* Dummy output to avoid compiler warning about uninitialized + * variables */ + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + + if (param->cipher_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + if (param->auth_range_in_bits) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP || + param->op_type == ODP_CRYPTO_OP_TYPE_BASIC_AND_OOP) { + *status = ODP_CRYPTO_SES_ERR_PARAMS; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + + /* Allocate memory for this session */ + session = alloc_session(); + if (NULL == session) { + *status = ODP_CRYPTO_SES_ERR_ENOMEM; + goto err; + } + + /* Copy parameters */ + session->p = *param; + + if (session->p.cipher_iv_len > ARM_CRYPTO_MAX_IV_LENGTH) { + _ODP_DBG("Maximum IV length exceeded\n"); + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + + if (session->p.auth_iv_len > ARM_CRYPTO_MAX_IV_LENGTH) { + _ODP_DBG("Maximum auth IV length exceeded\n"); + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + + /* Process based on cipher */ + switch (param->cipher_alg) { + case ODP_CIPHER_ALG_NULL: + session->func = null_crypto_routine; + rc = 0; + break; + case ODP_CIPHER_ALG_AES_GCM: + { + /* Set cipher mode for AES-GCM */ + armv8_cipher_mode_t mode = 0; + + switch (param->cipher_key.length) { + case 16: + mode = AES_GCM_128; + break; + case 24: + mode = AES_GCM_192; + break; + case 32: + mode = AES_GCM_256; + break; + default: + rc = -1; + break; + } + + /* AES-GCM requires to do both auth and + * cipher at the same time */ + if (param->auth_alg != ODP_AUTH_ALG_AES_GCM) { + rc = -1; + } else if (mode == AES_GCM_128 || mode == AES_GCM_192 || + mode == AES_GCM_256) { + if (armv8_aes_gcm_set_constants(mode, + session->p.auth_digest_len, + session->p.cipher_key.data, + &session->cc) != 0) { + _ODP_DBG("ARM Crypto: Failure in setting constants\n"); + rc = -1; + } + rc = process_aes_gcm_param(session); + } else { + rc = -1; + } + break; + } + default: + rc = -1; + } + + /* Check result */ + if (rc) { + *status = ODP_CRYPTO_SES_ERR_CIPHER; + goto err; + } + + /* Process based on auth */ + switch (param->auth_alg) { + case ODP_AUTH_ALG_NULL: + if (param->cipher_alg == ODP_CIPHER_ALG_NULL) + rc = 0; + else + rc = -1; + break; + case ODP_AUTH_ALG_AES_GCM: + /* AES-GCM requires to do both auth and + * cipher at the same time */ + if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM) { + rc = 0; + } else { + rc = -1; + } + break; + default: + rc = -1; + } + + /* Check result */ + if (rc) { + *status = ODP_CRYPTO_SES_ERR_AUTH; + goto err; + } + + /* We're happy */ + *session_out = (intptr_t)session; + *status = ODP_CRYPTO_SES_ERR_NONE; + return 0; + +err: + /* error status should be set at this moment */ + if (session != NULL) + free_session(session); + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; +} + +int odp_crypto_session_destroy(odp_crypto_session_t session) +{ + odp_crypto_generic_session_t *generic; + + generic = (odp_crypto_generic_session_t *)(intptr_t)session; + memset(generic, 0, sizeof(*generic)); + free_session(generic); + return 0; +} + +int _odp_crypto_init_global(void) +{ + size_t mem_size; + odp_shm_t shm; + int idx; + + if (odp_global_ro.disable.crypto) { + _ODP_PRINT("\nODP crypto is DISABLED\n"); + return 0; + } + + /* Calculate the memory size we need */ + mem_size = sizeof(odp_crypto_global_t); + + /* Allocate our globally shared memory */ + shm = odp_shm_reserve("_odp_crypto_pool_armv8crypto", mem_size, + ODP_CACHE_LINE_SIZE, + 0); + if (ODP_SHM_INVALID == shm) { + _ODP_ERR("unable to allocate crypto pool\n"); + return -1; + } + + global = odp_shm_addr(shm); + + /* Clear it out */ + memset(global, 0, mem_size); + + /* Initialize free list and lock */ + for (idx = 0; idx < MAX_SESSIONS; idx++) { + global->sessions[idx].next = global->free; + global->free = &global->sessions[idx]; + } + odp_spinlock_init(&global->lock); + + return 0; +} + +int _odp_crypto_term_global(void) +{ + int rc = 0; + int ret; + int count = 0; + odp_crypto_generic_session_t *session; + + if (odp_global_ro.disable.crypto) + return 0; + + for (session = global->free; session != NULL; session = session->next) + count++; + if (count != MAX_SESSIONS) { + _ODP_ERR("crypto sessions still active\n"); + rc = -1; + } + + ret = odp_shm_free(odp_shm_lookup("_odp_crypto_pool_armv8crypto")); + if (ret < 0) { + _ODP_ERR("shm free failed for _odp_crypto_pool_armv8crypto\n"); + rc = -1; + } + + return rc; +} + +int _odp_crypto_init_local(void) +{ + if (odp_global_ro.disable.crypto) + return 0; + + memset(&local, 0, sizeof(local)); + + return 0; +} + +int _odp_crypto_term_local(void) +{ + return 0; +} + +void odp_crypto_session_param_init(odp_crypto_session_param_t *param) +{ + memset(param, 0, sizeof(odp_crypto_session_param_t)); +} + +uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl) +{ + return (uint64_t)hdl; +} + +static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) +{ + int md_copy; + int rc; + + md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), + odp_packet_pool(src)); + if (odp_unlikely(md_copy < 0)) { + _ODP_ERR("Unable to copy packet metadata\n"); + return -1; + } + + rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); + if (odp_unlikely(rc < 0)) { + _ODP_ERR("Unable to copy packet data\n"); + return -1; + } + + _odp_packet_copy_md(packet_hdr(dst), packet_hdr(src), md_copy); + return 0; +} + +static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *session, + odp_packet_t pkt_in, + odp_packet_t pkt_out) +{ + int rc; + + if (odp_likely(pkt_in == pkt_out)) + return pkt_out; + + if (pkt_out == ODP_PACKET_INVALID) { + odp_pool_t pool = session->p.output_pool; + + _ODP_ASSERT(pool != ODP_POOL_INVALID); + if (pool == odp_packet_pool(pkt_in)) { + pkt_out = pkt_in; + } else { + pkt_out = odp_packet_copy(pkt_in, pool); + if (odp_likely(pkt_out != ODP_PACKET_INVALID)) + odp_packet_free(pkt_in); + } + return pkt_out; + } + rc = copy_data_and_metadata(pkt_out, pkt_in); + if (odp_unlikely(rc < 0)) + return ODP_PACKET_INVALID; + + odp_packet_free(pkt_in); + return pkt_out; +} + +static +int crypto_int(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_generic_session_t *session; + odp_packet_t out_pkt; + + session = (odp_crypto_generic_session_t *)(intptr_t)param->session; + + if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC)) { + out_pkt = pkt_in; + } else { + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; + } + + if (odp_unlikely(session->p.null_crypto_enable && + param->null_crypto)) + goto out; + + /* Invoke the crypto function */ + session->func(out_pkt, param, session); + +out: + packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); + + /* Synchronous, simply return results */ + *pkt_out = out_pkt; + + return 0; +} + +int odp_crypto_op(const odp_packet_t pkt_in[], + odp_packet_t pkt_out[], + const odp_crypto_packet_op_param_t param[], + int num_pkt) +{ + int i, rc; + odp_crypto_generic_session_t *session; + + for (i = 0; i < num_pkt; i++) { + session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; + _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode); + + rc = crypto_int(pkt_in[i], &pkt_out[i], ¶m[i]); + if (rc < 0) + break; + } + + return i; +} + +int odp_crypto_op_enq(const odp_packet_t pkt_in[], + const odp_packet_t pkt_out[], + const odp_crypto_packet_op_param_t param[], + int num_pkt) +{ + odp_packet_t pkt; + odp_event_t event; + odp_crypto_generic_session_t *session; + int i, rc; + + for (i = 0; i < num_pkt; i++) { + session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session; + _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode); + _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue); + + if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC) + pkt = pkt_out[i]; + + rc = crypto_int(pkt_in[i], &pkt, ¶m[i]); + if (rc < 0) + break; + + event = odp_packet_to_event(pkt); + if (odp_queue_enq(session->p.compl_queue, event)) { + odp_event_free(event); + break; + } + } + + return i; +} diff --git a/platform/linux-generic/arch/aarch64/odp_random.h b/platform/linux-generic/arch/aarch64/odp_random.h new file mode 100644 index 000000000..023e6c455 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_random.h @@ -0,0 +1,166 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_AARCH64_RANDOM_H_ +#define ODP_AARCH64_RANDOM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/random.h> +#include <odp/autoheader_internal.h> + +#include <stdint.h> + +odp_random_kind_t _odp_random_max_kind_generic(void); +int32_t _odp_random_true_data_generic(uint8_t *buf, uint32_t len); +int32_t _odp_random_crypto_data_generic(uint8_t *buf, uint32_t len); + +#ifdef __ARM_FEATURE_RNG + +#if __ARM_FEATURE_UNALIGNED != 1 +#error This implementation requires unaligned access +#endif + +static inline int _odp_random_max_kind(void) +{ + return ODP_RANDOM_TRUE; +} + +static inline int _odp_rndr(uint64_t *v) +{ + int pass; + + /* Return a 64-bit random number which is reseeded from the True Random + * Number source. If the hardware returns a genuine random number, + * PSTATE.NZCV is set to 0b0000. The NZCV condition flag is checked via + * the CSET instruction. If the hardware cannot return a genuine random + * number in a reasonable period of time, PSTATE.NZCV is set to 0b0100 + * and the data value returned is 0. */ + __asm__ volatile("mrs %0, s3_3_c2_c4_0\n\t" + "cset %w[pass], ne" + : "=r" (*v), [pass] "=r" (pass) + : + : "cc"); + + return pass; +} + +static inline int _odp_rndrrs(uint64_t *v) +{ + int pass; + + /* Return a 64-bit random number which is reseeded from the True Random + * Number source immediately before the read of the random number. + * If the hardware returns a genuine random number, PSTATE.NZCV is + * set to 0b0000. The NZCV condition flag is checked via the CSET + * instruction. If the hardware cannot return a genuine random number + * in a reasonable period of time, PSTATE.NZCV is set to 0b0100 and the + * data value returned is 0. */ + __asm__ volatile("mrs %0, s3_3_c2_c4_1\n\t" + "cset %w[pass], ne" + : "=r" (*v), [pass] "=r" (pass) + : + : "cc"); + + return pass; +} + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ + uint64_t temp; + + for (uint32_t i = 0; i < len / 8; i++) { + while (!_odp_rndr(&temp)) + ; + + *(uint64_t *)(uintptr_t)buf = temp; + buf += 8; + } + + if (len & 7) { + while (!_odp_rndr(&temp)) + ; + + if (len & 4) { + *(uint32_t *)(uintptr_t)buf = temp & 0xffffffff; + temp >>= 32; + buf += 4; + } + + if (len & 2) { + *(uint16_t *)(uintptr_t)buf = temp & 0xffff; + temp >>= 16; + buf += 2; + } + + if (len & 1) + *buf = temp & 0xff; + } + + return len; +} + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ + uint64_t temp; + + for (uint32_t i = 0; i < len / 8; i++) { + while (!_odp_rndrrs(&temp)) + ; + + *(uint64_t *)(uintptr_t)buf = temp; + buf += 8; + } + + if (len & 7) { + while (!_odp_rndrrs(&temp)) + ; + + if (len & 4) { + *(uint32_t *)(uintptr_t)buf = temp & 0xffffffff; + temp >>= 32; + buf += 4; + } + + if (len & 2) { + *(uint16_t *)(uintptr_t)buf = temp & 0xffff; + temp >>= 16; + buf += 2; + } + + if (len & 1) + *buf = temp & 0xff; + } + + return len; +} + +#else + +static inline int _odp_random_max_kind(void) +{ + return _odp_random_max_kind_generic(); +} + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_crypto_data_generic(buf, len); +} + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_true_data_generic(buf, len); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c new file mode 100644 index 000000000..f242c845e --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_sysinfo_parse.c @@ -0,0 +1,396 @@ +/* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2020-2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +#include <odp/api/hints.h> +#include <odp_global_data.h> +#include <odp_sysinfo_internal.h> +#include <odp_debug_internal.h> +#include "cpu_flags.h" + +#define TMP_STR_LEN 64 + +static void aarch64_impl_str(char *str, int maxlen, int implementer) +{ + switch (implementer) { + case 0x41: + snprintf(str, maxlen, "ARM Limited"); + return; + case 0x42: + snprintf(str, maxlen, "Broadcom Corporation"); + return; + case 0x43: + snprintf(str, maxlen, "Marvell (Cavium) Inc."); + return; + case 0x44: + snprintf(str, maxlen, "Digital Equipment Corporation"); + return; + case 0x46: + snprintf(str, maxlen, "Fujitsu Ltd."); + return; + case 0x49: + snprintf(str, maxlen, "Infineon Technologies AG"); + return; + case 0x4d: + snprintf(str, maxlen, "Freescale Semiconductor Inc."); + return; + case 0x4e: + snprintf(str, maxlen, "NVIDIA Corporation"); + return; + case 0x50: + snprintf(str, maxlen, "Applied Micro Circuits Corporation"); + return; + case 0x51: + snprintf(str, maxlen, "Qualcomm Inc."); + return; + case 0x56: + snprintf(str, maxlen, "Marvell International Ltd."); + return; + case 0x69: + snprintf(str, maxlen, "Intel Corporation"); + return; + case 0xc0: + snprintf(str, maxlen, "Ampere Computing"); + return; + default: + break; + } + + snprintf(str, maxlen, "UNKNOWN (0x%x)", implementer); +} + +static void aarch64_part_info(char *str, int maxlen, odp_cpu_arch_arm_t *cpu_isa, int implementer, + int part, int variant, int revision) +{ + *cpu_isa = ODP_CPU_ARCH_ARM_UNKNOWN; + + if (implementer == 0x41) { + /* Part numbers are specified in Main ID Register (MIDR_EL1) documentation */ + switch (part) { + case 0xd02: + snprintf(str, maxlen, "Cortex-A34"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_0; + return; + case 0xd04: + snprintf(str, maxlen, "Cortex-A35"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_0; + return; + case 0xd03: + snprintf(str, maxlen, "Cortex-A53"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_0; + return; + case 0xd05: + snprintf(str, maxlen, "Cortex-A55"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd07: + snprintf(str, maxlen, "Cortex-A57"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_0; + return; + case 0xd06: + snprintf(str, maxlen, "Cortex-A65"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd08: + snprintf(str, maxlen, "Cortex-A72"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_0; + return; + case 0xd09: + snprintf(str, maxlen, "Cortex-A73"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_0; + return; + case 0xd0a: + snprintf(str, maxlen, "Cortex-A75"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd0b: + snprintf(str, maxlen, "Cortex-A76"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd0c: + snprintf(str, maxlen, "Neoverse N1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd0e: + snprintf(str, maxlen, "Cortex-A76AE"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd0d: + snprintf(str, maxlen, "Cortex-A77"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd40: + snprintf(str, maxlen, "Neoverse V1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_4; + return; + case 0xd41: + snprintf(str, maxlen, "Cortex-A78"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd42: + snprintf(str, maxlen, "Cortex-A78AE"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd44: + snprintf(str, maxlen, "Cortex-X1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd46: + snprintf(str, maxlen, "Cortex-A510"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_0; + return; + case 0xd47: + snprintf(str, maxlen, "Cortex-A710"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_0; + return; + case 0xd48: + snprintf(str, maxlen, "Cortex-X2"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_0; + return; + case 0xd49: + snprintf(str, maxlen, "Neoverse N2"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_0; + return; + case 0xd4a: + snprintf(str, maxlen, "Neoverse E1"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd4b: + snprintf(str, maxlen, "Cortex-A78C"); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xd4d: + snprintf(str, maxlen, "Cortex-A715"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_0; + return; + case 0xd80: + snprintf(str, maxlen, "Cortex-A520"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_2; + return; + case 0xd81: + snprintf(str, maxlen, "Cortex-A720"); + *cpu_isa = ODP_CPU_ARCH_ARMV9_2; + return; + default: + break; + } + } else if (implementer == 0x43) { + switch (part) { + case 0xa1: + snprintf(str, maxlen, "CN88XX, Pass %i.%i", variant + 1, revision); + *cpu_isa = ODP_CPU_ARCH_ARMV8_1; + return; + case 0xa2: + snprintf(str, maxlen, "CN81XX, Pass %i.%i", variant + 1, revision); + *cpu_isa = ODP_CPU_ARCH_ARMV8_1; + return; + case 0xa3: + snprintf(str, maxlen, "CN83XX, Pass %i.%i", variant + 1, revision); + *cpu_isa = ODP_CPU_ARCH_ARMV8_1; + return; + case 0xaf: + snprintf(str, maxlen, "CN99XX, Rev %c%i", 'A' + variant, revision); + *cpu_isa = ODP_CPU_ARCH_ARMV8_1; + return; + case 0xb1: + snprintf(str, maxlen, "CN98XX, Rev %c%i", 'A' + variant, revision); + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + case 0xb2: + /* Handle B0 errata: variant and revision numbers show up as A1 */ + if (variant == 0 && revision == 1) + snprintf(str, maxlen, "CN96XX, Rev B0"); + else + snprintf(str, maxlen, "CN96XX, Rev %c%i", 'A' + variant, revision); + + *cpu_isa = ODP_CPU_ARCH_ARMV8_2; + return; + default: + break; + } + } + + snprintf(str, maxlen, "part 0x%x, var 0x%x, rev 0x%x", + part, variant, revision); +} + +static odp_cpu_arch_arm_t arm_isa_version(void) +{ +#if defined(__ARM_ARCH) + if (__ARM_ARCH == 8) { + #ifdef __ARM_FEATURE_QRDMX + /* v8.1 or higher */ + return ODP_CPU_ARCH_ARMV8_1; + #else + return ODP_CPU_ARCH_ARMV8_0; + #endif + } + + if (__ARM_ARCH == 9) { + /* v9.0 or higher */ + return ODP_CPU_ARCH_ARMV9_0; + } + + if (__ARM_ARCH >= 800) { + /* ACLE 2018 defines that from v8.1 onwards the value includes + * the minor version number: __ARM_ARCH = X * 100 + Y + * E.g. for Armv8.1 __ARM_ARCH = 801 */ + int major = __ARM_ARCH / 100; + int minor = __ARM_ARCH - (major * 100); + + if (major == 8) { + switch (minor) { + case 0: + return ODP_CPU_ARCH_ARMV8_0; + case 1: + return ODP_CPU_ARCH_ARMV8_1; + case 2: + return ODP_CPU_ARCH_ARMV8_2; + case 3: + return ODP_CPU_ARCH_ARMV8_3; + case 4: + return ODP_CPU_ARCH_ARMV8_4; + case 5: + return ODP_CPU_ARCH_ARMV8_5; + case 6: + return ODP_CPU_ARCH_ARMV8_6; + case 7: + return ODP_CPU_ARCH_ARMV8_7; + case 8: + return ODP_CPU_ARCH_ARMV8_8; + case 9: + return ODP_CPU_ARCH_ARMV8_9; + default: + return ODP_CPU_ARCH_ARM_UNKNOWN; + } + } else if (major == 9) { + switch (minor) { + case 0: + return ODP_CPU_ARCH_ARMV9_0; + case 1: + return ODP_CPU_ARCH_ARMV9_1; + case 2: + return ODP_CPU_ARCH_ARMV9_2; + case 3: + return ODP_CPU_ARCH_ARMV9_3; + default: + return ODP_CPU_ARCH_ARM_UNKNOWN; + } + } + } +#endif + return ODP_CPU_ARCH_ARM_UNKNOWN; +} + +int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) +{ + char str[1024]; + char impl_str[TMP_STR_LEN]; + char part_str[TMP_STR_LEN]; + const char *cur; + long int impl, arch, var, part, rev; + int id; + + sysinfo->cpu_arch = ODP_CPU_ARCH_ARM; + sysinfo->cpu_isa_sw.arm = arm_isa_version(); + /* Linux cpuinfo does not have detailed ISA version number (CPU architecture: 8) */ + sysinfo->cpu_isa_hw.arm = ODP_CPU_ARCH_ARM_UNKNOWN; + + strcpy(sysinfo->cpu_arch_str, "aarch64"); + + memset(impl_str, 0, sizeof(impl_str)); + memset(part_str, 0, sizeof(part_str)); + + impl = 0; + arch = 0; + var = 0; + part = 0; + rev = 0; + id = 0; + + while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU_IDS) { + /* Parse line by line a block of cpuinfo */ + cur = strstr(str, "CPU implementer"); + + if (cur) { + cur = strchr(cur, ':'); + impl = strtol(cur + 1, NULL, 16); + aarch64_impl_str(impl_str, TMP_STR_LEN, impl); + continue; + } + + cur = strstr(str, "CPU architecture"); + + if (cur) { + cur = strchr(cur, ':'); + arch = strtol(cur + 1, NULL, 10); + continue; + } + + cur = strstr(str, "CPU variant"); + + if (cur) { + cur = strchr(cur, ':'); + var = strtol(cur + 1, NULL, 16); + continue; + } + + cur = strstr(str, "CPU part"); + + if (cur) { + cur = strchr(cur, ':'); + part = strtol(cur + 1, NULL, 16); + continue; + } + + cur = strstr(str, "CPU revision"); + + if (cur) { + odp_cpu_arch_arm_t cpu_isa; + + cur = strchr(cur, ':'); + rev = strtol(cur + 1, NULL, 10); + + aarch64_part_info(part_str, TMP_STR_LEN, &cpu_isa, impl, part, var, rev); + sysinfo->cpu_isa_hw.arm = cpu_isa; + + /* This is the last line about this cpu, update + * model string. */ + snprintf(sysinfo->model_str[id], + sizeof(sysinfo->model_str[id]), + "%s, %s, arch %li", + impl_str, part_str, arch); + + /* Some CPUs do not support cpufreq, use a dummy + * max freq. */ + if (sysinfo->cpu_hz_max[id] == 0) { + uint64_t hz = sysinfo->default_cpu_hz_max; + + _ODP_WARN("CPU[%i] uses default max frequency of %" PRIu64 " " + "Hz from config file\n", id, hz); + sysinfo->cpu_hz_max[id] = hz; + } + + id++; + } + } + + return 0; +} + +void _odp_sys_info_print_arch(void) +{ + _odp_cpu_flags_print_all(); +} + +uint64_t odp_cpu_arch_hz_current(int id ODP_UNUSED) +{ + return odp_global_ro.system_info.default_cpu_hz; +} diff --git a/platform/linux-generic/arch/aarch64/odp_wait_until.h b/platform/linux-generic/arch/aarch64/odp_wait_until.h new file mode 100644 index 000000000..eca3f9ce5 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_wait_until.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017 ARM Limited + * Copyright (c) 2017-2018 Linaro Limited + * Copyright (c) 2024 Nokia + */ + +#ifndef ODP_AARCH64_WAIT_UNTIL_H_ +#define ODP_AARCH64_WAIT_UNTIL_H_ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#error This file should not be included directly, please include odp_cpu.h +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/cpu.h> + +#include <odp_cpu.h> + +#include <stdint.h> + +static inline void _odp_sevl(void) +{ + __asm__ volatile("sevl" : : : ); +} + +static inline int _odp_wfe(void) +{ + __asm__ volatile("wfe" : : : "memory"); + return 1; +} + +#define _odp_monitor_u8(addr, mo) ll8((addr), (mo)) +#define _odp_monitor_u32(addr, mo) ll32((addr), (mo)) +#define _odp_monitor_u64(addr, mo) ll64((addr), (mo)) +#define _odp_monitor_u128(addr, mo) lld((addr), (mo)) + +#if ATOM_BITSET_SIZE <= 32 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u32(bs, mo); +} +#elif ATOM_BITSET_SIZE <= 64 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u64(bs, mo); +} +#elif ATOM_BITSET_SIZE <= 128 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u128(bs, mo); +} +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +/** + * The _odp_wait_until_eq_*() functions defined in this header are intended to + * be used only with the scalable scheduler and queue implementations. Even + * though these functions use standard non-atomic parameter types, the + * parameters must only be operated using atomic operations. If new functions + * are added to this file, they should use _odp_wait_until_equal_*() prefix and + * atomic parameter types. + */ + +static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_bitset_monitor(val, __ATOMIC_RELAXED != expected)) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u8(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/arm/odp/api/abi/cpu.h b/platform/linux-generic/arch/arm/odp/api/abi/cpu.h new file mode 100644 index 000000000..9224af9a0 --- /dev/null +++ b/platform/linux-generic/arch/arm/odp/api/abi/cpu.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define ODP_CACHE_LINE_SIZE 64 + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/arm/odp/api/cpu_arch.h b/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h index 7c75a690e..bf44806a0 100644 --- a/platform/linux-generic/arch/arm/odp/api/cpu_arch.h +++ b/platform/linux-generic/arch/arm/odp/api/abi/cpu_inlines.h @@ -1,19 +1,18 @@ -/* Copyright (c) 2016, Linaro Limited +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_PLAT_CPU_ARCH_H_ -#define ODP_PLAT_CPU_ARCH_H_ +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ #ifdef __cplusplus extern "C" { #endif -#define _ODP_CACHE_LINE_SIZE 64 - -static inline void odp_cpu_pause(void) +static inline void _odp_cpu_pause(void) { /* YIELD hints the CPU to switch to another thread if possible * and executes as a NOP otherwise. @@ -23,6 +22,9 @@ static inline void odp_cpu_pause(void) __asm volatile("isb" ::: "memory"); } +/* Use generic implementations for the rest of the functions */ +#include <odp/api/abi/cpu_generic.h> + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h new file mode 100644 index 000000000..6b2674736 --- /dev/null +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -0,0 +1,87 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#define PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H + +#if !defined(__arm__) +#error Use this file only when compiling for ARM architecture +#endif + +#include <odp_debug_internal.h> + +/* + * Use LLD/SCD atomic primitives instead of lock-based code path in llqueue + * LLD/SCD is on ARM the fastest way to enqueue and dequeue elements from a + * linked list queue. + */ +#define CONFIG_LLDSCD + +/* + * Use DMB;STR instead of STRL on ARM + * On early ARMv8 implementations (e.g. Cortex-A57) this is noticeably more + * performant than using store-release. + * This also allows for load-only barriers (DMB ISHLD) which are much cheaper + * than a full barrier + */ +#define CONFIG_DMBSTR + +static inline uint64_t lld(uint64_t *var, int mm) +{ + uint64_t old; + + __asm__ volatile("ldrexd %0, %H0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("dmb" : : : "memory"); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + /* Barrier before a releasing store */ + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("dmb" : : : "memory"); + __asm__ volatile("strexd %0, %1, %H1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +#ifdef CONFIG_DMBSTR + +#define atomic_store_release(loc, val, ro) \ +do { \ + __atomic_thread_fence(__ATOMIC_RELEASE); \ + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ +} while (0) + +#else + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +#endif /* CONFIG_DMBSTR */ + +#include "../default/odp_atomic.h" +#include "../default/odp_wait_until.h" + +#ifdef __ARM_FEATURE_UNALIGNED +#define _ODP_UNALIGNED 1 +#else +#define _ODP_UNALIGNED 0 +#endif + +#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */ diff --git a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c index 53e2aaeaf..4cbe46d7c 100644 --- a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c @@ -1,27 +1,33 @@ -/* Copyright (c) 2016, Linaro Limited +/* Copyright (c) 2020, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_internal.h> -#include <odp_debug_internal.h> -#include <string.h> +#include <odp_global_data.h> +#include <odp_sysinfo_internal.h> -int cpuinfo_parser(FILE *file ODP_UNUSED, system_info_t *sysinfo) +int _odp_cpuinfo_parser(FILE *file ODP_UNUSED, system_info_t *sysinfo) { - int i; + sysinfo->cpu_arch = ODP_CPU_ARCH_ARM; + sysinfo->cpu_isa_sw.arm = ODP_CPU_ARCH_ARM_UNKNOWN; + sysinfo->cpu_isa_hw.arm = ODP_CPU_ARCH_ARM_UNKNOWN; - ODP_DBG("Warning: use dummy values for freq and model string\n"); - for (i = 0; i < MAX_CPU_NUMBER; i++) { - sysinfo->cpu_hz_max[i] = 1400000000; - strcpy(sysinfo->model_str[i], "UNKNOWN"); - } +#if defined(__ARM_ARCH) + if (__ARM_ARCH == 6) + sysinfo->cpu_isa_sw.arm = ODP_CPU_ARCH_ARMV6; + else if (__ARM_ARCH == 7) + sysinfo->cpu_isa_sw.arm = ODP_CPU_ARCH_ARMV7; +#endif - return 0; + return _odp_dummy_cpuinfo(sysinfo); } -uint64_t odp_cpu_hz_current(int id ODP_UNUSED) +void _odp_sys_info_print_arch(void) { - return 0; +} + +uint64_t odp_cpu_arch_hz_current(int id ODP_UNUSED) +{ + return odp_global_ro.system_info.default_cpu_hz; } diff --git a/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h new file mode 100644 index 000000000..553114666 --- /dev/null +++ b/platform/linux-generic/arch/common/odp/api/abi/time_cpu_inlines.h @@ -0,0 +1,100 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_CPU_INLINES_H_ +#define ODP_ARCH_TIME_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> + +#include <stdint.h> + +#define _ODP_TIME_GIGA_HZ 1000000000ULL + +typedef struct _odp_time_global_t { + uint64_t freq_hz; + uint64_t start_time; + uint64_t start_time_ns; + +} _odp_time_global_t; + +extern _odp_time_global_t _odp_time_glob; + +static inline odp_time_t _odp_time_cur(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global(); + return time; +} + +static inline odp_time_t _odp_time_cur_strict(void) +{ + odp_time_t time; + + time.count = _odp_time_cpu_global_strict(); + return time; +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + uint64_t nsec; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t count = time.count; + uint64_t sec = 0; + + if (count >= freq_hz) { + sec = count / freq_hz; + count = count - sec * freq_hz; + } + + nsec = (_ODP_TIME_GIGA_HZ * count) / freq_hz; + + return (sec * _ODP_TIME_GIGA_HZ) + nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + uint64_t count; + uint64_t freq_hz = _odp_time_glob.freq_hz; + uint64_t sec = 0; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } + + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + + return time; +} + +static inline uint64_t _odp_time_res(void) +{ + return _odp_time_glob.freq_hz; +} + +static inline void _odp_time_startup(odp_time_startup_t *startup) +{ + startup->global.count = _odp_time_glob.start_time; + startup->global_ns = _odp_time_glob.start_time_ns; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/common/odp_time_cpu.c b/platform/linux-generic/arch/common/odp_time_cpu.c new file mode 100644 index 000000000..3c392de0c --- /dev/null +++ b/platform/linux-generic/arch/common/odp_time_cpu.c @@ -0,0 +1,74 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> +#include <odp/api/abi/time_cpu_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <inttypes.h> +#include <stdint.h> +#include <string.h> + +#define YEAR_IN_SEC (365 * 24 * 3600) + +#include <odp/visibility_begin.h> + +_odp_time_global_t _odp_time_glob; + +#include <odp/visibility_end.h> + +int _odp_time_init_global(void) +{ + uint64_t count, diff, years; + odp_time_t time; + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + if (!_odp_time_cpu_global_freq_is_const()) + return -1; + + global->freq_hz = _odp_time_cpu_global_freq(); + if (global->freq_hz == 0) + return -1; + + _ODP_PRINT("HW time counter freq: %" PRIu64 " hz\n\n", global->freq_hz); + + count = _odp_time_cpu_global(); + time.count = count; + global->start_time = count; + global->start_time_ns = _odp_time_to_ns(time); + + /* Make sure that counters will not wrap */ + diff = UINT64_MAX - count; + years = (diff / global->freq_hz) / YEAR_IN_SEC; + + if (years < 10) { + _ODP_ERR("Time counter would wrap in 10 years: %" PRIu64 "\n", count); + return -1; + } + + diff = UINT64_MAX - global->start_time_ns; + years = (diff / ODP_TIME_SEC_IN_NS) / YEAR_IN_SEC; + + if (years < 10) { + _ODP_ERR("Time in nsec would wrap in 10 years: %" PRIu64 "\n", + global->start_time_ns); + return -1; + } + + return 0; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h new file mode 100644 index 000000000..c6ed86363 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -0,0 +1,276 @@ +/* Copyright (c) 2021, ARM Limited + * Copyright (c) 2021-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_ATOMIC_GENERIC_H_ +#define ODP_API_ABI_ATOMIC_GENERIC_H_ + +#include <odp/api/atomic.h> + +static inline void _odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_inc_u32(odp_atomic_u32_t *atom) +{ + (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_dec_u32(odp_atomic_u32_t *atom) +{ + (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_val) +{ + uint32_t old_val; + + old_val = __atomic_load_n(&atom->v, __ATOMIC_RELAXED); + + while (new_val > old_val) { + if (__atomic_compare_exchange_n(&atom->v, &old_val, new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + break; + } +} + +static inline void _odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_val) +{ + uint32_t old_val; + + old_val = __atomic_load_n(&atom->v, __ATOMIC_RELAXED); + + while (new_val < old_val) { + if (__atomic_compare_exchange_n(&atom->v, &old_val, new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + break; + } +} + +static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_inc_u64(odp_atomic_u64_t *atom) +{ + (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_dec_u64(odp_atomic_u64_t *atom) +{ + (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +} + +static inline void _odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_val) +{ + uint64_t old_val; + + old_val = __atomic_load_n(&atom->v, __ATOMIC_RELAXED); + + while (new_val > old_val) { + if (__atomic_compare_exchange_n(&atom->v, &old_val, new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + break; + } +} + +static inline void _odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_val) +{ + uint64_t old_val; + + old_val = __atomic_load_n(&atom->v, __ATOMIC_RELAXED); + + while (new_val < old_val) { + if (__atomic_compare_exchange_n(&atom->v, &old_val, new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) + break; + } +} + +#ifndef ODP_ATOMIC_U64_LOCK +static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); +} + +static inline void _odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); +} +#endif + +#ifdef __SIZEOF_INT128__ + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->v = val; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + union { + odp_u128_t val; + __int128_t i; + } u; + + u.i = __atomic_load_n((__int128_t *)&atom->v, __ATOMIC_RELAXED); + return u.val; +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_store_n((__int128_t *)&atom->v, *(__int128_t *)&val, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_RELEASE, __ATOMIC_RELAXED); +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return __atomic_compare_exchange_n((__int128_t *)&atom->v, (__int128_t *)old_val, + *(__int128_t *)&new_val, 0 /* strong */, + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); +} + +#else /* Lock-based implementation */ + +/** + * @internal + * 128 bit store operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_STORE_OP_128(new_val) \ +({ \ + (_atom)->v = (new_val); \ +}) + +/** + * @internal + * 128 bit CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ +__extension__ ({ \ + int *_ret_ptr = ret_ptr; \ + odp_u128_t *_cas_old = old_val; \ + odp_u128_t _cas_new = new_val; \ + if (((_atom)->v.u64[0] == (_cas_old)->u64[0]) && \ + ((_atom)->v.u64[1] == (_cas_old)->u64[1])) { \ + (_atom)->v = (_cas_new); \ + *(_ret_ptr) = 1; \ + } else { \ + *(_ret_ptr) = 0; \ + } \ +}) + +/** + * @internal + * Helper macro for lock-based atomic operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param expr Expression used update the variable. + * @return The old value of the variable. + */ +#define ATOMIC_OP_128(atom, expr) \ +__extension__ ({ \ + odp_u128_t _old_val; \ + odp_atomic_u128_t *_atom = atom; \ + /* Loop while lock is already taken, stop when lock becomes clear */ \ + while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \ + (void)0; \ + _old_val = (_atom)->v; \ + (expr); /* Perform whatever update is desired */ \ + __atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \ + _old_val; /* Return old value */ \ +}) + +static inline void _odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->v.u64[0] = val.u64[0]; + atom->v.u64[1] = val.u64[1]; + atom->lock = 0; +} + +static inline odp_u128_t _odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + return ATOMIC_OP_128(atom, (void)0); +} + +static inline void _odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static inline int _odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, new_val)); + return ret; +} + +static inline int _odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} + +static inline int _odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} + +static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val) +{ + return _odp_atomic_cas_u128(atom, old_val, new_val); +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h new file mode 100644 index 000000000..f1072d11f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/atomic_generic.h> diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu.h b/platform/linux-generic/arch/default/odp/api/abi/cpu.h new file mode 100644 index 000000000..e09efdfcf --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2018-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define ODP_CACHE_LINE_SIZE 64 + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h new file mode 100644 index 000000000..b75e65717 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_generic.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_GENERIC_H_ +#define ODP_API_ABI_CPU_GENERIC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +uint64_t _odp_cpu_cycles(void); +int _odp_cpu_cycles_init_global(void); + +static inline uint64_t _odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +static inline uint64_t _odp_cpu_cycles_resolution(void) +{ + return 1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..54aeae946 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/cpu_inlines.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_cpu_pause(void) +{ +} + +#include <odp/api/abi/cpu_generic.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/hash_crc32.h b/platform/linux-generic/arch/default/odp/api/abi/hash_crc32.h new file mode 100644 index 000000000..8759ed948 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/hash_crc32.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_HASH_CRC32_H_ +#define ODP_API_ABI_HASH_CRC32_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +uint32_t _odp_hash_crc32_generic(const void *data, uint32_t data_len, + uint32_t init_val); +uint32_t _odp_hash_crc32c_generic(const void *data, uint32_t data_len, + uint32_t init_val); + +static inline uint32_t _odp_hash_crc32(const void *data, uint32_t data_len, + uint32_t init_val) +{ + return _odp_hash_crc32_generic(data, data_len, init_val); +} + +static inline uint32_t _odp_hash_crc32c(const void *data, uint32_t data_len, + uint32_t init_val) +{ + return _odp_hash_crc32c_generic(data, data_len, init_val); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bfbb3039f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_load(void) +{ + __sync_synchronize(); +} + +static inline void _odp_mb_sync_store(void) +{ + __sync_synchronize(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..ed0ffdb3f --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/time_inlines.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_INLINES_H_ +#define ODP_ARCH_TIME_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/time_types.h> + +#include <stdint.h> + +odp_time_t _odp_time_cur(void); +uint64_t _odp_time_res(void); +void _odp_time_startup(odp_time_startup_t *startup); + +static inline odp_time_t _odp_time_cur_strict(void) +{ + return _odp_time_cur(); +} + +static inline uint64_t _odp_time_to_ns(odp_time_t time) +{ + return time.nsec; +} + +static inline odp_time_t _odp_time_from_ns(uint64_t ns) +{ + odp_time_t time; + + time.nsec = ns; + + return time; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/abi/wait_until.h b/platform/linux-generic/arch/default/odp/api/abi/wait_until.h new file mode 100644 index 000000000..35e8d2566 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/wait_until.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 ARM Limited + */ + +#include <odp/api/abi/wait_until_generic.h> diff --git a/platform/linux-generic/arch/default/odp/api/abi/wait_until_generic.h b/platform/linux-generic/arch/default/odp/api/abi/wait_until_generic.h new file mode 100644 index 000000000..3d3fce175 --- /dev/null +++ b/platform/linux-generic/arch/default/odp/api/abi/wait_until_generic.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 ARM Limited + */ + +#ifndef ODP_API_ABI_WAIT_UNTIL_GENERIC_H_ +#define ODP_API_ABI_WAIT_UNTIL_GENERIC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/atomic.h> + +static inline void +_odp_wait_until_equal_acq_u32(odp_atomic_u32_t *addr, uint32_t expected) +{ + while (odp_atomic_load_acq_u32(addr) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp/api/cpu_arch.h b/platform/linux-generic/arch/default/odp/api/cpu_arch.h deleted file mode 100644 index 22b1da2dd..000000000 --- a/platform/linux-generic/arch/default/odp/api/cpu_arch.h +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright (c) 2016, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_CPU_ARCH_H_ -#define ODP_PLAT_CPU_ARCH_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#define _ODP_CACHE_LINE_SIZE 64 - -static inline void odp_cpu_pause(void) -{ -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/default/odp_atomic.c b/platform/linux-generic/arch/default/odp_atomic.c new file mode 100644 index 000000000..36fc5e8ea --- /dev/null +++ b/platform/linux-generic/arch/default/odp_atomic.c @@ -0,0 +1,47 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/atomic.h> + +int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) +{ +#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#else + /* All operations are lock-free */ + if (atomic_op) { + atomic_op->all_bits = ~((uint32_t)0); + atomic_op->op.init = 0; + } + + return 2; +#endif +} + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef __SIZEOF_INT128__ + if (__atomic_is_lock_free(16, NULL)) { + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + return 2; + } +#endif + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +} diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h new file mode 100644 index 000000000..4cfc6b4bd --- /dev/null +++ b/platform/linux-generic/arch/default/odp_atomic.h @@ -0,0 +1,114 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_DEFAULT_ATOMIC_H_ +#define ODP_DEFAULT_ATOMIC_H_ + +#include <odp_types_internal.h> + +#ifdef __SIZEOF_INT128__ + +static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic) +{ + return __atomic_load_n(atomic, __ATOMIC_RELAXED); +} + +static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, + _odp_u128_t old_val, + _odp_u128_t new_val) +{ + return __atomic_compare_exchange_n(atomic, &old_val, new_val, + 0 /* strong */, + __ATOMIC_ACQ_REL, + __ATOMIC_RELAXED); +} + +static inline int lockfree_check_u128(void) +{ + return __atomic_is_lock_free(16, NULL); +} + +#endif + +#include <limits.h> + +/** Atomic bit set operations with memory ordering */ +#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) + +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) +#endif + +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (_odp_u128_t)(1ULL << (bit - 64)) << 64; +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ + return __atomic_load_n(bs, mo); +} + +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); +} + +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ + (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); +} + +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ + return __atomic_exchange_n(bs, neu, mo); +} + +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ + return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, + mo_failure); +} + +#endif diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h new file mode 100644 index 000000000..6b10966c6 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_DEFAULT_CPU_H_ +#define ODP_DEFAULT_CPU_H_ + +#ifndef _ODP_UNALIGNED +#define _ODP_UNALIGNED 0 +#endif + +/****************************************************************************** + * Atomics + *****************************************************************************/ + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +#include "odp_atomic.h" +#include "odp_wait_until.h" + +#endif diff --git a/platform/linux-generic/arch/default/odp_cpu_arch.c b/platform/linux-generic/arch/default/odp_cpu_arch.c deleted file mode 100644 index 2ac223e07..000000000 --- a/platform/linux-generic/arch/default/odp_cpu_arch.c +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright (c) 2015, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp_posix_extensions.h> - -#include <stdlib.h> -#include <time.h> - -#include <odp/api/cpu.h> -#include <odp/api/hints.h> -#include <odp/api/system_info.h> -#include <odp_debug_internal.h> - -#define GIGA 1000000000 - -uint64_t odp_cpu_cycles(void) -{ - struct timespec time; - uint64_t sec, ns, hz, cycles; - int ret; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time); - - if (ret != 0) - ODP_ABORT("clock_gettime failed\n"); - - hz = odp_cpu_hz_max(); - sec = (uint64_t)time.tv_sec; - ns = (uint64_t)time.tv_nsec; - - cycles = sec * hz; - cycles += (ns * hz) / GIGA; - - return cycles; -} - -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} diff --git a/platform/linux-generic/arch/arm/odp_cpu_arch.c b/platform/linux-generic/arch/default/odp_cpu_cycles.c index 2ac223e07..41436a672 100644 --- a/platform/linux-generic/arch/arm/odp_cpu_arch.c +++ b/platform/linux-generic/arch/default/odp_cpu_cycles.c @@ -1,4 +1,5 @@ -/* Copyright (c) 2015, Linaro Limited +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -6,17 +7,19 @@ #include <odp_posix_extensions.h> +#include <stdint.h> #include <stdlib.h> #include <time.h> -#include <odp/api/cpu.h> -#include <odp/api/hints.h> -#include <odp/api/system_info.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> #define GIGA 1000000000 -uint64_t odp_cpu_cycles(void) +#include <odp/api/abi/cpu_generic.h> + +uint64_t _odp_cpu_cycles(void) { struct timespec time; uint64_t sec, ns, hz, cycles; @@ -25,9 +28,10 @@ uint64_t odp_cpu_cycles(void) ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time); if (ret != 0) - ODP_ABORT("clock_gettime failed\n"); + _ODP_ABORT("clock_gettime failed\n"); + + hz = odp_global_ro.system_info.cpu_hz_max[0]; - hz = odp_cpu_hz_max(); sec = (uint64_t)time.tv_sec; ns = (uint64_t)time.tv_nsec; @@ -37,12 +41,7 @@ uint64_t odp_cpu_cycles(void) return cycles; } -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) +int _odp_cpu_cycles_init_global(void) { - return 1; + return 0; } diff --git a/platform/linux-generic/arch/default/odp_hash_crc32.c b/platform/linux-generic/arch/default/odp_hash_crc32.c new file mode 100644 index 000000000..f71c11909 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_hash_crc32.c @@ -0,0 +1,496 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2013 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <odp/api/align.h> +#include <odp/api/std_types.h> + +#include <odp/api/abi/hash_crc32.h> + +#include <stddef.h> +#include <stdint.h> + +/* Table generated with odp_hash_crc_gen64() */ +static const uint32_t crc32_table[256] ODP_ALIGNED_CACHE = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, + 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, + 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, + 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, + 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, + 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, + 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, + 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, + 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, + 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, + 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, + 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, + 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, + 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, + 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, + 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, + 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, + 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +static const uint32_t crc32c_tables[8][256] = {{ + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, + 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, + 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, + 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, + 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, + 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, + 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, + 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, + 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, + 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, + 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, + 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, + 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, + 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, + 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, + 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, + 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, + 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, + 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, + 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, + 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, + 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 +}, +{ + 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, + 0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, + 0x3FC5F181, 0x2C6769F6, 0x1880C16F, 0x0B225918, 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, + 0xA2D13239, 0xB173AA4E, 0x859402D7, 0x96369AA0, 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, + 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, + 0xE29F20BA, 0xF13DB8CD, 0xC5DA1054, 0xD6788823, 0xAC154166, 0xBFB7D911, 0x8B507188, 0x98F2E9FF, + 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, 0x0EC4735F, 0x1D66EB28, 0x298143B1, 0x3A23DBC6, + 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, 0x93D0B0E7, 0x80722890, 0xB4958009, 0xA737187E, + 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, + 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, 0x2C896460, 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, + 0xC0D23785, 0xD370AFF2, 0xE797076B, 0xF4359F1C, 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, + 0x5DC6F43D, 0x4E646C4A, 0x7A83C4D3, 0x69215CA4, 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, + 0x809C2506, 0x933EBD71, 0xA7D915E8, 0xB47B8D9F, 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, + 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, + 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, 0xF1D3B55B, 0xE2712D2C, 0xD69685B5, 0xC5341DC2, + 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, 0x6CC776E3, 0x7F65EE94, 0x4B82460D, 0x5820DE7A, + 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, 0xB5499B25, 0xA6EB0352, 0x920CABCB, 0x81AE33BC, + 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, + 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, + 0x5912C8C0, 0x4AB050B7, 0x7E57F82E, 0x6DF56059, 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, + 0x844819FB, 0x97EA818C, 0xA30D2915, 0xB0AFB162, 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, + 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, 0x2DBB72DA, 0x57D6BB9F, 0x447423E8, 0x70938B71, 0x63311306, + 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, + 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, 0x68134A1E, 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, + 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, 0x4A5E5D21, 0x59FCC556, 0x6D1B6DCF, 0x7EB9F5B8, + 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, 0xE3AD3600, + 0x3B11CD7C, 0x28B3550B, 0x1C54FD92, 0x0FF665E5, 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, + 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, + 0x7B5FDFFF, 0x68FD4788, 0x5C1AEF11, 0x4FB87766, 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, + 0xE64B1C47, 0xF5E98430, 0xC10E2CA9, 0xD2ACB4DE, 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, + 0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, 0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, + 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483 +}, +{ + 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, + 0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, + 0x70A27D8A, 0xD5E3EFF4, 0x3FCD2F87, 0x9A8CBDF9, 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, + 0x48F3434F, 0xEDB2D131, 0x079C1142, 0xA2DD833C, 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, + 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, + 0xD915C5D1, 0x7C5457AF, 0x967A97DC, 0x333B05A2, 0x47CB61CB, 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, + 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, 0x0F382284, 0xAA79B0FA, 0x40577089, 0xE516E2F7, + 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, 0x37691C41, 0x92288E3F, 0x78064E4C, 0xDD47DC32, + 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, + 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, 0x61EA1A06, 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, + 0xB7C7FD53, 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, + 0x8F96C396, 0x2AD751E8, 0xC0F9919B, 0x65B803E5, 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, + 0x26217BCD, 0x8360E9B3, 0x694E29C0, 0xCC0FBBBE, 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, + 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, + 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, 0xC85DA25D, 0x6D1C3023, 0x8732F050, 0x2273622E, + 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, 0xF00C9C98, 0x554D0EE6, 0xBF63CE95, 0x1A225CEB, + 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, 0x15F9D359, 0xB0B84127, 0x5A968154, 0xFFD7132A, + 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, + 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, + 0xC3D4340C, 0x6695A672, 0x8CBB6601, 0x29FAF47F, 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, + 0x6A638C57, 0xCF221E29, 0x250CDE5A, 0x804D4C24, 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, + 0x5232B292, 0xF77320EC, 0x1D5DE09F, 0xB81C72E1, 0xCCEC1688, 0x69AD84F6, 0x83834485, 0x26C2D6FB, + 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, + 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, 0xBC4E6B02, 0x190FF97C, 0xF321390F, 0x5660AB71, + 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, 0xD29C5380, 0x77DDC1FE, 0x9DF3018D, 0x38B293F3, + 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, 0x00E3AD36, + 0x3CE08A10, 0x99A1186E, 0x738FD81D, 0xD6CE4A63, 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, + 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, + 0xAD060C8E, 0x08479EF0, 0xE2695E83, 0x4728CCFD, 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, + 0x9557324B, 0x3016A035, 0xDA386046, 0x7F79F238, 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, + 0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, 0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, + 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8 +}, +{ + 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, + 0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, + 0xE964B13D, 0x34211B85, 0x560392BC, 0x8B463804, 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, + 0x1F20D2DB, 0xC2657863, 0xA047F15A, 0x7D025BE2, 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, + 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, + 0x2161776D, 0xFC24DDD5, 0x9E0654EC, 0x4343FE54, 0x5A43469E, 0x8706EC26, 0xE524651F, 0x3861CFA7, + 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, 0x45639445, 0x98263EFD, 0xFA04B7C4, 0x27411D7C, + 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, 0xB327F7A3, 0x6E625D1B, 0x0C40D422, 0xD1057E9A, + 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, + 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, 0x26C00DF2, 0xFB85A74A, 0x99A72E73, 0x44E284CB, + 0x42C2EEDA, 0x9F874462, 0xFDA5CD5B, 0x20E067E3, 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, + 0xB4868D3C, 0x69C32784, 0x0BE1AEBD, 0xD6A40405, 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, + 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, 0x1EA1C255, 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, + 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, + 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, 0xEEC5CBA2, 0x3380611A, 0x51A2E823, 0x8CE7429B, + 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, 0x1881A844, 0xC5C402FC, 0xA7E68BC5, 0x7AA3217D, + 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, 0x4BA071F5, + 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, + 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, + 0x4D801BE4, 0x90C5B15C, 0xF2E73865, 0x2FA292DD, 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, + 0x8585DDB4, 0x58C0770C, 0x3AE2FE35, 0xE7A7548D, 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, + 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, 0x11E3376B, 0x08E38FA1, 0xD5A62519, 0xB784AC20, 0x6AC10698, + 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, + 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, 0xE1873E9C, 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, + 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, 0x8224A72B, 0x5F610D93, 0x3D4384AA, 0xE0062E12, + 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, 0x7460C4CD, 0xA9256E75, 0xCB07E74C, 0x16424DF4, + 0x106227E5, 0xCD278D5D, 0xAF050464, 0x7240AEDC, 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, + 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, + 0x2E238253, 0xF36628EB, 0x9144A1D2, 0x4C010B6A, 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, + 0xD867E1B5, 0x05224B0D, 0x6700C234, 0xBA45688C, 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, + 0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, 0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, + 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842 +}, +{ + 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, + 0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, + 0x8F2261D3, 0xB7330E7F, 0xFF00BE8B, 0xC711D127, 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, + 0x4A456A42, 0x725405EE, 0x3A67B51A, 0x0276DAB6, 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, + 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, + 0xDECFBEC6, 0xE6DED16A, 0xAEED619E, 0x96FC0E32, 0x3E8A0076, 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, + 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, 0x74CF6A34, 0x4CDE0598, 0x04EDB56C, 0x3CFCDAC0, + 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, 0xF99BD151, + 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, + 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, 0x1273DF8F, 0x2A62B023, 0x625100D7, 0x5A406F7B, + 0xB8730B7D, 0x806264D1, 0xC851D425, 0xF040BB89, 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, + 0x7D1400EC, 0x45056F40, 0x0D36DFB4, 0x3527B018, 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, + 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, 0x64CA6F0D, 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, + 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, + 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, 0x439E009A, 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, + 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, 0x86F90B0B, 0xBEE864A7, 0xF6DBD453, 0xCECABBFF, + 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, 0xC6D4DB18, + 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, + 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, + 0x24E7BF1E, 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, + 0x750A600B, 0x4D1B0FA7, 0x0528BF53, 0x3D39D0FF, 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, + 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, 0xF85EDB6E, 0x5028D52A, 0x6839BA86, 0x200A0A72, 0x181B65DE, + 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, + 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, 0xDF0AB4F9, 0xE71BDB55, 0xAF286BA1, 0x9739040D, + 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, 0xB9B60142, 0x81A76EEE, 0xC994DE1A, 0xF185B1B6, + 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, 0x34E2BA27, + 0xD6D1DE21, 0xEEC0B18D, 0xA6F30179, 0x9EE26ED5, 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, + 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, + 0x425B0AA5, 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, + 0x873C0134, 0xBF2D6E98, 0xF71EDE6C, 0xCF0FB1C0, 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, + 0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, 0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, + 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3 +}, +{ + 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, + 0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, + 0xC00C303E, 0x2F3C5B27, 0x1B8090FD, 0xF4B0FBE4, 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, + 0xA00A2821, 0x4F3A4338, 0x7B8688E2, 0x94B6E3FB, 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, + 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, + 0xE5F20E92, 0x0AC2658B, 0x3E7EAE51, 0xD14EC548, 0x570739E5, 0xB83752FC, 0x8C8B9926, 0x63BBF23F, + 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, 0xF70D11C4, 0x183D7ADD, 0x2C81B107, 0xC3B1DA1E, + 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, 0x970B09DB, 0x783B62C2, 0x4C87A918, 0xA3B7C201, + 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, + 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, 0xDCF77483, 0x33C71F9A, 0x077BD440, 0xE84BBF59, + 0xCE086BD5, 0x213800CC, 0x1584CB16, 0xFAB4A00F, 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, + 0xAE0E73CA, 0x413E18D3, 0x7582D309, 0x9AB2B810, 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, + 0x8BF04D66, 0x64C0267F, 0x507CEDA5, 0xBF4C86BC, 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, + 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, + 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, 0xF9094A2F, 0x16392136, 0x2285EAEC, 0xCDB581F5, + 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, 0x990F5230, 0x763F3929, 0x4283F2F3, 0xADB399EA, + 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, 0xAEFD80A1, 0x41CDEBB8, 0x75712062, 0x9A414B7B, + 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, + 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, + 0xBC029FF7, 0x5332F4EE, 0x678E3F34, 0x88BE542D, 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, + 0x99FCA15B, 0x76CCCA42, 0x42700198, 0xAD406A81, 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, + 0xF9FAB944, 0x16CAD25D, 0x22761987, 0xCD46729E, 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, 0x7FB345E9, + 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, + 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, 0x8B03BE0D, 0x6433D514, 0x508F1ECE, 0xBFBF75D7, + 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, 0xA0F9DB4A, 0x4FC9B053, 0x7B757B89, 0x94451090, + 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, 0xC0FFC355, 0x2FCFA84C, 0x1B736396, 0xF443088F, + 0xD200DC03, 0x3D30B71A, 0x098C7CC0, 0xE6BC17D9, 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, + 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, + 0x97F8FAB0, 0x78C891A9, 0x4C745A73, 0xA344316A, 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, + 0xF7FEE2AF, 0x18CE89B6, 0x2C72426C, 0xC3422975, 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, + 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, 0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, + 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C +}, +{ + 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, + 0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, + 0x9C5BFAA6, 0xF458D66E, 0x4C5DA336, 0x245E8FFE, 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, + 0xD27607F5, 0xBA752B3D, 0x02705E65, 0x6A7372AD, 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, + 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, + 0x73767EEE, 0x1B755226, 0xA370277E, 0xCB730BB6, 0xD696BB3F, 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, + 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, 0x04E0BCCA, 0x6CE39002, 0xD4E6E55A, 0xBCE5C992, + 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, 0xF2C834C1, + 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, + 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, 0x917A3FF8, 0xF9791330, 0x417C6668, 0x297F4AA0, + 0xE6ECFDDC, 0x8EEFD114, 0x36EAA44C, 0x5EE98884, 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, + 0xA8C1008F, 0xC0C22C47, 0x78C7591F, 0x10C475D7, 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, + 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, 0xFFE9F19F, 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, + 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, + 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, 0x7E57BBB0, 0x16549778, 0xAE51E220, 0xC652CEE8, + 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, 0x307A46E3, 0x58796A2B, 0xE07C1F73, 0x887F33BB, + 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, 0x508ECB25, 0x388DE7ED, 0x808892B5, 0xE88BBE7D, + 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, + 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, + 0x27180901, 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, + 0xC8358D49, 0xA036A181, 0x1833D4D9, 0x7030F811, 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, + 0x8618701A, 0xEE1B5CD2, 0x561E298A, 0x3E1D0542, 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, 0x9BFDC093, + 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, + 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, 0xBFA34F6D, 0xD7A063A5, 0x6FA516FD, 0x07A63A35, + 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, 0x2A39CC5F, 0x423AE097, 0xFA3F95CF, 0x923CB907, + 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, 0x6414310C, 0x0C171DC4, 0xB412689C, 0xDC114454, + 0x1382F328, 0x7B81DFE0, 0xC384AAB8, 0xAB878670, 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, + 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, + 0xB2828A33, 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, + 0xFCAF7760, 0x94AC5BA8, 0x2CA92EF0, 0x44AA0238, 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, + 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, 0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, + 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F +}, +{ + 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, + 0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, + 0x847609B4, 0xCD4A7493, 0x160EF3FA, 0x5F328EDD, 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, + 0xC64D0D6E, 0x8F717049, 0x5435F720, 0x1D098A07, 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, + 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, + 0x4F3B6143, 0x06071C64, 0xDD439B0D, 0x947FE62A, 0x6E26E32E, 0x271A9E09, 0xFC5E1960, 0xB5626447, + 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, 0xA86BEE40, 0xE1579367, 0x3A13140E, 0x732F6929, + 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, 0xEA50EA9A, 0xA36C97BD, 0x782810D4, 0x31146DF3, + 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, + 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, 0x79264D85, 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, + 0x9E76C286, 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, + 0xDC4DC65C, 0x9571BB7B, 0x4E353C12, 0x07094135, 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, + 0x1700AEAB, 0x5E3CD38C, 0x857854E5, 0xCC4429C2, 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, + 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, + 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, 0xB26B2572, 0xFB575855, 0x2013DF3C, 0x692FA21B, + 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, 0xF05021A8, 0xB96C5C8F, 0x6228DBE6, 0x2B14A6C1, + 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, 0x151C1409, 0x5C20692E, 0x8764EE47, 0xCE589360, + 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, + 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, + 0xF24C9B0A, 0xBB70E62D, 0x60346144, 0x29081C63, 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, + 0x3901F3FD, 0x703D8EDA, 0xAB7909B3, 0xE2457494, 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, + 0x7B3AF727, 0x32068A00, 0xE9420D69, 0xA07E704E, 0x5A27754A, 0x131B086D, 0xC85F8F04, 0x8163F223, + 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, + 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, 0xDE517CFE, 0x976D01D9, 0x4C2986B0, 0x0515FB97, + 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, 0x0F1CDF3B, 0x4620A21C, 0x9D642575, 0xD4585852, + 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, 0x96635C88, + 0xAA7754E2, 0xE34B29C5, 0x380FAEAC, 0x7133D38B, 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, + 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, + 0x230138CF, 0x6A3D45E8, 0xB179C281, 0xF845BFA6, 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, + 0x613A3C15, 0x28064132, 0xF342C65B, 0xBA7EBB7C, 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, + 0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, 0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, + 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 +} }; + +#define CRC32_UPD(crc, n) \ + (crc32c_tables[(n)][(crc) & 0xff] ^ \ + crc32c_tables[(n) - 1][((crc) >> 8) & 0xff]) + +static inline uint32_t crc32c_u8(uint8_t data, uint32_t init_val) +{ + uint32_t crc; + + crc = init_val; + crc ^= data; + + return crc32c_tables[0][crc & 0xff] ^ (crc >> 8); +} + +static inline uint32_t crc32c_u16(uint16_t data, uint32_t init_val) +{ + uint32_t crc; + + crc = init_val; + crc ^= data; + + crc = CRC32_UPD(crc, 1) ^ (crc >> 16); + + return crc; +} + +static inline uint32_t crc32c_u32(uint32_t data, uint32_t init_val) +{ + uint32_t crc, term1, term2; + + crc = init_val; + crc ^= data; + + term1 = CRC32_UPD(crc, 3); + term2 = crc >> 16; + crc = term1 ^ CRC32_UPD(term2, 1); + + return crc; +} + +static inline uint32_t crc32c_u64(uint64_t data, uint32_t init_val) +{ + uint32_t crc, term1, term2; + + union { + uint64_t u64; + uint32_t u32[2]; + } d; + + d.u64 = data; + + crc = init_val; + crc ^= d.u32[0]; + + term1 = CRC32_UPD(crc, 7); + term2 = crc >> 16; + crc = term1 ^ CRC32_UPD(term2, 5); + term1 = CRC32_UPD(d.u32[1], 3); + term2 = d.u32[1] >> 16; + crc ^= term1 ^ CRC32_UPD(term2, 1); + + return crc; +} + +#include <odp/visibility_begin.h> + +uint32_t _odp_hash_crc32_generic(const void *data_ptr, uint32_t data_len, + uint32_t init_val) +{ + uint32_t i, crc; + const uint8_t *byte = data_ptr; + + crc = init_val; + + for (i = 0; i < data_len; i++) + crc = crc32_table[(crc ^ byte[i]) & 0xff] ^ (crc >> 8); + + return crc; +} + +uint32_t _odp_hash_crc32c_generic(const void *data, uint32_t data_len, + uint32_t init_val) +{ + uint32_t i; + uintptr_t pd = (uintptr_t)data; + + for (i = 0; i < data_len / 8; i++) { + init_val = crc32c_u64(*(const uint64_t *)pd, init_val); + pd += 8; + } + + if (data_len & 0x4) { + init_val = crc32c_u32(*(const uint32_t *)pd, init_val); + pd += 4; + } + + if (data_len & 0x2) { + init_val = crc32c_u16(*(const uint16_t *)pd, init_val); + pd += 2; + } + + if (data_len & 0x1) + init_val = crc32c_u8(*(const uint8_t *)pd, init_val); + + return init_val; +} + +#include <odp/visibility_end.h> diff --git a/platform/linux-generic/arch/default/odp_random.c b/platform/linux-generic/arch/default/odp_random.c new file mode 100644 index 000000000..18d2a45d2 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_random.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_random.h> +#include <odp/api/spec/random.h> + +#include <odp/visibility_begin.h> + +odp_random_kind_t _odp_random_max_kind_generic(void) +{ + return ODP_RANDOM_BASIC; +} + +int32_t _odp_random_true_data_generic(uint8_t *buf, uint32_t len) +{ + (void)buf; + (void)len; + + return -1; +} + +int32_t _odp_random_crypto_data_generic(uint8_t *buf, uint32_t len) +{ + (void)buf; + (void)len; + + return -1; +} + +#include <odp/visibility_end.h> diff --git a/platform/linux-generic/arch/default/odp_random.h b/platform/linux-generic/arch/default/odp_random.h new file mode 100644 index 000000000..215eb6d93 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_random.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_DEFAULT_RANDOM_H_ +#define ODP_DEFAULT_RANDOM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/random.h> + +#include <stdint.h> + +odp_random_kind_t _odp_random_max_kind_generic(void); +int32_t _odp_random_true_data_generic(uint8_t *buf, uint32_t len); +int32_t _odp_random_crypto_data_generic(uint8_t *buf, uint32_t len); + +static inline odp_random_kind_t _odp_random_max_kind(void) +{ + return _odp_random_max_kind_generic(); +} + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_true_data_generic(buf, len); +} + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_crypto_data_generic(buf, len); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/default/odp_sysinfo_parse.c b/platform/linux-generic/arch/default/odp_sysinfo_parse.c index 53e2aaeaf..11d33d576 100644 --- a/platform/linux-generic/arch/default/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/default/odp_sysinfo_parse.c @@ -1,27 +1,22 @@ -/* Copyright (c) 2016, Linaro Limited +/* Copyright (c) 2016-2018, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_internal.h> -#include <odp_debug_internal.h> -#include <string.h> +#include <odp_global_data.h> +#include <odp_sysinfo_internal.h> -int cpuinfo_parser(FILE *file ODP_UNUSED, system_info_t *sysinfo) +int _odp_cpuinfo_parser(FILE *file ODP_UNUSED, system_info_t *sysinfo) { - int i; - - ODP_DBG("Warning: use dummy values for freq and model string\n"); - for (i = 0; i < MAX_CPU_NUMBER; i++) { - sysinfo->cpu_hz_max[i] = 1400000000; - strcpy(sysinfo->model_str[i], "UNKNOWN"); - } + return _odp_dummy_cpuinfo(sysinfo); +} - return 0; +void _odp_sys_info_print_arch(void) +{ } -uint64_t odp_cpu_hz_current(int id ODP_UNUSED) +uint64_t odp_cpu_arch_hz_current(int id ODP_UNUSED) { - return 0; + return odp_global_ro.system_info.default_cpu_hz; } diff --git a/platform/linux-generic/arch/default/odp_time.c b/platform/linux-generic/arch/default/odp_time.c new file mode 100644 index 000000000..664a5deae --- /dev/null +++ b/platform/linux-generic/arch/default/odp_time.c @@ -0,0 +1,112 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2020-2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> + +#include <odp/api/align.h> +#include <odp/api/hints.h> +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_inlines.h> + +#include <odp_debug_internal.h> +#include <odp_init_internal.h> + +#include <inttypes.h> +#include <stdint.h> +#include <string.h> +#include <time.h> + +#define YEAR_IN_SEC (365 * 24 * 3600) + +typedef struct _odp_time_global_t { + struct timespec start_time; + uint64_t start_time_ns; + +} _odp_time_global_t; + +_odp_time_global_t _odp_time_glob; + +static inline uint64_t time_nsec(struct timespec *t) +{ + uint64_t nsec = (t->tv_sec * ODP_TIME_SEC_IN_NS) + t->tv_nsec; + + return nsec; +} + +#include <odp/visibility_begin.h> + +odp_time_t _odp_time_cur(void) +{ + int ret; + odp_time_t time; + struct timespec sys_time; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_gettime() failed\n"); + + time.nsec = time_nsec(&sys_time); + + return time; +} + +uint64_t _odp_time_res(void) +{ + int ret; + struct timespec tres; + + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + _ODP_ABORT("clock_getres() failed\n"); + + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; +} + +void _odp_time_startup(odp_time_startup_t *startup) +{ + startup->global.nsec = _odp_time_glob.start_time_ns; + startup->global_ns = _odp_time_glob.start_time_ns; +} + +#include <odp/visibility_end.h> + +int _odp_time_init_global(void) +{ + struct timespec *start_time; + uint64_t diff, years; + int ret = 0; + _odp_time_global_t *global = &_odp_time_glob; + + memset(global, 0, sizeof(_odp_time_global_t)); + + start_time = &global->start_time; + start_time->tv_sec = 0; + start_time->tv_nsec = 0; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, start_time); + if (ret) + _ODP_ERR("clock_gettime() failed: %d\n", ret); + + global->start_time_ns = time_nsec(start_time); + + diff = UINT64_MAX - global->start_time_ns; + years = (diff / ODP_TIME_SEC_IN_NS) / YEAR_IN_SEC; + + if (years < 10) { + _ODP_ERR("Time in nsec would wrap in 10 years: %" PRIu64 "\n", + global->start_time_ns); + return -1; + } + + return ret; +} + +int _odp_time_term_global(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/default/odp_wait_until.h b/platform/linux-generic/arch/default/odp_wait_until.h new file mode 100644 index 000000000..c51f4355e --- /dev/null +++ b/platform/linux-generic/arch/default/odp_wait_until.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#ifndef ODP_DEFAULT_WAIT_UNTIL_H_ +#define ODP_DEFAULT_WAIT_UNTIL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/plat/cpu_inlines.h> + +#include <stdint.h> + +/** + * The _odp_wait_until_eq_*() functions defined in this header are intended to + * be used only with the scalable scheduler and queue implementations. Even + * though these functions use standard non-atomic parameter types, the + * parameters must only be operated using atomic operations. If new functions + * are added to this file, they should use _odp_wait_until_equal_*() prefix and + * atomic parameter types. + */ + +static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/mips64/odp/api/cpu_arch.h b/platform/linux-generic/arch/mips64/odp/api/cpu_arch.h deleted file mode 100644 index 3582b129b..000000000 --- a/platform/linux-generic/arch/mips64/odp/api/cpu_arch.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2016, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_CPU_ARCH_H_ -#define ODP_PLAT_CPU_ARCH_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined __OCTEON__ -#define _ODP_CACHE_LINE_SIZE 128 -#else -#error Please add support for your arch in cpu_arch.h -#endif - -static inline void odp_cpu_pause(void) -{ - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); - __asm__ __volatile__ ("nop"); -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/mips64/odp_cpu_arch.c b/platform/linux-generic/arch/mips64/odp_cpu_arch.c deleted file mode 100644 index 646acf9c1..000000000 --- a/platform/linux-generic/arch/mips64/odp_cpu_arch.c +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2015, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/cpu.h> -#include <odp/api/hints.h> -#include <odp/api/system_info.h> - -uint64_t odp_cpu_cycles(void) -{ - #define CVMX_TMP_STR(x) CVMX_TMP_STR2(x) - #define CVMX_TMP_STR2(x) #x - uint64_t cycle; - - __asm__ __volatile__ ("rdhwr %[rt],$" CVMX_TMP_STR(31) : - [rt] "=d" (cycle) : : "memory"); - - return cycle; -} - -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} diff --git a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c deleted file mode 100644 index 407264b7f..000000000 --- a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright (c) 2016, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp_internal.h> -#include <string.h> - -int cpuinfo_parser(FILE *file, system_info_t *sysinfo) -{ - char str[1024]; - char *pos; - double mhz = 0.0; - uint64_t hz; - int model = 0; - int count = 2; - int id = 0; - - strcpy(sysinfo->cpu_arch_str, "mips64"); - while (fgets(str, sizeof(str), file) != NULL && id < MAX_CPU_NUMBER) { - if (!mhz) { - pos = strstr(str, "BogoMIPS"); - - if (pos) - if (sscanf(pos, "BogoMIPS : %lf", &mhz) == 1) { - /* bogomips seems to be 2x freq */ - hz = (uint64_t)(mhz * 1000000.0 / 2.0); - sysinfo->cpu_hz_max[id] = hz; - count--; - } - } - - if (!model) { - pos = strstr(str, "cpu model"); - - if (pos) { - int len; - - pos = strchr(str, ':'); - strncpy(sysinfo->model_str[id], pos + 2, - sizeof(sysinfo->model_str[id]) - 1); - len = strlen(sysinfo->model_str[id]); - sysinfo->model_str[id][len - 1] = 0; - model = 1; - count--; - } - } - - if (count == 0) { - mhz = 0.0; - model = 0; - count = 2; - id++; - } - } - - return 0; -} - -uint64_t odp_cpu_hz_current(int id ODP_UNUSED) -{ - return 0; -} diff --git a/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h new file mode 100644 index 000000000..ecf56e82e --- /dev/null +++ b/platform/linux-generic/arch/powerpc/odp/api/abi/cpu.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define ODP_CACHE_LINE_SIZE 128 + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h b/platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h deleted file mode 100644 index 22b1da2dd..000000000 --- a/platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright (c) 2016, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_CPU_ARCH_H_ -#define ODP_PLAT_CPU_ARCH_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#define _ODP_CACHE_LINE_SIZE 64 - -static inline void odp_cpu_pause(void) -{ -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c deleted file mode 100644 index 2ac223e07..000000000 --- a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright (c) 2015, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp_posix_extensions.h> - -#include <stdlib.h> -#include <time.h> - -#include <odp/api/cpu.h> -#include <odp/api/hints.h> -#include <odp/api/system_info.h> -#include <odp_debug_internal.h> - -#define GIGA 1000000000 - -uint64_t odp_cpu_cycles(void) -{ - struct timespec time; - uint64_t sec, ns, hz, cycles; - int ret; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time); - - if (ret != 0) - ODP_ABORT("clock_gettime failed\n"); - - hz = odp_cpu_hz_max(); - sec = (uint64_t)time.tv_sec; - ns = (uint64_t)time.tv_nsec; - - cycles = sec * hz; - cycles += (ns * hz) / GIGA; - - return cycles; -} - -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} diff --git a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c index 3b88d55b6..2049cc42f 100644 --- a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c @@ -1,13 +1,14 @@ -/* Copyright (c) 2016, Linaro Limited +/* Copyright (c) 2016-2018, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_internal.h> +#include <odp_global_data.h> +#include <odp_sysinfo_internal.h> #include <string.h> -int cpuinfo_parser(FILE *file, system_info_t *sysinfo) +int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) { char str[1024]; char *pos; @@ -17,8 +18,12 @@ int cpuinfo_parser(FILE *file, system_info_t *sysinfo) int count = 2; int id = 0; + sysinfo->cpu_arch = ODP_CPU_ARCH_PPC; + sysinfo->cpu_isa_sw.ppc = ODP_CPU_ARCH_PPC_UNKNOWN; + sysinfo->cpu_isa_hw.ppc = ODP_CPU_ARCH_PPC_UNKNOWN; + strcpy(sysinfo->cpu_arch_str, "powerpc"); - while (fgets(str, sizeof(str), file) != NULL && id < MAX_CPU_NUMBER) { + while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU_IDS) { if (!mhz) { pos = strstr(str, "clock"); @@ -38,7 +43,7 @@ int cpuinfo_parser(FILE *file, system_info_t *sysinfo) pos = strchr(str, ':'); strncpy(sysinfo->model_str[id], pos + 2, - sizeof(sysinfo->model_str[id]) - 1); + MODEL_STR_SIZE - 1); len = strlen(sysinfo->model_str[id]); sysinfo->model_str[id][len - 1] = 0; model = 1; @@ -57,7 +62,11 @@ int cpuinfo_parser(FILE *file, system_info_t *sysinfo) return 0; } -uint64_t odp_cpu_hz_current(int id ODP_UNUSED) +void _odp_sys_info_print_arch(void) { - return 0; +} + +uint64_t odp_cpu_arch_hz_current(int id ODP_UNUSED) +{ + return odp_global_ro.system_info.default_cpu_hz; } diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c new file mode 100644 index 000000000..9211df002 --- /dev/null +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -0,0 +1,378 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2015 Intel Corporation + */ + +#include "cpu_flags.h" + +#include <odp/api/abi/time_cpu.h> + +#include <odp_debug_internal.h> +#include <odp_global_data.h> + +#include <cpuid.h> +#include <errno.h> +#include <stdio.h> +#include <stdint.h> + +enum rte_cpu_flag_t { + /* (EAX 01h) ECX features*/ + RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */ + RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */ + RTE_CPUFLAG_DTES64, /**< DTES64 */ + RTE_CPUFLAG_MONITOR, /**< MONITOR */ + RTE_CPUFLAG_DS_CPL, /**< DS_CPL */ + RTE_CPUFLAG_VMX, /**< VMX */ + RTE_CPUFLAG_SMX, /**< SMX */ + RTE_CPUFLAG_EIST, /**< EIST */ + RTE_CPUFLAG_TM2, /**< TM2 */ + RTE_CPUFLAG_SSSE3, /**< SSSE3 */ + RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */ + RTE_CPUFLAG_FMA, /**< FMA */ + RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */ + RTE_CPUFLAG_XTPR, /**< XTPR */ + RTE_CPUFLAG_PDCM, /**< PDCM */ + RTE_CPUFLAG_PCID, /**< PCID */ + RTE_CPUFLAG_DCA, /**< DCA */ + RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */ + RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */ + RTE_CPUFLAG_X2APIC, /**< X2APIC */ + RTE_CPUFLAG_MOVBE, /**< MOVBE */ + RTE_CPUFLAG_POPCNT, /**< POPCNT */ + RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */ + RTE_CPUFLAG_AES, /**< AES */ + RTE_CPUFLAG_XSAVE, /**< XSAVE */ + RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */ + RTE_CPUFLAG_AVX, /**< AVX */ + RTE_CPUFLAG_F16C, /**< F16C */ + RTE_CPUFLAG_RDRAND, /**< RDRAND */ + RTE_CPUFLAG_HYPERVISOR, /**< Running in a VM */ + + /* (EAX 01h) EDX features */ + RTE_CPUFLAG_FPU, /**< FPU */ + RTE_CPUFLAG_VME, /**< VME */ + RTE_CPUFLAG_DE, /**< DE */ + RTE_CPUFLAG_PSE, /**< PSE */ + RTE_CPUFLAG_TSC, /**< TSC */ + RTE_CPUFLAG_MSR, /**< MSR */ + RTE_CPUFLAG_PAE, /**< PAE */ + RTE_CPUFLAG_MCE, /**< MCE */ + RTE_CPUFLAG_CX8, /**< CX8 */ + RTE_CPUFLAG_APIC, /**< APIC */ + RTE_CPUFLAG_SEP, /**< SEP */ + RTE_CPUFLAG_MTRR, /**< MTRR */ + RTE_CPUFLAG_PGE, /**< PGE */ + RTE_CPUFLAG_MCA, /**< MCA */ + RTE_CPUFLAG_CMOV, /**< CMOV */ + RTE_CPUFLAG_PAT, /**< PAT */ + RTE_CPUFLAG_PSE36, /**< PSE36 */ + RTE_CPUFLAG_PSN, /**< PSN */ + RTE_CPUFLAG_CLFSH, /**< CLFSH */ + RTE_CPUFLAG_DS, /**< DS */ + RTE_CPUFLAG_ACPI, /**< ACPI */ + RTE_CPUFLAG_MMX, /**< MMX */ + RTE_CPUFLAG_FXSR, /**< FXSR */ + RTE_CPUFLAG_SSE, /**< SSE */ + RTE_CPUFLAG_SSE2, /**< SSE2 */ + RTE_CPUFLAG_SS, /**< SS */ + RTE_CPUFLAG_HTT, /**< HTT */ + RTE_CPUFLAG_TM, /**< TM */ + RTE_CPUFLAG_PBE, /**< PBE */ + + /* (EAX 06h) EAX features */ + RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */ + RTE_CPUFLAG_TRBOBST, /**< TRBOBST */ + RTE_CPUFLAG_ARAT, /**< ARAT */ + RTE_CPUFLAG_PLN, /**< PLN */ + RTE_CPUFLAG_ECMD, /**< ECMD */ + RTE_CPUFLAG_PTM, /**< PTM */ + + /* (EAX 06h) ECX features */ + RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */ + RTE_CPUFLAG_ACNT2, /**< ACNT2 */ + RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */ + + /* (EAX 07h, ECX 0h) EBX features */ + RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */ + RTE_CPUFLAG_BMI1, /**< BMI1 */ + RTE_CPUFLAG_HLE, /**< Hardware Lock elision */ + RTE_CPUFLAG_AVX2, /**< AVX2 */ + RTE_CPUFLAG_SMEP, /**< SMEP */ + RTE_CPUFLAG_BMI2, /**< BMI2 */ + RTE_CPUFLAG_ERMS, /**< ERMS */ + RTE_CPUFLAG_INVPCID, /**< INVPCID */ + RTE_CPUFLAG_RTM, /**< Transactional memory */ + RTE_CPUFLAG_AVX512F, /**< AVX512F */ + RTE_CPUFLAG_RDSEED, /**< RDSEED instruction */ + + /* (EAX 80000001h) ECX features */ + RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ + RTE_CPUFLAG_LZCNT, /**< LZCNT */ + + /* (EAX 80000001h) EDX features */ + RTE_CPUFLAG_SYSCALL, /**< SYSCALL */ + RTE_CPUFLAG_XD, /**< XD */ + RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */ + RTE_CPUFLAG_RDTSCP, /**< RDTSCP */ + RTE_CPUFLAG_EM64T, /**< EM64T */ + + /* (EAX 80000007h) EDX features */ + RTE_CPUFLAG_INVTSC, /**< INVTSC */ + + RTE_CPUFLAG_AVX512DQ, /**< AVX512 Doubleword and Quadword */ + RTE_CPUFLAG_AVX512IFMA, /**< AVX512 Integer Fused Multiply-Add */ + RTE_CPUFLAG_AVX512CD, /**< AVX512 Conflict Detection*/ + RTE_CPUFLAG_AVX512BW, /**< AVX512 Byte and Word */ + RTE_CPUFLAG_AVX512VL, /**< AVX512 Vector Length */ + RTE_CPUFLAG_AVX512VBMI, /**< AVX512 Vector Bit Manipulation */ + RTE_CPUFLAG_AVX512VBMI2, /**< AVX512 Vector Bit Manipulation 2 */ + RTE_CPUFLAG_GFNI, /**< Galois Field New Instructions */ + RTE_CPUFLAG_VAES, /**< Vector AES */ + RTE_CPUFLAG_VPCLMULQDQ, /**< Vector Carry-less Multiply */ + RTE_CPUFLAG_AVX512VNNI, + /**< AVX512 Vector Neural Network Instructions */ + RTE_CPUFLAG_AVX512BITALG, /**< AVX512 Bit Algorithms */ + RTE_CPUFLAG_AVX512VPOPCNTDQ, /**< AVX512 Vector Popcount */ + RTE_CPUFLAG_CLDEMOTE, /**< Cache Line Demote */ + RTE_CPUFLAG_MOVDIRI, /**< Direct Store Instructions */ + RTE_CPUFLAG_MOVDIR64B, /**< Direct Store Instructions 64B */ + RTE_CPUFLAG_AVX512VP2INTERSECT, /**< AVX512 Two Register Intersection */ + + RTE_CPUFLAG_WAITPKG, /**< UMONITOR/UMWAIT/TPAUSE */ + + /* The last item */ + RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ +}; + +enum cpu_register_t { + RTE_REG_EAX = 0, + RTE_REG_EBX, + RTE_REG_ECX, + RTE_REG_EDX, +}; + +typedef uint32_t cpuid_registers_t[4]; + +/** + * Struct to hold a processor feature entry + */ +struct feature_entry { + uint32_t leaf; /**< cpuid leaf */ + uint32_t subleaf; /**< cpuid subleaf */ + uint32_t reg; /**< cpuid register */ + uint32_t bit; /**< cpuid register bit */ +#define CPU_FLAG_NAME_MAX_LEN 64 + char name[CPU_FLAG_NAME_MAX_LEN]; /**< String for printing */ +}; + +#define FEAT_DEF(name, leaf, subleaf, reg, bit) \ + [RTE_CPUFLAG_##name] = {leaf, subleaf, reg, bit, #name }, + +static const struct feature_entry cpu_feature_table[] = { + FEAT_DEF(SSE3, 0x00000001, 0, RTE_REG_ECX, 0) + FEAT_DEF(PCLMULQDQ, 0x00000001, 0, RTE_REG_ECX, 1) + FEAT_DEF(DTES64, 0x00000001, 0, RTE_REG_ECX, 2) + FEAT_DEF(MONITOR, 0x00000001, 0, RTE_REG_ECX, 3) + FEAT_DEF(DS_CPL, 0x00000001, 0, RTE_REG_ECX, 4) + FEAT_DEF(VMX, 0x00000001, 0, RTE_REG_ECX, 5) + FEAT_DEF(SMX, 0x00000001, 0, RTE_REG_ECX, 6) + FEAT_DEF(EIST, 0x00000001, 0, RTE_REG_ECX, 7) + FEAT_DEF(TM2, 0x00000001, 0, RTE_REG_ECX, 8) + FEAT_DEF(SSSE3, 0x00000001, 0, RTE_REG_ECX, 9) + FEAT_DEF(CNXT_ID, 0x00000001, 0, RTE_REG_ECX, 10) + FEAT_DEF(FMA, 0x00000001, 0, RTE_REG_ECX, 12) + FEAT_DEF(CMPXCHG16B, 0x00000001, 0, RTE_REG_ECX, 13) + FEAT_DEF(XTPR, 0x00000001, 0, RTE_REG_ECX, 14) + FEAT_DEF(PDCM, 0x00000001, 0, RTE_REG_ECX, 15) + FEAT_DEF(PCID, 0x00000001, 0, RTE_REG_ECX, 17) + FEAT_DEF(DCA, 0x00000001, 0, RTE_REG_ECX, 18) + FEAT_DEF(SSE4_1, 0x00000001, 0, RTE_REG_ECX, 19) + FEAT_DEF(SSE4_2, 0x00000001, 0, RTE_REG_ECX, 20) + FEAT_DEF(X2APIC, 0x00000001, 0, RTE_REG_ECX, 21) + FEAT_DEF(MOVBE, 0x00000001, 0, RTE_REG_ECX, 22) + FEAT_DEF(POPCNT, 0x00000001, 0, RTE_REG_ECX, 23) + FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, RTE_REG_ECX, 24) + FEAT_DEF(AES, 0x00000001, 0, RTE_REG_ECX, 25) + FEAT_DEF(XSAVE, 0x00000001, 0, RTE_REG_ECX, 26) + FEAT_DEF(OSXSAVE, 0x00000001, 0, RTE_REG_ECX, 27) + FEAT_DEF(AVX, 0x00000001, 0, RTE_REG_ECX, 28) + FEAT_DEF(F16C, 0x00000001, 0, RTE_REG_ECX, 29) + FEAT_DEF(RDRAND, 0x00000001, 0, RTE_REG_ECX, 30) + FEAT_DEF(HYPERVISOR, 0x00000001, 0, RTE_REG_ECX, 31) + + FEAT_DEF(FPU, 0x00000001, 0, RTE_REG_EDX, 0) + FEAT_DEF(VME, 0x00000001, 0, RTE_REG_EDX, 1) + FEAT_DEF(DE, 0x00000001, 0, RTE_REG_EDX, 2) + FEAT_DEF(PSE, 0x00000001, 0, RTE_REG_EDX, 3) + FEAT_DEF(TSC, 0x00000001, 0, RTE_REG_EDX, 4) + FEAT_DEF(MSR, 0x00000001, 0, RTE_REG_EDX, 5) + FEAT_DEF(PAE, 0x00000001, 0, RTE_REG_EDX, 6) + FEAT_DEF(MCE, 0x00000001, 0, RTE_REG_EDX, 7) + FEAT_DEF(CX8, 0x00000001, 0, RTE_REG_EDX, 8) + FEAT_DEF(APIC, 0x00000001, 0, RTE_REG_EDX, 9) + FEAT_DEF(SEP, 0x00000001, 0, RTE_REG_EDX, 11) + FEAT_DEF(MTRR, 0x00000001, 0, RTE_REG_EDX, 12) + FEAT_DEF(PGE, 0x00000001, 0, RTE_REG_EDX, 13) + FEAT_DEF(MCA, 0x00000001, 0, RTE_REG_EDX, 14) + FEAT_DEF(CMOV, 0x00000001, 0, RTE_REG_EDX, 15) + FEAT_DEF(PAT, 0x00000001, 0, RTE_REG_EDX, 16) + FEAT_DEF(PSE36, 0x00000001, 0, RTE_REG_EDX, 17) + FEAT_DEF(PSN, 0x00000001, 0, RTE_REG_EDX, 18) + FEAT_DEF(CLFSH, 0x00000001, 0, RTE_REG_EDX, 19) + FEAT_DEF(DS, 0x00000001, 0, RTE_REG_EDX, 21) + FEAT_DEF(ACPI, 0x00000001, 0, RTE_REG_EDX, 22) + FEAT_DEF(MMX, 0x00000001, 0, RTE_REG_EDX, 23) + FEAT_DEF(FXSR, 0x00000001, 0, RTE_REG_EDX, 24) + FEAT_DEF(SSE, 0x00000001, 0, RTE_REG_EDX, 25) + FEAT_DEF(SSE2, 0x00000001, 0, RTE_REG_EDX, 26) + FEAT_DEF(SS, 0x00000001, 0, RTE_REG_EDX, 27) + FEAT_DEF(HTT, 0x00000001, 0, RTE_REG_EDX, 28) + FEAT_DEF(TM, 0x00000001, 0, RTE_REG_EDX, 29) + FEAT_DEF(PBE, 0x00000001, 0, RTE_REG_EDX, 31) + + FEAT_DEF(DIGTEMP, 0x00000006, 0, RTE_REG_EAX, 0) + FEAT_DEF(TRBOBST, 0x00000006, 0, RTE_REG_EAX, 1) + FEAT_DEF(ARAT, 0x00000006, 0, RTE_REG_EAX, 2) + FEAT_DEF(PLN, 0x00000006, 0, RTE_REG_EAX, 4) + FEAT_DEF(ECMD, 0x00000006, 0, RTE_REG_EAX, 5) + FEAT_DEF(PTM, 0x00000006, 0, RTE_REG_EAX, 6) + + FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, RTE_REG_ECX, 0) + FEAT_DEF(ACNT2, 0x00000006, 0, RTE_REG_ECX, 1) + FEAT_DEF(ENERGY_EFF, 0x00000006, 0, RTE_REG_ECX, 3) + + FEAT_DEF(FSGSBASE, 0x00000007, 0, RTE_REG_EBX, 0) + FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 3) + FEAT_DEF(HLE, 0x00000007, 0, RTE_REG_EBX, 4) + FEAT_DEF(AVX2, 0x00000007, 0, RTE_REG_EBX, 5) + FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 7) + FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 9) + FEAT_DEF(INVPCID, 0x00000007, 0, RTE_REG_EBX, 10) + FEAT_DEF(RTM, 0x00000007, 0, RTE_REG_EBX, 11) + FEAT_DEF(AVX512F, 0x00000007, 0, RTE_REG_EBX, 16) + FEAT_DEF(AVX512DQ, 0x00000007, 0, RTE_REG_EBX, 17) + FEAT_DEF(RDSEED, 0x00000007, 0, RTE_REG_EBX, 18) + FEAT_DEF(AVX512IFMA, 0x00000007, 0, RTE_REG_EBX, 21) + FEAT_DEF(AVX512CD, 0x00000007, 0, RTE_REG_EBX, 28) + FEAT_DEF(AVX512BW, 0x00000007, 0, RTE_REG_EBX, 30) + FEAT_DEF(AVX512VL, 0x00000007, 0, RTE_REG_EBX, 31) + + FEAT_DEF(AVX512VBMI, 0x00000007, 0, RTE_REG_ECX, 1) + FEAT_DEF(WAITPKG, 0x00000007, 0, RTE_REG_ECX, 5) + FEAT_DEF(AVX512VBMI2, 0x00000007, 0, RTE_REG_ECX, 6) + FEAT_DEF(GFNI, 0x00000007, 0, RTE_REG_ECX, 8) + FEAT_DEF(VAES, 0x00000007, 0, RTE_REG_ECX, 9) + FEAT_DEF(VPCLMULQDQ, 0x00000007, 0, RTE_REG_ECX, 10) + FEAT_DEF(AVX512VNNI, 0x00000007, 0, RTE_REG_ECX, 11) + FEAT_DEF(AVX512BITALG, 0x00000007, 0, RTE_REG_ECX, 12) + FEAT_DEF(AVX512VPOPCNTDQ, 0x00000007, 0, RTE_REG_ECX, 14) + FEAT_DEF(CLDEMOTE, 0x00000007, 0, RTE_REG_ECX, 25) + FEAT_DEF(MOVDIRI, 0x00000007, 0, RTE_REG_ECX, 27) + FEAT_DEF(MOVDIR64B, 0x00000007, 0, RTE_REG_ECX, 28) + + FEAT_DEF(AVX512VP2INTERSECT, 0x00000007, 0, RTE_REG_EDX, 8) + + FEAT_DEF(LAHF_SAHF, 0x80000001, 0, RTE_REG_ECX, 0) + FEAT_DEF(LZCNT, 0x80000001, 0, RTE_REG_ECX, 4) + + FEAT_DEF(SYSCALL, 0x80000001, 0, RTE_REG_EDX, 11) + FEAT_DEF(XD, 0x80000001, 0, RTE_REG_EDX, 20) + FEAT_DEF(1GB_PG, 0x80000001, 0, RTE_REG_EDX, 26) + FEAT_DEF(RDTSCP, 0x80000001, 0, RTE_REG_EDX, 27) + FEAT_DEF(EM64T, 0x80000001, 0, RTE_REG_EDX, 29) + + FEAT_DEF(INVTSC, 0x80000007, 0, RTE_REG_EDX, 8) +}; + +static int cpu_get_flag_enabled(enum rte_cpu_flag_t feature) +{ + const struct feature_entry *feat; + cpuid_registers_t regs; + unsigned int maxleaf; + + if (feature >= RTE_CPUFLAG_NUMFLAGS) + /* Flag does not match anything in the feature tables */ + return -ENOENT; + + feat = &cpu_feature_table[feature]; + + if (!feat->leaf) + /* This entry in the table wasn't filled out! */ + return -EFAULT; + + maxleaf = __get_cpuid_max(feat->leaf & 0x80000000, NULL); + + if (maxleaf < feat->leaf) + return 0; + + __cpuid_count(feat->leaf, feat->subleaf, + regs[RTE_REG_EAX], regs[RTE_REG_EBX], + regs[RTE_REG_ECX], regs[RTE_REG_EDX]); + + /* check if the feature is enabled */ + return (regs[feat->reg] >> feat->bit) & 1; +} + +static const char *cpu_get_flag_name(enum rte_cpu_flag_t feature) +{ + if (feature >= RTE_CPUFLAG_NUMFLAGS) + return NULL; + return cpu_feature_table[feature].name; +} + +void _odp_cpu_flags_print_all(void) +{ + int len, i; + int max_str = 1024; + int max_len = max_str - 1; + char str[max_str]; + + len = snprintf(str, max_len, "\nCPU features supported:\n"); + + for (i = 0; i < RTE_CPUFLAG_NUMFLAGS; i++) { + if (cpu_get_flag_enabled(i) > 0) + len += snprintf(&str[len], max_len - len, "%s ", + cpu_get_flag_name(i)); + } + + len += snprintf(&str[len], max_len - len, + "\n\nCPU features NOT supported:\n"); + + for (i = 0; i < RTE_CPUFLAG_NUMFLAGS; i++) { + if (cpu_get_flag_enabled(i) <= 0) + len += snprintf(&str[len], max_len - len, "%s ", + cpu_get_flag_name(i)); + } + + len += snprintf(&str[len], max_len - len, "\n\n"); + + str[len] = '\0'; + _ODP_PRINT("%s", str); +} + +int _odp_time_cpu_global_freq_is_const(void) +{ + if (odp_global_ro.system_info.cpu_constant_tsc || + cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) + return 1; + + _ODP_ERR("WARN: assuming constant TSC based on CPU arch, but could not confirm from CPU " + "flags\n"); + + return 1; +} + +int _odp_cpu_flags_has_rdtsc(void) +{ + if (cpu_get_flag_enabled(RTE_CPUFLAG_TSC) > 0) + return 1; + + return 0; +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.h b/platform/linux-generic/arch/x86/cpu_flags.h new file mode 100644 index 000000000..8d485dbfa --- /dev/null +++ b/platform/linux-generic/arch/x86/cpu_flags.h @@ -0,0 +1,21 @@ +/* Copyright (c) 2017-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_FLAGS_H_ +#define ODP_PLAT_CPU_FLAGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void _odp_cpu_flags_print_all(void); +int _odp_cpu_flags_has_rdtsc(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu.h new file mode 100644 index 000000000..9224af9a0 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2016-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_CPU_H_ +#define ODP_API_ABI_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define ODP_CACHE_LINE_SIZE 64 + +/* Inlined functions for non-ABI compat mode */ +#include <odp/api/plat/cpu_inlines.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h new file mode 100644 index 000000000..4b542a577 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu_inlines.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_CPU_INLINES_H_ +#define ODP_ARCH_CPU_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> +#include <odp/api/abi/cpu_rdtsc.h> + +static inline void _odp_cpu_pause(void) +{ +#ifdef __SSE2__ + __asm__ __volatile__ ("pause"); +#else + __asm__ __volatile__ ("rep; nop"); +#endif +} + +static inline uint64_t _odp_cpu_cycles(void) +{ + return _odp_cpu_rdtsc(); +} + +static inline uint64_t _odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +static inline uint64_t _odp_cpu_cycles_resolution(void) +{ + return 1; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp_cpu_arch.c b/platform/linux-generic/arch/x86/odp/api/abi/cpu_rdtsc.h index c8cf27b65..ccc5f0f36 100644 --- a/platform/linux-generic/arch/x86/odp_cpu_arch.c +++ b/platform/linux-generic/arch/x86/odp/api/abi/cpu_rdtsc.h @@ -1,11 +1,15 @@ -/* Copyright (c) 2015, Linaro Limited +/* Copyright (c) 2018, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp/api/cpu.h> -uint64_t odp_cpu_cycles(void) +#ifndef ODP_ARCH_CPU_RDTSC_H_ +#define ODP_ARCH_CPU_RDTSC_H_ + +#include <stdint.h> + +static inline uint64_t _odp_cpu_rdtsc(void) { union { uint64_t tsc_64; @@ -22,12 +26,4 @@ uint64_t odp_cpu_cycles(void) return tsc.tsc_64; } -uint64_t odp_cpu_cycles_max(void) -{ - return UINT64_MAX; -} - -uint64_t odp_cpu_cycles_resolution(void) -{ - return 1; -} +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/hash_crc32.h b/platform/linux-generic/arch/x86/odp/api/abi/hash_crc32.h new file mode 100644 index 000000000..c2c71bcb7 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/hash_crc32.h @@ -0,0 +1,77 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_API_ABI_HASH_CRC32_H_ +#define ODP_API_ABI_HASH_CRC32_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +uint32_t _odp_hash_crc32_generic(const void *data, uint32_t data_len, + uint32_t init_val); +uint32_t _odp_hash_crc32c_generic(const void *data, uint32_t data_len, + uint32_t init_val); + +static inline uint32_t _odp_hash_crc32(const void *data, uint32_t data_len, + uint32_t init_val) +{ + return _odp_hash_crc32_generic(data, data_len, init_val); +} + +#ifdef __SSE4_2__ + +static inline uint32_t _odp_hash_crc32c(const void *data, uint32_t data_len, + uint32_t init_val) +{ + uint32_t i; + uintptr_t pd = (uintptr_t)data; + +#ifdef __x86_64__ + for (i = 0; i < data_len / 8; i++) { + init_val = (uint32_t)__builtin_ia32_crc32di(init_val, *(const uint64_t *)pd); + pd += 8; + } + + if (data_len & 0x4) { + init_val = __builtin_ia32_crc32si(init_val, *(const uint32_t *)pd); + pd += 4; + } +#else + for (i = 0; i < data_len / 4; i++) { + init_val = __builtin_ia32_crc32si(init_val, *(const uint32_t *)pd); + pd += 4; + } +#endif + + if (data_len & 0x2) { + init_val = __builtin_ia32_crc32hi(init_val, *(const uint16_t *)pd); + pd += 2; + } + + if (data_len & 0x1) + init_val = __builtin_ia32_crc32qi(init_val, *(const uint8_t *)pd); + + return init_val; +} + +#else + +static inline uint32_t _odp_hash_crc32c(const void *data, uint32_t data_len, + uint32_t init_val) +{ + return _odp_hash_crc32c_generic(data, data_len, init_val); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h new file mode 100644 index 000000000..bebe6b571 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/sync_inlines.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ARCH_SYNC_INLINES_H_ +#define ODP_ARCH_SYNC_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void _odp_mb_sync(void) +{ + __asm__ volatile("mfence" ::: "memory"); +} + +static inline void _odp_mb_sync_load(void) +{ + __asm__ volatile("lfence" ::: "memory"); +} + +static inline void _odp_mb_sync_store(void) +{ + __asm__ volatile("sfence" ::: "memory"); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h new file mode 100644 index 000000000..baf79ad3f --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_cpu.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ARCH_TIME_CPU_H_ +#define ODP_ARCH_TIME_CPU_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> +#include <odp/api/abi/cpu_rdtsc.h> + +static inline uint64_t _odp_time_cpu_global(void) +{ + return _odp_cpu_rdtsc(); +} + +static inline uint64_t _odp_time_cpu_global_strict(void) +{ + __atomic_thread_fence(__ATOMIC_SEQ_CST); + return _odp_cpu_rdtsc(); +} + +int _odp_time_cpu_global_freq_is_const(void); +uint64_t _odp_time_cpu_global_freq(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h new file mode 100644 index 000000000..331d1996f --- /dev/null +++ b/platform/linux-generic/arch/x86/odp/api/abi/time_inlines.h @@ -0,0 +1,7 @@ +/* Copyright (c) 2023, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/abi/time_cpu_inlines.h> diff --git a/platform/linux-generic/arch/x86/odp/api/cpu_arch.h b/platform/linux-generic/arch/x86/odp/api/cpu_arch.h deleted file mode 100644 index 44e6b30ed..000000000 --- a/platform/linux-generic/arch/x86/odp/api/cpu_arch.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (c) 2016, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_CPU_ARCH_H_ -#define ODP_PLAT_CPU_ARCH_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#define _ODP_CACHE_LINE_SIZE 64 - -static inline void odp_cpu_pause(void) -{ -#ifdef __SSE2__ - __asm__ __volatile__ ("pause"); -#else - __asm__ __volatile__ ("rep; nop"); -#endif -} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/arch/x86/odp_cpu.h b/platform/linux-generic/arch/x86/odp_cpu.h new file mode 100644 index 000000000..8f8f22daf --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_cpu.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_X86_CPU_H_ +#define ODP_X86_CPU_H_ + +#define _ODP_UNALIGNED 1 + +#include <default/odp_cpu.h> + +#endif diff --git a/platform/linux-generic/arch/x86/odp_cpu_cycles.c b/platform/linux-generic/arch/x86/odp_cpu_cycles.c new file mode 100644 index 000000000..2624af0f6 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_cpu_cycles.c @@ -0,0 +1,21 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/cpu.h> + +#include "cpu_flags.h" +#include <odp_init_internal.h> +#include <odp_debug_internal.h> + +int _odp_cpu_cycles_init_global(void) +{ + if (_odp_cpu_flags_has_rdtsc() == 0) { + _ODP_ERR("RDTSC instruction not supported\n"); + return -1; + } + + return 0; +} diff --git a/platform/linux-generic/arch/x86/odp_random.h b/platform/linux-generic/arch/x86/odp_random.h new file mode 100644 index 000000000..54628038e --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_random.h @@ -0,0 +1,160 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * These functions implement ODP_RANDOM_CRYPTO random data using rdrand [1], + * and ODP_RANDOM_TRUE random data using rdseed [1], via compiler builtin + * functions. + * + * Note that there may be issues with the quality or security of rdrand and + * rdseed. [2] + * + * [1] Intel Digital Random Number Generator (DRNG) Software Implementation + * Guide. John P Mechalas, 17 October 2018. + * https://www.intel.com/content/www/us/en/developer/articles/guide/intel-digital-random-number-generator-drng-software-implementation-guide.html + * + * [2] RDRAND. Wikipedia, 29 September 2021. + * https://en.wikipedia.org/wiki/RDRAND#Reception + */ + +#ifndef ODP_X86_RANDOM_H_ +#define ODP_X86_RANDOM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/random.h> + +#include <stdint.h> + +odp_random_kind_t _odp_random_max_kind_generic(void); +int32_t _odp_random_true_data_generic(uint8_t *buf, uint32_t len); +int32_t _odp_random_crypto_data_generic(uint8_t *buf, uint32_t len); + +#ifdef __RDRND__ + +static inline int _odp_random_max_kind(void) +{ +#ifdef __RDSEED__ + return ODP_RANDOM_TRUE; +#else + return ODP_RANDOM_CRYPTO; +#endif +} + +#else + +static inline int _odp_random_max_kind(void) +{ + return _odp_random_max_kind_generic(); +} + +#endif + +#ifdef __RDSEED__ + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ +#ifdef __x86_64__ + for (uint32_t i = 0; i < len / 8; i++) { + while (!__builtin_ia32_rdseed_di_step((unsigned long long *)buf)) + ; + buf += 8; + } + + if (len & 4) { + while (!__builtin_ia32_rdseed_si_step((unsigned int *)buf)) + ; + buf += 4; + } +#else + for (uint32_t i = 0; i < len / 4; i++) { + while (!__builtin_ia32_rdseed_si_step((unsigned int *)buf)) + ; + buf += 4; + } +#endif + if (len & 2) { + while (!__builtin_ia32_rdseed_hi_step((unsigned short int *)buf)) + ; + buf += 2; + } + + if (len & 1) { + uint16_t w; + + while (!__builtin_ia32_rdseed_hi_step(&w)) + ; + *((uint8_t *)buf) = w & 0xff; + } + + return len; +} + +#else + +static inline int32_t _odp_random_true_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_true_data_generic(buf, len); +} + +#endif + +#ifdef __RDRND__ + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ +#ifdef __x86_64__ + for (uint32_t i = 0; i < len / 8; i++) { + while (!__builtin_ia32_rdrand64_step((unsigned long long *)buf)) + ; + buf += 8; + } + + if (len & 4) { + while (!__builtin_ia32_rdrand32_step((unsigned int *)buf)) + ; + buf += 4; + } +#else + for (uint32_t i = 0; i < len / 4; i++) { + while (!__builtin_ia32_rdrand32_step((unsigned int *)buf)) + ; + buf += 4; + } +#endif + if (len & 2) { + while (!__builtin_ia32_rdrand16_step((unsigned short int *)buf)) + ; + buf += 2; + } + + if (len & 1) { + uint16_t w; + + while (!__builtin_ia32_rdrand16_step(&w)) + ; + *((uint8_t *)buf) = w & 0xff; + } + + return len; +} + +#else + +static inline int32_t _odp_random_crypto_data(uint8_t *buf, uint32_t len) +{ + return _odp_random_crypto_data_generic(buf, len); +} + +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index 96127ec67..3cbdb2037 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -1,44 +1,94 @@ -/* Copyright (c) 2016, Linaro Limited +/* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp_internal.h> +#include <odp_sysinfo_internal.h> +#include "cpu_flags.h" #include <string.h> -int cpuinfo_parser(FILE *file, system_info_t *sysinfo) +int _odp_cpuinfo_parser(FILE *file, system_info_t *sysinfo) { char str[1024]; - char *pos; + char *pos, *pos_end; double ghz = 0.0; + double mhz = 0.0; uint64_t hz; int id = 0; + bool freq_set = false; + + sysinfo->cpu_arch = ODP_CPU_ARCH_X86; + sysinfo->cpu_isa_sw.x86 = ODP_CPU_ARCH_X86_UNKNOWN; + sysinfo->cpu_isa_hw.x86 = ODP_CPU_ARCH_X86_UNKNOWN; + + #if defined __x86_64 || defined __x86_64__ + sysinfo->cpu_isa_sw.x86 = ODP_CPU_ARCH_X86_64; + #elif defined __i686 || defined __i686__ + sysinfo->cpu_isa_sw.x86 = ODP_CPU_ARCH_X86_I686; + #endif strcpy(sysinfo->cpu_arch_str, "x86"); - while (fgets(str, sizeof(str), file) != NULL && id < MAX_CPU_NUMBER) { + while (fgets(str, sizeof(str), file) != NULL && id < CONFIG_NUM_CPU_IDS) { + if (strstr(str, "flags") && strstr(str, "constant_tsc")) { + sysinfo->cpu_constant_tsc = 1; + continue; + } + pos = strstr(str, "model name"); if (pos) { - pos = strchr(str, ':'); + freq_set = false; + + /* Copy model name between : and @ characters */ + pos = strchr(str, ':'); + pos_end = strchr(str, '@'); + if (pos == NULL) + continue; + + if (pos_end != NULL) + *(pos_end - 1) = '\0'; + strncpy(sysinfo->model_str[id], pos + 2, - sizeof(sysinfo->model_str[id]) - 1); - - pos = strchr(sysinfo->model_str[id], '@'); - if (pos) { - *(pos - 1) = '\0'; - if (sscanf(pos, "@ %lfGHz", &ghz) == 1) { - hz = (uint64_t)(ghz * 1000000000.0); - sysinfo->cpu_hz_max[id] = hz; - } + MODEL_STR_SIZE - 1); + + if (sysinfo->cpu_hz_max[id]) { + freq_set = true; + id++; + continue; + } + + /* max frequency needs to be set */ + if (pos_end != NULL && + sscanf(pos_end, "@ %lfGHz", &ghz) == 1) { + hz = (uint64_t)(ghz * 1000000000.0); + sysinfo->cpu_hz_max[id++] = hz; + freq_set = true; + } + } else if (!freq_set && + strstr(str, "bogomips") != NULL) { + pos = strchr(str, ':'); + if (pos == NULL) + continue; + + if (sscanf(pos + 2, "%lf", &mhz) == 1) { + /* On typical x86 BogoMIPS is freq * 2 */ + hz = (uint64_t)(mhz * 1000000.0 / 2); + sysinfo->cpu_hz_max[id++] = hz; + freq_set = true; } - id++; } } return 0; } -uint64_t odp_cpu_hz_current(int id) +void _odp_sys_info_print_arch(void) +{ + _odp_cpu_flags_print_all(); +} + +uint64_t odp_cpu_arch_hz_current(int id) { char str[1024]; FILE *file; @@ -47,6 +97,8 @@ uint64_t odp_cpu_hz_current(int id) double mhz = 0.0; file = fopen("/proc/cpuinfo", "rt"); + if (!file) + return 0; /* find the correct processor instance */ while (fgets(str, sizeof(str), file) != NULL) { diff --git a/platform/linux-generic/arch/x86/odp_time_cpu.c b/platform/linux-generic/arch/x86/odp_time_cpu.c new file mode 100644 index 000000000..ab897296d --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_time_cpu.c @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2024 Nokia + */ + +#include <odp_posix_extensions.h> + +#include <odp/api/hints.h> +#include <odp/api/time_types.h> + +#include <odp/api/abi/time_cpu.h> + +#include <odp_debug_internal.h> + +#include <time.h> +#include <errno.h> +#include <string.h> + +static int nwait(uint64_t nsec) +{ + struct timespec ts1, ts2; + uint64_t diff; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) + return 1; + + do { + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) + return 1; + + diff = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS + + ts2.tv_nsec - ts1.tv_nsec; + } while (diff < nsec); + + return 0; +} + +static void sort(uint64_t values[], int num) +{ + for (int n = 0; n < num; n++) { + for (int i = n + 1; i < num; i++) { + if (values[i] < values[n]) { + uint64_t tmp = values[i]; + + values[i] = values[n]; + values[n] = tmp; + } + } + } +} + +static uint64_t median(uint64_t values[], int num) +{ + sort(values, num); + if (num % 2 == 0) + return (values[num / 2 - 1] + values[num / 2]) / 2; + else + return values[num / 2]; +} + +/* Measure TSC frequency. */ +uint64_t _odp_time_cpu_global_freq(void) +{ + struct timespec ts1, ts2; + uint64_t t1, t2, ts_nsec, cycles; + int i; + const int rounds = 6; /* first round is warmup */ + int warm_up = 1; + uint64_t hz[rounds]; + + for (i = 0; i < rounds; i++) { + uint64_t wait_nsec = ODP_TIME_SEC_IN_NS / 50; + + if (warm_up) + wait_nsec = ODP_TIME_SEC_IN_NS / 1000; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) + goto err_out; + + t1 = _odp_time_cpu_global(); + + if (nwait(wait_nsec)) + goto err_out; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) + goto err_out; + + t2 = _odp_time_cpu_global(); + + ts_nsec = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS; + ts_nsec += ts2.tv_nsec - ts1.tv_nsec; + + cycles = t2 - t1; + + hz[i] = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; + + if (warm_up) + warm_up = 0; + } + + return median(&hz[1], rounds - 1); + +err_out: + _ODP_ERR("clock_gettime() failed (%s)\n", strerror(errno)); + return 0; +} |