diff options
Diffstat (limited to 'platform/linux-generic')
50 files changed, 1942 insertions, 1300 deletions
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 9e8a0064e..8dcdebd29 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -9,6 +9,7 @@ AM_CFLAGS += -I$(top_srcdir)/include AM_CFLAGS += -I$(top_srcdir)/include/odp/arch/@ARCH_ABI@ AM_CFLAGS += -I$(top_builddir)/include AM_CFLAGS += -Iinclude +AM_CFLAGS += -D_ODP_PKTIO_IPC include_HEADERS = \ $(top_srcdir)/include/odp.h \ @@ -32,6 +33,7 @@ odpapiinclude_HEADERS = \ $(srcdir)/include/odp/api/cpumask.h \ $(srcdir)/include/odp/api/crypto.h \ $(srcdir)/include/odp/api/debug.h \ + $(srcdir)/include/odp/api/deprecated.h \ $(srcdir)/include/odp/api/errno.h \ $(srcdir)/include/odp/api/event.h \ $(srcdir)/include/odp/api/hash.h \ @@ -145,6 +147,7 @@ noinst_HEADERS = \ ${srcdir}/include/odp_schedule_if.h \ ${srcdir}/include/odp_sorted_list_internal.h \ ${srcdir}/include/odp_shm_internal.h \ + ${srcdir}/include/odp_time_internal.h \ ${srcdir}/include/odp_timer_internal.h \ ${srcdir}/include/odp_timer_wheel_internal.h \ ${srcdir}/include/odp_traffic_mngr_internal.h \ @@ -218,6 +221,12 @@ __LIB__libodp_linux_la_SOURCES = \ arch/@ARCH_DIR@/odp_cpu_arch.c \ arch/@ARCH_DIR@/odp_sysinfo_parse.c +__LIB__libodp_linux_la_LIBADD = $(ATOMIC_LIBS) + +if ARCH_IS_X86 +__LIB__libodp_linux_la_SOURCES += arch/@ARCH_DIR@/cpu_flags.c +endif + if HAVE_PCAP __LIB__libodp_linux_la_SOURCES += pktio/pcap.c endif @@ -225,9 +234,9 @@ endif # Create symlink for ABI header files. Application does not need to use the arch # specific include path for installed files. install-data-hook: - if [ -h $(prefix)/include/odp/api/abi ]; then \ - : \ + if [ -h $(DESTDIR)$(prefix)/include/odp/api/abi ]; then \ + : ; \ else \ - $(LN_S) -rf $(prefix)/include/odp/arch/@ARCH_ABI@/odp/api/abi \ - $(prefix)/include/odp/api/abi; \ + $(LN_S) -rf $(DESTDIR)$(prefix)/include/odp/arch/@ARCH_ABI@/odp/api/abi \ + $(DESTDIR)$(prefix)/include/odp/api/abi; \ fi diff --git a/platform/linux-generic/arch/arm/odp_cpu_arch.c b/platform/linux-generic/arch/arm/odp_cpu_arch.c index 2ac223e07..c31f90844 100644 --- a/platform/linux-generic/arch/arm/odp_cpu_arch.c +++ b/platform/linux-generic/arch/arm/odp_cpu_arch.c @@ -13,6 +13,7 @@ #include <odp/api/hints.h> #include <odp/api/system_info.h> #include <odp_debug_internal.h> +#include <odp_time_internal.h> #define GIGA 1000000000 @@ -46,3 +47,18 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +int cpu_has_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time_freq(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c index 53e2aaeaf..8ae2022ac 100644 --- a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c @@ -25,3 +25,7 @@ uint64_t odp_cpu_hz_current(int id ODP_UNUSED) { return 0; } + +void sys_info_print_arch(void) +{ +} diff --git a/platform/linux-generic/arch/default/odp_cpu_arch.c b/platform/linux-generic/arch/default/odp_cpu_arch.c index 2ac223e07..c31f90844 100644 --- a/platform/linux-generic/arch/default/odp_cpu_arch.c +++ b/platform/linux-generic/arch/default/odp_cpu_arch.c @@ -13,6 +13,7 @@ #include <odp/api/hints.h> #include <odp/api/system_info.h> #include <odp_debug_internal.h> +#include <odp_time_internal.h> #define GIGA 1000000000 @@ -46,3 +47,18 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +int cpu_has_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time_freq(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/default/odp_sysinfo_parse.c b/platform/linux-generic/arch/default/odp_sysinfo_parse.c index 53e2aaeaf..8ae2022ac 100644 --- a/platform/linux-generic/arch/default/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/default/odp_sysinfo_parse.c @@ -25,3 +25,7 @@ uint64_t odp_cpu_hz_current(int id ODP_UNUSED) { return 0; } + +void sys_info_print_arch(void) +{ +} diff --git a/platform/linux-generic/arch/mips64/odp_cpu_arch.c b/platform/linux-generic/arch/mips64/odp_cpu_arch.c index 646acf9c1..f7eafa0fb 100644 --- a/platform/linux-generic/arch/mips64/odp_cpu_arch.c +++ b/platform/linux-generic/arch/mips64/odp_cpu_arch.c @@ -7,6 +7,7 @@ #include <odp/api/cpu.h> #include <odp/api/hints.h> #include <odp/api/system_info.h> +#include <odp_time_internal.h> uint64_t odp_cpu_cycles(void) { @@ -29,3 +30,18 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +int cpu_has_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time_freq(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c index 407264b7f..d6f75f283 100644 --- a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c @@ -62,3 +62,7 @@ uint64_t odp_cpu_hz_current(int id ODP_UNUSED) { return 0; } + +void sys_info_print_arch(void) +{ +} diff --git a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c index 2ac223e07..c31f90844 100644 --- a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c +++ b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c @@ -13,6 +13,7 @@ #include <odp/api/hints.h> #include <odp/api/system_info.h> #include <odp_debug_internal.h> +#include <odp_time_internal.h> #define GIGA 1000000000 @@ -46,3 +47,18 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +int cpu_has_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time_freq(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c index 3b88d55b6..bd4b9b429 100644 --- a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c @@ -61,3 +61,7 @@ uint64_t odp_cpu_hz_current(int id ODP_UNUSED) { return 0; } + +void sys_info_print_arch(void) +{ +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c new file mode 100644 index 000000000..a492a35bf --- /dev/null +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -0,0 +1,368 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <arch/x86/cpu_flags.h> +#include <odp_debug_internal.h> +#include <odp_time_internal.h> +#include <stdio.h> +#include <stdint.h> + +enum rte_cpu_flag_t { + /* (EAX 01h) ECX features*/ + RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */ + RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */ + RTE_CPUFLAG_DTES64, /**< DTES64 */ + RTE_CPUFLAG_MONITOR, /**< MONITOR */ + RTE_CPUFLAG_DS_CPL, /**< DS_CPL */ + RTE_CPUFLAG_VMX, /**< VMX */ + RTE_CPUFLAG_SMX, /**< SMX */ + RTE_CPUFLAG_EIST, /**< EIST */ + RTE_CPUFLAG_TM2, /**< TM2 */ + RTE_CPUFLAG_SSSE3, /**< SSSE3 */ + RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */ + RTE_CPUFLAG_FMA, /**< FMA */ + RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */ + RTE_CPUFLAG_XTPR, /**< XTPR */ + RTE_CPUFLAG_PDCM, /**< PDCM */ + RTE_CPUFLAG_PCID, /**< PCID */ + RTE_CPUFLAG_DCA, /**< DCA */ + RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */ + RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */ + RTE_CPUFLAG_X2APIC, /**< X2APIC */ + RTE_CPUFLAG_MOVBE, /**< MOVBE */ + RTE_CPUFLAG_POPCNT, /**< POPCNT */ + RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */ + RTE_CPUFLAG_AES, /**< AES */ + RTE_CPUFLAG_XSAVE, /**< XSAVE */ + RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */ + RTE_CPUFLAG_AVX, /**< AVX */ + RTE_CPUFLAG_F16C, /**< F16C */ + RTE_CPUFLAG_RDRAND, /**< RDRAND */ + + /* (EAX 01h) EDX features */ + RTE_CPUFLAG_FPU, /**< FPU */ + RTE_CPUFLAG_VME, /**< VME */ + RTE_CPUFLAG_DE, /**< DE */ + RTE_CPUFLAG_PSE, /**< PSE */ + RTE_CPUFLAG_TSC, /**< TSC */ + RTE_CPUFLAG_MSR, /**< MSR */ + RTE_CPUFLAG_PAE, /**< PAE */ + RTE_CPUFLAG_MCE, /**< MCE */ + RTE_CPUFLAG_CX8, /**< CX8 */ + RTE_CPUFLAG_APIC, /**< APIC */ + RTE_CPUFLAG_SEP, /**< SEP */ + RTE_CPUFLAG_MTRR, /**< MTRR */ + RTE_CPUFLAG_PGE, /**< PGE */ + RTE_CPUFLAG_MCA, /**< MCA */ + RTE_CPUFLAG_CMOV, /**< CMOV */ + RTE_CPUFLAG_PAT, /**< PAT */ + RTE_CPUFLAG_PSE36, /**< PSE36 */ + RTE_CPUFLAG_PSN, /**< PSN */ + RTE_CPUFLAG_CLFSH, /**< CLFSH */ + RTE_CPUFLAG_DS, /**< DS */ + RTE_CPUFLAG_ACPI, /**< ACPI */ + RTE_CPUFLAG_MMX, /**< MMX */ + RTE_CPUFLAG_FXSR, /**< FXSR */ + RTE_CPUFLAG_SSE, /**< SSE */ + RTE_CPUFLAG_SSE2, /**< SSE2 */ + RTE_CPUFLAG_SS, /**< SS */ + RTE_CPUFLAG_HTT, /**< HTT */ + RTE_CPUFLAG_TM, /**< TM */ + RTE_CPUFLAG_PBE, /**< PBE */ + + /* (EAX 06h) EAX features */ + RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */ + RTE_CPUFLAG_TRBOBST, /**< TRBOBST */ + RTE_CPUFLAG_ARAT, /**< ARAT */ + RTE_CPUFLAG_PLN, /**< PLN */ + RTE_CPUFLAG_ECMD, /**< ECMD */ + RTE_CPUFLAG_PTM, /**< PTM */ + + /* (EAX 06h) ECX features */ + RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */ + RTE_CPUFLAG_ACNT2, /**< ACNT2 */ + RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */ + + /* (EAX 07h, ECX 0h) EBX features */ + RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */ + RTE_CPUFLAG_BMI1, /**< BMI1 */ + RTE_CPUFLAG_HLE, /**< Hardware Lock elision */ + RTE_CPUFLAG_AVX2, /**< AVX2 */ + RTE_CPUFLAG_SMEP, /**< SMEP */ + RTE_CPUFLAG_BMI2, /**< BMI2 */ + RTE_CPUFLAG_ERMS, /**< ERMS */ + RTE_CPUFLAG_INVPCID, /**< INVPCID */ + RTE_CPUFLAG_RTM, /**< Transactional memory */ + RTE_CPUFLAG_AVX512F, /**< AVX512F */ + + /* (EAX 80000001h) ECX features */ + RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ + RTE_CPUFLAG_LZCNT, /**< LZCNT */ + + /* (EAX 80000001h) EDX features */ + RTE_CPUFLAG_SYSCALL, /**< SYSCALL */ + RTE_CPUFLAG_XD, /**< XD */ + RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */ + RTE_CPUFLAG_RDTSCP, /**< RDTSCP */ + RTE_CPUFLAG_EM64T, /**< EM64T */ + + /* (EAX 80000007h) EDX features */ + RTE_CPUFLAG_INVTSC, /**< INVTSC */ + + /* The last item */ + RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ +}; + +enum cpu_register_t { + RTE_REG_EAX = 0, + RTE_REG_EBX, + RTE_REG_ECX, + RTE_REG_EDX, +}; + +typedef uint32_t cpuid_registers_t[4]; + +/** + * Struct to hold a processor feature entry + */ +struct feature_entry { + uint32_t leaf; /**< cpuid leaf */ + uint32_t subleaf; /**< cpuid subleaf */ + uint32_t reg; /**< cpuid register */ + uint32_t bit; /**< cpuid register bit */ +#define CPU_FLAG_NAME_MAX_LEN 64 + char name[CPU_FLAG_NAME_MAX_LEN]; /**< String for printing */ +}; + +#define FEAT_DEF(name, leaf, subleaf, reg, bit) \ + [RTE_CPUFLAG_##name] = {leaf, subleaf, reg, bit, #name }, + +static const struct feature_entry cpu_feature_table[] = { + FEAT_DEF(SSE3, 0x00000001, 0, RTE_REG_ECX, 0) + FEAT_DEF(PCLMULQDQ, 0x00000001, 0, RTE_REG_ECX, 1) + FEAT_DEF(DTES64, 0x00000001, 0, RTE_REG_ECX, 2) + FEAT_DEF(MONITOR, 0x00000001, 0, RTE_REG_ECX, 3) + FEAT_DEF(DS_CPL, 0x00000001, 0, RTE_REG_ECX, 4) + FEAT_DEF(VMX, 0x00000001, 0, RTE_REG_ECX, 5) + FEAT_DEF(SMX, 0x00000001, 0, RTE_REG_ECX, 6) + FEAT_DEF(EIST, 0x00000001, 0, RTE_REG_ECX, 7) + FEAT_DEF(TM2, 0x00000001, 0, RTE_REG_ECX, 8) + FEAT_DEF(SSSE3, 0x00000001, 0, RTE_REG_ECX, 9) + FEAT_DEF(CNXT_ID, 0x00000001, 0, RTE_REG_ECX, 10) + FEAT_DEF(FMA, 0x00000001, 0, RTE_REG_ECX, 12) + FEAT_DEF(CMPXCHG16B, 0x00000001, 0, RTE_REG_ECX, 13) + FEAT_DEF(XTPR, 0x00000001, 0, RTE_REG_ECX, 14) + FEAT_DEF(PDCM, 0x00000001, 0, RTE_REG_ECX, 15) + FEAT_DEF(PCID, 0x00000001, 0, RTE_REG_ECX, 17) + FEAT_DEF(DCA, 0x00000001, 0, RTE_REG_ECX, 18) + FEAT_DEF(SSE4_1, 0x00000001, 0, RTE_REG_ECX, 19) + FEAT_DEF(SSE4_2, 0x00000001, 0, RTE_REG_ECX, 20) + FEAT_DEF(X2APIC, 0x00000001, 0, RTE_REG_ECX, 21) + FEAT_DEF(MOVBE, 0x00000001, 0, RTE_REG_ECX, 22) + FEAT_DEF(POPCNT, 0x00000001, 0, RTE_REG_ECX, 23) + FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, RTE_REG_ECX, 24) + FEAT_DEF(AES, 0x00000001, 0, RTE_REG_ECX, 25) + FEAT_DEF(XSAVE, 0x00000001, 0, RTE_REG_ECX, 26) + FEAT_DEF(OSXSAVE, 0x00000001, 0, RTE_REG_ECX, 27) + FEAT_DEF(AVX, 0x00000001, 0, RTE_REG_ECX, 28) + FEAT_DEF(F16C, 0x00000001, 0, RTE_REG_ECX, 29) + FEAT_DEF(RDRAND, 0x00000001, 0, RTE_REG_ECX, 30) + + FEAT_DEF(FPU, 0x00000001, 0, RTE_REG_EDX, 0) + FEAT_DEF(VME, 0x00000001, 0, RTE_REG_EDX, 1) + FEAT_DEF(DE, 0x00000001, 0, RTE_REG_EDX, 2) + FEAT_DEF(PSE, 0x00000001, 0, RTE_REG_EDX, 3) + FEAT_DEF(TSC, 0x00000001, 0, RTE_REG_EDX, 4) + FEAT_DEF(MSR, 0x00000001, 0, RTE_REG_EDX, 5) + FEAT_DEF(PAE, 0x00000001, 0, RTE_REG_EDX, 6) + FEAT_DEF(MCE, 0x00000001, 0, RTE_REG_EDX, 7) + FEAT_DEF(CX8, 0x00000001, 0, RTE_REG_EDX, 8) + FEAT_DEF(APIC, 0x00000001, 0, RTE_REG_EDX, 9) + FEAT_DEF(SEP, 0x00000001, 0, RTE_REG_EDX, 11) + FEAT_DEF(MTRR, 0x00000001, 0, RTE_REG_EDX, 12) + FEAT_DEF(PGE, 0x00000001, 0, RTE_REG_EDX, 13) + FEAT_DEF(MCA, 0x00000001, 0, RTE_REG_EDX, 14) + FEAT_DEF(CMOV, 0x00000001, 0, RTE_REG_EDX, 15) + FEAT_DEF(PAT, 0x00000001, 0, RTE_REG_EDX, 16) + FEAT_DEF(PSE36, 0x00000001, 0, RTE_REG_EDX, 17) + FEAT_DEF(PSN, 0x00000001, 0, RTE_REG_EDX, 18) + FEAT_DEF(CLFSH, 0x00000001, 0, RTE_REG_EDX, 19) + FEAT_DEF(DS, 0x00000001, 0, RTE_REG_EDX, 21) + FEAT_DEF(ACPI, 0x00000001, 0, RTE_REG_EDX, 22) + FEAT_DEF(MMX, 0x00000001, 0, RTE_REG_EDX, 23) + FEAT_DEF(FXSR, 0x00000001, 0, RTE_REG_EDX, 24) + FEAT_DEF(SSE, 0x00000001, 0, RTE_REG_EDX, 25) + FEAT_DEF(SSE2, 0x00000001, 0, RTE_REG_EDX, 26) + FEAT_DEF(SS, 0x00000001, 0, RTE_REG_EDX, 27) + FEAT_DEF(HTT, 0x00000001, 0, RTE_REG_EDX, 28) + FEAT_DEF(TM, 0x00000001, 0, RTE_REG_EDX, 29) + FEAT_DEF(PBE, 0x00000001, 0, RTE_REG_EDX, 31) + + FEAT_DEF(DIGTEMP, 0x00000006, 0, RTE_REG_EAX, 0) + FEAT_DEF(TRBOBST, 0x00000006, 0, RTE_REG_EAX, 1) + FEAT_DEF(ARAT, 0x00000006, 0, RTE_REG_EAX, 2) + FEAT_DEF(PLN, 0x00000006, 0, RTE_REG_EAX, 4) + FEAT_DEF(ECMD, 0x00000006, 0, RTE_REG_EAX, 5) + FEAT_DEF(PTM, 0x00000006, 0, RTE_REG_EAX, 6) + + FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, RTE_REG_ECX, 0) + FEAT_DEF(ACNT2, 0x00000006, 0, RTE_REG_ECX, 1) + FEAT_DEF(ENERGY_EFF, 0x00000006, 0, RTE_REG_ECX, 3) + + FEAT_DEF(FSGSBASE, 0x00000007, 0, RTE_REG_EBX, 0) + FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 2) + FEAT_DEF(HLE, 0x00000007, 0, RTE_REG_EBX, 4) + FEAT_DEF(AVX2, 0x00000007, 0, RTE_REG_EBX, 5) + FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 6) + FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 7) + FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(INVPCID, 0x00000007, 0, RTE_REG_EBX, 10) + FEAT_DEF(RTM, 0x00000007, 0, RTE_REG_EBX, 11) + FEAT_DEF(AVX512F, 0x00000007, 0, RTE_REG_EBX, 16) + + FEAT_DEF(LAHF_SAHF, 0x80000001, 0, RTE_REG_ECX, 0) + FEAT_DEF(LZCNT, 0x80000001, 0, RTE_REG_ECX, 4) + + FEAT_DEF(SYSCALL, 0x80000001, 0, RTE_REG_EDX, 11) + FEAT_DEF(XD, 0x80000001, 0, RTE_REG_EDX, 20) + FEAT_DEF(1GB_PG, 0x80000001, 0, RTE_REG_EDX, 26) + FEAT_DEF(RDTSCP, 0x80000001, 0, RTE_REG_EDX, 27) + FEAT_DEF(EM64T, 0x80000001, 0, RTE_REG_EDX, 29) + + FEAT_DEF(INVTSC, 0x80000007, 0, RTE_REG_EDX, 8) +}; + +/* + * Execute CPUID instruction and get contents of a specific register + * + * This function, when compiled with GCC, will generate architecture-neutral + * code, as per GCC manual. + */ +static void cpu_get_features(uint32_t leaf, uint32_t subleaf, + cpuid_registers_t out) +{ +#if defined(__i386__) && defined(__PIC__) + /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ + __asm__ __volatile__("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" + : "=r" (out[RTE_REG_EBX]), + "=a" (out[RTE_REG_EAX]), + "=c" (out[RTE_REG_ECX]), + "=d" (out[RTE_REG_EDX]) + : "a" (leaf), "c" (subleaf)); +#else + __asm__ __volatile__("cpuid" + : "=a" (out[RTE_REG_EAX]), + "=b" (out[RTE_REG_EBX]), + "=c" (out[RTE_REG_ECX]), + "=d" (out[RTE_REG_EDX]) + : "a" (leaf), "c" (subleaf)); +#endif +} + +static int cpu_get_flag_enabled(enum rte_cpu_flag_t feature) +{ + const struct feature_entry *feat; + cpuid_registers_t regs; + + if (feature >= RTE_CPUFLAG_NUMFLAGS) + /* Flag does not match anything in the feature tables */ + return -1; + + feat = &cpu_feature_table[feature]; + + if (!feat->leaf) + /* This entry in the table wasn't filled out! */ + return -1; + + cpu_get_features(feat->leaf & 0xffff0000, 0, regs); + if (((regs[RTE_REG_EAX] ^ feat->leaf) & 0xffff0000) || + regs[RTE_REG_EAX] < feat->leaf) + return 0; + + /* get the cpuid leaf containing the desired feature */ + cpu_get_features(feat->leaf, feat->subleaf, regs); + + /* check if the feature is enabled */ + return (regs[feat->reg] >> feat->bit) & 1; +} + +static const char *cpu_get_flag_name(enum rte_cpu_flag_t feature) +{ + if (feature >= RTE_CPUFLAG_NUMFLAGS) + return NULL; + return cpu_feature_table[feature].name; +} + +void cpu_flags_print_all(void) +{ + int len, i; + int max_str = 1024; + int max_len = max_str - 1; + char str[max_str]; + + len = snprintf(str, max_len, "\nCPU features supported:\n"); + + for (i = 0; i < RTE_CPUFLAG_NUMFLAGS; i++) { + if (cpu_get_flag_enabled(i) > 0) + len += snprintf(&str[len], max_len - len, "%s ", + cpu_get_flag_name(i)); + } + + len += snprintf(&str[len], max_len - len, + "\n\nCPU features NOT supported:\n"); + + for (i = 0; i < RTE_CPUFLAG_NUMFLAGS; i++) { + if (cpu_get_flag_enabled(i) <= 0) + len += snprintf(&str[len], max_len - len, "%s ", + cpu_get_flag_name(i)); + } + + len += snprintf(&str[len], max_len - len, "\n\n"); + + str[len] = '\0'; + ODP_PRINT("%s", str); +} + +int cpu_has_global_time(void) +{ + if (cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) + return 1; + + return 0; +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.h b/platform/linux-generic/arch/x86/cpu_flags.h new file mode 100644 index 000000000..f709ca08b --- /dev/null +++ b/platform/linux-generic/arch/x86/cpu_flags.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_FLAGS_H_ +#define ODP_PLAT_CPU_FLAGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void cpu_flags_print_all(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp_cpu_arch.c b/platform/linux-generic/arch/x86/odp_cpu_arch.c index c8cf27b65..b1da428b9 100644 --- a/platform/linux-generic/arch/x86/odp_cpu_arch.c +++ b/platform/linux-generic/arch/x86/odp_cpu_arch.c @@ -3,7 +3,14 @@ * * SPDX-License-Identifier: BSD-3-Clause */ + +#include <odp_posix_extensions.h> + #include <odp/api/cpu.h> +#include <odp_time_internal.h> +#include <odp_debug_internal.h> + +#include <time.h> uint64_t odp_cpu_cycles(void) { @@ -31,3 +38,64 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +uint64_t cpu_global_time(void) +{ + return odp_cpu_cycles(); +} + +#define SEC_IN_NS 1000000000ULL + +/* Measure TSC frequency. Frequency information registers are defined for x86, + * but those are often not enumerated. */ +uint64_t cpu_global_time_freq(void) +{ + struct timespec sleep, ts1, ts2; + uint64_t t1, t2, ts_nsec, cycles, hz; + int i; + uint64_t avg = 0; + int rounds = 3; + int warm_up = 1; + + for (i = 0; i < rounds; i++) { + sleep.tv_sec = 0; + + if (warm_up) + sleep.tv_nsec = SEC_IN_NS / 1000; + else + sleep.tv_nsec = SEC_IN_NS / 4; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { + ODP_DBG("clock_gettime failed\n"); + return 0; + } + + t1 = cpu_global_time(); + + if (nanosleep(&sleep, NULL) < 0) { + ODP_DBG("nanosleep failed\n"); + return 0; + } + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { + ODP_DBG("clock_gettime failed\n"); + return 0; + } + + t2 = cpu_global_time(); + + ts_nsec = (ts2.tv_sec - ts1.tv_sec) * SEC_IN_NS; + ts_nsec += ts2.tv_nsec - ts1.tv_nsec; + + cycles = t2 - t1; + + hz = (cycles * SEC_IN_NS) / ts_nsec; + + if (warm_up) + warm_up = 0; + else + avg += hz; + } + + return avg / (rounds - 1); +} diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index 96127ec67..d77165a41 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -5,6 +5,7 @@ */ #include <odp_internal.h> +#include <arch/x86/cpu_flags.h> #include <string.h> int cpuinfo_parser(FILE *file, system_info_t *sysinfo) @@ -73,3 +74,8 @@ uint64_t odp_cpu_hz_current(int id) return 0; } + +void sys_info_print_arch(void) +{ + cpu_flags_print_all(); +} diff --git a/platform/linux-generic/include/odp/api/debug.h b/platform/linux-generic/include/odp/api/debug.h index 7db143395..bef2fd0eb 100644 --- a/platform/linux-generic/include/odp/api/debug.h +++ b/platform/linux-generic/include/odp/api/debug.h @@ -19,17 +19,44 @@ extern "C" { #include <odp/api/spec/debug.h> -#if defined(__GNUC__) && !defined(__clang__) - -#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) - /** - * @internal _Static_assert was only added in GCC 4.6. Provide a weak replacement - * for previous versions. + * @internal _Static_assert was only added in GCC 4.6 and the C++ version + * static_assert for g++ 6 and above. Provide a weak replacement for previous + * versions. */ -#define _Static_assert(e, s) (extern int (*static_assert_checker(void)) \ - [sizeof(struct { unsigned int error_if_negative:(e) ? 1 : -1; })]) +#define _odp_merge(a, b) a##b +/** @internal */ +#define _odp_label(a) _odp_merge(_ODP_SASSERT_, a) +/** @internal */ +#define _ODP_SASSERT _odp_label(__COUNTER__) +/** @internal */ +#define _ODP_SASSERT_ENUM(e) { _ODP_SASSERT = 1 / !!(e) } +/** @internal */ +#define _odp_static_assert(e, s) enum _ODP_SASSERT_ENUM(e) + +#if defined(__clang__) +#if defined(__cplusplus) +#if !__has_feature(cxx_static_assert) && !defined(static_assert) +/** @internal */ +#define static_assert(e, s) _odp_static_assert(e, s) +#endif +#elif !__has_feature(c_static_assert) && !defined(_Static_assert) +/** @internal */ +#define _Static_assert(e, s) _odp_static_assert(e, s) +#endif +#elif defined(__GNUC__) +#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) || \ + (__GNUC__ < 6 && defined(__cplusplus)) +#if defined(__cplusplus) +#if !defined(static_assert) +/** @intenral */ +#define static_assert(e, s) _odp_static_assert(e, s) +#endif +#elif !defined(_Static_assert) +/** @internal */ +#define _Static_assert(e, s) _odp_static_assert(e, s) +#endif #endif #endif @@ -39,9 +66,11 @@ extern "C" { * if condition 'cond' is false. Macro definition is empty when compiler is not * supported or the compiler does not support static assertion. */ -#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) +#ifndef __cplusplus +#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) -#ifdef __cplusplus +#else +#define ODP_STATIC_ASSERT(cond, msg) static_assert(cond, msg) } #endif diff --git a/platform/linux-generic/include/odp/api/deprecated.h b/platform/linux-generic/include/odp/api/deprecated.h new file mode 100644 index 000000000..82797ebc4 --- /dev/null +++ b/platform/linux-generic/include/odp/api/deprecated.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * Control deprecated API definitions + */ + +#ifndef ODP_PLAT_DEPRECATED_H_ +#define ODP_PLAT_DEPRECATED_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/deprecated.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 7e3c51e6c..a209c759f 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -105,8 +105,8 @@ typedef union { /** All input flags */ uint64_t all; + /** Individual input flags */ struct { - uint64_t parsed_l2:1; /**< L2 parsed */ uint64_t dst_queue:1; /**< Dst queue present */ uint64_t flow_hash:1; /**< Flow hash present */ diff --git a/platform/linux-generic/include/odp/api/plat/time_types.h b/platform/linux-generic/include/odp/api/plat/time_types.h index 4847f3b1f..e7111c8c5 100644 --- a/platform/linux-generic/include/odp/api/plat/time_types.h +++ b/platform/linux-generic/include/odp/api/plat/time_types.h @@ -22,15 +22,25 @@ extern "C" { **/ /** - * @internal Time structure used to isolate odp-linux implementation from - * the linux timespec structure, which is dependent on POSIX extension level. + * @internal Time structure used for both POSIX timespec and HW counter + * implementations. */ typedef struct odp_time_t { - int64_t tv_sec; /**< @internal Seconds */ - int64_t tv_nsec; /**< @internal Nanoseconds */ + /** @internal Variant mappings for time type */ + union { + /** @internal Used with generic 64 bit operations */ + uint64_t u64; + + /** @internal Nanoseconds */ + uint64_t nsec; + + /** @internal HW timer counter value */ + uint64_t count; + + }; } odp_time_t; -#define ODP_TIME_NULL ((odp_time_t){0, 0}) +#define ODP_TIME_NULL ((odp_time_t){.u64 = 0}) /** * @} diff --git a/platform/linux-generic/include/odp/api/plat/traffic_mngr_types.h b/platform/linux-generic/include/odp/api/plat/traffic_mngr_types.h index b766afecd..f47a13f6f 100644 --- a/platform/linux-generic/include/odp/api/plat/traffic_mngr_types.h +++ b/platform/linux-generic/include/odp/api/plat/traffic_mngr_types.h @@ -168,7 +168,7 @@ typedef odp_tm_handle_t odp_tm_wred_t; */ #define ODP_TM_ROOT ((odp_tm_handle_t)-1) -/** Get printable format of odp_queue_t */ +/** @internal Get printable format of odp_tm_handle_t @param hdl @return */ static inline uint64_t odp_tm_handle_to_u64(odp_tm_handle_t hdl) { return hdl; diff --git a/platform/linux-generic/include/odp_crypto_internal.h b/platform/linux-generic/include/odp_crypto_internal.h index f85b76eaa..c3b70b231 100644 --- a/platform/linux-generic/include/odp_crypto_internal.h +++ b/platform/linux-generic/include/odp_crypto_internal.h @@ -11,8 +11,6 @@ extern "C" { #endif -#include <openssl/des.h> -#include <openssl/aes.h> #include <openssl/evp.h> #define MAX_IV_LEN 64 @@ -42,34 +40,17 @@ struct odp_crypto_generic_session { struct { /* Copy of session IV data */ uint8_t iv_data[MAX_IV_LEN]; + uint8_t key_data[EVP_MAX_KEY_LENGTH]; - union { - struct { - DES_key_schedule ks1; - DES_key_schedule ks2; - DES_key_schedule ks3; - } des; - struct { - AES_KEY key; - } aes; - struct { - EVP_CIPHER_CTX *ctx; - } aes_gcm; - } data; + const EVP_CIPHER *evp_cipher; crypto_func_t func; } cipher; struct { - union { - struct { - uint8_t key[16]; - uint32_t bytes; - } md5; - struct { - uint8_t key[32]; - uint32_t bytes; - } sha256; - } data; + uint8_t key[EVP_MAX_KEY_LENGTH]; + uint32_t key_length; + uint32_t bytes; + const EVP_MD *evp_md; crypto_func_t func; } auth; }; diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index e1267cff7..8bae028d9 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -128,6 +128,7 @@ int _odp_ishm_term_local(void); int cpuinfo_parser(FILE *file, system_info_t *sysinfo); uint64_t odp_cpu_hz_current(int id); +void sys_info_print_arch(void); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 0a9f17799..a480a7484 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -80,18 +80,6 @@ ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), "OUTPUT_FLAGS_SIZE_ERROR"); /** - * Protocol stack layers - */ -typedef enum { - LAYER_NONE = 0, - LAYER_L1, - LAYER_L2, - LAYER_L3, - LAYER_L4, - LAYER_ALL -} layer_t; - -/** * Packet parser metadata */ typedef struct { @@ -102,14 +90,6 @@ typedef struct { uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ - - uint32_t l3_len; /**< Layer 3 length */ - uint32_t l4_len; /**< Layer 4 length */ - - uint16_t ethtype; /**< EtherType */ - uint8_t ip_proto; /**< IP protocol */ - uint8_t parsed_layers; /**< Highest parsed protocol stack layer */ - } packet_parser_t; /** @@ -203,16 +183,6 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->frame_len = len; } -static inline int packet_parse_l2_not_done(packet_parser_t *prs) -{ - return !prs->input_flags.parsed_l2; -} - -static inline int packet_parse_not_complete(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->p.parsed_layers != LAYER_ALL; -} - /* Forward declarations */ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); @@ -220,11 +190,9 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], int max_num); -/* Fill in parser metadata for L2 */ -void packet_parse_l2(packet_parser_t *prs, uint32_t frame_len); - /* Perform packet parse up to a given protocol layer */ -int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer); +int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, + odp_pktio_parser_layer_t layer); /* Reset parser metadata for a new parse */ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr); @@ -264,10 +232,17 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts) } int packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr, - uint32_t pkt_len, uint32_t seg_len, layer_t layer); + uint32_t pkt_len, uint32_t seg_len, + odp_pktio_parser_layer_t layer); int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr); +int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset, + uint8_t c, uint32_t len); + +int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, + const void *s, uint32_t len); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_socket.h b/platform/linux-generic/include/odp_packet_socket.h index dbfc9f1f8..0e61f6f0c 100644 --- a/platform/linux-generic/include/odp_packet_socket.h +++ b/platform/linux-generic/include/odp_packet_socket.h @@ -27,11 +27,6 @@ * Packet socket config: */ -/** Max receive (Rx) burst size*/ -#define ODP_PACKET_SOCKET_MAX_BURST_RX 32 -/** Max transmit (Tx) burst size*/ -#define ODP_PACKET_SOCKET_MAX_BURST_TX 32 - /* * This makes sure that building for kernels older than 3.1 works * and a fanout requests fails (for invalid packet socket option) @@ -47,8 +42,6 @@ typedef struct { odp_pool_t pool; /**< pool to alloc packets from */ uint32_t mtu; /**< maximum transmission unit */ unsigned char if_mac[ETH_ALEN]; /**< IF eth mac addr */ - uint8_t *cache_ptr[ODP_PACKET_SOCKET_MAX_BURST_RX]; - odp_shm_t shm; } pkt_sock_t; /** packet mmap ring */ diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index 55fedeb3a..44b83c603 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -57,7 +57,7 @@ static inline uint32_t ring_deq(ring_t *ring, uint32_t mask) /* Move reader head. This thread owns data at the new head. */ do { - tail = odp_atomic_load_u32(&ring->w_tail); + tail = odp_atomic_load_acq_u32(&ring->w_tail); if (head == tail) return RING_EMPTY; @@ -90,7 +90,7 @@ static inline uint32_t ring_deq_multi(ring_t *ring, uint32_t mask, /* Move reader head. This thread owns data at the new head. */ do { - tail = odp_atomic_load_u32(&ring->w_tail); + tail = odp_atomic_load_acq_u32(&ring->w_tail); /* Ring is empty */ if (head == tail) diff --git a/platform/linux-generic/include/odp_time_internal.h b/platform/linux-generic/include/odp_time_internal.h new file mode 100644 index 000000000..99ac79772 --- /dev/null +++ b/platform/linux-generic/include/odp_time_internal.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_TIME_INTERNAL_H_ +#define ODP_TIME_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +int cpu_has_global_time(void); +uint64_t cpu_global_time(void); +uint64_t cpu_global_time_freq(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/protocols/ip.h b/platform/linux-generic/include/protocols/ip.h index 20041f1c0..2b34a753f 100644 --- a/platform/linux-generic/include/protocols/ip.h +++ b/platform/linux-generic/include/protocols/ip.h @@ -157,13 +157,14 @@ typedef struct ODP_PACKED { * IP protocol values (IPv4:'proto' or IPv6:'next_hdr') * @{*/ #define _ODP_IPPROTO_HOPOPTS 0x00 /**< IPv6 hop-by-hop options */ -#define _ODP_IPPROTO_ICMP 0x01 /**< Internet Control Message Protocol (1) */ +#define _ODP_IPPROTO_ICMPv4 0x01 /**< Internet Control Message Protocol (1) */ #define _ODP_IPPROTO_TCP 0x06 /**< Transmission Control Protocol (6) */ #define _ODP_IPPROTO_UDP 0x11 /**< User Datagram Protocol (17) */ #define _ODP_IPPROTO_ROUTE 0x2B /**< IPv6 Routing header (43) */ #define _ODP_IPPROTO_FRAG 0x2C /**< IPv6 Fragment (44) */ #define _ODP_IPPROTO_AH 0x33 /**< Authentication Header (51) */ #define _ODP_IPPROTO_ESP 0x32 /**< Encapsulating Security Payload (50) */ +#define _ODP_IPPROTO_ICMPv6 0x3A /**< Internet Control Message Protocol (58) */ #define _ODP_IPPROTO_SCTP 0x84 /**< Stream Control Transmission protocol (132) */ #define _ODP_IPPROTO_INVALID 0xFF /**< Reserved invalid by IANA */ diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index d3e5528c1..e1197f606 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -28,12 +28,56 @@ AC_LINK_IFELSE( echo "Use newer version. For gcc > 4.7.0" exit -1) +dnl Check whether -latomic is needed +use_libatomic=no + +AC_MSG_CHECKING(whether -latomic is needed for 64-bit atomic built-ins) +AC_LINK_IFELSE( + [AC_LANG_SOURCE([[ + static int loc; + int main(void) + { + int prev = __atomic_exchange_n(&loc, 7, __ATOMIC_RELAXED); + return 0; + } + ]])], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT(yes) + AC_CHECK_LIB( + [atomic], [__atomic_exchange_8], + [use_libatomic=yes], + [AC_MSG_FAILURE([__atomic_exchange_8 is not available])]) + ]) + +AC_MSG_CHECKING(whether -latomic is needed for 128-bit atomic built-ins) +AC_LINK_IFELSE( + [AC_LANG_SOURCE([[ + static __int128 loc; + int main(void) + { + __int128 prev; + prev = __atomic_exchange_n(&loc, 7, __ATOMIC_RELAXED); + return 0; + } + ]])], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT(yes) + AC_CHECK_LIB( + [atomic], [__atomic_exchange_16], + [use_libatomic=yes], + [AC_MSG_CHECKING([cannot detect support for 128-bit atomics])]) + ]) + +if test "x$use_libatomic" = "xyes"; then + ATOMIC_LIBS="-latomic" +fi +AC_SUBST([ATOMIC_LIBS]) + m4_include([platform/linux-generic/m4/odp_pthread.m4]) m4_include([platform/linux-generic/m4/odp_openssl.m4]) m4_include([platform/linux-generic/m4/odp_pcap.m4]) m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) -m4_include([platform/linux-generic/m4/odp_ipc.m4]) m4_include([platform/linux-generic/m4/odp_schedule.m4]) AC_CONFIG_FILES([platform/linux-generic/Makefile diff --git a/platform/linux-generic/m4/odp_dpdk.m4 b/platform/linux-generic/m4/odp_dpdk.m4 index 30347dce8..58d14727b 100644 --- a/platform/linux-generic/m4/odp_dpdk.m4 +++ b/platform/linux-generic/m4/odp_dpdk.m4 @@ -2,22 +2,10 @@ # Enable DPDK support ########################################################################## pktio_dpdk_support=no -AC_ARG_ENABLE([dpdk_support], - [ --enable-dpdk-support include dpdk IO support], - [if test x$enableval = xyes; then - pktio_dpdk_support=yes - fi]) - -########################################################################## -# Set optional DPDK path -########################################################################## AC_ARG_WITH([dpdk-path], -AC_HELP_STRING([--with-dpdk-path=DIR path to dpdk build directory], - [(or in the default path if not specified).]), +AC_HELP_STRING([--with-dpdk-path=DIR path to dpdk build directory]), [DPDK_PATH=$withval AM_CPPFLAGS="$AM_CPPFLAGS -msse4.2 -isystem $DPDK_PATH/include" - AM_LDFLAGS="$AM_LDFLAGS -L$DPDK_PATH/lib" - LIBS="$LIBS -ldpdk -ldl -lpcap" pktio_dpdk_support=yes],[]) ########################################################################## @@ -28,12 +16,31 @@ CPPFLAGS="$AM_CPPFLAGS $CPPFLAGS" ########################################################################## # Check for DPDK availability +# +# DPDK pmd drivers are not linked unless the --whole-archive option is +# used. No spaces are allowed between the --whole-arhive flags. ########################################################################## if test x$pktio_dpdk_support = xyes then AC_CHECK_HEADERS([rte_config.h], [], [AC_MSG_FAILURE(["can't find DPDK header"])]) + + DPDK_PMD=--whole-archive, + for filename in $with_dpdk_path/lib/*.a; do + cur_driver=`echo $(basename "$filename" .a) | \ + sed -n 's/^\(librte_pmd_\)/-lrte_pmd_/p' | sed -n 's/$/,/p'` + # rte_pmd_nfp has external dependencies which break linking + if test "$cur_driver" = "-lrte_pmd_nfp,"; then + echo "skip linking rte_pmd_nfp" + else + DPDK_PMD+=$cur_driver + fi + done + DPDK_PMD+=--no-whole-archive + ODP_CFLAGS="$ODP_CFLAGS -DODP_PKTIO_DPDK" + AM_LDFLAGS="$AM_LDFLAGS -L$DPDK_PATH/lib -Wl,$DPDK_PMD" + LIBS="$LIBS -ldpdk -ldl -lpcap" else pktio_dpdk_support=no fi diff --git a/platform/linux-generic/m4/odp_ipc.m4 b/platform/linux-generic/m4/odp_ipc.m4 deleted file mode 100644 index 78217e221..000000000 --- a/platform/linux-generic/m4/odp_ipc.m4 +++ /dev/null @@ -1,9 +0,0 @@ -########################################################################## -# Enable IPC pktio support -########################################################################## -AC_ARG_ENABLE([pktio_ipc_support], - [ --enable-pktio_ipc-support include ipc IO support], - [if test x$enableval = xyes; then - pktio_ipc_support=yes - ODP_CFLAGS="$ODP_CFLAGS -D_ODP_PKTIO_IPC" - fi]) diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 5d96b00b3..7ebc47d7d 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -790,10 +790,6 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry, cls = &entry->s.cls; default_cos = cls->default_cos; - /* Check for lazy parse needed */ - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - /* Return error cos for error packet */ if (pkt_hdr->p.error_flags.all) return cls->error_cos; @@ -838,7 +834,8 @@ int cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, packet_parse_reset(pkt_hdr); packet_set_len(pkt_hdr, pkt_len); - packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len, LAYER_ALL); + packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len, + ODP_PKTIO_PARSER_LAYER_ALL); cos = cls_select_cos(entry, base, pkt_hdr); if (cos == NULL) diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 54b222fd2..6fc1907d9 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -34,6 +34,9 @@ * * Keep sorted: first by key length, then by IV length */ +static const odp_crypto_cipher_capability_t cipher_capa_null[] = { +{.key_len = 0, .iv_len = 0} }; + static const odp_crypto_cipher_capability_t cipher_capa_des[] = { {.key_len = 24, .iv_len = 8} }; @@ -51,11 +54,24 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = { * * Keep sorted: first by digest length, then by key length */ +static const odp_crypto_auth_capability_t auth_capa_null[] = { +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + static const odp_crypto_auth_capability_t auth_capa_md5_hmac[] = { -{.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +{.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 16, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + +static const odp_crypto_auth_capability_t auth_capa_sha1_hmac[] = { +{.digest_len = 12, .key_len = 20, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 20, .key_len = 20, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; static const odp_crypto_auth_capability_t auth_capa_sha256_hmac[] = { -{.digest_len = 16, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +{.digest_len = 16, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 32, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + +static const odp_crypto_auth_capability_t auth_capa_sha512_hmac[] = { +{.digest_len = 32, .key_len = 64, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 64, .key_len = 64, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = { {.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } }; @@ -86,8 +102,10 @@ odp_crypto_generic_session_t *alloc_session(void) odp_spinlock_lock(&global->lock); session = global->free; - if (session) + if (session) { global->free = session->next; + session->next = NULL; + } odp_spinlock_unlock(&global->lock); return session; @@ -110,128 +128,74 @@ null_crypto_routine(odp_crypto_op_param_t *param ODP_UNUSED, } static -odp_crypto_alg_err_t md5_gen(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +void packet_hmac(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session, + uint8_t *hash) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint8_t *icv = data; + odp_packet_t pkt = param->out_pkt; + uint32_t offset = param->auth_range.offset; uint32_t len = param->auth_range.length; - uint8_t hash[EVP_MAX_MD_SIZE]; + HMAC_CTX ctx; - /* Adjust pointer for beginning of area to auth */ - data += param->auth_range.offset; - icv += param->hash_result_offset; + ODP_ASSERT(offset + len <= odp_packet_len(pkt)); /* Hash it */ - HMAC(EVP_md5(), - session->auth.data.md5.key, - 16, - data, - len, - hash, - NULL); - - /* Copy to the output location */ - memcpy(icv, hash, session->auth.data.md5.bytes); - - return ODP_CRYPTO_ALG_ERR_NONE; -} - -static -odp_crypto_alg_err_t md5_check(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) -{ - uint8_t *data = odp_packet_data(param->out_pkt); - uint8_t *icv = data; - uint32_t len = param->auth_range.length; - uint32_t bytes = session->auth.data.md5.bytes; - uint8_t hash_in[EVP_MAX_MD_SIZE]; - uint8_t hash_out[EVP_MAX_MD_SIZE]; - - /* Adjust pointer for beginning of area to auth */ - data += param->auth_range.offset; - icv += param->hash_result_offset; - - /* Copy current value out and clear it before authentication */ - memset(hash_in, 0, sizeof(hash_in)); - memcpy(hash_in, icv, bytes); - memset(icv, 0, bytes); - memset(hash_out, 0, sizeof(hash_out)); - - /* Hash it */ - HMAC(EVP_md5(), - session->auth.data.md5.key, - 16, - data, - len, - hash_out, - NULL); - - /* Verify match */ - if (0 != memcmp(hash_in, hash_out, bytes)) - return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + HMAC_CTX_init(&ctx); + HMAC_Init_ex(&ctx, + session->auth.key, + session->auth.key_length, + session->auth.evp_md, + NULL); + + while (len > 0) { + uint32_t seglen = 0; /* GCC */ + void *mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); + uint32_t maclen = len > seglen ? seglen : len; + + HMAC_Update(&ctx, mapaddr, maclen); + offset += maclen; + len -= maclen; + } - /* Matched */ - return ODP_CRYPTO_ALG_ERR_NONE; + HMAC_Final(&ctx, hash, NULL); + HMAC_CTX_cleanup(&ctx); } static -odp_crypto_alg_err_t sha256_gen(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t auth_gen(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint8_t *icv = data; - uint32_t len = param->auth_range.length; uint8_t hash[EVP_MAX_MD_SIZE]; - /* Adjust pointer for beginning of area to auth */ - data += param->auth_range.offset; - icv += param->hash_result_offset; - /* Hash it */ - HMAC(EVP_sha256(), - session->auth.data.sha256.key, - 32, - data, - len, - hash, - NULL); + packet_hmac(param, session, hash); /* Copy to the output location */ - memcpy(icv, hash, session->auth.data.sha256.bytes); + odp_packet_copy_from_mem(param->out_pkt, + param->hash_result_offset, + session->auth.bytes, + hash); return ODP_CRYPTO_ALG_ERR_NONE; } static -odp_crypto_alg_err_t sha256_check(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t auth_check(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint8_t *icv = data; - uint32_t len = param->auth_range.length; - uint32_t bytes = session->auth.data.sha256.bytes; + uint32_t bytes = session->auth.bytes; uint8_t hash_in[EVP_MAX_MD_SIZE]; uint8_t hash_out[EVP_MAX_MD_SIZE]; - /* Adjust pointer for beginning of area to auth */ - data += param->auth_range.offset; - icv += param->hash_result_offset; - /* Copy current value out and clear it before authentication */ - memset(hash_in, 0, sizeof(hash_in)); - memcpy(hash_in, icv, bytes); - memset(icv, 0, bytes); - memset(hash_out, 0, sizeof(hash_out)); + odp_packet_copy_to_mem(param->out_pkt, param->hash_result_offset, + bytes, hash_in); + + _odp_packet_set_data(param->out_pkt, param->hash_result_offset, + 0, bytes); /* Hash it */ - HMAC(EVP_sha256(), - session->auth.data.sha256.key, - 32, - data, - len, - hash_out, - NULL); + packet_hmac(param, session, hash_out); /* Verify match */ if (0 != memcmp(hash_in, hash_out, bytes)) @@ -242,102 +206,122 @@ odp_crypto_alg_err_t sha256_check(odp_crypto_op_param_t *param, } static -odp_crypto_alg_err_t aes_encrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +int internal_encrypt(EVP_CIPHER_CTX *ctx, odp_crypto_op_param_t *param) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t len = param->cipher_range.length; - unsigned char iv_enc[AES_BLOCK_SIZE]; - void *iv_ptr; + odp_packet_t pkt = param->out_pkt; + unsigned in_pos = param->cipher_range.offset; + unsigned out_pos = param->cipher_range.offset; + unsigned in_len = param->cipher_range.length; + uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; + unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); + int cipher_len; + int ret; - if (param->override_iv_ptr) - iv_ptr = param->override_iv_ptr; - else if (session->p.iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; + while (in_len > 0) { + uint32_t seglen = 0; /* GCC */ + uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, + &seglen, NULL); + unsigned inseglen = in_len < seglen ? in_len : seglen; - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); + /* There should be at least 1 additional block in out buffer */ + if (inseglen > block_len) { + unsigned part = inseglen - block_len; - /* Adjust pointer for beginning of area to cipher */ - data += param->cipher_range.offset; - /* Encrypt it */ - AES_cbc_encrypt(data, data, len, &session->cipher.data.aes.key, - iv_enc, AES_ENCRYPT); + EVP_EncryptUpdate(ctx, insegaddr, &cipher_len, + insegaddr, part); + in_pos += part; + in_len -= part; + insegaddr += part; + inseglen -= part; - return ODP_CRYPTO_ALG_ERR_NONE; + out_pos += cipher_len; + } + + /* Use temporal storage */ + if (inseglen > 0) { + unsigned part = inseglen; + + EVP_EncryptUpdate(ctx, block, &cipher_len, + insegaddr, part); + in_pos += part; + in_len -= part; + insegaddr += part; + inseglen -= part; + + odp_packet_copy_from_mem(pkt, out_pos, + cipher_len, block); + out_pos += cipher_len; + } + } + + ret = EVP_EncryptFinal_ex(ctx, block, &cipher_len); + odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); + + return ret; } static -odp_crypto_alg_err_t aes_decrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +int internal_decrypt(EVP_CIPHER_CTX *ctx, odp_crypto_op_param_t *param) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t len = param->cipher_range.length; - unsigned char iv_enc[AES_BLOCK_SIZE]; - void *iv_ptr; + odp_packet_t pkt = param->out_pkt; + unsigned in_pos = param->cipher_range.offset; + unsigned out_pos = param->cipher_range.offset; + unsigned in_len = param->cipher_range.length; + uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; + unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); + int cipher_len; + int ret; - if (param->override_iv_ptr) - iv_ptr = param->override_iv_ptr; - else if (session->p.iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; + while (in_len > 0) { + uint32_t seglen = 0; /* GCC */ + uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, + &seglen, NULL); + unsigned inseglen = in_len < seglen ? in_len : seglen; - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); + /* There should be at least 1 additional block in out buffer */ + if (inseglen > block_len) { + unsigned part = inseglen - block_len; - /* Adjust pointer for beginning of area to cipher */ - data += param->cipher_range.offset; - /* Encrypt it */ - AES_cbc_encrypt(data, data, len, &session->cipher.data.aes.key, - iv_enc, AES_DECRYPT); + EVP_DecryptUpdate(ctx, insegaddr, &cipher_len, + insegaddr, part); + in_pos += part; + in_len -= part; + insegaddr += part; + inseglen -= part; - return ODP_CRYPTO_ALG_ERR_NONE; -} + out_pos += cipher_len; + } -static int process_aes_param(odp_crypto_generic_session_t *session) -{ - /* Verify IV len is either 0 or 16 */ - if (!((0 == session->p.iv.length) || (16 == session->p.iv.length))) - return -1; + /* Use temporal storage */ + if (inseglen > 0) { + unsigned part = inseglen; - /* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - session->cipher.func = aes_encrypt; - AES_set_encrypt_key(session->p.cipher_key.data, 128, - &session->cipher.data.aes.key); - } else { - session->cipher.func = aes_decrypt; - AES_set_decrypt_key(session->p.cipher_key.data, 128, - &session->cipher.data.aes.key); + EVP_DecryptUpdate(ctx, block, &cipher_len, + insegaddr, part); + in_pos += part; + in_len -= part; + insegaddr += part; + inseglen -= part; + + odp_packet_copy_from_mem(pkt, out_pos, + cipher_len, block); + out_pos += cipher_len; + } } - return 0; + ret = EVP_DecryptFinal_ex(ctx, block, &cipher_len); + odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); + + return ret; } static -odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t cipher_encrypt(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t plain_len = param->cipher_range.length; - uint8_t *aad_head = data + param->auth_range.offset; - uint8_t *aad_tail = data + param->cipher_range.offset + - param->cipher_range.length; - uint32_t auth_len = param->auth_range.length; - unsigned char iv_enc[AES_BLOCK_SIZE]; + EVP_CIPHER_CTX *ctx; void *iv_ptr; - uint8_t *tag = data + param->hash_result_offset; + int ret; if (param->override_iv_ptr) iv_ptr = param->override_iv_ptr; @@ -346,63 +330,28 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* All cipher data must be part of the authentication */ - if (param->auth_range.offset > param->cipher_range.offset || - param->auth_range.offset + auth_len < - param->cipher_range.offset + plain_len) - return ODP_CRYPTO_ALG_ERR_DATA_SIZE; - - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); - - /* Adjust pointer for beginning of area to cipher/auth */ - uint8_t *plaindata = data + param->cipher_range.offset; - /* Encrypt it */ - EVP_CIPHER_CTX *ctx = session->cipher.data.aes_gcm.ctx; - int cipher_len = 0; - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_enc); - - /* Authenticate header data (if any) without encrypting them */ - if (aad_head < plaindata) { - EVP_EncryptUpdate(ctx, NULL, &cipher_len, - aad_head, plaindata - aad_head); - } + ctx = EVP_CIPHER_CTX_new(); + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_CIPHER_CTX_set_padding(ctx, 0); - EVP_EncryptUpdate(ctx, plaindata, &cipher_len, - plaindata, plain_len); - cipher_len = plain_len; + ret = internal_encrypt(ctx, param); - /* Authenticate footer data (if any) without encrypting them */ - if (aad_head + auth_len > plaindata + plain_len) { - EVP_EncryptUpdate(ctx, NULL, NULL, aad_tail, - auth_len - (aad_tail - aad_head)); - } + EVP_CIPHER_CTX_free(ctx); - EVP_EncryptFinal_ex(ctx, plaindata + cipher_len, &cipher_len); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag); - - return ODP_CRYPTO_ALG_ERR_NONE; + return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : + ODP_CRYPTO_ALG_ERR_NONE; } static -odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t cipher_decrypt(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t cipher_len = param->cipher_range.length; - uint8_t *aad_head = data + param->auth_range.offset; - uint8_t *aad_tail = data + param->cipher_range.offset + - param->cipher_range.length; - uint32_t auth_len = param->auth_range.length; - unsigned char iv_enc[AES_BLOCK_SIZE]; + EVP_CIPHER_CTX *ctx; void *iv_ptr; - uint8_t *tag = data + param->hash_result_offset; + int ret; if (param->override_iv_ptr) iv_ptr = param->override_iv_ptr; @@ -411,90 +360,59 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* All cipher data must be part of the authentication */ - if (param->auth_range.offset > param->cipher_range.offset || - param->auth_range.offset + auth_len < - param->cipher_range.offset + cipher_len) - return ODP_CRYPTO_ALG_ERR_DATA_SIZE; - - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); - - /* Adjust pointer for beginning of area to cipher/auth */ - uint8_t *cipherdata = data + param->cipher_range.offset; - /* Encrypt it */ - EVP_CIPHER_CTX *ctx = session->cipher.data.aes_gcm.ctx; - int plain_len = 0; - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_enc); - - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag); - - /* Authenticate header data (if any) without encrypting them */ - if (aad_head < cipherdata) { - EVP_DecryptUpdate(ctx, NULL, &plain_len, - aad_head, cipherdata - aad_head); - } - - EVP_DecryptUpdate(ctx, cipherdata, &plain_len, - cipherdata, cipher_len); - plain_len = cipher_len; + /* Decrypt it */ + ctx = EVP_CIPHER_CTX_new(); + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_CIPHER_CTX_set_padding(ctx, 0); - /* Authenticate footer data (if any) without encrypting them */ - if (aad_head + auth_len > cipherdata + cipher_len) { - EVP_DecryptUpdate(ctx, NULL, NULL, aad_tail, - auth_len - (aad_tail - aad_head)); - } + ret = internal_decrypt(ctx, param); - if (EVP_DecryptFinal_ex(ctx, cipherdata + cipher_len, &plain_len) < 0) - return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + EVP_CIPHER_CTX_free(ctx); - return ODP_CRYPTO_ALG_ERR_NONE; + return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : + ODP_CRYPTO_ALG_ERR_NONE; } -static int process_aes_gcm_param(odp_crypto_generic_session_t *session) +static int process_cipher_param(odp_crypto_generic_session_t *session, + const EVP_CIPHER *cipher) { - /* Verify Key len is 16 */ - if (session->p.cipher_key.length != 16) + /* Verify Key len is valid */ + if ((uint32_t)EVP_CIPHER_key_length(cipher) != + session->p.cipher_key.length) return -1; - /* Set function */ - EVP_CIPHER_CTX *ctx = - session->cipher.data.aes_gcm.ctx = EVP_CIPHER_CTX_new(); + /* Verify IV len is correct */ + if (!((0 == session->p.iv.length) || + ((uint32_t)EVP_CIPHER_iv_length(cipher) == session->p.iv.length))) + return -1; - if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - session->cipher.func = aes_gcm_encrypt; - EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL); - } else { - session->cipher.func = aes_gcm_decrypt; - EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL); - } + session->cipher.evp_cipher = cipher; - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); - if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - EVP_EncryptInit_ex(ctx, NULL, NULL, - session->p.cipher_key.data, NULL); - } else { - EVP_DecryptInit_ex(ctx, NULL, NULL, - session->p.cipher_key.data, NULL); - } + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); + + /* Set function */ + if (ODP_CRYPTO_OP_ENCODE == session->p.op) + session->cipher.func = cipher_encrypt; + else + session->cipher.func = cipher_decrypt; return 0; } static -odp_crypto_alg_err_t des_encrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t len = param->cipher_range.length; - DES_cblock iv; + EVP_CIPHER_CTX *ctx; + const uint8_t *aad_head = param->aad.ptr; + uint32_t aad_len = param->aad.length; void *iv_ptr; + int dummy_len = 0; + uint8_t block[EVP_MAX_MD_SIZE]; + int ret; if (param->override_iv_ptr) iv_ptr = param->override_iv_ptr; @@ -503,36 +421,44 @@ odp_crypto_alg_err_t des_encrypt(odp_crypto_op_param_t *param, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv, iv_ptr, sizeof(iv)); - - /* Adjust pointer for beginning of area to cipher */ - data += param->cipher_range.offset; /* Encrypt it */ - DES_ede3_cbc_encrypt(data, - data, - len, - &session->cipher.data.des.ks1, - &session->cipher.data.des.ks2, - &session->cipher.data.des.ks3, - &iv, - 1); + ctx = EVP_CIPHER_CTX_new(); + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_CIPHER_CTX_set_padding(ctx, 0); - return ODP_CRYPTO_ALG_ERR_NONE; + /* Authenticate header data (if any) without encrypting them */ + if (aad_len > 0) + EVP_EncryptUpdate(ctx, NULL, &dummy_len, + aad_head, aad_len); + + ret = internal_encrypt(ctx, param); + + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, + session->p.auth_digest_len, block); + odp_packet_copy_from_mem(param->out_pkt, param->hash_result_offset, + session->p.auth_digest_len, block); + + EVP_CIPHER_CTX_free(ctx); + + return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : + ODP_CRYPTO_ALG_ERR_NONE; } static -odp_crypto_alg_err_t des_decrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t len = param->cipher_range.length; - DES_cblock iv; + EVP_CIPHER_CTX *ctx; + const uint8_t *aad_head = param->aad.ptr; + uint32_t aad_len = param->aad.length; + int dummy_len = 0; void *iv_ptr; + uint8_t block[EVP_MAX_MD_SIZE]; + int ret; if (param->override_iv_ptr) iv_ptr = param->override_iv_ptr; @@ -541,84 +467,76 @@ odp_crypto_alg_err_t des_decrypt(odp_crypto_op_param_t *param, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv, iv_ptr, sizeof(iv)); + /* Decrypt it */ + ctx = EVP_CIPHER_CTX_new(); + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_CIPHER_CTX_set_padding(ctx, 0); - /* Adjust pointer for beginning of area to cipher */ - data += param->cipher_range.offset; + odp_packet_copy_to_mem(param->out_pkt, param->hash_result_offset, + session->p.auth_digest_len, block); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, + session->p.auth_digest_len, block); - /* Decrypt it */ - DES_ede3_cbc_encrypt(data, - data, - len, - &session->cipher.data.des.ks1, - &session->cipher.data.des.ks2, - &session->cipher.data.des.ks3, - &iv, - 0); + /* Authenticate header data (if any) without encrypting them */ + if (aad_len > 0) + EVP_DecryptUpdate(ctx, NULL, &dummy_len, + aad_head, aad_len); - return ODP_CRYPTO_ALG_ERR_NONE; + ret = internal_decrypt(ctx, param); + + EVP_CIPHER_CTX_free(ctx); + + return ret <= 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : + ODP_CRYPTO_ALG_ERR_NONE; } -static int process_des_param(odp_crypto_generic_session_t *session) +static int process_aes_gcm_param(odp_crypto_generic_session_t *session, + const EVP_CIPHER *cipher) { - /* Verify IV len is either 0 or 8 */ - if (!((0 == session->p.iv.length) || (8 == session->p.iv.length))) + /* Verify Key len is valid */ + if ((uint32_t)EVP_CIPHER_key_length(cipher) != + session->p.cipher_key.length) return -1; - /* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->cipher.func = des_encrypt; - else - session->cipher.func = des_decrypt; + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); - /* Convert keys */ - DES_set_key((DES_cblock *)&session->p.cipher_key.data[0], - &session->cipher.data.des.ks1); - DES_set_key((DES_cblock *)&session->p.cipher_key.data[8], - &session->cipher.data.des.ks2); - DES_set_key((DES_cblock *)&session->p.cipher_key.data[16], - &session->cipher.data.des.ks3); + session->cipher.evp_cipher = cipher; - return 0; -} - -static int process_md5_param(odp_crypto_generic_session_t *session, - uint32_t bits) -{ /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->auth.func = md5_gen; + session->cipher.func = aes_gcm_encrypt; else - session->auth.func = md5_check; - - /* Number of valid bytes */ - session->auth.data.md5.bytes = bits / 8; - - /* Convert keys */ - memcpy(session->auth.data.md5.key, session->p.auth_key.data, 16); + session->cipher.func = aes_gcm_decrypt; return 0; } -static int process_sha256_param(odp_crypto_generic_session_t *session, - uint32_t bits) +static int process_auth_param(odp_crypto_generic_session_t *session, + uint32_t key_length, + const EVP_MD *evp_md) { /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->auth.func = sha256_gen; + session->auth.func = auth_gen; else - session->auth.func = sha256_check; + session->auth.func = auth_check; + + session->auth.evp_md = evp_md; /* Number of valid bytes */ - session->auth.data.sha256.bytes = bits / 8; + session->auth.bytes = session->p.auth_digest_len; + if (session->auth.bytes < (unsigned)EVP_MD_size(evp_md) / 2) + return -1; /* Convert keys */ - memcpy(session->auth.data.sha256.key, session->p.auth_key.data, 32); + session->auth.key_length = key_length; + memcpy(session->auth.key, session->p.auth_key.data, + session->auth.key_length); return 0; } @@ -639,15 +557,18 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->auths.bit.null = 1; capa->auths.bit.md5_hmac = 1; + capa->auths.bit.sha1_hmac = 1; capa->auths.bit.sha256_hmac = 1; + capa->auths.bit.sha512_hmac = 1; capa->auths.bit.aes_gcm = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API capa->ciphers.bit.aes128_cbc = 1; capa->ciphers.bit.aes128_gcm = 1; capa->auths.bit.md5_96 = 1; capa->auths.bit.sha256_128 = 1; capa->auths.bit.aes128_gcm = 1; +#endif capa->max_sessions = MAX_SESSIONS; @@ -664,8 +585,8 @@ int odp_crypto_cipher_capability(odp_cipher_alg_t cipher, switch (cipher) { case ODP_CIPHER_ALG_NULL: - src = NULL; - num = 0; + src = cipher_capa_null; + num = sizeof(cipher_capa_null) / size; break; case ODP_CIPHER_ALG_DES: src = cipher_capa_des; @@ -704,17 +625,25 @@ int odp_crypto_auth_capability(odp_auth_alg_t auth, switch (auth) { case ODP_AUTH_ALG_NULL: - src = NULL; - num = 0; + src = auth_capa_null; + num = sizeof(auth_capa_null) / size; break; case ODP_AUTH_ALG_MD5_HMAC: src = auth_capa_md5_hmac; num = sizeof(auth_capa_md5_hmac) / size; break; + case ODP_AUTH_ALG_SHA1_HMAC: + src = auth_capa_sha1_hmac; + num = sizeof(auth_capa_sha1_hmac) / size; + break; case ODP_AUTH_ALG_SHA256_HMAC: src = auth_capa_sha256_hmac; num = sizeof(auth_capa_sha256_hmac) / size; break; + case ODP_AUTH_ALG_SHA512_HMAC: + src = auth_capa_sha512_hmac; + num = sizeof(auth_capa_sha512_hmac) / size; + break; case ODP_AUTH_ALG_AES_GCM: src = auth_capa_aes_gcm; num = sizeof(auth_capa_aes_gcm) / size; @@ -738,6 +667,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, { int rc; odp_crypto_generic_session_t *session; + int aes_gcm = 0; /* Default to successful result */ *status = ODP_CRYPTO_SES_CREATE_ERR_NONE; @@ -752,16 +682,16 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, /* Copy parameters */ session->p = *param; - /* Copy IV data */ - if (session->p.iv.data) { - if (session->p.iv.length > MAX_IV_LEN) { - ODP_DBG("Maximum IV length exceeded\n"); - return -1; - } + if (session->p.iv.length > MAX_IV_LEN) { + ODP_DBG("Maximum IV length exceeded\n"); + free_session(session); + return -1; + } + /* Copy IV data */ + if (session->p.iv.data) memcpy(session->cipher.iv_data, session->p.iv.data, session->p.iv.length); - } /* Derive order */ if (ODP_CRYPTO_OP_ENCODE == param->op) @@ -777,21 +707,25 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, break; case ODP_CIPHER_ALG_DES: case ODP_CIPHER_ALG_3DES_CBC: - rc = process_des_param(session); + rc = process_cipher_param(session, EVP_des_ede3_cbc()); break; case ODP_CIPHER_ALG_AES_CBC: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_CIPHER_ALG_AES128_CBC: - rc = process_aes_param(session); +#endif + rc = process_cipher_param(session, EVP_aes_128_cbc()); break; - case ODP_CIPHER_ALG_AES_GCM: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_CIPHER_ALG_AES128_GCM: + if (param->auth_alg == ODP_AUTH_ALG_AES128_GCM) + aes_gcm = 1; + /* Fallthrough */ +#endif + case ODP_CIPHER_ALG_AES_GCM: /* AES-GCM requires to do both auth and * cipher at the same time */ - if (param->auth_alg == ODP_AUTH_ALG_AES_GCM || - param->auth_alg == ODP_AUTH_ALG_AES128_GCM) - rc = process_aes_gcm_param(session); + if (param->auth_alg == ODP_AUTH_ALG_AES_GCM || aes_gcm) + rc = process_aes_gcm_param(session, EVP_aes_128_gcm()); else rc = -1; break; @@ -802,32 +736,54 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + free_session(session); return -1; } + aes_gcm = 0; + /* Process based on auth */ switch (param->auth_alg) { case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; rc = 0; break; - case ODP_AUTH_ALG_MD5_HMAC: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_MD5_96: - rc = process_md5_param(session, 96); + /* Fixed digest tag length with deprecated algo */ + session->p.auth_digest_len = 96 / 8; + /* Fallthrough */ +#endif + case ODP_AUTH_ALG_MD5_HMAC: + rc = process_auth_param(session, 16, EVP_md5()); break; - case ODP_AUTH_ALG_SHA256_HMAC: - /* deprecated */ + case ODP_AUTH_ALG_SHA1_HMAC: + rc = process_auth_param(session, 20, EVP_sha1()); + break; +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_SHA256_128: - rc = process_sha256_param(session, 128); + /* Fixed digest tag length with deprecated algo */ + session->p.auth_digest_len = 128 / 8; + /* Fallthrough */ +#endif + case ODP_AUTH_ALG_SHA256_HMAC: + rc = process_auth_param(session, 32, EVP_sha256()); break; - case ODP_AUTH_ALG_AES_GCM: - /* deprecated */ + case ODP_AUTH_ALG_SHA512_HMAC: + rc = process_auth_param(session, 64, EVP_sha512()); + break; +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_AES128_GCM: + if (param->cipher_alg == ODP_CIPHER_ALG_AES128_GCM) + aes_gcm = 1; + /* Fixed digest tag length with deprecated algo */ + session->p.auth_digest_len = 16; + /* Fallthrough */ +#endif + case ODP_AUTH_ALG_AES_GCM: /* AES-GCM requires to do both auth and * cipher at the same time */ - if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || - param->cipher_alg == ODP_CIPHER_ALG_AES128_GCM) { + if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || aes_gcm) { session->auth.func = null_crypto_routine; rc = 0; } else { @@ -841,6 +797,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH; + free_session(session); return -1; } @@ -854,9 +811,6 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) odp_crypto_generic_session_t *generic; generic = (odp_crypto_generic_session_t *)(intptr_t)session; - if (generic->p.cipher_alg == ODP_CIPHER_ALG_AES128_GCM || - generic->p.cipher_alg == ODP_CIPHER_ALG_AES_GCM) - EVP_CIPHER_CTX_free(generic->cipher.data.aes_gcm.ctx); memset(generic, 0, sizeof(*generic)); free_session(generic); return 0; @@ -871,14 +825,17 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE; odp_crypto_generic_session_t *session; odp_crypto_op_result_t local_result; + odp_bool_t allocated = false; session = (odp_crypto_generic_session_t *)(intptr_t)param->session; /* Resolve output buffer */ if (ODP_PACKET_INVALID == param->out_pkt && - ODP_POOL_INVALID != session->p.output_pool) + ODP_POOL_INVALID != session->p.output_pool) { param->out_pkt = odp_packet_alloc(session->p.output_pool, odp_packet_len(param->pkt)); + allocated = true; + } if (odp_unlikely(ODP_PACKET_INVALID == param->out_pkt)) { ODP_DBG("Alloc failed.\n"); @@ -886,11 +843,16 @@ odp_crypto_operation(odp_crypto_op_param_t *param, } if (param->pkt != param->out_pkt) { - (void)odp_packet_copy_from_pkt(param->out_pkt, + int ret; + + ret = odp_packet_copy_from_pkt(param->out_pkt, 0, param->pkt, 0, odp_packet_len(param->pkt)); + if (odp_unlikely(ret < 0)) + goto err; + _odp_packet_copy_md_to_packet(param->pkt, param->out_pkt); odp_packet_free(param->pkt); param->pkt = ODP_PACKET_INVALID; @@ -932,7 +894,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param, op_result->result = local_result; if (odp_queue_enq(session->p.compl_queue, completion_event)) { odp_event_free(completion_event); - return -1; + goto err; } /* Indicate to caller operation was async */ @@ -940,13 +902,21 @@ odp_crypto_operation(odp_crypto_op_param_t *param, } else { /* Synchronous, simply return results */ if (!result) - return -1; + goto err; *result = local_result; /* Indicate to caller operation was sync */ *posted = 0; } return 0; + +err: + if (allocated) { + odp_packet_free(param->out_pkt); + param->out_pkt = ODP_PACKET_INVALID; + } + + return -1; } static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 17a51b069..eb66af2d3 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -24,6 +24,8 @@ /* Initial packet segment data length */ #define BASE_LEN CONFIG_PACKET_MAX_SEG_LEN +#include <odp/visibility_begin.h> + /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .data = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].data), @@ -43,6 +45,8 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { }; +#include <odp/visibility_end.h> + static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) { return (odp_packet_hdr_t *)(uintptr_t)pkt; @@ -216,16 +220,9 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, return addr; } -static inline void packet_parse_disable(odp_packet_hdr_t *pkt_hdr) -{ - pkt_hdr->p.input_flags.parsed_l2 = 1; - pkt_hdr->p.parsed_layers = LAYER_ALL; -} - void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) { /* Reset parser metadata before new parse */ - pkt_hdr->p.parsed_layers = LAYER_NONE; pkt_hdr->p.error_flags.all = 0; pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.output_flags.all = 0; @@ -237,8 +234,7 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) /** * Initialize packet */ -static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, - int parse) +static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) { uint32_t seg_len; int num = pkt_hdr->buf_hdr.segcount; @@ -253,7 +249,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, pkt_hdr->buf_hdr.seg[num - 1].len = seg_len; } - pkt_hdr->p.parsed_layers = LAYER_NONE; pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.output_flags.all = 0; pkt_hdr->p.error_flags.all = 0; @@ -262,10 +257,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - /* Disable lazy parsing on user allocated packets */ - if (!parse) - packet_parse_disable(pkt_hdr); - /* * Packet headroom is set from the pool's headroom * Packet tailroom is rounded up to fill the last @@ -481,7 +472,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, } static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, - int num_seg, odp_packet_t *pkt, int parse) + int num_seg, odp_packet_t *pkt) { int num_buf, i; int num = max_pkt; @@ -514,7 +505,7 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, pkt[i] = packet_handle(hdr); init_segments(&pkt_hdr[i * num_seg], num_seg); - packet_init(hdr, len, parse); + packet_init(hdr, len); } return num; @@ -527,7 +518,7 @@ int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, int num, num_seg; num_seg = num_segments(len); - num = packet_alloc(pool, len, max_num, num_seg, pkt, 1); + num = packet_alloc(pool, len, max_num, num_seg, pkt); return num; } @@ -547,7 +538,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) return ODP_PACKET_INVALID; num_seg = num_segments(len); - num = packet_alloc(pool, len, 1, num_seg, &pkt, 0); + num = packet_alloc(pool, len, 1, num_seg, &pkt); if (odp_unlikely(num == 0)) return ODP_PACKET_INVALID; @@ -570,7 +561,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, return -1; num_seg = num_segments(len); - num = packet_alloc(pool, len, max_num, num_seg, pkt, 0); + num = packet_alloc(pool, len, max_num, num_seg, pkt); return num; } @@ -590,11 +581,16 @@ void odp_packet_free(odp_packet_t pkt) void odp_packet_free_multi(const odp_packet_t pkt[], int num) { + odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; + int i; + if (CONFIG_PACKET_MAX_SEGS == 1) { - buffer_free_multi((const odp_buffer_t * const)pkt, num); + for (i = 0; i < num; i++) + buf[i] = buffer_handle(packet_hdr(pkt[i])); + + buffer_free_multi(buf, num); } else { - odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; - int i, j; + int j; int bufs = 0; for (i = 0; i < num; i++) { @@ -626,7 +622,7 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (len > pool->headroom + pool->data_size + pool->tailroom) return -1; - packet_init(pkt_hdr, len, 0); + packet_init(pkt_hdr, len); return 0; } @@ -795,7 +791,8 @@ static inline int move_data_to_head(odp_packet_hdr_t *pkt_hdr, int segs) free_len = BASE_LEN - len; - for (src_seg = dst_seg + 1; src_seg < segs; src_seg++) { + for (src_seg = dst_seg + 1; CONFIG_PACKET_MAX_SEGS > 1 && + src_seg < segs; src_seg++) { len = fill_seg_head(pkt_hdr, dst_seg, src_seg, free_len); moved += len; @@ -919,7 +916,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, pkt_hdr = new_hdr; *pkt = packet_handle(pkt_hdr); - } else if (free_segs) { + } else if (CONFIG_PACKET_MAX_SEGS > 1 && free_segs) { new_hdr = pkt_hdr->buf_hdr.seg[free_segs].hdr; packet_seg_copy_md(new_hdr, pkt_hdr); @@ -1237,8 +1234,6 @@ void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); return packet_map(pkt_hdr, pkt_hdr->p.l3_offset, len, NULL); } @@ -1246,8 +1241,6 @@ uint32_t odp_packet_l3_offset(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); return pkt_hdr->p.l3_offset; } @@ -1258,8 +1251,6 @@ int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) if (offset >= pkt_hdr->frame_len) return -1; - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); pkt_hdr->p.l3_offset = offset; return 0; } @@ -1268,8 +1259,6 @@ void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); return packet_map(pkt_hdr, pkt_hdr->p.l4_offset, len, NULL); } @@ -1277,8 +1266,6 @@ uint32_t odp_packet_l4_offset(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); return pkt_hdr->p.l4_offset; } @@ -1289,8 +1276,6 @@ int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) if (offset >= pkt_hdr->frame_len) return -1; - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); pkt_hdr->p.l4_offset = offset; return 0; } @@ -1655,6 +1640,54 @@ int odp_packet_move_data(odp_packet_t pkt, uint32_t dst_offset, pkt, src_offset, len); } +int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset, + uint8_t c, uint32_t len) +{ + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t setlen; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (offset + len > pkt_hdr->frame_len) + return -1; + + while (len > 0) { + mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + setlen = len > seglen ? seglen : len; + memset(mapaddr, c, setlen); + offset += setlen; + len -= setlen; + } + + return 0; +} + +int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, + const void *s, uint32_t len) +{ + const uint8_t *ptr = s; + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t cmplen; + int ret; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + ODP_ASSERT(offset + len <= pkt_hdr->frame_len); + + while (len > 0) { + mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + cmplen = len > seglen ? seglen : len; + ret = memcmp(mapaddr, ptr, cmplen); + if (ret != 0) + return ret; + offset += cmplen; + len -= cmplen; + ptr += cmplen; + } + + return 0; +} + /* * * Debugging @@ -1771,12 +1804,11 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); uint16_t frag_offset; uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); - - prs->l3_len = odp_be_to_cpu_16(ipv4->tot_len); + uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len); if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN) || odp_unlikely(ver != 4) || - (prs->l3_len > frame_len - *offset)) { + (l3_len > frame_len - *offset)) { prs->error_flags.ip_err = 1; return 0; } @@ -1813,13 +1845,12 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; const _odp_ipv6hdr_ext_t *ipv6ext; uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]); - - prs->l3_len = odp_be_to_cpu_16(ipv6->payload_len) + - _ODP_IPV6HDR_LEN; + uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) + + _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - prs->l3_len > frame_len - *offset) { + l3_len > frame_len - *offset) { prs->error_flags.ip_err = 1; return 0; } @@ -1880,9 +1911,6 @@ static inline void parse_tcp(packet_parser_t *prs, else if ((uint32_t)tcp->hl * 4 > sizeof(_odp_tcphdr_t)) prs->input_flags.tcpopt = 1; - prs->l4_len = prs->l3_len + - prs->l3_offset - prs->l4_offset; - if (offset) *offset += (uint32_t)tcp->hl * 4; *parseptr += (uint32_t)tcp->hl * 4; @@ -1897,13 +1925,8 @@ static inline void parse_udp(packet_parser_t *prs, const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; uint32_t udplen = odp_be_to_cpu_16(udp->length); - if (udplen < sizeof(_odp_udphdr_t) || - udplen > (prs->l3_len + - prs->l4_offset - prs->l3_offset)) { + if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) prs->error_flags.udp_err = 1; - } - - prs->l4_len = udplen; if (offset) *offset += sizeof(_odp_udphdr_t); @@ -1911,215 +1934,170 @@ static inline void parse_udp(packet_parser_t *prs, } /** - * Initialize L2 related parser flags and metadata - */ -void packet_parse_l2(packet_parser_t *prs, uint32_t frame_len) -{ - /* Packet alloc or reset have already init other offsets and flags */ - - /* We only support Ethernet for now */ - prs->input_flags.eth = 1; - - /* Detect jumbo frames */ - if (frame_len > _ODP_ETH_LEN_MAX) - prs->input_flags.jumbo = 1; - - /* Assume valid L2 header, no CRC/FCS check in SW */ - prs->input_flags.l2 = 1; - - prs->input_flags.parsed_l2 = 1; -} - -/** * Parse common packet headers up to given layer * * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be * available from the ptr. */ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, - uint32_t frame_len, uint32_t seg_len, layer_t layer) + uint32_t frame_len, uint32_t seg_len, + odp_pktio_parser_layer_t layer) { uint32_t offset; + uint16_t ethtype; const uint8_t *parseptr; + uint8_t ip_proto; + const _odp_ethhdr_t *eth; + uint16_t macaddr0, macaddr2, macaddr4; + const _odp_vlanhdr_t *vlan; - switch (prs->parsed_layers) { - case LAYER_NONE: - /* Fall through */ - - case LAYER_L2: - { - const _odp_ethhdr_t *eth; - uint16_t macaddr0, macaddr2, macaddr4; - const _odp_vlanhdr_t *vlan; - - offset = sizeof(_odp_ethhdr_t); - if (packet_parse_l2_not_done(prs)) - packet_parse_l2(prs, frame_len); - - eth = (const _odp_ethhdr_t *)ptr; - - /* Handle Ethernet broadcast/multicast addresses */ - macaddr0 = odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth)); - prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100; - - if (macaddr0 == 0xffff) { - macaddr2 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 1)); - macaddr4 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 2)); - prs->input_flags.eth_bcast = - (macaddr2 == 0xffff) && (macaddr4 == 0xffff); - } else { - prs->input_flags.eth_bcast = 0; - } + if (layer == ODP_PKTIO_PARSER_LAYER_NONE) + return 0; - /* Get Ethertype */ - prs->ethtype = odp_be_to_cpu_16(eth->type); - parseptr = (const uint8_t *)(eth + 1); + /* We only support Ethernet for now */ + prs->input_flags.eth = 1; + /* Assume valid L2 header, no CRC/FCS check in SW */ + prs->input_flags.l2 = 1; + /* Detect jumbo frames */ + if (frame_len > _ODP_ETH_LEN_MAX) + prs->input_flags.jumbo = 1; - /* Check for SNAP vs. DIX */ - if (prs->ethtype < _ODP_ETH_LEN_MAX) { - prs->input_flags.snap = 1; - if (prs->ethtype > frame_len - offset) { - prs->error_flags.snap_len = 1; - goto parse_exit; - } - prs->ethtype = odp_be_to_cpu_16(*((const uint16_t *) - (uintptr_t) - (parseptr + 6))); - offset += 8; - parseptr += 8; - } + offset = sizeof(_odp_ethhdr_t); + eth = (const _odp_ethhdr_t *)ptr; + + /* Handle Ethernet broadcast/multicast addresses */ + macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); + prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100; + + if (macaddr0 == 0xffff) { + macaddr2 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 1)); + macaddr4 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 2)); + prs->input_flags.eth_bcast = + (macaddr2 == 0xffff) && (macaddr4 == 0xffff); + } else { + prs->input_flags.eth_bcast = 0; + } - /* Parse the VLAN header(s), if present */ - if (prs->ethtype == _ODP_ETHTYPE_VLAN_OUTER) { - prs->input_flags.vlan_qinq = 1; - prs->input_flags.vlan = 1; + /* Get Ethertype */ + ethtype = odp_be_to_cpu_16(eth->type); + parseptr = (const uint8_t *)(eth + 1); - vlan = (const _odp_vlanhdr_t *)parseptr; - prs->ethtype = odp_be_to_cpu_16(vlan->type); - offset += sizeof(_odp_vlanhdr_t); - parseptr += sizeof(_odp_vlanhdr_t); + /* Check for SNAP vs. DIX */ + if (ethtype < _ODP_ETH_LEN_MAX) { + prs->input_flags.snap = 1; + if (ethtype > frame_len - offset) { + prs->error_flags.snap_len = 1; + goto parse_exit; } + ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) + (parseptr + 6))); + offset += 8; + parseptr += 8; + } - if (prs->ethtype == _ODP_ETHTYPE_VLAN) { - prs->input_flags.vlan = 1; - vlan = (const _odp_vlanhdr_t *)parseptr; - prs->ethtype = odp_be_to_cpu_16(vlan->type); - offset += sizeof(_odp_vlanhdr_t); - parseptr += sizeof(_odp_vlanhdr_t); - } + /* Parse the VLAN header(s), if present */ + if (ethtype == _ODP_ETHTYPE_VLAN_OUTER) { + prs->input_flags.vlan_qinq = 1; + prs->input_flags.vlan = 1; - prs->l3_offset = offset; - prs->parsed_layers = LAYER_L2; - if (layer == LAYER_L2) - return prs->error_flags.all != 0; + vlan = (const _odp_vlanhdr_t *)parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + offset += sizeof(_odp_vlanhdr_t); + parseptr += sizeof(_odp_vlanhdr_t); } - /* Fall through */ - case LAYER_L3: - { - offset = prs->l3_offset; - parseptr = (const uint8_t *)(ptr + offset); - /* Set l3_offset+flag only for known ethtypes */ - prs->input_flags.l3 = 1; - - /* Parse Layer 3 headers */ - switch (prs->ethtype) { - case _ODP_ETHTYPE_IPV4: - prs->input_flags.ipv4 = 1; - prs->ip_proto = parse_ipv4(prs, &parseptr, &offset, - frame_len); - break; + if (ethtype == _ODP_ETHTYPE_VLAN) { + prs->input_flags.vlan = 1; + vlan = (const _odp_vlanhdr_t *)parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + offset += sizeof(_odp_vlanhdr_t); + parseptr += sizeof(_odp_vlanhdr_t); + } - case _ODP_ETHTYPE_IPV6: - prs->input_flags.ipv6 = 1; - prs->ip_proto = parse_ipv6(prs, &parseptr, &offset, - frame_len, seg_len); - break; + if (layer == ODP_PKTIO_PARSER_LAYER_L2) + return prs->error_flags.all != 0; - case _ODP_ETHTYPE_ARP: - prs->input_flags.arp = 1; - prs->ip_proto = 255; /* Reserved invalid by IANA */ - break; + /* Set l3_offset+flag only for known ethtypes */ + prs->l3_offset = offset; + prs->input_flags.l3 = 1; - default: - prs->input_flags.l3 = 0; - prs->l3_offset = ODP_PACKET_OFFSET_INVALID; - prs->ip_proto = 255; /* Reserved invalid by IANA */ - } + /* Parse Layer 3 headers */ + switch (ethtype) { + case _ODP_ETHTYPE_IPV4: + prs->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len); + break; - /* Set l4_offset+flag only for known ip_proto */ - prs->l4_offset = offset; - prs->parsed_layers = LAYER_L3; - if (layer == LAYER_L3) - return prs->error_flags.all != 0; - } - /* Fall through */ + case _ODP_ETHTYPE_IPV6: + prs->input_flags.ipv6 = 1; + ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, + seg_len); + break; - case LAYER_L4: - { - offset = prs->l4_offset; - parseptr = (const uint8_t *)(ptr + offset); - prs->input_flags.l4 = 1; + case _ODP_ETHTYPE_ARP: + prs->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; - /* Parse Layer 4 headers */ - switch (prs->ip_proto) { - case _ODP_IPPROTO_ICMP: - prs->input_flags.icmp = 1; - break; + default: + prs->input_flags.l3 = 0; + prs->l3_offset = ODP_PACKET_OFFSET_INVALID; + ip_proto = 255; /* Reserved invalid by IANA */ + } - case _ODP_IPPROTO_TCP: - if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) - return -1; - prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, NULL); - break; + if (layer == ODP_PKTIO_PARSER_LAYER_L3) + return prs->error_flags.all != 0; - case _ODP_IPPROTO_UDP: - if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) - return -1; - prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, NULL); - break; + /* Set l4_offset+flag only for known ip_proto */ + prs->l4_offset = offset; + prs->input_flags.l4 = 1; - case _ODP_IPPROTO_AH: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_ah = 1; - break; + /* Parse Layer 4 headers */ + switch (ip_proto) { + case _ODP_IPPROTO_ICMPv4: + /* Fall through */ - case _ODP_IPPROTO_ESP: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_esp = 1; - break; + case _ODP_IPPROTO_ICMPv6: + prs->input_flags.icmp = 1; + break; - case _ODP_IPPROTO_SCTP: - prs->input_flags.sctp = 1; - break; + case _ODP_IPPROTO_TCP: + if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) + return -1; + prs->input_flags.tcp = 1; + parse_tcp(prs, &parseptr, NULL); + break; - default: - prs->input_flags.l4 = 0; - prs->l4_offset = ODP_PACKET_OFFSET_INVALID; - break; - } + case _ODP_IPPROTO_UDP: + if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) + return -1; + prs->input_flags.udp = 1; + parse_udp(prs, &parseptr, NULL); + break; - prs->parsed_layers = LAYER_L4; + case _ODP_IPPROTO_AH: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_ah = 1; + break; + + case _ODP_IPPROTO_ESP: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_esp = 1; break; - } - case LAYER_ALL: + case _ODP_IPPROTO_SCTP: + prs->input_flags.sctp = 1; break; default: - ODP_ERR("Invalid parse layer: %d\n", (int)layer); - return -1; + prs->input_flags.l4 = 0; + prs->l4_offset = ODP_PACKET_OFFSET_INVALID; + break; } - - prs->parsed_layers = LAYER_ALL; - parse_exit: return prs->error_flags.all != 0; } @@ -2127,7 +2105,8 @@ parse_exit: /** * Simple packet parser */ -int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer) +int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, + odp_pktio_parser_layer_t layer) { uint32_t seg_len = packet_first_seg_len(pkt_hdr); void *base = packet_data(pkt_hdr); diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index ea9a22710..72df1ecfe 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -8,27 +8,21 @@ #include <odp/api/packet_flags.h> #include <odp_packet_internal.h> -#define retflag(pkt, x, layer) do { \ +#define retflag(pkt, x) do { \ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \ - if (pkt_hdr->p.parsed_layers < layer) \ - packet_parse_layer(pkt_hdr, layer); \ return pkt_hdr->p.x; \ } while (0) -#define setflag(pkt, x, v, layer) do { \ +#define setflag(pkt, x, v) do { \ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \ - if (pkt_hdr->p.parsed_layers < layer) \ - packet_parse_layer(pkt_hdr, layer); \ - pkt_hdr->p.x = v & 1; \ + pkt_hdr->p.x = (v) & 1; \ } while (0) int odp_packet_has_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - return odp_packet_hdr(pkt)->p.error_flags.all != 0; + return pkt_hdr->p.error_flags.all != 0; } /* Get Input Flags */ @@ -45,126 +39,117 @@ int odp_packet_has_l2_error(odp_packet_t pkt) int odp_packet_has_l3(odp_packet_t pkt) { - retflag(pkt, input_flags.l3, LAYER_L3); + retflag(pkt, input_flags.l3); } int odp_packet_has_l3_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); - return pkt_hdr->p.error_flags.ip_err; } int odp_packet_has_l4(odp_packet_t pkt) { - retflag(pkt, input_flags.l4, LAYER_L4); + retflag(pkt, input_flags.l4); } int odp_packet_has_l4_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); - return pkt_hdr->p.error_flags.tcp_err | pkt_hdr->p.error_flags.udp_err; } int odp_packet_has_eth_bcast(odp_packet_t pkt) { - retflag(pkt, input_flags.eth_bcast, LAYER_L2); + retflag(pkt, input_flags.eth_bcast); } int odp_packet_has_eth_mcast(odp_packet_t pkt) { - retflag(pkt, input_flags.eth_mcast, LAYER_L2); + retflag(pkt, input_flags.eth_mcast); } int odp_packet_has_vlan(odp_packet_t pkt) { - retflag(pkt, input_flags.vlan, LAYER_L2); + retflag(pkt, input_flags.vlan); } int odp_packet_has_vlan_qinq(odp_packet_t pkt) { - retflag(pkt, input_flags.vlan_qinq, LAYER_L2); + retflag(pkt, input_flags.vlan_qinq); } int odp_packet_has_arp(odp_packet_t pkt) { - retflag(pkt, input_flags.arp, LAYER_L3); + retflag(pkt, input_flags.arp); } int odp_packet_has_ipv4(odp_packet_t pkt) { - retflag(pkt, input_flags.ipv4, LAYER_L3); + retflag(pkt, input_flags.ipv4); } int odp_packet_has_ipv6(odp_packet_t pkt) { - retflag(pkt, input_flags.ipv6, LAYER_L3); + retflag(pkt, input_flags.ipv6); } int odp_packet_has_ip_bcast(odp_packet_t pkt) { - retflag(pkt, input_flags.ip_bcast, LAYER_L3); + retflag(pkt, input_flags.ip_bcast); } int odp_packet_has_ip_mcast(odp_packet_t pkt) { - retflag(pkt, input_flags.ip_mcast, LAYER_L3); + retflag(pkt, input_flags.ip_mcast); } int odp_packet_has_ipfrag(odp_packet_t pkt) { - retflag(pkt, input_flags.ipfrag, LAYER_L3); + retflag(pkt, input_flags.ipfrag); } int odp_packet_has_ipopt(odp_packet_t pkt) { - retflag(pkt, input_flags.ipopt, LAYER_L3); + retflag(pkt, input_flags.ipopt); } int odp_packet_has_ipsec(odp_packet_t pkt) { - retflag(pkt, input_flags.ipsec, LAYER_L4); + retflag(pkt, input_flags.ipsec); } int odp_packet_has_udp(odp_packet_t pkt) { - retflag(pkt, input_flags.udp, LAYER_L4); + retflag(pkt, input_flags.udp); } int odp_packet_has_tcp(odp_packet_t pkt) { - retflag(pkt, input_flags.tcp, LAYER_L4); + retflag(pkt, input_flags.tcp); } int odp_packet_has_sctp(odp_packet_t pkt) { - retflag(pkt, input_flags.sctp, LAYER_L4); + retflag(pkt, input_flags.sctp); } int odp_packet_has_icmp(odp_packet_t pkt) { - retflag(pkt, input_flags.icmp, LAYER_L4); + retflag(pkt, input_flags.icmp); } odp_packet_color_t odp_packet_color(odp_packet_t pkt) { - retflag(pkt, input_flags.color, LAYER_ALL); + retflag(pkt, input_flags.color); } void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - pkt_hdr->p.input_flags.color = color; } @@ -172,29 +157,23 @@ odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - return !pkt_hdr->p.input_flags.nodrop; } void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop) { - setflag(pkt, input_flags.nodrop, !drop, LAYER_ALL); + setflag(pkt, input_flags.nodrop, !drop); } int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) { - retflag(pkt, output_flags.shaper_len_adj, LAYER_ALL); + retflag(pkt, output_flags.shaper_len_adj); } void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - pkt_hdr->p.output_flags.shaper_len_adj = adj; } @@ -202,107 +181,107 @@ void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) void odp_packet_has_l2_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l2, val, LAYER_L2); + setflag(pkt, input_flags.l2, val); } void odp_packet_has_l3_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l3, val, LAYER_L3); + setflag(pkt, input_flags.l3, val); } void odp_packet_has_l4_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l4, val, LAYER_L4); + setflag(pkt, input_flags.l4, val); } void odp_packet_has_eth_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth, val, LAYER_L2); + setflag(pkt, input_flags.eth, val); } void odp_packet_has_eth_bcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth_bcast, val, LAYER_L2); + setflag(pkt, input_flags.eth_bcast, val); } void odp_packet_has_eth_mcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth_mcast, val, LAYER_L2); + setflag(pkt, input_flags.eth_mcast, val); } void odp_packet_has_jumbo_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.jumbo, val, LAYER_L2); + setflag(pkt, input_flags.jumbo, val); } void odp_packet_has_vlan_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.vlan, val, LAYER_L2); + setflag(pkt, input_flags.vlan, val); } void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.vlan_qinq, val, LAYER_L2); + setflag(pkt, input_flags.vlan_qinq, val); } void odp_packet_has_arp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.arp, val, LAYER_L3); + setflag(pkt, input_flags.arp, val); } void odp_packet_has_ipv4_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipv4, val, LAYER_L3); + setflag(pkt, input_flags.ipv4, val); } void odp_packet_has_ipv6_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipv6, val, LAYER_L3); + setflag(pkt, input_flags.ipv6, val); } void odp_packet_has_ip_bcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ip_bcast, val, LAYER_L3); + setflag(pkt, input_flags.ip_bcast, val); } void odp_packet_has_ip_mcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ip_mcast, val, LAYER_L3); + setflag(pkt, input_flags.ip_mcast, val); } void odp_packet_has_ipfrag_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipfrag, val, LAYER_L3); + setflag(pkt, input_flags.ipfrag, val); } void odp_packet_has_ipopt_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipopt, val, LAYER_L3); + setflag(pkt, input_flags.ipopt, val); } void odp_packet_has_ipsec_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipsec, val, LAYER_L4); + setflag(pkt, input_flags.ipsec, val); } void odp_packet_has_udp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.udp, val, LAYER_L4); + setflag(pkt, input_flags.udp, val); } void odp_packet_has_tcp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.tcp, val, LAYER_L4); + setflag(pkt, input_flags.tcp, val); } void odp_packet_has_sctp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.sctp, val, LAYER_L4); + setflag(pkt, input_flags.sctp, val); } void odp_packet_has_icmp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.icmp, val, LAYER_L4); + setflag(pkt, input_flags.icmp, val); } void odp_packet_has_flow_hash_clr(odp_packet_t pkt) diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 98460a566..877978ba2 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -206,6 +206,8 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, memcpy(&pktio_entry->s.param, param, sizeof(odp_pktio_param_t)); pktio_entry->s.handle = hdl; + odp_pktio_config_init(&pktio_entry->s.config); + for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { ret = pktio_if_ops[pktio_if]->open(hdl, pktio_entry, name, pool); @@ -586,6 +588,10 @@ int pktout_enqueue(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr) int len = 1; int nbr; + if (sched_fn->ord_enq_multi(qentry->s.index, (void **)buf_hdr, len, + &nbr)) + return (nbr == len ? 0 : -1); + nbr = odp_pktout_send(qentry->s.pktout, &pkt, len); return (nbr == len ? 0 : -1); } @@ -603,6 +609,10 @@ int pktout_enq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int nbr; int i; + if (sched_fn->ord_enq_multi(qentry->s.index, (void **)buf_hdr, num, + &nbr)) + return nbr; + for (i = 0; i < num; ++i) pkt_tbl[i] = _odp_packet_from_buffer(buf_hdr[i]->handle.handle); @@ -923,6 +933,8 @@ void odp_pktout_queue_param_init(odp_pktout_queue_param_t *param) void odp_pktio_config_init(odp_pktio_config_t *config) { memset(config, 0, sizeof(odp_pktio_config_t)); + + config->parser.layer = ODP_PKTIO_PARSER_LAYER_ALL; } int odp_pktio_info(odp_pktio_t hdl, odp_pktio_info_t *info) @@ -1098,6 +1110,7 @@ int odp_pktio_term_global(void) int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) { pktio_entry_t *entry; + int ret; entry = get_pktio_entry(pktio); if (entry == NULL) { @@ -1106,9 +1119,15 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) } if (entry->s.ops->capability) - return entry->s.ops->capability(entry, capa); + ret = entry->s.ops->capability(entry, capa); + else + ret = single_capability(capa); - return single_capability(capa); + /* The same parser is used for all pktios */ + if (ret == 0) + capa->config.parser.layer = ODP_PKTIO_PARSER_LAYER_ALL; + + return ret; } unsigned odp_pktio_max_index(void) diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index cf7c2c415..9dba73413 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -445,7 +445,8 @@ static int check_params(odp_pool_param_t *params) { odp_pool_capability_t capa; - odp_pool_capability(&capa); + if (odp_pool_capability(&capa) < 0) + return -1; switch (params->type) { case ODP_POOL_BUFFER: @@ -768,6 +769,8 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) pool_t *pool; int ret; + ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + pool = pool_entry_from_hdl(pool_hdl); ret = buffer_alloc_multi(pool, &buf, NULL, 1); @@ -781,6 +784,8 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) { pool_t *pool; + ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + pool = pool_entry_from_hdl(pool_hdl); return buffer_alloc_multi(pool, buf, NULL, num); diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index d4267c72f..450613394 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -95,6 +95,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.dequeue_multi = queue_deq_multi; queue->s.pktin = PKTIN_INVALID; + queue->s.pktout = PKTOUT_INVALID; queue->s.head = NULL; queue->s.tail = NULL; @@ -175,6 +176,8 @@ int odp_queue_capability(odp_queue_capability_t *capa) capa->max_ordered_locks = sched_fn->max_ordered_locks(); capa->max_sched_groups = sched_fn->num_grps(); capa->sched_prios = odp_schedule_num_prio(); + capa->plain.max_num = capa->max_queues; + capa->sched.max_num = capa->max_queues; return 0; } diff --git a/platform/linux-generic/odp_rwlock.c b/platform/linux-generic/odp_rwlock.c index 13c17a2c7..5bef13a45 100644 --- a/platform/linux-generic/odp_rwlock.c +++ b/platform/linux-generic/odp_rwlock.c @@ -33,9 +33,14 @@ void odp_rwlock_read_lock(odp_rwlock_t *rwlock) int odp_rwlock_read_trylock(odp_rwlock_t *rwlock) { - uint32_t zero = 0; + uint32_t cnt = odp_atomic_load_u32(&rwlock->cnt); + + while (cnt != (uint32_t)-1) { + if (odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1)) + return 1; + } - return odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)1); + return 0; } void odp_rwlock_read_unlock(odp_rwlock_t *rwlock) diff --git a/platform/linux-generic/odp_schedule.c b/platform/linux-generic/odp_schedule.c index cd5bf21bd..c4567d810 100644 --- a/platform/linux-generic/odp_schedule.c +++ b/platform/linux-generic/odp_schedule.c @@ -20,6 +20,7 @@ #include <odp_config_internal.h> #include <odp_align_internal.h> #include <odp/api/sync.h> +#include <odp/api/packet_io.h> #include <odp_ring_internal.h> #include <odp_queue_internal.h> @@ -34,11 +35,18 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && "normal_prio_is_not_between_highest_and_lowest"); /* Number of scheduling groups */ -#define NUM_SCHED_GRPS 256 +#define NUM_SCHED_GRPS 32 /* Priority queues per priority */ #define QUEUES_PER_PRIO 4 +/* A thread polls a non preferred sched queue every this many polls + * of the prefer queue. */ +#define PREFER_RATIO 64 + +/* Size of poll weight table */ +#define WEIGHT_TBL_SIZE ((QUEUES_PER_PRIO - 1) * PREFER_RATIO) + /* Packet input poll cmd queues */ #define PKTIO_CMD_QUEUES 4 @@ -142,7 +150,6 @@ typedef struct { int index; int pause; uint16_t round; - uint16_t prefer_offset; uint16_t pktin_polls; uint32_t queue_index; odp_queue_t queue; @@ -157,6 +164,12 @@ typedef struct { ordered_stash_t stash[MAX_ORDERED_STASH]; } ordered; + uint32_t grp_epoch; + int num_grp; + uint8_t grp[NUM_SCHED_GRPS]; + uint8_t weight_tbl[WEIGHT_TBL_SIZE]; + uint8_t grp_weight[WEIGHT_TBL_SIZE]; + } sched_local_t; /* Priority queue */ @@ -191,7 +204,7 @@ typedef struct { pri_mask_t pri_mask[NUM_PRIO]; odp_spinlock_t mask_lock; - prio_queue_t prio_q[NUM_PRIO][QUEUES_PER_PRIO]; + prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][QUEUES_PER_PRIO]; odp_spinlock_t poll_cmd_lock; /* Number of commands in a command queue */ @@ -206,8 +219,10 @@ typedef struct { odp_shm_t shm; uint32_t pri_count[NUM_PRIO][QUEUES_PER_PRIO]; - odp_spinlock_t grp_lock; - odp_thrmask_t mask_all; + odp_thrmask_t mask_all; + odp_spinlock_t grp_lock; + odp_atomic_u32_t grp_epoch; + struct { char name[ODP_SCHED_GROUP_NAME_LEN]; odp_thrmask_t mask; @@ -215,6 +230,7 @@ typedef struct { } sched_grp[NUM_SCHED_GRPS]; struct { + int grp; int prio; int queue_per_prio; } queue[ODP_CONFIG_QUEUES]; @@ -237,17 +253,35 @@ static inline void schedule_release_context(void); static void sched_local_init(void) { + int i; + uint8_t id; + uint8_t offset = 0; + memset(&sched_local, 0, sizeof(sched_local_t)); sched_local.thr = odp_thread_id(); sched_local.queue = ODP_QUEUE_INVALID; sched_local.queue_index = PRIO_QUEUE_EMPTY; + + id = sched_local.thr & (QUEUES_PER_PRIO - 1); + + for (i = 0; i < WEIGHT_TBL_SIZE; i++) { + sched_local.weight_tbl[i] = id; + + if (i % PREFER_RATIO == 0) { + offset++; + sched_local.weight_tbl[i] = (id + offset) & + (QUEUES_PER_PRIO - 1); + if (offset == QUEUES_PER_PRIO - 1) + offset = 0; + } + } } static int schedule_init_global(void) { odp_shm_t shm; - int i, j; + int i, j, grp; ODP_DBG("Schedule init ... "); @@ -267,15 +301,20 @@ static int schedule_init_global(void) sched->shm = shm; odp_spinlock_init(&sched->mask_lock); - for (i = 0; i < NUM_PRIO; i++) { - for (j = 0; j < QUEUES_PER_PRIO; j++) { - int k; + for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { + for (i = 0; i < NUM_PRIO; i++) { + for (j = 0; j < QUEUES_PER_PRIO; j++) { + prio_queue_t *prio_q; + int k; - ring_init(&sched->prio_q[i][j].ring); + prio_q = &sched->prio_q[grp][i][j]; + ring_init(&prio_q->ring); - for (k = 0; k < PRIO_QUEUE_RING_SIZE; k++) - sched->prio_q[i][j].queue_index[k] = - PRIO_QUEUE_EMPTY; + for (k = 0; k < PRIO_QUEUE_RING_SIZE; k++) { + prio_q->queue_index[k] = + PRIO_QUEUE_EMPTY; + } + } } } @@ -291,12 +330,17 @@ static int schedule_init_global(void) sched->pktio_cmd[i].cmd_index = PKTIO_CMD_FREE; odp_spinlock_init(&sched->grp_lock); + odp_atomic_init_u32(&sched->grp_epoch, 0); for (i = 0; i < NUM_SCHED_GRPS; i++) { memset(sched->sched_grp[i].name, 0, ODP_SCHED_GROUP_NAME_LEN); odp_thrmask_zero(&sched->sched_grp[i].mask); } + sched->sched_grp[ODP_SCHED_GROUP_ALL].allocated = 1; + sched->sched_grp[ODP_SCHED_GROUP_WORKER].allocated = 1; + sched->sched_grp[ODP_SCHED_GROUP_CONTROL].allocated = 1; + odp_thrmask_setall(&sched->mask_all); ODP_DBG("done\n"); @@ -304,29 +348,38 @@ static int schedule_init_global(void) return 0; } +static inline void queue_destroy_finalize(uint32_t qi) +{ + sched_cb_queue_destroy_finalize(qi); +} + static int schedule_term_global(void) { int ret = 0; int rc = 0; - int i, j; + int i, j, grp; - for (i = 0; i < NUM_PRIO; i++) { - for (j = 0; j < QUEUES_PER_PRIO; j++) { - ring_t *ring = &sched->prio_q[i][j].ring; - uint32_t qi; + for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { + for (i = 0; i < NUM_PRIO; i++) { + for (j = 0; j < QUEUES_PER_PRIO; j++) { + ring_t *ring = &sched->prio_q[grp][i][j].ring; + uint32_t qi; - while ((qi = ring_deq(ring, PRIO_QUEUE_MASK)) != - RING_EMPTY) { - odp_event_t events[1]; - int num; + while ((qi = ring_deq(ring, PRIO_QUEUE_MASK)) != + RING_EMPTY) { + odp_event_t events[1]; + int num; - num = sched_cb_queue_deq_multi(qi, events, 1); + num = sched_cb_queue_deq_multi(qi, + events, + 1); - if (num < 0) - sched_cb_queue_destroy_finalize(qi); + if (num < 0) + queue_destroy_finalize(qi); - if (num > 0) - ODP_ERR("Queue not empty\n"); + if (num > 0) + ODP_ERR("Queue not empty\n"); + } } } } @@ -357,6 +410,40 @@ static int schedule_term_local(void) return 0; } +static inline void grp_update_mask(int grp, const odp_thrmask_t *new_mask) +{ + odp_thrmask_copy(&sched->sched_grp[grp].mask, new_mask); + odp_atomic_add_rel_u32(&sched->grp_epoch, 1); +} + +static inline int grp_update_tbl(void) +{ + int i; + int num = 0; + int thr = sched_local.thr; + + odp_spinlock_lock(&sched->grp_lock); + + for (i = 0; i < NUM_SCHED_GRPS; i++) { + if (sched->sched_grp[i].allocated == 0) + continue; + + if (odp_thrmask_isset(&sched->sched_grp[i].mask, thr)) { + sched_local.grp[num] = i; + num++; + } + } + + odp_spinlock_unlock(&sched->grp_lock); + + /* Update group weights. Round robin over all thread's groups. */ + for (i = 0; i < WEIGHT_TBL_SIZE; i++) + sched_local.grp_weight[i] = i % num; + + sched_local.num_grp = num; + return num; +} + static unsigned schedule_max_ordered_locks(void) { return MAX_ORDERED_LOCKS_PER_QUEUE; @@ -407,6 +494,7 @@ static int schedule_init_queue(uint32_t queue_index, int prio = sched_param->prio; pri_set_queue(queue_index, prio); + sched->queue[queue_index].grp = sched_param->group; sched->queue[queue_index].prio = prio; sched->queue[queue_index].queue_per_prio = queue_per_prio(queue_index); @@ -418,6 +506,7 @@ static void schedule_destroy_queue(uint32_t queue_index) int prio = sched->queue[queue_index].prio; pri_clr_queue(queue_index, prio); + sched->queue[queue_index].grp = 0; sched->queue[queue_index].prio = 0; sched->queue[queue_index].queue_per_prio = 0; } @@ -509,9 +598,10 @@ static void schedule_release_atomic(void) uint32_t qi = sched_local.queue_index; if (qi != PRIO_QUEUE_EMPTY && sched_local.num == 0) { - int prio = sched->queue[qi].prio; + int grp = sched->queue[qi].grp; + int prio = sched->queue[qi].prio; int queue_per_prio = sched->queue[qi].queue_per_prio; - ring_t *ring = &sched->prio_q[prio][queue_per_prio].ring; + ring_t *ring = &sched->prio_q[grp][prio][queue_per_prio].ring; /* Release current atomic queue */ ring_enq(ring, PRIO_QUEUE_MASK, qi); @@ -640,7 +730,9 @@ static int schedule_ord_enq_multi(uint32_t queue_index, void *buf_hdr[], return 0; } - if (odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { + /* Pktout may drop packets, so the operation cannot be stashed. */ + if (dst_queue->s.pktout.pktio != ODP_PKTIO_INVALID || + odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ wait_for_order(src_queue); @@ -662,54 +754,26 @@ static int schedule_ord_enq_multi(uint32_t queue_index, void *buf_hdr[], return 1; } -/* - * Schedule queues - */ -static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], - unsigned int max_num) +static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], + unsigned int max_num, int grp, int first) { int prio, i; int ret; int id; - int offset = 0; unsigned int max_deq = MAX_DEQ; uint32_t qi; - if (sched_local.num) { - ret = copy_events(out_ev, max_num); - - if (out_queue) - *out_queue = sched_local.queue; - - return ret; - } - - schedule_release_context(); - - if (odp_unlikely(sched_local.pause)) - return 0; - - /* Each thread prefers a priority queue. This offset avoids starvation - * of other priority queues on low thread counts. */ - if (odp_unlikely((sched_local.round & 0x3f) == 0)) { - offset = sched_local.prefer_offset; - sched_local.prefer_offset = (offset + 1) & - (QUEUES_PER_PRIO - 1); - } - - sched_local.round++; - /* Schedule events */ for (prio = 0; prio < NUM_PRIO; prio++) { if (sched->pri_mask[prio] == 0) continue; - id = (sched_local.thr + offset) & (QUEUES_PER_PRIO - 1); + /* Select the first ring based on weights */ + id = first; for (i = 0; i < QUEUES_PER_PRIO;) { int num; - int grp; int ordered; odp_queue_t handle; ring_t *ring; @@ -726,7 +790,7 @@ static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], } /* Get queue index from the priority queue */ - ring = &sched->prio_q[prio][id].ring; + ring = &sched->prio_q[grp][prio][id].ring; qi = ring_deq(ring, PRIO_QUEUE_MASK); /* Priority queue empty */ @@ -736,24 +800,10 @@ static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], continue; } - grp = sched_cb_queue_grp(qi); - - if (grp > ODP_SCHED_GROUP_ALL && - !odp_thrmask_isset(&sched->sched_grp[grp].mask, - sched_local.thr)) { - /* This thread is not eligible for work from - * this queue, so continue scheduling it. - */ - ring_enq(ring, PRIO_QUEUE_MASK, qi); - - i++; - id++; - continue; - } - /* Low priorities have smaller batch size to limit * head of line blocking latency. */ - if (odp_unlikely(prio > ODP_SCHED_PRIO_DEFAULT)) + if (odp_unlikely(MAX_DEQ > 1 && + prio > ODP_SCHED_PRIO_DEFAULT)) max_deq = MAX_DEQ / 2; ordered = sched_cb_queue_is_ordered(qi); @@ -818,6 +868,70 @@ static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], } } + return 0; +} + +/* + * Schedule queues + */ +static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], + unsigned int max_num) +{ + int i, num_grp; + int ret; + int id, first, grp_id; + uint16_t round; + uint32_t epoch; + + if (sched_local.num) { + ret = copy_events(out_ev, max_num); + + if (out_queue) + *out_queue = sched_local.queue; + + return ret; + } + + schedule_release_context(); + + if (odp_unlikely(sched_local.pause)) + return 0; + + /* Each thread prefers a priority queue. Poll weight table avoids + * starvation of other priority queues on low thread counts. */ + round = sched_local.round + 1; + + if (odp_unlikely(round == WEIGHT_TBL_SIZE)) + round = 0; + + sched_local.round = round; + first = sched_local.weight_tbl[round]; + + epoch = odp_atomic_load_acq_u32(&sched->grp_epoch); + num_grp = sched_local.num_grp; + + if (odp_unlikely(sched_local.grp_epoch != epoch)) { + num_grp = grp_update_tbl(); + sched_local.grp_epoch = epoch; + } + + grp_id = sched_local.grp_weight[round]; + + /* Schedule queues per group and priority */ + for (i = 0; i < num_grp; i++) { + int grp; + + grp = sched_local.grp[grp_id]; + ret = do_schedule_grp(out_queue, out_ev, max_num, grp, first); + + if (odp_likely(ret)) + return ret; + + grp_id++; + if (odp_unlikely(grp_id >= num_grp)) + grp_id = 0; + } + /* * Poll packet input when there are no events * * Each thread starts the search for a poll command from its @@ -1023,7 +1137,8 @@ static odp_schedule_group_t schedule_group_create(const char *name, ODP_SCHED_GROUP_NAME_LEN - 1); grp_name[ODP_SCHED_GROUP_NAME_LEN - 1] = 0; } - odp_thrmask_copy(&sched->sched_grp[i].mask, mask); + + grp_update_mask(i, mask); group = (odp_schedule_group_t)i; sched->sched_grp[i].allocated = 1; break; @@ -1036,13 +1151,16 @@ static odp_schedule_group_t schedule_group_create(const char *name, static int schedule_group_destroy(odp_schedule_group_t group) { + odp_thrmask_t zero; int ret; + odp_thrmask_zero(&zero); + odp_spinlock_lock(&sched->grp_lock); if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && sched->sched_grp[group].allocated) { - odp_thrmask_zero(&sched->sched_grp[group].mask); + grp_update_mask(group, &zero); memset(sched->sched_grp[group].name, 0, ODP_SCHED_GROUP_NAME_LEN); sched->sched_grp[group].allocated = 0; @@ -1082,9 +1200,11 @@ static int schedule_group_join(odp_schedule_group_t group, if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && sched->sched_grp[group].allocated) { - odp_thrmask_or(&sched->sched_grp[group].mask, - &sched->sched_grp[group].mask, - mask); + odp_thrmask_t new_mask; + + odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, mask); + grp_update_mask(group, &new_mask); + ret = 0; } else { ret = -1; @@ -1097,18 +1217,19 @@ static int schedule_group_join(odp_schedule_group_t group, static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t *mask) { + odp_thrmask_t new_mask; int ret; + odp_thrmask_xor(&new_mask, mask, &sched->mask_all); + odp_spinlock_lock(&sched->grp_lock); if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && sched->sched_grp[group].allocated) { - odp_thrmask_t leavemask; + odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, + &new_mask); + grp_update_mask(group, &new_mask); - odp_thrmask_xor(&leavemask, mask, &sched->mask_all); - odp_thrmask_and(&sched->sched_grp[group].mask, - &sched->sched_grp[group].mask, - &leavemask); ret = 0; } else { ret = -1; @@ -1159,12 +1280,19 @@ static int schedule_group_info(odp_schedule_group_t group, static int schedule_thr_add(odp_schedule_group_t group, int thr) { + odp_thrmask_t mask; + odp_thrmask_t new_mask; + if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; + odp_thrmask_zero(&mask); + odp_thrmask_set(&mask, thr); + odp_spinlock_lock(&sched->grp_lock); - odp_thrmask_set(&sched->sched_grp[group].mask, thr); + odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, &mask); + grp_update_mask(group, &new_mask); odp_spinlock_unlock(&sched->grp_lock); @@ -1173,12 +1301,20 @@ static int schedule_thr_add(odp_schedule_group_t group, int thr) static int schedule_thr_rem(odp_schedule_group_t group, int thr) { + odp_thrmask_t mask; + odp_thrmask_t new_mask; + if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; + odp_thrmask_zero(&mask); + odp_thrmask_set(&mask, thr); + odp_thrmask_xor(&new_mask, &mask, &sched->mask_all); + odp_spinlock_lock(&sched->grp_lock); - odp_thrmask_clr(&sched->sched_grp[group].mask, thr); + odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, &new_mask); + grp_update_mask(group, &new_mask); odp_spinlock_unlock(&sched->grp_lock); @@ -1192,9 +1328,10 @@ static void schedule_prefetch(int num ODP_UNUSED) static int schedule_sched_queue(uint32_t queue_index) { + int grp = sched->queue[queue_index].grp; int prio = sched->queue[queue_index].prio; int queue_per_prio = sched->queue[queue_index].queue_per_prio; - ring_t *ring = &sched->prio_q[prio][queue_per_prio].ring; + ring_t *ring = &sched->prio_q[grp][prio][queue_per_prio].ring; ring_enq(ring, PRIO_QUEUE_MASK, queue_index); return 0; diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index b69245788..75470aff8 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -21,6 +21,7 @@ #include <odp/api/hints.h> #include <odp/api/cpu.h> #include <odp/api/thrmask.h> +#include <odp/api/packet_io.h> #include <odp_config_internal.h> /* Number of priority levels */ @@ -1176,7 +1177,9 @@ static int schedule_ord_enq_multi(uint32_t queue_index, void *buf_hdr[], return 0; } - if (odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { + /* Pktout may drop packets, so the operation cannot be stashed. */ + if (dst_queue->s.pktout.pktio != ODP_PKTIO_INVALID || + odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ wait_for_order(src_queue); @@ -1457,7 +1460,7 @@ static inline int consume_queue(int prio, unsigned int queue_index) /* Low priorities have smaller batch size to limit * head of line blocking latency. */ - if (odp_unlikely(prio > ODP_SCHED_PRIO_DEFAULT)) + if (odp_unlikely(MAX_DEQ > 1 && prio > ODP_SCHED_PRIO_DEFAULT)) max = MAX_DEQ / 2; /* For ordered queues we want consecutive events to diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 18c61dbe7..40ffca078 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -14,6 +14,7 @@ #include <odp_posix_extensions.h> #include <odp/api/system_info.h> +#include <odp/api/version.h> #include <odp_internal.h> #include <odp_debug_internal.h> #include <odp/api/align.h> @@ -405,3 +406,32 @@ int odp_cpu_count(void) { return odp_global_data.system_info.cpu_count; } + +void odp_sys_info_print(void) +{ + int len; + int max_len = 512; + char str[max_len]; + + len = snprintf(str, max_len, "\n" + "ODP system info\n" + "---------------\n" + "ODP API version: %s\n" + "ODP impl name: %s\n" + "CPU model: %s\n" + "CPU freq (hz): %" PRIu64 "\n" + "Cache line size: %i\n" + "CPU count: %i\n" + "\n", + odp_version_api_str(), + odp_version_impl_name(), + odp_cpu_model_str(), + odp_cpu_hz_max(), + odp_sys_cache_line_size(), + odp_cpu_count()); + + str[len] = '\0'; + ODP_PRINT("%s", str); + + sys_info_print_arch(); +} diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c index 81e05224c..2bbe56664 100644 --- a/platform/linux-generic/odp_time.c +++ b/platform/linux-generic/odp_time.c @@ -10,36 +10,43 @@ #include <odp/api/time.h> #include <odp/api/hints.h> #include <odp_debug_internal.h> +#include <odp_time_internal.h> +#include <string.h> +#include <inttypes.h> -static odp_time_t start_time; +typedef struct time_global_t { + struct timespec spec_start; + int use_hw; + uint64_t hw_start; + uint64_t hw_freq_hz; +} time_global_t; -static inline -uint64_t time_to_ns(odp_time_t time) -{ - uint64_t ns; - - ns = time.tv_sec * ODP_TIME_SEC_IN_NS; - ns += time.tv_nsec; +static time_global_t global; - return ns; -} +/* + * Posix timespec based functions + */ -static inline odp_time_t time_diff(odp_time_t t2, odp_time_t t1) +static inline uint64_t time_spec_diff_nsec(struct timespec *t2, + struct timespec *t1) { - odp_time_t time; + struct timespec diff; + uint64_t nsec; - time.tv_sec = t2.tv_sec - t1.tv_sec; - time.tv_nsec = t2.tv_nsec - t1.tv_nsec; + diff.tv_sec = t2->tv_sec - t1->tv_sec; + diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - if (time.tv_nsec < 0) { - time.tv_nsec += ODP_TIME_SEC_IN_NS; - --time.tv_sec; + if (diff.tv_nsec < 0) { + diff.tv_nsec += ODP_TIME_SEC_IN_NS; + diff.tv_sec -= 1; } - return time; + nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; + + return nsec; } -static inline odp_time_t time_local(void) +static inline odp_time_t time_spec_cur(void) { int ret; odp_time_t time; @@ -49,82 +56,176 @@ static inline odp_time_t time_local(void) if (odp_unlikely(ret != 0)) ODP_ABORT("clock_gettime failed\n"); - time.tv_sec = sys_time.tv_sec; - time.tv_nsec = sys_time.tv_nsec; + time.nsec = time_spec_diff_nsec(&sys_time, &global.spec_start); - return time_diff(time, start_time); + return time; } -static inline int time_cmp(odp_time_t t2, odp_time_t t1) +static inline uint64_t time_spec_res(void) { - if (t2.tv_sec < t1.tv_sec) - return -1; + int ret; + struct timespec tres; - if (t2.tv_sec > t1.tv_sec) - return 1; + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + ODP_ABORT("clock_getres failed\n"); - return t2.tv_nsec - t1.tv_nsec; + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; } -static inline odp_time_t time_sum(odp_time_t t1, odp_time_t t2) +static inline uint64_t time_spec_to_ns(odp_time_t time) +{ + return time.nsec; +} + +static inline odp_time_t time_spec_from_ns(uint64_t ns) { odp_time_t time; - time.tv_sec = t2.tv_sec + t1.tv_sec; - time.tv_nsec = t2.tv_nsec + t1.tv_nsec; + time.nsec = ns; + + return time; +} + +/* + * HW time counter based functions + */ - if (time.tv_nsec >= (long)ODP_TIME_SEC_IN_NS) { - time.tv_nsec -= ODP_TIME_SEC_IN_NS; - ++time.tv_sec; +static inline odp_time_t time_hw_cur(void) +{ + odp_time_t time; + + time.count = cpu_global_time() - global.hw_start; + + return time; +} + +static inline uint64_t time_hw_res(void) +{ + /* Promise a bit lower resolution than average cycle counter + * frequency */ + return global.hw_freq_hz / 10; +} + +static inline uint64_t time_hw_to_ns(odp_time_t time) +{ + uint64_t nsec; + uint64_t freq_hz = global.hw_freq_hz; + uint64_t count = time.count; + uint64_t sec = 0; + + if (count >= freq_hz) { + sec = count / freq_hz; + count = count - sec * freq_hz; } + nsec = (ODP_TIME_SEC_IN_NS * count) / freq_hz; + + return (sec * ODP_TIME_SEC_IN_NS) + nsec; +} + +static inline odp_time_t time_hw_from_ns(uint64_t ns) +{ + odp_time_t time; + uint64_t count; + uint64_t freq_hz = global.hw_freq_hz; + uint64_t sec = 0; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } + + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + return time; } -static inline odp_time_t time_local_from_ns(uint64_t ns) +/* + * Common functions + */ + +static inline odp_time_t time_cur(void) +{ + if (global.use_hw) + return time_hw_cur(); + + return time_spec_cur(); +} + +static inline uint64_t time_res(void) +{ + if (global.use_hw) + return time_hw_res(); + + return time_spec_res(); +} + +static inline int time_cmp(odp_time_t t2, odp_time_t t1) +{ + if (odp_likely(t2.u64 > t1.u64)) + return 1; + + if (t2.u64 < t1.u64) + return -1; + + return 0; +} + +static inline odp_time_t time_sum(odp_time_t t1, odp_time_t t2) { odp_time_t time; - time.tv_sec = ns / ODP_TIME_SEC_IN_NS; - time.tv_nsec = ns - time.tv_sec * ODP_TIME_SEC_IN_NS; + time.u64 = t1.u64 + t2.u64; return time; } -static inline void time_wait_until(odp_time_t time) +static inline uint64_t time_to_ns(odp_time_t time) { - odp_time_t cur; + if (global.use_hw) + return time_hw_to_ns(time); - do { - cur = time_local(); - } while (time_cmp(time, cur) > 0); + return time_spec_to_ns(time); } -static inline uint64_t time_local_res(void) +static inline odp_time_t time_from_ns(uint64_t ns) { - int ret; - struct timespec tres; + if (global.use_hw) + return time_hw_from_ns(ns); - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_getres failed\n"); + return time_spec_from_ns(ns); +} - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; +static inline void time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = time_cur(); + } while (time_cmp(time, cur) > 0); } odp_time_t odp_time_local(void) { - return time_local(); + return time_cur(); } odp_time_t odp_time_global(void) { - return time_local(); + return time_cur(); } odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1) { - return time_diff(t2, t1); + odp_time_t time; + + time.u64 = t2.u64 - t1.u64; + + return time; } uint64_t odp_time_to_ns(odp_time_t time) @@ -134,12 +235,12 @@ uint64_t odp_time_to_ns(odp_time_t time) odp_time_t odp_time_local_from_ns(uint64_t ns) { - return time_local_from_ns(ns); + return time_from_ns(ns); } odp_time_t odp_time_global_from_ns(uint64_t ns) { - return time_local_from_ns(ns); + return time_from_ns(ns); } int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -154,18 +255,18 @@ odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) uint64_t odp_time_local_res(void) { - return time_local_res(); + return time_res(); } uint64_t odp_time_global_res(void) { - return time_local_res(); + return time_res(); } void odp_time_wait_ns(uint64_t ns) { - odp_time_t cur = time_local(); - odp_time_t wait = time_local_from_ns(ns); + odp_time_t cur = time_cur(); + odp_time_t wait = time_from_ns(ns); odp_time_t end_time = time_sum(cur, wait); time_wait_until(end_time); @@ -176,34 +277,31 @@ void odp_time_wait_until(odp_time_t time) return time_wait_until(time); } -uint64_t odp_time_to_u64(odp_time_t time) +int odp_time_init_global(void) { - int ret; - struct timespec tres; - uint64_t resolution; + int ret = 0; - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_getres failed\n"); + memset(&global, 0, sizeof(time_global_t)); - resolution = (uint64_t)tres.tv_nsec; + if (cpu_has_global_time()) { + global.use_hw = 1; + global.hw_freq_hz = cpu_global_time_freq(); - return time_to_ns(time) / resolution; -} + if (global.hw_freq_hz == 0) + return -1; -int odp_time_init_global(void) -{ - int ret; - struct timespec time; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time); - if (ret) { - start_time = ODP_TIME_NULL; - } else { - start_time.tv_sec = time.tv_sec; - start_time.tv_nsec = time.tv_nsec; + printf("HW time counter freq: %" PRIu64 " hz\n\n", + global.hw_freq_hz); + + global.hw_start = cpu_global_time(); + return 0; } + global.spec_start.tv_sec = 0; + global.spec_start.tv_nsec = 0; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &global.spec_start); + return ret; } diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 4e9358b96..a93b3ba90 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -3238,7 +3238,7 @@ static void tm_sched_params_cvt_to(odp_tm_sched_params_t *odp_sched_params, if (weight == 0) inv_weight = 0; else - inv_weight = 0x10000 / weight; + inv_weight = 0xFFFF / weight; tm_sched_params->sched_modes[priority] = sched_mode; tm_sched_params->inverted_weights[priority] = inv_weight; @@ -3254,7 +3254,7 @@ static void tm_sched_params_cvt_from(tm_sched_params_t *tm_sched_params, for (priority = 0; priority < ODP_TM_MAX_PRIORITIES; priority++) { sched_mode = tm_sched_params->sched_modes[priority]; inv_weight = tm_sched_params->inverted_weights[priority]; - weight = 0x10000 / inv_weight; + weight = 0xFFFF / inv_weight; odp_sched_params->sched_modes[priority] = sched_mode; odp_sched_params->sched_weights[priority] = weight; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 1922109a7..c52cd09d1 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -34,49 +34,6 @@ static int disable_pktio; /** !0 this pktio disabled, 0 enabled */ /* Has dpdk_pktio_init() been called */ static odp_bool_t dpdk_initialized; -#define PMD_EXT(drv) \ -extern void devinitfn_##drv(void) - -PMD_EXT(aesni_gcm_pmd_drv); -PMD_EXT(cryptodev_aesni_mb_pmd_drv); -PMD_EXT(cryptodev_kasumi_pmd_drv); -PMD_EXT(cryptodev_null_pmd_drv); -PMD_EXT(cryptodev_snow3g_pmd_drv); -PMD_EXT(pmd_qat_drv); -PMD_EXT(pmd_af_packet_drv); -PMD_EXT(rte_bnx2x_driver); -PMD_EXT(rte_bnx2xvf_driver); -PMD_EXT(bnxt_pmd_drv); -PMD_EXT(bond_drv); -PMD_EXT(rte_cxgbe_driver); -PMD_EXT(em_pmd_drv); -PMD_EXT(pmd_igb_drv); -PMD_EXT(pmd_igbvf_drv); -PMD_EXT(ena_pmd_drv); -PMD_EXT(rte_enic_driver); -PMD_EXT(rte_fm10k_driver); -PMD_EXT(rte_i40e_driver); -PMD_EXT(rte_i40evf_driver); -PMD_EXT(rte_ixgbe_driver); -PMD_EXT(rte_ixgbevf_driver); -PMD_EXT(rte_mlx4_driver); -PMD_EXT(rte_mlx5_driver); -PMD_EXT(pmd_mpipe_xgbe_drv); -PMD_EXT(pmd_mpipe_gbe_drv); -PMD_EXT(rte_nfp_net_driver); -PMD_EXT(pmd_null_drv); -PMD_EXT(pmd_pcap_drv); -PMD_EXT(rte_qede_driver); -PMD_EXT(rte_qedevf_driver); -PMD_EXT(pmd_ring_drv); -PMD_EXT(pmd_szedata2_drv); -PMD_EXT(rte_nicvf_driver); -PMD_EXT(pmd_vhost_drv); -PMD_EXT(rte_virtio_driver); -PMD_EXT(virtio_user_driver); -PMD_EXT(rte_vmxnet3_driver); -PMD_EXT(pmd_xenvirt_drv); - #define MEMPOOL_OPS(hdl) \ extern void mp_hdlr_init_##hdl(void) @@ -89,116 +46,11 @@ MEMPOOL_OPS(ops_stack); /* * This function is not called from anywhere, it's only purpose is to make sure * that if ODP and DPDK are statically linked to an application, the GCC - * constructors of the PMDs are linked as well. Otherwise the linker would omit - * them. It's not an issue with dynamic linking. */ + * constructors of mempool handlers are linked as well. Otherwise the linker + * would omit them. It's not an issue with dynamic linking. */ void refer_constructors(void); void refer_constructors(void) { -#ifdef RTE_LIBRTE_PMD_AESNI_GCM - devinitfn_aesni_gcm_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_AESNI_MB - devinitfn_cryptodev_aesni_mb_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_KASUMI - devinitfn_cryptodev_kasumi_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_NULL_CRYPTO - devinitfn_cryptodev_null_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_SNOW3G - devinitfn_cryptodev_snow3g_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_QAT - devinitfn_pmd_qat_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_AF_PACKET - devinitfn_pmd_af_packet_drv(); -#endif -#ifdef RTE_LIBRTE_BNX2X_PMD - devinitfn_rte_bnx2x_driver(); - devinitfn_rte_bnx2xvf_driver(); -#endif -#ifdef RTE_LIBRTE_BNXT_PMD - devinitfn_bnxt_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_BOND - devinitfn_bond_drv(); -#endif -#ifdef RTE_LIBRTE_CXGBE_PMD - devinitfn_rte_cxgbe_driver(); -#endif -#ifdef RTE_LIBRTE_EM_PMD - devinitfn_em_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_IGB_PMD - devinitfn_pmd_igb_drv(); - devinitfn_pmd_igbvf_drv(); -#endif -#ifdef RTE_LIBRTE_ENA_PMD - devinitfn_ena_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_ENIC_PMD - devinitfn_rte_enic_driver(); -#endif -#ifdef RTE_LIBRTE_FM10K_PMD - devinitfn_rte_fm10k_driver(); -#endif -#ifdef RTE_LIBRTE_I40E_PMD - devinitfn_rte_i40e_driver(); - devinitfn_rte_i40evf_driver(); -#endif -#ifdef RTE_LIBRTE_IXGBE_PMD - devinitfn_rte_ixgbe_driver(); - devinitfn_rte_ixgbevf_driver(); -#endif -#ifdef RTE_LIBRTE_MLX4_PMD - devinitfn_rte_mlx4_driver(); -#endif -#ifdef RTE_LIBRTE_MLX5_PMD - devinitfn_rte_mlx5_driver(); -#endif -#ifdef RTE_LIBRTE_MPIPE_PMD - devinitfn_pmd_mpipe_xgbe_drv() - devinitfn_pmd_mpipe_gbe_drv() -#endif -#ifdef RTE_LIBRTE_NFP_PMD - devinitfn_rte_nfp_net_driver(); -#endif -#ifdef RTE_LIBRTE_PMD_NULL - devinitfn_pmd_null_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_PCAP - devinitfn_pmd_pcap_drv(); -#endif -#ifdef RTE_LIBRTE_QEDE_PMD - devinitfn_rte_qede_driver(); - devinitfn_rte_qedevf_driver(); -#endif -#ifdef RTE_LIBRTE_PMD_RING - devinitfn_pmd_ring_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_SZEDATA2 - devinitfn_pmd_szedata2_drv(); -#endif -#ifdef RTE_LIBRTE_THUNDERX_NICVF_PMD - devinitfn_rte_nicvf_driver(); -#endif -#ifdef RTE_LIBRTE_PMD_VHOST - devinitfn_pmd_vhost_drv(); -#endif -#ifdef RTE_LIBRTE_VIRTIO_PMD - devinitfn_rte_virtio_driver(); -#endif -#ifdef RTE_VIRTIO_USER - devinitfn_rte_virtio_driver(); -#endif -#ifdef RTE_LIBRTE_VMXNET3_PMD - devinitfn_rte_vmxnet3_driver(); -#endif -#ifdef RTE_LIBRTE_PMD_XENVIRT - devinitfn_pmd_xenvirt_drv(); -#endif mp_hdlr_init_ops_mp_mc(); mp_hdlr_init_ops_sp_sc(); mp_hdlr_init_ops_mp_sc(); @@ -801,7 +653,8 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else - packet_parse_l2(&pkt_hdr->p, pkt_len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); if (mbuf->ol_flags & PKT_RX_RSS_HASH) odp_packet_flow_hash_set(pkt, mbuf->hash.rss); diff --git a/platform/linux-generic/pktio/ethtool.c b/platform/linux-generic/pktio/ethtool.c index 1b0f25b2c..d8f9e12cb 100644 --- a/platform/linux-generic/pktio/ethtool.c +++ b/platform/linux-generic/pktio/ethtool.c @@ -158,6 +158,7 @@ int ethtool_stats_get_fd(int fd, const char *name, odp_pktio_stats_t *stats) { struct ifreq ifr; + memset(&ifr, 0, sizeof(ifr)); snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); return ethtool_stats(fd, &ifr, stats); diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 70962839f..e9ad22ba2 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -132,7 +132,8 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else - packet_parse_l2(&pkt_hdr->p, pkt_len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); packet_set_ts(pkt_hdr, ts); @@ -176,7 +177,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pktio_entry->s.stats.out_octets += bytes; } else { ODP_DBG("queue enqueue failed %i\n", ret); - return -1; + ret = -1; } odp_ticketlock_unlock(&pktio_entry->s.txl); diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index ae3db34d9..928bb00af 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -663,7 +663,8 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else - packet_parse_l2(&pkt_hdr->p, len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); packet_set_ts(pkt_hdr, ts); } diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index e54a56f5f..a467b6402 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -252,7 +252,8 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, break; } - packet_parse_l2(&pkt_hdr->p, pkt_len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); pktio_entry->s.stats.in_octets += pkt_hdr->frame_len; packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/pktio/ring.c b/platform/linux-generic/pktio/ring.c index aeda04b26..e3c73d1cb 100644 --- a/platform/linux-generic/pktio/ring.c +++ b/platform/linux-generic/pktio/ring.c @@ -263,8 +263,8 @@ int ___ring_mp_do_enqueue(_ring_t *r, void * const *obj_table, /* Reset n to the initial burst count */ n = max; - prod_head = r->prod.head; - cons_tail = r->cons.tail; + prod_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED); + cons_tail = __atomic_load_n(&r->cons.tail, __ATOMIC_ACQUIRE); /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * prod_head > cons_tail). So 'free_entries' is always between 0 @@ -306,12 +306,12 @@ int ___ring_mp_do_enqueue(_ring_t *r, void * const *obj_table, * If there are other enqueues in progress that preceded us, * we need to wait for them to complete */ - while (odp_unlikely(r->prod.tail != prod_head)) + while (odp_unlikely(__atomic_load_n(&r->prod.tail, __ATOMIC_RELAXED) != + prod_head)) odp_cpu_pause(); /* Release our entries and the memory they refer to */ - __atomic_thread_fence(__ATOMIC_RELEASE); - r->prod.tail = prod_next; + __atomic_store_n(&r->prod.tail, prod_next, __ATOMIC_RELEASE); return ret; } @@ -328,7 +328,7 @@ int ___ring_sp_do_enqueue(_ring_t *r, void * const *obj_table, int ret; prod_head = r->prod.head; - cons_tail = r->cons.tail; + cons_tail = __atomic_load_n(&r->cons.tail, __ATOMIC_ACQUIRE); /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * prod_head > cons_tail). So 'free_entries' is always between 0 @@ -361,8 +361,7 @@ int ___ring_sp_do_enqueue(_ring_t *r, void * const *obj_table, } /* Release our entries and the memory they refer to */ - __atomic_thread_fence(__ATOMIC_RELEASE); - r->prod.tail = prod_next; + __atomic_store_n(&r->prod.tail, prod_next, __ATOMIC_RELEASE); return ret; } @@ -385,8 +384,8 @@ int ___ring_mc_do_dequeue(_ring_t *r, void **obj_table, /* Restore n as it may change every loop */ n = max; - cons_head = r->cons.head; - prod_tail = r->prod.tail; + cons_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED); + prod_tail = __atomic_load_n(&r->prod.tail, __ATOMIC_ACQUIRE); /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * cons_head > prod_tail). So 'entries' is always between 0 @@ -419,12 +418,12 @@ int ___ring_mc_do_dequeue(_ring_t *r, void **obj_table, * If there are other dequeues in progress that preceded us, * we need to wait for them to complete */ - while (odp_unlikely(r->cons.tail != cons_head)) + while (odp_unlikely(__atomic_load_n(&r->cons.tail, __ATOMIC_RELAXED) != + cons_head)) odp_cpu_pause(); /* Release our entries and the memory they refer to */ - __atomic_thread_fence(__ATOMIC_RELEASE); - r->cons.tail = cons_next; + __atomic_store_n(&r->cons.tail, cons_next, __ATOMIC_RELEASE); return behavior == _RING_QUEUE_FIXED ? 0 : n; } @@ -441,7 +440,7 @@ int ___ring_sc_do_dequeue(_ring_t *r, void **obj_table, uint32_t mask = r->prod.mask; cons_head = r->cons.head; - prod_tail = r->prod.tail; + prod_tail = __atomic_load_n(&r->prod.tail, __ATOMIC_ACQUIRE); /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * cons_head > prod_tail). So 'entries' is always between 0 @@ -461,11 +460,10 @@ int ___ring_sc_do_dequeue(_ring_t *r, void **obj_table, r->cons.head = cons_next; /* Acquire the pointers and the memory they refer to */ - __atomic_thread_fence(__ATOMIC_ACQUIRE); /* copy in table */ DEQUEUE_PTRS(); - r->cons.tail = cons_next; + __atomic_store_n(&r->cons.tail, cons_next, __ATOMIC_RELEASE); return behavior == _RING_QUEUE_FIXED ? 0 : n; } diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 7d2396866..a383adc6a 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -234,6 +234,7 @@ static inline int get_rss_hash_options(int fd, const char *name, struct ifreq ifr; struct ethtool_rxnfc rsscmd; + memset(&ifr, 0, sizeof(ifr)); memset(&rsscmd, 0, sizeof(rsscmd)); *options = 0; @@ -461,8 +462,6 @@ static int sock_close(pktio_entry_t *pktio_entry) return -1; } - odp_shm_free(pkt_sock->shm); - return 0; } @@ -474,13 +473,11 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, { int sockfd; int err; - int i; unsigned int if_idx; struct ifreq ethreq; struct sockaddr_ll sa_ll; char shm_name[ODP_SHM_NAME_LEN]; pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock; - uint8_t *addr; odp_pktio_stats_t cur_stats; /* Init pktio entry */ @@ -494,18 +491,6 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, snprintf(shm_name, ODP_SHM_NAME_LEN, "%s-%s", "pktio", netdev); shm_name[ODP_SHM_NAME_LEN - 1] = '\0'; - pkt_sock->shm = odp_shm_reserve(shm_name, PACKET_JUMBO_LEN, - PACKET_JUMBO_LEN * - ODP_PACKET_SOCKET_MAX_BURST_RX, 0); - if (pkt_sock->shm == ODP_SHM_INVALID) - return -1; - - addr = odp_shm_addr(pkt_sock->shm); - for (i = 0; i < ODP_PACKET_SOCKET_MAX_BURST_RX; i++) { - pkt_sock->cache_ptr[i] = addr; - addr += PACKET_JUMBO_LEN; - } - sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sockfd == -1) { __odp_errno = errno; @@ -617,19 +602,17 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_t pkt_table[], int len) { pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock; + odp_pool_t pool = pkt_sock->pool; odp_time_t ts_val; odp_time_t *ts = NULL; const int sockfd = pkt_sock->sockfd; - int msgvec_len; - struct mmsghdr msgvec[ODP_PACKET_SOCKET_MAX_BURST_RX]; + struct mmsghdr msgvec[len]; + struct iovec iovecs[len][MAX_SEGS]; int nb_rx = 0; + int nb_pkts; int recv_msgs; - uint8_t **recv_cache; int i; - if (odp_unlikely(len > ODP_PACKET_SOCKET_MAX_BURST_RX)) - return -1; - odp_ticketlock_lock(&pktio_entry->s.rxl); if (pktio_entry->s.config.pktin.bit.ts_all || @@ -637,131 +620,72 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ts = &ts_val; memset(msgvec, 0, sizeof(msgvec)); - recv_cache = pkt_sock->cache_ptr; - if (pktio_cls_enabled(pktio_entry)) { - struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX]; + nb_pkts = packet_alloc_multi(pool, pkt_sock->mtu, pkt_table, len); + for (i = 0; i < nb_pkts; i++) { + msgvec[i].msg_hdr.msg_iovlen = + _rx_pkt_to_iovec(pkt_table[i], iovecs[i]); + msgvec[i].msg_hdr.msg_iov = iovecs[i]; + } + + recv_msgs = recvmmsg(sockfd, msgvec, nb_pkts, MSG_DONTWAIT, NULL); - for (i = 0; i < (int)len; i++) { - msgvec[i].msg_hdr.msg_iovlen = 1; - iovecs[i].iov_base = recv_cache[i]; - iovecs[i].iov_len = PACKET_JUMBO_LEN; - msgvec[i].msg_hdr.msg_iov = &iovecs[i]; - } - msgvec_len = i; - - recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, - MSG_DONTWAIT, NULL); - - if (ts != NULL) - ts_val = odp_time_global(); - - for (i = 0; i < recv_msgs; i++) { - odp_packet_hdr_t *pkt_hdr; - odp_packet_t pkt; - odp_pool_t pool = pkt_sock->pool; - odp_packet_hdr_t parsed_hdr; - void *base = msgvec[i].msg_hdr.msg_iov->iov_base; - struct ethhdr *eth_hdr = base; - uint16_t pkt_len = msgvec[i].msg_len; - int num; - - /* Don't receive packets sent by ourselves */ - if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, - eth_hdr->h_source))) - continue; + if (ts != NULL) + ts_val = odp_time_global(); - if (cls_classify_packet(pktio_entry, base, pkt_len, - pkt_len, &pool, &parsed_hdr)) - continue; + for (i = 0; i < recv_msgs; i++) { + void *base = msgvec[i].msg_hdr.msg_iov->iov_base; + struct ethhdr *eth_hdr = base; + odp_packet_t pkt = pkt_table[i]; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint16_t pkt_len = msgvec[i].msg_len; + int ret; - num = packet_alloc_multi(pool, pkt_len, &pkt, 1); - if (num != 1) - continue; + if (pktio_cls_enabled(pktio_entry)) { + uint16_t seg_len = pkt_len; - pkt_hdr = odp_packet_hdr(pkt); + if (msgvec[i].msg_hdr.msg_iov->iov_len < pkt_len) + seg_len = msgvec[i].msg_hdr.msg_iov->iov_len; - if (odp_packet_copy_from_mem(pkt, 0, pkt_len, - base) != 0) { + if (cls_classify_packet(pktio_entry, base, pkt_len, + seg_len, &pool, pkt_hdr)) { + ODP_ERR("cls_classify_packet failed"); odp_packet_free(pkt); continue; } - pkt_hdr->input = pktio_entry->s.handle; - copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); - packet_set_ts(pkt_hdr, ts); - - pkt_table[nb_rx++] = pkt; } - } else { - struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX] - [MAX_SEGS]; - - for (i = 0; i < (int)len; i++) { - int num; - num = packet_alloc_multi(pkt_sock->pool, pkt_sock->mtu, - &pkt_table[i], 1); - - if (odp_unlikely(num != 1)) { - pkt_table[i] = ODP_PACKET_INVALID; - break; - } - - msgvec[i].msg_hdr.msg_iovlen = - _rx_pkt_to_iovec(pkt_table[i], iovecs[i]); - - msgvec[i].msg_hdr.msg_iov = iovecs[i]; + /* Don't receive packets sent by ourselves */ + if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, + eth_hdr->h_source))) { + odp_packet_free(pkt); + continue; } - /* number of successfully allocated pkt buffers */ - msgvec_len = i; - - recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, - MSG_DONTWAIT, NULL); - - if (ts != NULL) - ts_val = odp_time_global(); - - for (i = 0; i < recv_msgs; i++) { - void *base = msgvec[i].msg_hdr.msg_iov->iov_base; - struct ethhdr *eth_hdr = base; - odp_packet_hdr_t *pkt_hdr; - odp_packet_t pkt; - int ret; - - pkt = pkt_table[i]; - - /* Don't receive packets sent by ourselves */ - if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, - eth_hdr->h_source))) { - odp_packet_free(pkt); - continue; - } + ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, + NULL, NULL); + if (ret < 0) { + ODP_ERR("trunk_tail failed"); + odp_packet_free(pkt); + continue; + } - /* Parse and set packet header data */ - ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - - msgvec[i].msg_len, - NULL, NULL); - if (ret < 0) { - ODP_ERR("trunk_tail failed"); - odp_packet_free(pkt); - continue; - } + pkt_hdr->input = pktio_entry->s.handle; - pkt_hdr = odp_packet_hdr(pkt); - packet_parse_l2(&pkt_hdr->p, pkt_hdr->frame_len); - packet_set_ts(pkt_hdr, ts); - pkt_hdr->input = pktio_entry->s.handle; + if (!pktio_cls_enabled(pktio_entry)) + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); - pkt_table[nb_rx] = pkt; - nb_rx++; - } + pkt_hdr->input = pktio_entry->s.handle; + packet_set_ts(pkt_hdr, ts); - /* Free unused pkt buffers */ - for (; i < msgvec_len; i++) - odp_packet_free(pkt_table[i]); + pkt_table[nb_rx++] = pkt; } + /* Free unused pkt buffers */ + for (; i < nb_pkts; i++) + odp_packet_free(pkt_table[i]); + odp_ticketlock_unlock(&pktio_entry->s.rxl); return nb_rx; @@ -793,15 +717,12 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, const odp_packet_t pkt_table[], int len) { pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock; - struct mmsghdr msgvec[ODP_PACKET_SOCKET_MAX_BURST_TX]; - struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_TX][MAX_SEGS]; + struct mmsghdr msgvec[len]; + struct iovec iovecs[len][MAX_SEGS]; int ret; int sockfd; int n, i; - if (odp_unlikely(len > ODP_PACKET_SOCKET_MAX_BURST_TX)) - return -1; - odp_ticketlock_lock(&pktio_entry->s.txl); sockfd = pkt_sock->sockfd; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 666aae6af..2dba7b08f 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -231,7 +231,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, hdr); else - packet_parse_l2(&hdr->p, pkt_len); + packet_parse_layer(hdr, + pktio_entry->s.config.parser.layer); packet_set_ts(hdr, ts); @@ -454,11 +455,11 @@ static int mmap_sock(pkt_sock_mmap_t *pkt_sock) return 0; } -static void mmap_unmap_sock(pkt_sock_mmap_t *pkt_sock) +static int mmap_unmap_sock(pkt_sock_mmap_t *pkt_sock) { - munmap(pkt_sock->mmap_base, pkt_sock->mmap_len); free(pkt_sock->rx_ring.rd); free(pkt_sock->tx_ring.rd); + return munmap(pkt_sock->mmap_base, pkt_sock->mmap_len); } static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) @@ -486,8 +487,14 @@ static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) static int sock_mmap_close(pktio_entry_t *entry) { pkt_sock_mmap_t *const pkt_sock = &entry->s.pkt_sock_mmap; + int ret; + + ret = mmap_unmap_sock(pkt_sock); + if (ret != 0) { + ODP_ERR("mmap_unmap_sock() %s\n", strerror(errno)); + return -1; + } - mmap_unmap_sock(pkt_sock); if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { __odp_errno = errno; ODP_ERR("close(sockfd): %s\n", strerror(errno)); diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index ac2045606..650c12a77 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -213,7 +213,8 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else - packet_parse_l2(&pkt_hdr->p, len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); packet_set_ts(pkt_hdr, ts); pkt_hdr->input = pktio_entry->s.handle; |